From 80df46dbc87b4c20629a768e18517f86bafd6f56 Mon Sep 17 00:00:00 2001 From: vagarwal-viant Date: Thu, 10 Jul 2025 14:58:18 -0700 Subject: [PATCH 001/279] ENG-00000 remove trim of nongraphic characters --- gateway/router/marshal/json/marshaller_strings.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gateway/router/marshal/json/marshaller_strings.go b/gateway/router/marshal/json/marshaller_strings.go index bb2a8ebd8..322dd53a0 100644 --- a/gateway/router/marshal/json/marshaller_strings.go +++ b/gateway/router/marshal/json/marshaller_strings.go @@ -5,7 +5,6 @@ import ( "github.com/viant/tagly/format" "github.com/viant/xunsafe" "strings" - "unicode" "unsafe" ) @@ -51,9 +50,12 @@ func (i *stringMarshaller) ensureReplacer() { } func marshallString(asString string, sb *MarshallSession, replacer *strings.Replacer) { - asString = strings.TrimFunc(asString, func(r rune) bool { - return !unicode.IsGraphic(r) - }) + // This removes all /n characters at begining and end of log lines in CI_EVENT Table + /* + asString = strings.TrimFunc(asString, func(r rune) bool { + return !unicode.IsGraphic(r) + }) + */ sb.WriteByte('"') sb.WriteString(replacer.Replace(asString)) From 41fc97d1d1c6b5d97061f50d4d0b75708690ea25 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 22 Aug 2025 10:47:06 -0700 Subject: [PATCH 002/279] fixed nil pointer --- go.sum | 2 -- internal/translator/service.go | 14 +++++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/go.sum b/go.sum index 736b0b41e..5d442daa1 100644 --- a/go.sum +++ b/go.sum @@ -1141,8 +1141,6 @@ github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns= github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.16.6 h1:3/D1/c3E8cMaUWTUBW56Gg/1vW4QMMWm42HkSAbzSZQ= -github.com/viant/sqlx v0.16.6/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/sqlx v0.17.6 h1:6uMZVWk+WJl/y8coEh4F4mqbTHbtzWkLVEQdrk+m7sE= github.com/viant/sqlx v0.17.6/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.6.1 h1:Forza+RF/1tmlQFk9ABNhu+IQ8vMAqbYM6FOsYtGh9E= diff --git a/internal/translator/service.go b/internal/translator/service.go index 66c9ccabb..d53f4e6a3 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -4,6 +4,12 @@ import ( "context" "database/sql" "fmt" + "net/http" + spath "path" + "reflect" + "strings" + "time" + "github.com/viant/afs" "github.com/viant/afs/file" "github.com/viant/afs/url" @@ -27,11 +33,6 @@ import ( "github.com/viant/xreflect" "golang.org/x/mod/modfile" "gopkg.in/yaml.v3" - "net/http" - spath "path" - "reflect" - "strings" - "time" ) type Service struct { @@ -449,6 +450,9 @@ func (s *Service) adjustView(viewlet *Viewlet, resource *Resource, mode view.Mod } if viewlet.TypeDefinition != nil { if viewlet.TypeDefinition.Cardinality == state.Many { + if viewlet.View.View.Schema == nil { + viewlet.View.View.Schema = &state.Schema{} + } viewlet.View.View.Schema.Cardinality = viewlet.TypeDefinition.Cardinality } viewlet.TypeDefinition.Cardinality = "" From 69bf4ff272e6c68373a5216a3801187125648d1e Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 22 Aug 2025 12:15:22 -0700 Subject: [PATCH 003/279] fixed logger merge issue --- gateway/router.go | 9 +- gateway/router/handler.go | 9 +- repository/contract/dispatcher.go | 9 + .../component/dispatcher/disptacher.go | 1 + service/executor/handler/executor.go | 11 +- service/executor/handler/options.go | 8 + service/session/option.go | 7 + shared/logging/logger.go | 410 ++++++++++++++++++ 8 files changed, 459 insertions(+), 5 deletions(-) create mode 100644 shared/logging/logger.go diff --git a/gateway/router.go b/gateway/router.go index 5da821bd0..a63bf8a98 100644 --- a/gateway/router.go +++ b/gateway/router.go @@ -18,11 +18,14 @@ import ( "github.com/viant/datly/repository/path" "github.com/viant/datly/service/operator" "github.com/viant/datly/service/session" + "github.com/viant/datly/shared/logging" "github.com/viant/datly/view" vcontext "github.com/viant/datly/view/context" + "github.com/viant/datly/view/state/kind/locator" "github.com/viant/gmetric" serverproto "github.com/viant/mcp-protocol/server" "github.com/viant/xdatly/handler/async" + "github.com/viant/xdatly/handler/logger" hstate "github.com/viant/xdatly/handler/state" "net/http" @@ -38,6 +41,7 @@ type ( repository *repository.Service operator *operator.Service config *Config + logger logger.Logger OpenAPIInfo openapi3.Info metrics *gmetric.Service statusHandler http.Handler @@ -78,6 +82,7 @@ func NewRouter(ctx context.Context, components *repository.Service, config *Conf operator: operator.New(), apiKeyMatcher: newApiKeyMatcher(config.APIKeys), mcpRegistry: mcpRegistry, + logger: logging.New(logging.INFO, nil), } return r, r.init(ctx) } @@ -154,8 +159,10 @@ func (r *Router) HandleJob(ctx context.Context, aJob *async.Job) error { request := &http.Request{Method: aJob.Method, URL: URL, RequestURI: aPath.URI} unmarshal := aComponent.UnmarshalFunc(request) locatorOptions := append(aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal)) + locatorOptions = append(locatorOptions, locator.WithLogger(r.logger)) aSession := session.New(aComponent.View, session.WithAuth(r.repository.Auth()), + session.WithLogger(r.logger), session.WithComponent(aComponent), session.WithLocatorOptions(locatorOptions...), session.WithOperate(r.operator.Operate)) @@ -342,7 +349,7 @@ func (r *Router) newMatcher(ctx context.Context) (*matcher.Matcher, []*contract. } r.EnsureCors(aPath) - aRoute := r.NewRouteHandler(router.New(aPath, provider, r.repository.Registry(), r.repository.Auth(), r.config.Version, r.config.Logging)) + aRoute := r.NewRouteHandler(router.New(aPath, provider, r.repository.Registry(), r.repository.Auth(), r.config.Version, r.config.Logging, r.logger)) routes = append(routes, aRoute) if aPath.Cors != nil { optionsPaths[aPath.URI] = append(optionsPaths[aPath.URI], aPath) diff --git a/gateway/router/handler.go b/gateway/router/handler.go index b45917b65..ce7398a41 100644 --- a/gateway/router/handler.go +++ b/gateway/router/handler.go @@ -27,7 +27,9 @@ import ( "github.com/viant/datly/view" vcontext "github.com/viant/datly/view/context" "github.com/viant/datly/view/state" + "github.com/viant/datly/view/state/kind/locator" "github.com/viant/xdatly/handler/exec" + "github.com/viant/xdatly/handler/logger" "github.com/viant/xdatly/handler/response" hstate "github.com/viant/xdatly/handler/state" "io" @@ -54,6 +56,7 @@ type ( registry *repository.Registry auth *auth.Service logging logging.Config + logger logger.Logger } ) @@ -87,7 +90,7 @@ func (r *Handler) AuthorizeRequest(request *http.Request, aPath *path.Path) erro return nil } -func New(aPath *path.Path, provider *repository.Provider, registry *repository.Registry, authService *auth.Service, version string, config logging.Config) *Handler { +func New(aPath *path.Path, provider *repository.Provider, registry *repository.Registry, authService *auth.Service, version string, config logging.Config, logger logger.Logger) *Handler { ret := &Handler{ Path: aPath, Provider: provider, @@ -96,6 +99,7 @@ func New(aPath *path.Path, provider *repository.Provider, registry *repository.R auth: authService, Version: version, logging: config, + logger: logger, } return ret } @@ -390,11 +394,14 @@ func (r *Handler) handleComponent(ctx context.Context, request *http.Request, aC anOperator := operator.New() unmarshal := aComponent.UnmarshalFunc(request) locatorOptions := append(aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal)) + locatorOptions = append(locatorOptions, locator.WithLogger(r.logger)) aSession := session.New(aComponent.View, session.WithAuth(r.auth), + session.WithLogger(r.logger), session.WithComponent(aComponent), session.WithLocatorOptions(locatorOptions...), session.WithRegistry(r.registry), + session.WithOperate(anOperator.Operate)) err := aSession.InitKinds(state.KindComponent, state.KindHeader, state.KindRequestBody, state.KindForm, state.KindQuery) if err != nil { diff --git a/repository/contract/dispatcher.go b/repository/contract/dispatcher.go index 22afc3d26..6da3f9f72 100644 --- a/repository/contract/dispatcher.go +++ b/repository/contract/dispatcher.go @@ -2,6 +2,7 @@ package contract import ( "context" + "github.com/viant/xdatly/handler/logger" hstate "github.com/viant/xdatly/handler/state" "net/http" "net/url" @@ -16,6 +17,7 @@ type ( Header http.Header Form *hstate.Form Request *http.Request + Logger logger.Logger } //Option represents a dispatcher option Option func(o *Options) @@ -77,3 +79,10 @@ func WithRequest(request *http.Request) Option { o.Request = request } } + +// WithLogger adds path parameters +func WithLogger(loger logger.Logger) Option { + return func(o *Options) { + o.Logger = loger + } +} diff --git a/repository/locator/component/dispatcher/disptacher.go b/repository/locator/component/dispatcher/disptacher.go index 0ea2fbd3b..0c13a136f 100644 --- a/repository/locator/component/dispatcher/disptacher.go +++ b/repository/locator/component/dispatcher/disptacher.go @@ -47,6 +47,7 @@ func (d *Dispatcher) Dispatch(ctx context.Context, path *contract.Path, opts ... aSession := session.New(aComponent.View, session.WithLocatorOptions(options...), session.WithAuth(d.auth), session.WithRegistry(d.registry), + session.WithLogger(cOptions.Logger), session.WithComponent(aComponent), session.WithOperate(d.service.Operate)) ctx = aSession.Context(ctx, true) diff --git a/service/executor/handler/executor.go b/service/executor/handler/executor.go index 4d5e719ae..ce20877d0 100644 --- a/service/executor/handler/executor.go +++ b/service/executor/handler/executor.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "fmt" + "net/http" + "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" executor "github.com/viant/datly/service/executor" @@ -20,7 +22,6 @@ import ( "github.com/viant/xdatly/handler/sqlx" hstate "github.com/viant/xdatly/handler/state" "github.com/viant/xdatly/handler/validator" - "net/http" ) type ( @@ -126,6 +127,9 @@ func (e *Executor) newSession(aSession *session.Session, opts ...Option) *extens if options.auth != nil { e.auth = options.auth } + if e.logger == nil { + e.logger = options.logger + } res := e.view.GetResource() sess := extension.NewSession( extension.WithTemplateFlush(func(ctx context.Context) error { @@ -135,6 +139,7 @@ func (e *Executor) newSession(aSession *session.Session, opts ...Option) *extens extension.WithRedirect(e.redirect), extension.WithSql(e.newSqlService), extension.WithHttp(e.newHttp), + extension.WithLogger(e.logger), extension.WithAuth(e.newAuth), extension.WithMessageBus(res.MessageBuses), ) @@ -262,7 +267,6 @@ func (e *Executor) redirect(ctx context.Context, route *http2.Route, opts ...hst request.Header = originalRequest.Header } stateOptions := hstate.NewOptions(opts...) - unmarshal := aComponent.UnmarshalFunc(request) locatorOptions := append(aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal)) if stateOptions.Query() != nil { @@ -286,6 +290,7 @@ func (e *Executor) redirect(ctx context.Context, route *http2.Route, opts ...hst session.WithOperate(e.session.Options.Operate()), session.WithTypes(&aComponent.Contract.Input.Type, &aComponent.Contract.Output.Type), session.WithComponent(aComponent), + session.WithLogger(e.logger), session.WithRegistry(registry), ) @@ -295,7 +300,7 @@ func (e *Executor) redirect(ctx context.Context, route *http2.Route, opts ...hst } ctx = aSession.Context(ctx, true) anExecutor := NewExecutor(aComponent.View, aSession) - return anExecutor.NewHandlerSession(ctx) + return anExecutor.NewHandlerSession(ctx, WithLogger(aSession.Logger())) } func (e *Executor) newHttp() http2.Http { diff --git a/service/executor/handler/options.go b/service/executor/handler/options.go index 538e130df..e1221318b 100644 --- a/service/executor/handler/options.go +++ b/service/executor/handler/options.go @@ -4,6 +4,7 @@ import ( "embed" "github.com/viant/datly/service/auth" "github.com/viant/datly/view/state" + "github.com/viant/xdatly/handler/logger" ) type options struct { @@ -11,6 +12,7 @@ type options struct { embedFS *embed.FS opts []Option auth *auth.Service + logger logger.Logger } func (o *options) Clone(opts []Option) *options { @@ -37,6 +39,12 @@ func WithTypes(types ...*state.Type) Option { } } +func WithLogger(logger logger.Logger) Option { + return func(o *options) { + o.logger = logger + } +} + func WithAuth(auth *auth.Service) Option { return func(o *options) { o.auth = auth diff --git a/service/session/option.go b/service/session/option.go index 0fc7ea6a0..148242cb4 100644 --- a/service/session/option.go +++ b/service/session/option.go @@ -3,6 +3,7 @@ package session import ( "context" "embed" + "github.com/viant/datly/repository" "github.com/viant/datly/service/auth" "github.com/viant/datly/view" @@ -183,3 +184,9 @@ func WithRegistry(registry *repository.Registry) Option { s.registry = registry } } + +func WithLogger(logger logger.Logger) Option { + return func(s *Options) { + s.logger = logger + } +} diff --git a/shared/logging/logger.go b/shared/logging/logger.go new file mode 100644 index 000000000..aea30470f --- /dev/null +++ b/shared/logging/logger.go @@ -0,0 +1,410 @@ +package logging + +import ( + "context" + "encoding/json" + "fmt" + "github.com/aws/aws-lambda-go/events" + "github.com/viant/xdatly/handler/logger" + "io" + "log/slog" + "os" + regexp "regexp" + "runtime" + strings "strings" +) + +const ( + ReqId = "RequestId" + OpenTelemetryTraceId = "OpenTelemetryTraceId" + DEBUG = "DEBUG" + INFO = "INFO" + WARN = "WARN" + ERROR = "ERROR" + UNKNOWN = "UNKNOWN" // Indicate other environment +) + +type slogger struct { + logger *slog.Logger + level slog.Level +} + +// Init creates an ISLogger instance, a structured logger using the JSON Handler. +// Creating this logger sets this as the default logger, so any logging after this +// which goes through the standard logging package will also produce JSON structured +// logs. +func New(level string, dest io.Writer) logger.Logger { + if dest == nil { + dest = os.Stdout + } + + logLevel := slog.LevelInfo + switch strings.ToUpper(level) { + case DEBUG: + logLevel = slog.LevelDebug + case WARN: + logLevel = slog.LevelWarn + case ERROR: + logLevel = slog.LevelError + } + + handler := slog.NewJSONHandler(dest, &slog.HandlerOptions{ + AddSource: false, + Level: logLevel, + ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { + // Rename the time key to "timestamp" + if a.Key == slog.TimeKey { + a.Key = "timestamp" + } + return a + }, + }) + sl := slog.New(handler) + slog.SetDefault(sl) + logger := &slogger{sl, logLevel} + + return logger +} + +func (s *slogger) IsDebugEnabled() bool { + return s.level.Level() <= slog.LevelDebug +} + +func (s *slogger) IsInfoEnabled() bool { + return s.level.Level() <= slog.LevelInfo +} + +func (s *slogger) IsWarnEnabled() bool { + return s.level.Level() <= slog.LevelWarn +} + +func (s *slogger) IsErrorEnabled() bool { + return s.level.Level() <= slog.LevelError +} + +// getCallerInfo uses runtime to get the caller's program counter +// and extract info from the stack frame to get the function name, etc. +func (s *slogger) getCallerInfo() []any { + callers := make([]uintptr, 1) + count := runtime.Callers(3, callers[:]) // skip to actual caller + if count == 0 { + slog.Warn("getCallerInfo: no frames, exiting") + return nil + } + + frames := runtime.CallersFrames(callers) + var frame runtime.Frame + var more bool + for { + frame, more = frames.Next() + if !more { + break + } + } + + attr := []any{ + "function", frame.Function, "file", frame.File, "line", frame.Line, + } + + return attr +} + +// getContextValues retrieves "known" logging values from the Context. +// These values can be added to the Context using the provided utility functions. +func (s *slogger) getContextValues(ctx context.Context) []any { + var values []any + if ctx == nil { + slog.Warn("getContextValues: ctx is nil") + return nil + } + + openTelemetryTraceId := ctx.Value(OpenTelemetryTraceId) + if openTelemetryTraceId != nil { + values = append(values, "OpenTelemetryTraceId", openTelemetryTraceId) + } + return values +} + +// Info wraps a call to slog.Info, inserting details for the calling function. +func (s *slogger) Info(msg string, args ...any) { + if !s.IsInfoEnabled() { + return + } + caller := s.getCallerInfo() + caller = append(caller, args...) + s.logger.Info(msg, caller...) +} + +// Debug wraps a call to slog.Debug, inserting details for the calling function. +func (s *slogger) Debug(msg string, args ...any) { + if !s.IsDebugEnabled() { + return + } + caller := s.getCallerInfo() + caller = append(caller, args...) + s.logger.Debug(msg, caller...) +} + +// Warn wraps a call to slog.Warn, inserting details for the calling function. +func (s *slogger) Warn(msg string, args ...any) { + if !s.IsWarnEnabled() { + return + } + caller := s.getCallerInfo() + caller = append(caller, args...) + s.logger.Warn(msg, caller...) +} + +// Error wraps a call to slog.Error, inserting details for the calling function. +func (s *slogger) Error(msg string, args ...any) { + if !s.IsErrorEnabled() { + return + } + caller := s.getCallerInfo() + caller = append(caller, args...) + s.logger.Error(msg, caller...) +} + +// Infoc wraps a call to slog.Info, inserting details for the calling function, +// and retrieving known values from the context object. +func (s *slogger) Infoc(ctx context.Context, msg string, args ...any) { + if !s.IsInfoEnabled() { + return + } + caller := s.getCallerInfo() + values := s.getContextValues(ctx) + caller = append(caller, values...) + caller = append(caller, args...) + s.logger.Info(msg, caller...) +} + +func (s *slogger) Infos(ctx context.Context, msg string, attrs ...slog.Attr) { + if !s.IsInfoEnabled() { + return + } + caller := s.getCallerInfo() + values := s.getContextValues(ctx) + caller = append(caller, values...) + caller = append(caller, redactAttrs(attrs...)...) + + s.logger.Info(msg, caller...) +} + +// Debugc wraps a call to slog.Debug, inserting details for the calling function, +// and retrieving known values from the context object. +func (s *slogger) Debugc(ctx context.Context, msg string, args ...any) { + if !s.IsDebugEnabled() { + return + } + caller := s.getCallerInfo() + values := s.getContextValues(ctx) + caller = append(caller, values...) + caller = append(caller, args...) + s.logger.Debug(msg, caller...) +} + +func (s *slogger) Debugs(ctx context.Context, msg string, attrs ...slog.Attr) { + if !s.IsDebugEnabled() { + return + } + caller := s.getCallerInfo() + values := s.getContextValues(ctx) + caller = append(caller, values...) + caller = append(caller, redactAttrs(attrs...)...) + + s.logger.Debug(msg, caller...) +} + +// DebugJSONc wraps a call to slog.Debug, inserting details for the calling function, +// and retrieving known values from the context object. +func (s *slogger) DebugJSONc(ctx context.Context, msg string, obj any) { + caller := s.getCallerInfo() + values := s.getContextValues(ctx) + caller = append(caller, values...) + + // Initialize request and jsonData variables + var request events.APIGatewayProxyRequest + var jsonData []byte + // Marshal the object to JSON string + jsonString, _ := json.Marshal(obj) + // Unmarshal JSON string to APIGatewayProxyRequest + err := json.Unmarshal(jsonString, &request) + if err != nil { + return + } + + // Check if the request has an HTTP method + if len(request.HTTPMethod) > 0 { + if request.MultiValueHeaders == nil { + request.MultiValueHeaders = make(map[string][]string) + } + // Remove Authorization header + request.MultiValueHeaders["Authorization"] = nil + request.Headers["Authorization"] = "" + // Marshal the modified request to JSON + jsonData, _ = json.Marshal(request) + } else { + jsonData = jsonString + } + msg = fmt.Sprintf("%s %s", msg, string(jsonData)) + s.Debugc(ctx, msg, caller...) +} + +// Warnc wraps a call to slog.Warn, inserting details for the calling function, +// and retrieving known values from the context object. +func (s *slogger) Warnc(ctx context.Context, msg string, args ...any) { + if !s.IsWarnEnabled() { + return + } + caller := s.getCallerInfo() + values := s.getContextValues(ctx) + caller = append(caller, values...) + caller = append(caller, args...) + s.logger.Warn(msg, caller...) +} + +func (s *slogger) Warns(ctx context.Context, msg string, attrs ...slog.Attr) { + if !s.IsWarnEnabled() { + return + } + caller := s.getCallerInfo() + values := s.getContextValues(ctx) + caller = append(caller, values...) + caller = append(caller, redactAttrs(attrs...)...) + + s.logger.Warn(msg, caller...) +} + +// Errorc wraps a call to slog.Error, inserting details for the calling function, +// and retrieving known values from the context object. +func (s *slogger) Errorc(ctx context.Context, msg string, args ...any) { + if !s.IsErrorEnabled() { + return + } + caller := s.getCallerInfo() + values := s.getContextValues(ctx) + caller = append(caller, values...) + caller = append(caller, args...) + s.logger.Error(msg, caller...) +} + +func (s *slogger) Errors(ctx context.Context, msg string, attrs ...slog.Attr) { + if !s.IsErrorEnabled() { + return + } + caller := s.getCallerInfo() + values := s.getContextValues(ctx) + caller = append(caller, values...) + caller = append(caller, redactAttrs(attrs...)...) + + s.logger.Error(msg, caller...) +} + +// Helper to get platform from environment suffix +func getPlatformFromEnv(environment string) string { + switch { + case strings.Contains(environment, "dev"): + return "development" + case strings.Contains(environment, "stage"): + return "stage" + case strings.Contains(environment, "prod"): + return "production" + default: + return UNKNOWN + } +} + +// redactAttrs applies redaction rules to slog.Attr list. +// Skip redactValue for primitive types to avoid unnecessary processing +// This avoids redundant type switch/marshalling cost in high-volume logging +func redactAttrs(attrs ...slog.Attr) []any { + var result []any + for _, attr := range attrs { + if isSensitiveKey(attr.Key) { + result = append(result, slog.String(attr.Key, "[REDACTED]")) + continue + } + val := attr.Value.Any() + switch val.(type) { + case int, int64, float64, bool, nil: + result = append(result, attr) + default: + redactedValue := slog.AnyValue(redactValue(val)) + result = append(result, slog.Attr{Key: attr.Key, Value: redactedValue}) + } + } + return result +} + +// redactValue recursively redacts sensitive info in maps, slices, or structs. +func redactValue(value any) any { + switch v := value.(type) { + case string: + // Redact sensitive information in strings + return redactSensitiveInfo(v) + case int, int64, float64, bool, nil: + // Return primitive values directly (skip JSON marshalling) + return v + case map[string]any: + // Redact value if key is sensitive (e.g., Authorization → [REDACTED]) + // Ensures map fields are redacted even if value doesn’t match regex + for key, val := range v { + if isSensitiveKey(key) { + v[key] = "[REDACTED]" + } else { + v[key] = redactValue(val) + } + } + return v + case []any: + // Recursively redact sensitive information in slices + for i, val := range v { + v[i] = redactValue(val) + } + return v + default: + // Only marshal/unmarshal if absolutely needed (structs, unknown). + jsonData, err := json.Marshal(v) // Converts struct to map to enable nested field redaction. + if err != nil { + return v // If marshal fails, skip redaction + } + var unmarshaled any + if err := json.Unmarshal(jsonData, &unmarshaled); err != nil { + return v // If unmarshal fails, skip redaction + } + return redactValue(unmarshaled) + } +} + +// redactSensitiveInfo redacts known patterns in a string (e.g., tokens in URLs). +func redactSensitiveInfo(value string) string { + sensitivePatterns := []*regexp.Regexp{ + // Redact key=value style + regexp.MustCompile(`(?i)(X-Amz-Security-Token|X-Amz-Signature|X-Amz-Credential|Authorization|password|token|apiKey)=([^&\s]+)`), + // Redact key: value or key value + regexp.MustCompile(`(?i)(Authorization|password|token|apiKey)[\s:=]+([^&\s]+)`), + // Redact URL with user:pass@host + regexp.MustCompile(`(?i)https?://[^/]+:[^@]+@`), + } + + redacted := value + for _, pattern := range sensitivePatterns { + redacted = pattern.ReplaceAllString(redacted, "$1=[REDACTED]") + } + return redacted +} + +// isSensitiveKey returns true if the key is known to contain sensitive data. +func isSensitiveKey(key string) bool { + sensitiveKeys := []string{ + "authorization", "token", "apikey", "password", + "credential", "secret", "access_key", "secret_key", + } + key = strings.ToLower(key) + for _, sk := range sensitiveKeys { + if key == sk { + return true + } + } + return false +} From ac2a99fa866b311096e368e932578311584417ee Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 22 Aug 2025 13:15:35 -0700 Subject: [PATCH 004/279] patched slicelen issue --- service/executor/expand/data_unit.go | 2 +- service/executor/expand/state.go | 1 + service/reader/service.go | 10 ++++++---- view/state/kind/locator/repeated.go | 7 ++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/service/executor/expand/data_unit.go b/service/executor/expand/data_unit.go index 66a694dad..ac7d62452 100644 --- a/service/executor/expand/data_unit.go +++ b/service/executor/expand/data_unit.go @@ -48,7 +48,7 @@ func (c *DataUnit) Reset() { c.mu.Lock() c.placeholderCounter = 0 if len(c.ParamsGroup) > 0 { - c.ParamsGroup = c.ParamsGroup[:0] + clear(c.ParamsGroup) } c.TemplateSQL = "" c.mu.Unlock() diff --git a/service/executor/expand/state.go b/service/executor/expand/state.go index f970ff679..a69767fdb 100644 --- a/service/executor/expand/state.go +++ b/service/executor/expand/state.go @@ -2,6 +2,7 @@ package expand import ( "context" + "github.com/viant/datly/service/executor/extension" "github.com/viant/datly/view/state/predicate" diff --git a/service/reader/service.go b/service/reader/service.go index 29f1fea7d..63ca09436 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -4,6 +4,11 @@ import ( "context" "database/sql" "fmt" + "reflect" + "sync" + "time" + "unsafe" + "github.com/google/uuid" "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/shared" @@ -19,10 +24,6 @@ import ( "github.com/viant/xdatly/handler" "github.com/viant/xdatly/handler/exec" "github.com/viant/xdatly/handler/response" - "reflect" - "sync" - "time" - "unsafe" ) // Service represents reader service @@ -183,6 +184,7 @@ func (s *Service) readAll(ctx context.Context, session *Session, collector *view } return } + // if onRelationalConcurrency > 1 , then only we call it concurrently concurrencyLimit := make(chan struct{}, onRelationerConcurrency) var onRelationWaitGroup sync.WaitGroup diff --git a/view/state/kind/locator/repeated.go b/view/state/kind/locator/repeated.go index d40972b4f..1d07fde50 100644 --- a/view/state/kind/locator/repeated.go +++ b/view/state/kind/locator/repeated.go @@ -3,12 +3,13 @@ package locator import ( "context" "fmt" - "github.com/viant/datly/view/state" - "github.com/viant/datly/view/state/kind" - "github.com/viant/xunsafe" "reflect" "sync" "sync/atomic" + + "github.com/viant/datly/view/state" + "github.com/viant/datly/view/state/kind" + "github.com/viant/xunsafe" ) type Repeated struct { From 4d4d7045c3161138b107afc522ce44a40426533d Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 22 Aug 2025 13:32:27 -0700 Subject: [PATCH 005/279] patched logger nil --- repository/locator/component/component.go | 26 ++++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/repository/locator/component/component.go b/repository/locator/component/component.go index 1db480628..9fd8f6dea 100644 --- a/repository/locator/component/component.go +++ b/repository/locator/component/component.go @@ -3,27 +3,31 @@ package component import ( "context" "fmt" + "net/http" + "net/url" + "reflect" + "github.com/viant/datly/repository/contract" "github.com/viant/datly/shared" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind" "github.com/viant/datly/view/state/kind/locator" + "github.com/viant/xdatly/handler/logger" "github.com/viant/xdatly/handler/response" hstate "github.com/viant/xdatly/handler/state" "github.com/viant/xunsafe" - "net/http" - "net/url" - "reflect" ) type componentLocator struct { - custom []interface{} - dispatch contract.Dispatcher - constants map[string]interface{} - path map[string]string - form *hstate.Form - query url.Values - header http.Header + custom []interface{} + dispatch contract.Dispatcher + constants map[string]interface{} + path map[string]string + form *hstate.Form + query url.Values + header http.Header + logger logger.Logger + getRequest func() (*http.Request, error) } @@ -43,6 +47,7 @@ func (l *componentLocator) Value(ctx context.Context, name string) (interface{}, contract.WithPath(l.path), contract.WithQuery(l.query), contract.WithForm(form), + contract.WithLogger(l.logger), contract.WithHeader(l.header), ) err = updateErrWithResponseStatus(err, value) @@ -102,6 +107,7 @@ func newComponentLocator(opts ...locator.Option) (kind.Locator, error) { dispatch: options.Dispatcher, constants: options.Constants, getRequest: options.GetRequest, + logger: options.Logger, form: options.Form, query: options.Query, header: options.Header, From 9939b7c7f330d4845fd5b58164cec3b8aab0c5d6 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 22 Aug 2025 13:42:32 -0700 Subject: [PATCH 006/279] patched logger nil --- service/operator/executor.go | 5 ++++- service/reader/handler/handler.go | 10 +++++++--- service/session/state.go | 1 + service/session/stater.go | 8 +++++--- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/service/operator/executor.go b/service/operator/executor.go index 94ba9c1b6..53915c0ba 100644 --- a/service/operator/executor.go +++ b/service/operator/executor.go @@ -3,12 +3,13 @@ package operator import ( "context" "fmt" + "time" + "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" "github.com/viant/datly/service/executor/handler" "github.com/viant/gmetric/counter" xhandler "github.com/viant/xdatly/handler" - "time" "github.com/viant/datly/service/session" "github.com/viant/datly/view/state/kind/locator" @@ -59,6 +60,8 @@ func (s *Service) execute(ctx context.Context, aComponent *repository.Component, status := contract.StatusSuccess(executorSession.TemplateState) if err := aSession.SetState(ctx, aComponent.Output.Type.Parameters, responseState, aSession.Indirect(true, locator.WithCustom(&status), + locator.WithLogger(aSession.Logger()), + locator.WithState(statelet.Template))); err != nil { return nil, fmt.Errorf("failed to set response %w", err) } diff --git a/service/reader/handler/handler.go b/service/reader/handler/handler.go index 83d47d11e..c3a485dc1 100644 --- a/service/reader/handler/handler.go +++ b/service/reader/handler/handler.go @@ -2,6 +2,7 @@ package handler import ( "context" + goJson "github.com/goccy/go-json" "github.com/viant/datly/gateway/router/status" _ "github.com/viant/datly/repository/locator/async" @@ -10,6 +11,9 @@ import ( _ "github.com/viant/datly/repository/locator/output" _ "github.com/viant/datly/service/executor/handler/locator" + "net/http" + "reflect" + reader "github.com/viant/datly/service/reader" "github.com/viant/datly/service/session" "github.com/viant/datly/utils/httputils" @@ -18,8 +22,6 @@ import ( "github.com/viant/datly/view/state/kind/locator" "github.com/viant/structology" "github.com/viant/xdatly/handler/response" - "net/http" - "reflect" ) type ( @@ -65,7 +67,9 @@ func (h *Handler) Handle(ctx context.Context, aView *view.View, aSession *sessio resultState := h.output.NewState() statelet := aSession.State().Lookup(aView) - var locatorOptions []locator.Option + var locatorOptions = []locator.Option{ + locator.WithLogger(aSession.Logger()), + } locatorOptions = append(locatorOptions, locator.WithParameterLookup(func(ctx context.Context, parameter *state.Parameter) (interface{}, bool, error) { return aSession.LookupValue(ctx, parameter, aSession.Indirect(true, locatorOptions...)) }), diff --git a/service/session/state.go b/service/session/state.go index 62a75f07f..7fcfb5c70 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -168,6 +168,7 @@ func (s *Session) viewLookupOptions(aView *view.View, parameters state.NamedPara if !opts.HasInputParameters() { result = append(result, locator.WithInputParameters(parameters)) } + result = append(result, locator.WithLogger(s.logger)) result = append(result, locator.WithReadInto(s.ReadInto)) viewState := s.state.Lookup(aView) result = append(result, locator.WithState(viewState.Template)) diff --git a/service/session/stater.go b/service/session/stater.go index 2c75bcbe9..ed50be0b3 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -2,11 +2,12 @@ package session import ( "context" + "reflect" + "github.com/viant/datly/utils/types" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind/locator" hstate "github.com/viant/xdatly/handler/state" - "reflect" ) func (s *Session) ValuesOf(ctx context.Context, any interface{}) (map[string]interface{}, error) { @@ -53,8 +54,9 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt hOptions := hstate.NewOptions(opts...) aState := stateType.Type().WithValue(dest) - var stateOptions []locator.Option - + var stateOptions = []locator.Option{ + locator.WithLogger(s.logger), + } var locatorsToRemove = []state.Kind{state.KindComponent} if hOptions.Constants() != nil { stateOptions = append(stateOptions, locator.WithConstants(hOptions.Constants())) From c91f5c207e9f218bdbbc6eb992458648a081bcb6 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 22 Aug 2025 13:57:38 -0700 Subject: [PATCH 007/279] patched logger nil --- service/operator/executor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/service/operator/executor.go b/service/operator/executor.go index 53915c0ba..94d7fab94 100644 --- a/service/operator/executor.go +++ b/service/operator/executor.go @@ -26,6 +26,7 @@ func (s *Service) execute(ctx context.Context, aComponent *repository.Component, if aComponent.Handler != nil { aSession.SetView(aComponent.View) sessionHandler, err := anExecutor.NewHandlerSession(ctx, + handler.WithLogger(aSession.Logger()), handler.WithTypes(aComponent.Types()...), handler.WithAuth(aSession.Auth())) if err != nil { return nil, err From c0e2ca2aac83b07f7591e20f8bd98333e22425dd Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 25 Aug 2025 09:58:50 -0700 Subject: [PATCH 008/279] added race condition safeguard --- service/executor/expand/data_unit.go | 20 +++++++++++++------- view/predicate.go | 12 ++++++++---- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/service/executor/expand/data_unit.go b/service/executor/expand/data_unit.go index ac7d62452..8af4668aa 100644 --- a/service/executor/expand/data_unit.go +++ b/service/executor/expand/data_unit.go @@ -17,18 +17,18 @@ import ( type ( DataUnit struct { - Columns codec.ColumnsSource - ParamsGroup []interface{} - Mock bool - TemplateSQL string - MetaSource Dber `velty:"-"` - Statements *Statements `velty:"-"` - + Columns codec.ColumnsSource + ParamsGroup []interface{} + Mock bool + TemplateSQL string + MetaSource Dber `velty:"-"` + Statements *Statements `velty:"-"` mu sync.Mutex `velty:"-"` placeholderCounter int `velty:"-"` sqlxValidator *validator.Service `velty:"-"` sliceIndex map[reflect.Type]*xunsafe.Slice `velty:"-"` ctx context.Context `velty:"-"` + EvalLock sync.Mutex } ExecutablesIndex map[string]*Executable @@ -187,6 +187,12 @@ func (c *DataUnit) addAll(args ...interface{}) { c.mu.Unlock() } +func (c *DataUnit) Shrink(offset int) { + c.mu.Lock() + c.ParamsGroup = c.ParamsGroup[:offset] + c.mu.Unlock() +} + func (c *DataUnit) IsServiceExec(SQL string) (*Executable, bool) { return c.Statements.LookupExecutable(SQL) } diff --git a/view/predicate.go b/view/predicate.go index 69eb438f4..b7193fa94 100644 --- a/view/predicate.go +++ b/view/predicate.go @@ -3,6 +3,10 @@ package view import ( "context" "fmt" + "reflect" + "strings" + "sync" + expand "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/utils/types" "github.com/viant/datly/view/extension" @@ -12,9 +16,6 @@ import ( "github.com/viant/xdatly/predicate" "github.com/viant/xreflect" "github.com/viant/xunsafe" - "reflect" - "strings" - "sync" ) type ( @@ -50,6 +51,9 @@ func (e *PredicateEvaluator) Compute(ctx context.Context, value interface{}) (*c panic("not found custom ctx") } + cuxtomCtx.DataUnit.EvalLock.Lock() + defer cuxtomCtx.DataUnit.EvalLock.Unlock() + val := ctx.Value(expand.PredicateState) aState := val.(*structology.State) offset := len(cuxtomCtx.DataUnit.ParamsGroup) @@ -64,7 +68,7 @@ func (e *PredicateEvaluator) Compute(ctx context.Context, value interface{}) (*c copy(values, evaluate.DataUnit.ParamsGroup[offset:]) } criteria := &codec.Criteria{Expression: evaluate.Buffer.String(), Placeholders: values} - cuxtomCtx.DataUnit.ParamsGroup = cuxtomCtx.DataUnit.ParamsGroup[:offset] + cuxtomCtx.DataUnit.Shrink(offset) return criteria, nil } From b8977dac36b85458e39b74f3021553bab884921f Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 25 Aug 2025 10:00:27 -0700 Subject: [PATCH 009/279] added race condition safeguard --- view/template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/view/template.go b/view/template.go index c0b523b6c..92bed575c 100644 --- a/view/template.go +++ b/view/template.go @@ -372,7 +372,7 @@ func (t *Template) Expand(placeholders *[]interface{}, SQL string, selector *Sta if value.Key == "?" { placeholder, err := sanitized.Next() if err != nil { - return "", fmt.Errorf("failed to get placeholder: %w, SQL: %v, values: %v\n", err, SQL, values) + return "", fmt.Errorf("failed to get placeholder: %w, SQL: %v, values: %+v\n", err, SQL, values) } *placeholders = append(*placeholders, placeholder) From abfa80bfa68181b49b15ea73a23470ef14c87b1d Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 25 Aug 2025 17:09:30 -0700 Subject: [PATCH 010/279] added race condition safeguard --- service/executor/expand/data_unit.go | 11 ----------- service/executor/expand/state.go | 2 -- view/template.go | 1 - 3 files changed, 14 deletions(-) diff --git a/service/executor/expand/data_unit.go b/service/executor/expand/data_unit.go index 8af4668aa..89fea3f55 100644 --- a/service/executor/expand/data_unit.go +++ b/service/executor/expand/data_unit.go @@ -43,17 +43,6 @@ func (c *DataUnit) WithLocation(loc string) interface{} { return opt } -// Reset clears binding-related state so DataUnit can be safely reused for a new evaluation -func (c *DataUnit) Reset() { - c.mu.Lock() - c.placeholderCounter = 0 - if len(c.ParamsGroup) > 0 { - clear(c.ParamsGroup) - } - c.TemplateSQL = "" - c.mu.Unlock() -} - func (c *DataUnit) Validate(dest interface{}, opts ...interface{}) (*validator.Validation, error) { db, err := c.MetaSource.Db() if err != nil { diff --git a/service/executor/expand/state.go b/service/executor/expand/state.go index a69767fdb..1a6fffeb9 100644 --- a/service/executor/expand/state.go +++ b/service/executor/expand/state.go @@ -104,8 +104,6 @@ func (s *State) Init(templateState *est.State, predicates []*PredicateConfig, op if s.DataUnit == nil { s.DataUnit = NewDataUnit(nil) } - // Ensure bindings/cursor are reset for a fresh evaluation cycle - s.DataUnit.Reset() if s.Http == nil { s.Http = &Http{} diff --git a/view/template.go b/view/template.go index 92bed575c..e98ff37a2 100644 --- a/view/template.go +++ b/view/template.go @@ -373,7 +373,6 @@ func (t *Template) Expand(placeholders *[]interface{}, SQL string, selector *Sta placeholder, err := sanitized.Next() if err != nil { return "", fmt.Errorf("failed to get placeholder: %w, SQL: %v, values: %+v\n", err, SQL, values) - } *placeholders = append(*placeholders, placeholder) continue From 3b2abc8ba085dcb7373dc792466cb898d2e8f8a8 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 26 Aug 2025 05:30:39 -0700 Subject: [PATCH 011/279] updated error handling --- go.mod | 2 +- go.sum | 2 ++ service/operator/reader.go | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 89bb2f2dd..c1a5bb6c7 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 - github.com/viant/sqlx v0.17.6 + github.com/viant/sqlx v0.17.7 github.com/viant/structql v0.5.2 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 diff --git a/go.sum b/go.sum index 5d442daa1..ccfd3ed70 100644 --- a/go.sum +++ b/go.sum @@ -1143,6 +1143,8 @@ github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.17.6 h1:6uMZVWk+WJl/y8coEh4F4mqbTHbtzWkLVEQdrk+m7sE= github.com/viant/sqlx v0.17.6/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= +github.com/viant/sqlx v0.17.7 h1:drUv3N8mOboq917gnmcT9zC4G9vj4jU11bO/SsLpmc8= +github.com/viant/sqlx v0.17.7/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.6.1 h1:Forza+RF/1tmlQFk9ABNhu+IQ8vMAqbYM6FOsYtGh9E= github.com/viant/structology v0.6.1/go.mod h1:63XfkzUyNw7wdi99HJIsH2Rg3d5AOumqbWLUYytOkxU= github.com/viant/structql v0.5.2 h1:0dAratszxC6AD/TNaV8BnLQQprNO5GJHaKjmszrIoeY= diff --git a/service/operator/reader.go b/service/operator/reader.go index bca67d847..e882d0643 100644 --- a/service/operator/reader.go +++ b/service/operator/reader.go @@ -23,6 +23,10 @@ func (s *Service) runQuery(ctx context.Context, component *repository.Component, defer func() { if r := recover(); r != nil { panicMsg := fmt.Sprintf("Panic occurred: %v, Stack trace: %v", r, string(debug.Stack())) + logger := aSession.Logger() + if logger == nil { + panic(panicMsg) + } aSession.Logger().Errorc(ctx, panicMsg) err = response.NewError(http.StatusInternalServerError, "Internal server error") output = nil From 3f8777cdea57812e7ecea6662eabf796c9ba71be Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 26 Aug 2025 06:40:04 -0700 Subject: [PATCH 012/279] updated fs embeder handling --- view/resource.go | 14 +++++++++++++- view/state/parameter.go | 2 +- view/state/resource.go | 5 +++++ view/state/type.go | 4 ++++ view/tags/parser.go | 5 +++-- 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/view/resource.go b/view/resource.go index 94ca4db9a..94cc59435 100644 --- a/view/resource.go +++ b/view/resource.go @@ -72,7 +72,8 @@ type ( Substitutes Substitutes Docs *Documentation - FSEmbedder *state.FSEmbedder + + FSEmbedder *state.FSEmbedder modTime time.Time _doc docs.Service @@ -152,6 +153,17 @@ func (r *Resource) ReverseSubstitutes(text string) string { return r.Substitutes.ReverseReplace(text) } +func (r *Resource) EmbedFS() *embed.FS { + if r.FSEmbedder == nil { + return nil + } + return r.FSEmbedder.EmbedFS() +} + +func (r *Resource) SetFSEmbedder(embedder *state.FSEmbedder) { + r.FSEmbedder = embedder +} + func (r *Resource) SetFs(fs afs.Service) { r.fs = fs } diff --git a/view/state/parameter.go b/view/state/parameter.go index ac6461b57..93670cd9f 100644 --- a/view/state/parameter.go +++ b/view/state/parameter.go @@ -523,7 +523,7 @@ func (p *Parameter) initCodec(resource Resource) error { if !p.Output.Schema.IsNamed() { fieldTag := reflect.StructTag(p.Tag) - if stateTag, _ := tags.ParseStateTags(fieldTag, nil); stateTag != nil { + if stateTag, _ := tags.ParseStateTags(fieldTag, resource.EmbedFS()); stateTag != nil { stateTag.TypeName = SanitizeTypeName(p.Output.Schema.Name) p.Tag = string(stateTag.UpdateTag(fieldTag)) } diff --git a/view/state/resource.go b/view/state/resource.go index 7c39ad857..b80f1fe1d 100644 --- a/view/state/resource.go +++ b/view/state/resource.go @@ -2,6 +2,7 @@ package state import ( "context" + "embed" "github.com/viant/xdatly/codec" "github.com/viant/xreflect" ) @@ -22,5 +23,9 @@ type ( ExpandSubstitutes(text string) string ReverseSubstitutes(text string) string + + EmbedFS() *embed.FS + + SetFSEmbedder(embedder *FSEmbedder) } ) diff --git a/view/state/type.go b/view/state/type.go index d64c07281..d2ea7c0c2 100644 --- a/view/state/type.go +++ b/view/state/type.go @@ -110,6 +110,10 @@ func (t *Type) ensureEmbedder(reflect.Type) { t.embedder = NewFSEmbedder(nil) } t.embedder.SetType(reflect.TypeOf(t)) + if t.resource != nil && t.resource.EmbedFS() == nil { + t.resource.SetFSEmbedder(t.embedder) + } + } func (t *Type) adjustConstants() { diff --git a/view/tags/parser.go b/view/tags/parser.go index aa4c407a0..283d1eded 100644 --- a/view/tags/parser.go +++ b/view/tags/parser.go @@ -4,14 +4,15 @@ import ( "context" "embed" "fmt" + "reflect" + "strings" + "github.com/viant/afs" "github.com/viant/afs/storage" "github.com/viant/afs/url" "github.com/viant/tagly/format" "github.com/viant/tagly/tags" "github.com/viant/xreflect" - "reflect" - "strings" ) // ValueTag represents default value tag From a7553f7d326fda119f462c6f358808d12ebcf9b3 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 26 Aug 2025 10:03:27 -0700 Subject: [PATCH 013/279] updated fs embeder handling --- view/state/parameter.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/view/state/parameter.go b/view/state/parameter.go index 93670cd9f..a1ded9294 100644 --- a/view/state/parameter.go +++ b/view/state/parameter.go @@ -3,6 +3,11 @@ package state import ( "context" "fmt" + "net/http" + "reflect" + "strconv" + "strings" + "github.com/viant/datly/internal/setter" "github.com/viant/datly/shared" "github.com/viant/datly/utils/types" @@ -11,10 +16,6 @@ import ( "github.com/viant/structology" "github.com/viant/xreflect" "github.com/viant/xunsafe" - "net/http" - "reflect" - "strconv" - "strings" ) type ( @@ -512,7 +513,12 @@ func (p *Parameter) initCodec(resource Resource) error { if p.Output == nil { return nil } - + stateTag, _ := tags.ParseStateTags(reflect.StructTag(p.Tag), resource.EmbedFS()) + if stateTag != nil { + if stateTag.Codec != nil && stateTag.Codec.Body != "" { + p.Output.Body = stateTag.Codec.Body + } + } inputType := p.Schema.Type() if err := p.Output.Init(resource, inputType); err != nil { return err @@ -520,10 +526,9 @@ func (p *Parameter) initCodec(resource Resource) error { if p.Output.Schema == nil { return nil } - if !p.Output.Schema.IsNamed() { fieldTag := reflect.StructTag(p.Tag) - if stateTag, _ := tags.ParseStateTags(fieldTag, resource.EmbedFS()); stateTag != nil { + if stateTag != nil { stateTag.TypeName = SanitizeTypeName(p.Output.Schema.Name) p.Tag = string(stateTag.UpdateTag(fieldTag)) } From 5a9614b80d18324e92502037b54060b76ce1b76c Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 26 Aug 2025 10:16:31 -0700 Subject: [PATCH 014/279] updated fs embeder handling --- service/executor/expand/predicate.go | 4 ++++ view/predicate.go | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/service/executor/expand/predicate.go b/service/executor/expand/predicate.go index b67e83312..292bb29c0 100644 --- a/service/executor/expand/predicate.go +++ b/service/executor/expand/predicate.go @@ -129,6 +129,10 @@ func (p *Predicate) expand(group int, operator string) (string, error) { } ctx = vcontext.WithValue(ctx, PredicateCtx, p.ctx) ctx = vcontext.WithValue(ctx, PredicateState, p.state) + + p.ctx.DataUnit.EvalLock.Lock() + defer p.ctx.DataUnit.EvalLock.Unlock() + if p.ctx.Session != nil { aLogger := p.ctx.Session.Logger() ctx = vcontext.WithValue(ctx, logger.ContextKey, aLogger) diff --git a/view/predicate.go b/view/predicate.go index b7193fa94..189e656aa 100644 --- a/view/predicate.go +++ b/view/predicate.go @@ -51,9 +51,6 @@ func (e *PredicateEvaluator) Compute(ctx context.Context, value interface{}) (*c panic("not found custom ctx") } - cuxtomCtx.DataUnit.EvalLock.Lock() - defer cuxtomCtx.DataUnit.EvalLock.Unlock() - val := ctx.Value(expand.PredicateState) aState := val.(*structology.State) offset := len(cuxtomCtx.DataUnit.ParamsGroup) From 1325f81749e688218ef173849eeff67ff6d0410c Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 26 Aug 2025 12:33:17 -0700 Subject: [PATCH 015/279] updated fs embeder handling --- go.sum | 2 -- service/executor/expand/evaluator.go | 3 ++- service/session/stater.go | 8 +++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/go.sum b/go.sum index ccfd3ed70..9eda4cc46 100644 --- a/go.sum +++ b/go.sum @@ -1141,8 +1141,6 @@ github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns= github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.17.6 h1:6uMZVWk+WJl/y8coEh4F4mqbTHbtzWkLVEQdrk+m7sE= -github.com/viant/sqlx v0.17.6/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/sqlx v0.17.7 h1:drUv3N8mOboq917gnmcT9zC4G9vj4jU11bO/SsLpmc8= github.com/viant/sqlx v0.17.7/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.6.1 h1:Forza+RF/1tmlQFk9ABNhu+IQ8vMAqbYM6FOsYtGh9E= diff --git a/service/executor/expand/evaluator.go b/service/executor/expand/evaluator.go index d57b935b2..3b14cf832 100644 --- a/service/executor/expand/evaluator.go +++ b/service/executor/expand/evaluator.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "reflect" + "github.com/viant/datly/view/keywords" "github.com/viant/datly/view/state/predicate" "github.com/viant/godiff" @@ -12,7 +14,6 @@ import ( "github.com/viant/velty/est" "github.com/viant/velty/est/op" "github.com/viant/xreflect" - "reflect" ) type ( diff --git a/service/session/stater.go b/service/session/stater.go index ed50be0b3..c2d4555b5 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -116,10 +116,12 @@ func (s *Session) handleComponentpOutputType(ctx context.Context, dest interface destValue, err := s.operate(ctx, s, s.component) s.Options = sessionOpt - if destValue != nil { - reflect.ValueOf(dest).Elem().Set(reflect.ValueOf(destValue).Elem()) + reflectDestValue := reflect.ValueOf(destValue) + if reflectDestValue.Kind() == reflect.Ptr { + reflect.ValueOf(dest).Elem().Set(reflectDestValue.Elem()) + } else { + reflect.ValueOf(dest).Elem().Set(reflectDestValue) } - if err != nil { return err } From 09d33d3d0fbadca33321499edd3a7dd91633e246 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 26 Aug 2025 12:44:38 -0700 Subject: [PATCH 016/279] updated fs embeder handling --- service/session/stater.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/service/session/stater.go b/service/session/stater.go index c2d4555b5..f50f63cfb 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -2,11 +2,15 @@ package session import ( "context" + "fmt" + "net/http" "reflect" + "runtime/debug" "github.com/viant/datly/utils/types" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind/locator" + "github.com/viant/xdatly/handler/response" hstate "github.com/viant/xdatly/handler/state" ) @@ -36,6 +40,18 @@ func (s *Session) Into(ctx context.Context, dest interface{}, opts ...hstate.Opt } func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Option) (err error) { + defer func() { + if r := recover(); r != nil { + panicMsg := fmt.Sprintf("Panic occurred: %v, Stack trace: %v", r, string(debug.Stack())) + logger := s.Logger() + if logger == nil { + panic(panicMsg) + } + s.Logger().Errorc(ctx, panicMsg) + err = response.NewError(http.StatusInternalServerError, "Internal server error") + } + }() + destType := reflect.TypeOf(dest) sType := types.EnsureStruct(destType) stateType, ok := s.Types.Lookup(sType) From 8de645caaad7a89df8f0887893c671368e11aacc Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 26 Aug 2025 12:45:43 -0700 Subject: [PATCH 017/279] updated fs embeder handling --- service/session/stater.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/service/session/stater.go b/service/session/stater.go index f50f63cfb..3530b912e 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -133,10 +133,11 @@ func (s *Session) handleComponentpOutputType(ctx context.Context, dest interface s.Options = sessionOpt reflectDestValue := reflect.ValueOf(destValue) + destPtr := reflect.ValueOf(dest) if reflectDestValue.Kind() == reflect.Ptr { - reflect.ValueOf(dest).Elem().Set(reflectDestValue.Elem()) + destPtr.Elem().Set(reflectDestValue.Elem()) } else { - reflect.ValueOf(dest).Elem().Set(reflectDestValue) + destPtr.Elem().Set(reflectDestValue) } if err != nil { return err From e6519bec30ba5619bebaf883905b98ac5e27a39a Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 26 Aug 2025 13:01:58 -0700 Subject: [PATCH 018/279] updated fs embeder handling --- service/session/stater.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/service/session/stater.go b/service/session/stater.go index 3530b912e..d2bda7a34 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -130,17 +130,21 @@ func (s *Session) handleComponentpOutputType(ctx context.Context, dest interface sessionOpt := s.Options s.Options = *s.Indirect(true, stateOptions...) destValue, err := s.operate(ctx, s, s.component) + destPtr := reflect.ValueOf(dest) + if err != nil && destValue == nil { + if errorSetter, ok := dest.(response.StatusSetter); ok { + errorSetter.SetError(err) + return nil + } + return err + } s.Options = sessionOpt - reflectDestValue := reflect.ValueOf(destValue) - destPtr := reflect.ValueOf(dest) + if reflectDestValue.Kind() == reflect.Ptr { destPtr.Elem().Set(reflectDestValue.Elem()) } else { destPtr.Elem().Set(reflectDestValue) } - if err != nil { - return err - } return nil } From 3c912377621f1100270130b6099329838cdf57dd Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 27 Aug 2025 20:35:27 -0700 Subject: [PATCH 019/279] patched predicate racing --- service/executor/expand/data_unit.go | 2 ++ service/reader/sql.go | 6 ++++-- view/predicate.go | 23 +++++++++++++++-------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/service/executor/expand/data_unit.go b/service/executor/expand/data_unit.go index 89fea3f55..ad8516e76 100644 --- a/service/executor/expand/data_unit.go +++ b/service/executor/expand/data_unit.go @@ -34,6 +34,8 @@ type ( ExecutablesIndex map[string]*Executable ) +// + func (c *DataUnit) WithPresence() interface{} { var opt interface{} = validator.WithSetMarker() return opt diff --git a/service/reader/sql.go b/service/reader/sql.go index 33d7747ca..5e3ac5b35 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -3,14 +3,15 @@ package reader import ( "context" "fmt" + "strconv" + "strings" + "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/service/reader/metadata" "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/datly/view/keywords" "github.com/viant/sqlx/io/read/cache" - "strconv" - "strings" ) const ( @@ -50,6 +51,7 @@ func (b *Builder) Build(ctx context.Context, opts ...BuilderOption) (*cache.Parm parent := options.parent partitions := options.partition expander := options.expander + state, err := aView.Template.EvaluateSource(ctx, statelet.Template, parent, &batchData, expander) if err != nil { diff --git a/view/predicate.go b/view/predicate.go index 189e656aa..d6e5b3820 100644 --- a/view/predicate.go +++ b/view/predicate.go @@ -53,19 +53,26 @@ func (e *PredicateEvaluator) Compute(ctx context.Context, value interface{}) (*c val := ctx.Value(expand.PredicateState) aState := val.(*structology.State) - offset := len(cuxtomCtx.DataUnit.ParamsGroup) - evaluate, err := e.Evaluate(cuxtomCtx, aState, value) + // evaluate predicate with an isolated DataUnit to avoid + // mutating parent DataUnit and relying on Shrink/restore across nesting. + var metaSource expand.Dber + if cuxtomCtx.DataUnit != nil { + metaSource = cuxtomCtx.DataUnit.MetaSource + } + isolatedDU := expand.NewDataUnit(metaSource) + tmpCtx := *cuxtomCtx + tmpCtx.DataUnit = isolatedDU + + evaluate, err := e.Evaluate(&tmpCtx, aState, value) if err != nil { return nil, err } - placeholderLen := len(evaluate.DataUnit.ParamsGroup) - offset - var values = make([]interface{}, placeholderLen) - if placeholderLen > 0 { - copy(values, evaluate.DataUnit.ParamsGroup[offset:]) - } + // Collect placeholders from the isolated DataUnit and return them + // to the caller; do not mutate the parent DataUnit here. + values := make([]interface{}, len(isolatedDU.ParamsGroup)) + copy(values, isolatedDU.ParamsGroup) criteria := &codec.Criteria{Expression: evaluate.Buffer.String(), Placeholders: values} - cuxtomCtx.DataUnit.Shrink(offset) return criteria, nil } From 1ed2fee050a9b215999fc2692de8f276fef8c85f Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 29 Aug 2025 07:20:42 -0700 Subject: [PATCH 020/279] updated limit --- internal/inference/parameter.go | 11 ++++++----- repository/component.go | 10 +++++++--- view/view.go | 15 ++++++++++----- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/internal/inference/parameter.go b/internal/inference/parameter.go index d3403859f..30b3e6893 100644 --- a/internal/inference/parameter.go +++ b/internal/inference/parameter.go @@ -4,6 +4,12 @@ import ( "embed" _ "embed" "fmt" + "go/ast" + "path" + "reflect" + "strconv" + "strings" + "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/datly/view/tags" @@ -15,11 +21,6 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/xreflect" "github.com/viant/xunsafe" - "go/ast" - "path" - "reflect" - "strconv" - "strings" ) type ( diff --git a/repository/component.go b/repository/component.go index 31faed6cf..112f6ff7d 100644 --- a/repository/component.go +++ b/repository/component.go @@ -4,6 +4,10 @@ import ( "context" "embed" "fmt" + "net/http" + "reflect" + "strings" + "github.com/francoispqt/gojay" "github.com/viant/afs" "github.com/viant/datly/gateway/router/marshal" @@ -29,9 +33,6 @@ import ( xhandler "github.com/viant/xdatly/handler" hstate "github.com/viant/xdatly/handler/state" "github.com/viant/xreflect" - "net/http" - "reflect" - "strings" ) // Component represents abstract API view/handler based component @@ -424,6 +425,9 @@ func WithContract(inputType, outputType reflect.Type, embedFs *embed.FS, viewOpt aCache := &view.Cache{Reference: shared.Reference{Ref: aView.Cache}} viewOptions = append(viewOptions, view.WithCache(aCache)) } + if aView.Limit != nil { + viewOptions = append(viewOptions, view.WithLimit(aView.Limit)) + } if aTag.View.PublishParent { viewOptions = append(viewOptions, view.WithViewPublishParent(aTag.View.PublishParent)) diff --git a/view/view.go b/view/view.go index 42bcc1906..8226a9cd2 100644 --- a/view/view.go +++ b/view/view.go @@ -4,6 +4,12 @@ import ( "context" "database/sql" "fmt" + "net/http" + "path" + "reflect" + "strings" + "time" + "github.com/viant/afs/url" "github.com/viant/datly/gateway/router/marshal" "github.com/viant/datly/internal/setter" @@ -23,11 +29,6 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/xreflect" "github.com/viant/xunsafe" - "net/http" - "path" - "reflect" - "strings" - "time" ) const ( @@ -400,6 +401,10 @@ func (v *View) inheritRelationsFromTag(schema *state.Schema) error { refViewOptions = append(refViewOptions, WithCache(aCache)) } + if viewTag.Limit != nil { + viewOptions = append(viewOptions, WithLimit(viewTag.Limit)) + } + if viewTag.PublishParent { refViewOptions = append(refViewOptions, WithViewPublishParent(viewTag.PublishParent)) } From 1d3e0a271d6465128932b6f42bd7b227b0981b37 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 29 Aug 2025 09:30:01 -0700 Subject: [PATCH 021/279] updated limit --- view/view.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/view/view.go b/view/view.go index 8226a9cd2..d2f3bf567 100644 --- a/view/view.go +++ b/view/view.go @@ -478,6 +478,9 @@ func WithLimit(limit *int) Option { } view.Selector.Constraints.Limit = true view.Selector.Limit = *limit + if limit != nil { + view.Selector.NoLimit = *limit == 0 + } return nil } } From 17b3e4507af0740bd0e84b71f148875c2d5cc9cf Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 29 Aug 2025 10:51:55 -0700 Subject: [PATCH 022/279] patched limit --- internal/codegen/handler.go | 3 ++- internal/inference/parameter.go | 8 ++++++++ internal/translator/parser/declarations.go | 11 ++++++++--- internal/translator/resource.go | 16 +++++++++++++--- internal/translator/service.go | 2 +- repository/component.go | 4 ++++ 6 files changed, 36 insertions(+), 8 deletions(-) diff --git a/internal/codegen/handler.go b/internal/codegen/handler.go index 5c732a16e..dafbd99a8 100644 --- a/internal/codegen/handler.go +++ b/internal/codegen/handler.go @@ -2,11 +2,12 @@ package codegen import ( _ "embed" + "strings" + "github.com/viant/datly/cmd/options" "github.com/viant/datly/internal/codegen/ast" "github.com/viant/datly/internal/inference" "github.com/viant/datly/internal/plugin" - "strings" ) //go:embed tmpl/handler/handler.gox diff --git a/internal/inference/parameter.go b/internal/inference/parameter.go index 30b3e6893..eddb8efe5 100644 --- a/internal/inference/parameter.go +++ b/internal/inference/parameter.go @@ -35,6 +35,7 @@ type ( AssumedType bool Connector string Cache string + Limit *int InOutput bool Of string } @@ -116,6 +117,10 @@ func (p *Parameter) veltyDeclaration(builder *strings.Builder) { builder.WriteString(".WithCache('" + p.Cache + "')") } + if p.Limit != nil { + builder.WriteString(".WithLimit('" + strconv.Itoa(*p.Limit) + "')") + } + if p.Required != nil { if !*p.Required { builder.WriteString(".Optional()") @@ -305,6 +310,9 @@ func buildParameter(field *xunsafe.Field, aTag *tags.Tag, types *xreflect.Types, if aTag.View.Cache != "" { param.Cache = aTag.View.Cache } + if aTag.View.Limit != nil { + param.Limit = aTag.View.Limit + } } fType := field.Type diff --git a/internal/translator/parser/declarations.go b/internal/translator/parser/declarations.go index ca11710de..d5113539e 100644 --- a/internal/translator/parser/declarations.go +++ b/internal/translator/parser/declarations.go @@ -2,6 +2,10 @@ package parser import ( "fmt" + "reflect" + "strconv" + "strings" + "github.com/viant/datly/gateway/router/marshal" "github.com/viant/datly/internal/inference" "github.com/viant/datly/shared" @@ -12,9 +16,6 @@ import ( "github.com/viant/velty/ast/expr" "github.com/viant/velty/parser" "github.com/viant/xreflect" - "reflect" - "strconv" - "strings" ) type ( @@ -322,6 +323,10 @@ func (s *Declarations) parseShorthands(declaration *Declaration, cursor *parsly. declaration.InOutput = true case "WithCache": declaration.Cache = strings.Trim(args[0], `"'`) + case "WithLimit": + limit, _ := strconv.Atoi(strings.Trim(args[0], `"'`)) + declaration.Limit = &limit + case "Cacheable": literal := strings.Trim(args[0], `"'`) value, _ := strconv.ParseBool(literal) diff --git a/internal/translator/resource.go b/internal/translator/resource.go index eb92605b8..163684277 100644 --- a/internal/translator/resource.go +++ b/internal/translator/resource.go @@ -3,6 +3,10 @@ package translator import ( "context" "fmt" + "path" + "reflect" + "strings" + "github.com/viant/afs" "github.com/viant/afs/url" "github.com/viant/datly/cmd/options" @@ -22,9 +26,6 @@ import ( "github.com/viant/toolbox" "github.com/viant/xreflect" "golang.org/x/mod/modfile" - "path" - "reflect" - "strings" ) type ( @@ -352,6 +353,15 @@ func (r *Resource) buildParameterViews() { if parameter.Cache != "" { viewlet.View.Cache = &view.Cache{Reference: shared.Reference{Ref: parameter.Cache}} } + if parameter.Limit != nil { + if viewlet.View.Selector == nil { + viewlet.View.Selector = &view.Config{ + Constraints: &view.Constraints{Limit: true}, + } + } + viewlet.View.Selector.Limit = *parameter.Limit + viewlet.View.Selector.NoLimit = viewlet.View.Selector.Limit == 0 + } if viewlet.Connector == "" { viewlet.Connector = r.rootConnector } diff --git a/internal/translator/service.go b/internal/translator/service.go index d53f4e6a3..70e71d683 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -20,7 +20,7 @@ import ( "github.com/viant/datly/internal/plugin" "github.com/viant/datly/internal/setter" "github.com/viant/datly/internal/translator/parser" - signature "github.com/viant/datly/repository/contract/signature" + "github.com/viant/datly/repository/contract/signature" "github.com/viant/datly/repository/path" "github.com/viant/datly/service" "github.com/viant/datly/shared" diff --git a/repository/component.go b/repository/component.go index 112f6ff7d..109ded23a 100644 --- a/repository/component.go +++ b/repository/component.go @@ -445,6 +445,10 @@ func WithContract(inputType, outputType reflect.Type, embedFs *embed.FS, viewOpt if aTag.View.Batch != 0 { viewOptions = append(viewOptions, view.WithBatchSize(aTag.View.Batch)) } + if aTag.View.Limit != nil { + viewOptions = append(viewOptions, view.WithLimit(aTag.View.Limit)) + } + if aTag.View.RelationalConcurrency != 0 { viewOptions = append(viewOptions, view.WithRelationalConcurrency(aTag.View.RelationalConcurrency)) } From 09ce111ae2213c67699dc5cddaf369b1f4220f93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Filipowicz?= Date: Wed, 3 Sep 2025 14:09:09 +0200 Subject: [PATCH 023/279] updated ensureValidValue --- service/session/state.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/service/session/state.go b/service/session/state.go index 7fcfb5c70..f7b0e8ce5 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -347,10 +347,17 @@ func (s *Session) ensureValidValue(value interface{}, parameter *state.Parameter if valueType.Elem().Kind() == reflect.Struct && parameter.Schema.Type().Kind() == reflect.Slice { if parameter.Schema.CompType() == valueType { sliceValuePtr := reflect.New(parameterType) + + if isNil(value) { + empty := reflect.MakeSlice(parameterType, 0, 0) + sliceValuePtr.Elem().Set(empty) + return sliceValuePtr.Interface(), nil // []T{} + } + sliceValue := reflect.MakeSlice(parameterType, 1, 1) sliceValuePtr.Elem().Set(sliceValue) sliceValue.Index(0).Set(reflect.ValueOf(value)) - return sliceValuePtr.Interface(), nil + return sliceValuePtr.Interface(), nil // []T{value}` } } case reflect.Slice: From d51933a38e1e11714ea046be309f6dba9339610b Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 5 Sep 2025 14:29:33 -0700 Subject: [PATCH 024/279] updated error handling --- service/reader/sql.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/service/reader/sql.go b/service/reader/sql.go index 5e3ac5b35..58598ed12 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -57,6 +57,12 @@ func (b *Builder) Build(ctx context.Context, opts ...BuilderOption) (*cache.Parm if err != nil { return nil, err } + if state == nil { + return nil, fmt.Errorf("failed to evaluate state for view %v, state was nil", aView.Name) + } + if state.Expanded == "" { + return nil, fmt.Errorf("failed to evaluate expanded for view %vm statelet was nil", aView.Name) + } if len(state.Filters) > 0 { statelet.Filters = append(statelet.Filters, state.Filters...) } From fda81402bd1914fdc786aec2faab8dcb5c3d5426 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 5 Sep 2025 15:23:59 -0700 Subject: [PATCH 025/279] updated error handling --- service/reader/sql.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/service/reader/sql.go b/service/reader/sql.go index 58598ed12..7cc1f6fa1 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -45,13 +45,23 @@ func (b *Builder) Build(ctx context.Context, opts ...BuilderOption) (*cache.Parm options := newBuilderOptions(opts...) aView := options.view statelet := options.statelet - batchData := *options.batchData + // guard against nil batchData passed by callers + var batchData view.BatchData + if options.batchData != nil { + batchData = *options.batchData + } relation := options.relation exclude := options.exclude parent := options.parent partitions := options.partition expander := options.expander + // ensure non-nil statelet to avoid nil deref on Template usage + if statelet == nil { + statelet = view.NewStatelet() + statelet.Init(aView) + } + state, err := aView.Template.EvaluateSource(ctx, statelet.Template, parent, &batchData, expander) if err != nil { From c0f5411ef1cf0bd7607d15df6733478a0c0f3e1a Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 5 Sep 2025 16:27:03 -0700 Subject: [PATCH 026/279] updated error handling --- service/reader/sql.go | 2 +- view/state.go | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/service/reader/sql.go b/service/reader/sql.go index 7cc1f6fa1..bcb7aff93 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -74,7 +74,7 @@ func (b *Builder) Build(ctx context.Context, opts ...BuilderOption) (*cache.Parm return nil, fmt.Errorf("failed to evaluate expanded for view %vm statelet was nil", aView.Name) } if len(state.Filters) > 0 { - statelet.Filters = append(statelet.Filters, state.Filters...) + statelet.AppendFilters(state.Filters) } if aView.Template.IsActualTemplate() && aView.ShouldTryDiscover() { state.Expanded = metadata.EnrichWithDiscover(state.Expanded, true) diff --git a/view/state.go b/view/state.go index 3484de7f8..a7ff10ece 100644 --- a/view/state.go +++ b/view/state.go @@ -44,6 +44,7 @@ type ( initialized bool _columnNames map[string]bool result *cache.ParmetrizedQuery + filtersMu sync.Mutex } ) @@ -99,6 +100,16 @@ func (s *QuerySelector) SetCriteria(expanded string, placeholders []interface{}) s.Placeholders = placeholders } +// AppendFilters safely appends filters to the selector's Filters to avoid data races. +func (s *Statelet) AppendFilters(filters predicate.Filters) { + if len(filters) == 0 { + return + } + s.QuerySelector.filtersMu.Lock() + s.QuerySelector.Filters = append(s.QuerySelector.Filters, filters...) + s.QuerySelector.filtersMu.Unlock() +} + // NewStatelet creates a selector func NewStatelet() *Statelet { return &Statelet{ From 310295e0edd66d82cdfa1d80ff81b44e658ae364 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 8 Sep 2025 14:11:24 -0700 Subject: [PATCH 027/279] updated error handling --- service/executor/expand/evaluator.go | 2 +- service/executor/expand/predicate.go | 6 +++++- service/executor/expand/state.go | 6 +++--- view/predicate.go | 13 ++++++++++++- view/template.go | 4 ++++ 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/service/executor/expand/evaluator.go b/service/executor/expand/evaluator.go index 3b14cf832..f898b4ffa 100644 --- a/service/executor/expand/evaluator.go +++ b/service/executor/expand/evaluator.go @@ -253,7 +253,7 @@ func (e *Evaluator) ensureState(ctx *Context, options ...StateOption) *State { state.Context = ctx } - state.Init(e.stateProvider(), e.predicateConfigs, options...) + state.Init(e.stateProvider(), e.predicateConfigs, e.stateType, options...) return state } diff --git a/service/executor/expand/predicate.go b/service/executor/expand/predicate.go index 292bb29c0..aae51c8bb 100644 --- a/service/executor/expand/predicate.go +++ b/service/executor/expand/predicate.go @@ -38,7 +38,11 @@ type ( } ) -func NewPredicate(ctx *Context, state *structology.State, config []*PredicateConfig) *Predicate { +func NewPredicate(ctx *Context, state *structology.State, config []*PredicateConfig, stateType *structology.StateType) *Predicate { + // Initialize state if not provided, but never override an existing state + if state == nil && stateType != nil { + state = stateType.NewState() + } return &Predicate{ ctx: ctx, config: config, diff --git a/service/executor/expand/state.go b/service/executor/expand/state.go index 1a6fffeb9..6399be43d 100644 --- a/service/executor/expand/state.go +++ b/service/executor/expand/state.go @@ -84,7 +84,7 @@ func WithCustomContext(customContext *Variable) StateOption { } } -func (s *State) Init(templateState *est.State, predicates []*PredicateConfig, options ...StateOption) { +func (s *State) Init(templateState *est.State, predicates []*PredicateConfig, stateType *structology.StateType, options ...StateOption) { for _, option := range options { option(s) } @@ -121,7 +121,7 @@ func (s *State) Init(templateState *est.State, predicates []*PredicateConfig, op s.MessageBus = s.Session.MessageBus() } - s.Predicate = NewPredicate(s.Context, s.ParametersState, predicates) + s.Predicate = NewPredicate(s.Context, s.ParametersState, predicates, stateType) s.State = templateState } @@ -148,6 +148,6 @@ func StateWithSQL(ctx context.Context, SQL string) *State { Context: &Context{Context: ctx}, } - aState.Init(nil, nil) + aState.Init(nil, nil, nil) return aState } diff --git a/view/predicate.go b/view/predicate.go index d6e5b3820..415550c28 100644 --- a/view/predicate.go +++ b/view/predicate.go @@ -35,6 +35,7 @@ type ( state *expand.NamedVariable hasStateName *expand.NamedVariable handler codec.PredicateHandler + stateType *structology.StateType } PredicateEvaluator struct { @@ -42,6 +43,7 @@ type ( evaluator *expand.Evaluator valueState *expand.NamedVariable hasValueState *expand.NamedVariable + stateType *structology.StateType } ) @@ -52,7 +54,14 @@ func (e *PredicateEvaluator) Compute(ctx context.Context, value interface{}) (*c } val := ctx.Value(expand.PredicateState) - aState := val.(*structology.State) + var aState *structology.State + if s, ok := val.(*structology.State); ok { + aState = s + } + if aState == nil && e.stateType != nil { + // Initialize state if absent; do not override if provided. + aState = e.stateType.NewState() + } // evaluate predicate with an isolated DataUnit to avoid // mutating parent DataUnit and relying on Shrink/restore across nesting. var metaSource expand.Dber @@ -158,6 +167,7 @@ func (p *predicateEvaluatorProvider) new(predicateConfig *extension.PredicateCon evaluator: p.evaluator, valueState: p.state, hasValueState: p.hasStateName, + stateType: p.stateType, }, nil } @@ -215,5 +225,6 @@ func (p *predicateEvaluatorProvider) init(resource *Resource, predicateConfig *e p.signature = argsIndexed p.state = stateVariable p.hasStateName = hasVariable + p.stateType = stateType return nil } diff --git a/view/template.go b/view/template.go index e98ff37a2..ae4308723 100644 --- a/view/template.go +++ b/view/template.go @@ -231,6 +231,10 @@ func (t *Template) EvaluateState(ctx context.Context, parameterState *structolog } func (t *Template) EvaluateStateWithSession(ctx context.Context, parameterState *structology.State, parentParam *expand.ViewContext, batchData *BatchData, sess *extension.Session, options ...interface{}) (*expand.State, error) { + // Ensure parameter state is initialized when absent, but never override an existing one. + if parameterState == nil && t.stateType != nil { + parameterState = t.stateType.NewState() + } var expander expand.Expander var dataUnit *expand.DataUnit for _, option := range options { From e95c4d7311b92217ef43957001b949ea4e6c008e Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 8 Sep 2025 14:18:55 -0700 Subject: [PATCH 028/279] updated error handling --- repository/logging/logging.go | 4 ++-- service/reader/handler/handler.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/repository/logging/logging.go b/repository/logging/logging.go index 2383e2f11..c0054ea70 100644 --- a/repository/logging/logging.go +++ b/repository/logging/logging.go @@ -15,7 +15,7 @@ func Log(config *Config, execContext *exec.Context) { execContext.Metrics = execContext.Metrics.HideMetrics() } if config.IsAuditEnabled() { - data, _ := json.Marshal(execContext) + data, _ := json.MarshalNoEscape(execContext) fmt.Println("[AUDIT] " + string(data)) } if config.IsTracingEnabled() { @@ -42,7 +42,7 @@ func Log(config *Config, execContext *exec.Context) { } else { trace.Spans[0].SetStatusFromHTTPCode(execContext.StatusCode) } - traceData, _ := json.Marshal(trace) + traceData, _ := json.MarshalNoEscape(trace) fmt.Println("[TRACE] " + string(traceData)) } } diff --git a/service/reader/handler/handler.go b/service/reader/handler/handler.go index c3a485dc1..f85797711 100644 --- a/service/reader/handler/handler.go +++ b/service/reader/handler/handler.go @@ -139,7 +139,7 @@ func (h *Handler) publishViewSummaryIfNeeded(aView *view.View, ret *Response) { if templateMeta.Kind != view.MetaKindHeader { return } - data, err := goJson.Marshal(ret.Reader.DataSummary) + data, err := goJson.MarshalNoEscape(ret.Reader.DataSummary) if err != nil { ret.StatusCode = http.StatusInternalServerError ret.Status.Status = "error" @@ -157,7 +157,7 @@ func (h *Handler) publishMetricsIfNeeded(aSession *reader.Session, ret *Response if info.Executions == nil { continue } - data, err := goJson.Marshal(info) + data, err := goJson.MarshalNoEscape(info) if err != nil { continue } From c672e99e1072a4c845b4e65ac94ab4a5c6a1eab3 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 15 Sep 2025 11:23:01 -0700 Subject: [PATCH 029/279] updated dep --- go.mod | 7 +++---- go.sum | 14 ++++++-------- internal/translator/viewlets.go | 3 +++ mcp/server.go | 15 ++++++++------ service/executor/expand/data_unit.go | 11 +++++++++++ view/extension/init.go | 9 ++++++--- view/extension/predicates.go | 29 +++++++++++++++++++++++++++- view/tags/codec.go | 9 +++++---- view/tags/predicate.go | 8 +++++--- 9 files changed, 76 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index c1a5bb6c7..51eacc09c 100644 --- a/go.mod +++ b/go.mod @@ -49,9 +49,9 @@ require ( require ( github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d github.com/viant/firebase v0.1.1 - github.com/viant/jsonrpc v0.7.2 - github.com/viant/mcp v0.4.3 - github.com/viant/mcp-protocol v0.4.4 + github.com/viant/jsonrpc v0.7.5 + github.com/viant/mcp v0.5.2 + github.com/viant/mcp-protocol v0.5.7 github.com/viant/structology v0.6.1 github.com/viant/tagly v0.2.2 github.com/viant/xdatly v0.5.4-0.20250806192028-819cadf93282 @@ -106,7 +106,6 @@ require ( github.com/go-errors/errors v1.5.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/go.sum b/go.sum index 9eda4cc46..9508d64ec 100644 --- a/go.sum +++ b/go.sum @@ -786,8 +786,6 @@ github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= @@ -1127,12 +1125,12 @@ github.com/viant/govalidator v0.3.1 h1:V7f/KgfzbP8fVDc+Kj+jyPvfXxMr2N1x7srOlDV6l github.com/viant/govalidator v0.3.1/go.mod h1:D35Dwx0R8rR1knRxhlseoYvOkiqo24kpMg1/o977i9Y= github.com/viant/igo v0.2.0 h1:ygWmTCinnGPaeV7omJLiyneOpzYZ5kiw7oYz7mUJZVQ= github.com/viant/igo v0.2.0/go.mod h1:7V6AWsLhKWeGzXNTNH3AZiIEKa0m33DrQbdWtapsI74= -github.com/viant/jsonrpc v0.7.2 h1:FUzhfFN76E09ZbQOxReFOyPhsxYhE0fjWzPhattR9Dk= -github.com/viant/jsonrpc v0.7.2/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= -github.com/viant/mcp v0.4.3 h1:ykQ2XyS2l5xrxHY5peJgIWoH+n8ZSpiSifnO/UH6/3I= -github.com/viant/mcp v0.4.3/go.mod h1:3SnILtYVIT8PIWICMyzP9KfhepawoFRv+//FBU/hc7c= -github.com/viant/mcp-protocol v0.4.4 h1:jKuCHvXeNof1Of1UfUyJkrSSNfOBiN4pXKWv3J2NwFM= -github.com/viant/mcp-protocol v0.4.4/go.mod h1:EL4NY7yW2gge+XLorgJA7PIazQX3x4ZkutYihwBwINs= +github.com/viant/jsonrpc v0.7.5 h1:QiLEVl5nP7j5i55jDQp4HUURKJLf+ENPafTlJT13u38= +github.com/viant/jsonrpc v0.7.5/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= +github.com/viant/mcp v0.5.2 h1:m0Z7LdYOQOs7MhYg7Ql9nZlJDiedHfC86VNRzfKg3Gs= +github.com/viant/mcp v0.5.2/go.mod h1:ybylH9mD3/aVLrT8KQsakmsKxtfw7Kpo4A9hmTuetGw= +github.com/viant/mcp-protocol v0.5.7 h1:3ifypMAy+oUjQEAsq+XwrAhE/B/3eIes4yXdhoRF9Eo= +github.com/viant/mcp-protocol v0.5.7/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b h1:3q166tV28yFdbFV+tXXjH7ViKAmgAgGdoWzMtvhQv28= github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b/go.mod h1:85fneXJbErKMGhSQto3A5ElTQCwl3t74U9cSV0waBHw= github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= diff --git a/internal/translator/viewlets.go b/internal/translator/viewlets.go index eb1d24b7b..72707387c 100644 --- a/internal/translator/viewlets.go +++ b/internal/translator/viewlets.go @@ -138,6 +138,9 @@ func (n *Viewlets) addRelations(query *query.Select) error { parentNs := inference.ParentAlias(join) parentViewlet := n.Lookup(parentNs) + if parentViewlet == nil { + return fmt.Errorf("parent viewlet %v doesn't exist", parentNs) + } relation.Spec.Parent = parentViewlet.Spec cardinality := state.Many if inference.IsToOne(join) || relation.OutputSettings.IsToOne() { diff --git a/mcp/server.go b/mcp/server.go index 2c9cd306e..2409a38ea 100644 --- a/mcp/server.go +++ b/mcp/server.go @@ -3,6 +3,7 @@ package mcp import ( "context" "fmt" + "github.com/viant/afs" "github.com/viant/afs/http" "github.com/viant/afs/url" @@ -15,18 +16,20 @@ import ( "github.com/viant/mcp/client/auth/transport" authserver "github.com/viant/mcp/server/auth" - serverproto "github.com/viant/mcp-protocol/server" - "github.com/viant/scy/auth/flow" "os" "path" + serverproto "github.com/viant/mcp-protocol/server" + "github.com/viant/scy/auth/flow" + + "reflect" + "strconv" + "strings" + "github.com/viant/mcp/server" "github.com/viant/scy" "github.com/viant/scy/cred" "golang.org/x/oauth2" - "reflect" - "strconv" - "strings" ) type Server struct { @@ -40,7 +43,7 @@ func (s *Server) init() error { var newImplementer = extension.New(s.registry) var options = []server.Option{ server.WithNewHandler(newImplementer), - server.WithImplementation(schema.Implementation{"Datly", "0.1"}), + server.WithImplementation(schema.Implementation{Name: "Datly", Version: "0.1"}), } issuerURL := s.config.IssuerURL var oauth2Config *oauth2.Config diff --git a/service/executor/expand/data_unit.go b/service/executor/expand/data_unit.go index ad8516e76..784953fc9 100644 --- a/service/executor/expand/data_unit.go +++ b/service/executor/expand/data_unit.go @@ -275,6 +275,17 @@ func (c *DataUnit) Like(columnName string, args interface{}) (string, error) { func (c *DataUnit) NotLike(columnName string, args interface{}) (string, error) { return c.like(columnName, args, false) } +func (c *DataUnit) Expression(expr string, value interface{}) (string, error) { + return c.expression(expr, value) +} + +func (c *DataUnit) expression(expr string, value interface{}) (string, error) { + if value == "" { + return "", nil + } + c.addAll(value) + return expr, nil +} func (c *DataUnit) like(columnName string, args interface{}, inclusive bool) (string, error) { expander, err := bindingsCache.Lookup(args) diff --git a/view/extension/init.go b/view/extension/init.go index db515af42..f05ff7140 100644 --- a/view/extension/init.go +++ b/view/extension/init.go @@ -3,6 +3,8 @@ package extension import ( "encoding/json" "fmt" + "net/http" + dcodec "github.com/viant/datly/view/extension/codec" "github.com/viant/datly/view/extension/handler" "github.com/viant/datly/view/extension/marshaller" @@ -17,14 +19,14 @@ import ( "github.com/viant/xdatly/handler/response/tabular/tjson" "github.com/viant/xdatly/handler/response/tabular/xml" "github.com/viant/xdatly/handler/validator" - "net/http" + + "reflect" + "time" "github.com/viant/xdatly/predicate" "github.com/viant/xdatly/types/core" _ "github.com/viant/xdatly/types/custom" "github.com/viant/xreflect" - "reflect" - "time" ) const ( @@ -119,6 +121,7 @@ func InitRegistry() { PredicateGreaterOrEqual: NewGreaterOrEqualPredicate(), PredicateGreaterThan: NewGreaterThanPredicate(), PredicateLike: NewLikePredicate(), + PredicateExpr: NewExprPredicate(), PredicateNotLike: NewNotLikePredicate(), PredicateHandler: NewPredicateHandler(), PredicateContains: NewContainsPredicate(), diff --git a/view/extension/predicates.go b/view/extension/predicates.go index 04b0ba098..ae0276e95 100644 --- a/view/extension/predicates.go +++ b/view/extension/predicates.go @@ -2,12 +2,13 @@ package extension import ( "fmt" + "sync" + "github.com/viant/datly/utils/types" codec2 "github.com/viant/datly/view/extension/codec" "github.com/viant/xdatly/codec" "github.com/viant/xdatly/predicate" "github.com/viant/xreflect" - "sync" ) const ( @@ -32,6 +33,7 @@ const ( PredicateExists = "exists" PredicateNotExists = "not_exists" + PredicateExpr = "expr" PredicateCriteriaExists = "exists_criteria" PredicateCriteriaNotExists = "not_exists_criteria" PredicateCriteriaIn = "in_criteria" @@ -225,6 +227,10 @@ func NewEqualPredicate() *Predicate { return binaryPredicate(PredicateEqual, "=") } +func NewColumnExpressionPredicate() *Predicate { + return binaryPredicate(PredicateEqual, "=") +} + func NewLessOrEqualPredicate() *Predicate { return binaryPredicate(PredicateLessOrEqual, "<=") } @@ -333,6 +339,10 @@ func NewLikePredicate() *Predicate { return newLikePredicate(PredicateLike, true) } +func NewExprPredicate() *Predicate { + return newExprPredicate(PredicateExpr) +} + func NewNotLikePredicate() *Predicate { return newLikePredicate(PredicateNotLike, false) } @@ -362,6 +372,23 @@ func newLikePredicate(name string, inclusive bool) *Predicate { } } +func newExprPredicate(expr string) *Predicate { + args := []*predicate.NamedArgument{ + { + Name: "Expression", + Position: 0, + }, + } + criteria := fmt.Sprintf(`$criteria.Expression($Expression, $FilterValue)`) + return &Predicate{ + Template: &predicate.Template{ + Name: expr, + Source: " " + criteria, + Args: args, + }, + } +} + func NewContainsPredicate() *Predicate { return newContainsPredicate(PredicateContains, true) } diff --git a/view/tags/codec.go b/view/tags/codec.go index a16be7509..d606ed717 100644 --- a/view/tags/codec.go +++ b/view/tags/codec.go @@ -1,9 +1,9 @@ package tags import ( - "fmt" - "github.com/viant/tagly/tags" "strings" + + "github.com/viant/tagly/tags" ) // CodecTag codec tag @@ -31,10 +31,11 @@ func (t *Tag) updatedCodec(key string, value string) (err error) { } tag.Body = string(data) default: + expr := key if value != "" { - return fmt.Errorf("invalid argument %s", value) + expr += " =" + value } - tag.Arguments = append(tag.Arguments, key) + tag.Arguments = append(tag.Arguments, expr) } return err } diff --git a/view/tags/predicate.go b/view/tags/predicate.go index e66ef5e76..956a2bc5b 100644 --- a/view/tags/predicate.go +++ b/view/tags/predicate.go @@ -2,9 +2,10 @@ package tags import ( "fmt" - "github.com/viant/tagly/tags" "strconv" "strings" + + "github.com/viant/tagly/tags" ) // PredicateTag Predicate tag @@ -70,10 +71,11 @@ func (t *Tag) updatedPredicate(key string, value string) (err error) { return fmt.Errorf("invalid predicate ensure: %s %w", value, err) } default: + expr := key if value != "" { - return fmt.Errorf("invalid argument %s", value) + expr = key + "=" + value } - tag.Arguments = append(tag.Arguments, key) + tag.Arguments = append(tag.Arguments, expr) } return err } From 9b734aaee1343879abd236fa3d038f43f356f478 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 18 Sep 2025 14:30:31 -0700 Subject: [PATCH 030/279] updated deparemoved goccy josn --- gateway/route.go | 2 +- go.mod | 2 +- go.sum | 3 +-- repository/logging/logging.go | 6 +++--- service/reader/handler/handler.go | 10 +++++++--- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/gateway/route.go b/gateway/route.go index 24e682d3d..4bec8528a 100644 --- a/gateway/route.go +++ b/gateway/route.go @@ -2,7 +2,7 @@ package gateway import ( "context" - "github.com/goccy/go-json" + "encoding/json" "github.com/viant/afs/url" "github.com/viant/datly/gateway/router" "github.com/viant/datly/repository" diff --git a/go.mod b/go.mod index 51eacc09c..6b8b056c3 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/aws/aws-lambda-go v1.31.0 github.com/francoispqt/gojay v1.2.13 github.com/go-sql-driver/mysql v1.7.0 - github.com/goccy/go-json v0.10.5 github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/google/gops v0.3.23 github.com/google/uuid v1.6.0 @@ -106,6 +105,7 @@ require ( github.com/go-errors/errors v1.5.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/go.sum b/go.sum index 9508d64ec..717c1ab96 100644 --- a/go.sum +++ b/go.sum @@ -787,9 +787,8 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= diff --git a/repository/logging/logging.go b/repository/logging/logging.go index c0054ea70..00440ac60 100644 --- a/repository/logging/logging.go +++ b/repository/logging/logging.go @@ -1,8 +1,8 @@ package logging import ( + "encoding/json" "fmt" - "github.com/goccy/go-json" "github.com/viant/xdatly/handler/exec" "strconv" "time" @@ -15,7 +15,7 @@ func Log(config *Config, execContext *exec.Context) { execContext.Metrics = execContext.Metrics.HideMetrics() } if config.IsAuditEnabled() { - data, _ := json.MarshalNoEscape(execContext) + data, _ := json.Marshal(execContext) fmt.Println("[AUDIT] " + string(data)) } if config.IsTracingEnabled() { @@ -42,7 +42,7 @@ func Log(config *Config, execContext *exec.Context) { } else { trace.Spans[0].SetStatusFromHTTPCode(execContext.StatusCode) } - traceData, _ := json.MarshalNoEscape(trace) + traceData, _ := json.Marshal(trace) fmt.Println("[TRACE] " + string(traceData)) } } diff --git a/service/reader/handler/handler.go b/service/reader/handler/handler.go index f85797711..6e83335e3 100644 --- a/service/reader/handler/handler.go +++ b/service/reader/handler/handler.go @@ -2,8 +2,8 @@ package handler import ( "context" + "encoding/json" - goJson "github.com/goccy/go-json" "github.com/viant/datly/gateway/router/status" _ "github.com/viant/datly/repository/locator/async" _ "github.com/viant/datly/repository/locator/component" @@ -139,7 +139,11 @@ func (h *Handler) publishViewSummaryIfNeeded(aView *view.View, ret *Response) { if templateMeta.Kind != view.MetaKindHeader { return } - data, err := goJson.MarshalNoEscape(ret.Reader.DataSummary) + var data []byte + var err error + if ret.Reader.DataSummary != nil { + data, err = json.Marshal(ret.Reader.DataSummary) + } if err != nil { ret.StatusCode = http.StatusInternalServerError ret.Status.Status = "error" @@ -157,7 +161,7 @@ func (h *Handler) publishMetricsIfNeeded(aSession *reader.Session, ret *Response if info.Executions == nil { continue } - data, err := goJson.MarshalNoEscape(info) + data, err := json.Marshal(info) if err != nil { continue } From 198248fc665ad552d4299ec0cdd2d8ee0645e86e Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 23 Sep 2025 11:59:23 -0700 Subject: [PATCH 031/279] added retry on invalid connection --- service/reader/service.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/service/reader/service.go b/service/reader/service.go index 63ca09436..d3701f9bc 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -5,7 +5,9 @@ import ( "database/sql" "fmt" "reflect" + "strings" "sync" + "sync/atomic" "time" "unsafe" @@ -515,12 +517,25 @@ func (s *Service) queryWithHandler(ctx context.Context, session *Session, aView if session.DryRun { return []*response.SQLExecution{stats}, nil } + + retires := uint32(0) +BEGIN: reader, err := read.New(ctx, db, parametrizedSQL.SQL, collector.NewItem(), options...) + + isInvalidConnection := err != nil && strings.Contains(err.Error(), "invalid connection") + if isInvalidConnection && atomic.AddUint32(&retires, 1) < 3 { + db, err = aView.Connector.DB() + if err != nil { + return nil, fmt.Errorf("failed to connect to db: %w", err) + } + goto BEGIN + } if err != nil { stats.SetError(err) anExec, err := s.HandleSQLError(err, session, aView, parametrizedSQL, stats) return []*response.SQLExecution{anExec}, err } + defer func() { stmt := reader.Stmt() if stmt == nil { @@ -529,7 +544,17 @@ func (s *Service) queryWithHandler(ctx context.Context, session *Session, aView _ = stmt.Close() }() err = reader.QueryAll(ctx, handler, parametrizedSQL.Args...) + + isInvalidConnection = err != nil && strings.Contains(err.Error(), "invalid connection") + if isInvalidConnection && atomic.AddUint32(&retires, 1) < 3 { + db, err = aView.Connector.DB() + if err != nil { + return nil, fmt.Errorf("failed to connect to db: %w", err) + } + goto BEGIN + } end := time.Now() + aView.Logger.ReadingData(end.Sub(begin), parametrizedSQL.SQL, *readData, parametrizedSQL.Args, err) if err != nil { stats.SetError(err) From cf7d3954d83a8fdda7fd6057013a35e32af1b3f3 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 23 Sep 2025 13:35:27 -0700 Subject: [PATCH 032/279] added retry on invalid connection --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6b8b056c3..fa54285be 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 - github.com/viant/sqlx v0.17.7 + github.com/viant/sqlx v0.17.8 github.com/viant/structql v0.5.2 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 diff --git a/go.sum b/go.sum index 717c1ab96..3914ffdf7 100644 --- a/go.sum +++ b/go.sum @@ -1140,6 +1140,8 @@ github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.17.7 h1:drUv3N8mOboq917gnmcT9zC4G9vj4jU11bO/SsLpmc8= github.com/viant/sqlx v0.17.7/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= +github.com/viant/sqlx v0.17.8 h1:YxGTrXC2B1JmDz1qp8G+G9hGPk7XCRevWN1E4E+ZlCI= +github.com/viant/sqlx v0.17.8/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.6.1 h1:Forza+RF/1tmlQFk9ABNhu+IQ8vMAqbYM6FOsYtGh9E= github.com/viant/structology v0.6.1/go.mod h1:63XfkzUyNw7wdi99HJIsH2Rg3d5AOumqbWLUYytOkxU= github.com/viant/structql v0.5.2 h1:0dAratszxC6AD/TNaV8BnLQQprNO5GJHaKjmszrIoeY= From e7d7ab5b2a30b9602f7042d8bc6bb970acffa860 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 24 Sep 2025 07:53:00 -0700 Subject: [PATCH 033/279] enhanced marhsller --- gateway/router/marshal/json/cache.go | 26 ++++++++++++++++--- gateway/router/marshal/json/init.go | 2 ++ .../router/marshal/json/marshaller_custom.go | 2 +- gateway/router/marshal/json/option.go | 4 +-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/gateway/router/marshal/json/cache.go b/gateway/router/marshal/json/cache.go index cd79eba61..b5c562932 100644 --- a/gateway/router/marshal/json/cache.go +++ b/gateway/router/marshal/json/cache.go @@ -121,8 +121,11 @@ func (c *pathCache) getMarshaller(rType reflect.Type, config *config.IOConfig, p } aConfig := c.parseConfig(options) - if (aConfig == nil || !aConfig.ignoreCustomUnmarshaller) && rType.Implements(unmarshallerIntoType) { - return newCustomUnmarshaller(rType, config, path, outputPath, tag, c.parent) + // Keep UnmarshalerInto precedence for non-structs; structs handled below to honor gojay first. + if rType.Kind() != reflect.Struct { + if (aConfig == nil || !aConfig.IgnoreCustomUnmarshaller) && rType.Implements(unmarshallerIntoType) { + return newCustomUnmarshaller(rType, config, path, outputPath, tag, c.parent) + } } switch rType { @@ -212,12 +215,27 @@ func (c *pathCache) getMarshaller(rType reflect.Type, config *config.IOConfig, p return newTimeMarshaller(tag, config), nil } - marshaller, err := newStructMarshaller(config, rType, path, outputPath, tag, c.parent) + // Build base struct marshaller first. + base, err := newStructMarshaller(config, rType, path, outputPath, tag, c.parent) if err != nil { return nil, err } - return marshaller, nil + // If struct defines gojay interfaces, wrap the base. + if aConfig == nil || !aConfig.IgnoreCustomMarshaller { + hasMarshal := rType.Implements(marshalerJSONObjectType) || reflect.PtrTo(rType).Implements(marshalerJSONObjectType) + hasUnmarshal := rType.Implements(unmarshalerJSONObjectType) || reflect.PtrTo(rType).Implements(unmarshalerJSONObjectType) + if hasMarshal || hasUnmarshal { + return newGojayObjectMarshaller(getXType(rType), getXType(reflect.PtrTo(rType)), base, hasMarshal, hasUnmarshal), nil + } + } + + // Otherwise, allow custom unmarshaller on structs if defined. + if (aConfig == nil || !aConfig.IgnoreCustomUnmarshaller) && rType.Implements(unmarshallerIntoType) { + return newCustomUnmarshaller(rType, config, path, outputPath, tag, c.parent) + } + + return base, nil case reflect.Interface: marshaller, err := newInterfaceMarshaller(rType, config, path, outputPath, tag, c.parent) diff --git a/gateway/router/marshal/json/init.go b/gateway/router/marshal/json/init.go index cacb47602..31eebb40e 100644 --- a/gateway/router/marshal/json/init.go +++ b/gateway/router/marshal/json/init.go @@ -13,6 +13,8 @@ import ( var rawMessageType = reflect.TypeOf(json.RawMessage{}) var unmarshallerIntoType = reflect.TypeOf((*UnmarshalerInto)(nil)).Elem() +var marshalerJSONObjectType = reflect.TypeOf((*gojay.MarshalerJSONObject)(nil)).Elem() +var unmarshalerJSONObjectType = reflect.TypeOf((*gojay.UnmarshalerJSONObject)(nil)).Elem() var mapStringIfaceType = reflect.TypeOf(map[string]interface{}{}) var decData *xunsafe.Field var decCur *xunsafe.Field diff --git a/gateway/router/marshal/json/marshaller_custom.go b/gateway/router/marshal/json/marshaller_custom.go index eb73890c3..81ca8fbd8 100644 --- a/gateway/router/marshal/json/marshaller_custom.go +++ b/gateway/router/marshal/json/marshaller_custom.go @@ -21,7 +21,7 @@ type customMarshaller struct { } func newCustomUnmarshaller(rType reflect.Type, config *config.IOConfig, path string, outputPath string, tag *format.Tag, cache *marshallersCache) (marshaler, error) { - marshaller, err := cache.loadMarshaller(rType, config, path, outputPath, tag, &cacheConfig{ignoreCustomUnmarshaller: true}) + marshaller, err := cache.loadMarshaller(rType, config, path, outputPath, tag, &cacheConfig{IgnoreCustomUnmarshaller: true}) if err != nil { return nil, err } diff --git a/gateway/router/marshal/json/option.go b/gateway/router/marshal/json/option.go index cd1855380..82a8e2dd1 100644 --- a/gateway/router/marshal/json/option.go +++ b/gateway/router/marshal/json/option.go @@ -26,6 +26,6 @@ func (o Options) FormatTag() *format.Tag { } type cacheConfig struct { - ignoreCustomUnmarshaller bool - ignoreCustomMarshaller bool + IgnoreCustomUnmarshaller bool + IgnoreCustomMarshaller bool } From 148f00b058c2b1e6619ac7c074198f3ae86d1da5 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 24 Sep 2025 14:36:14 -0700 Subject: [PATCH 034/279] enhanced marhsller --- .../marshal/json/marshaller_gojay_object.go | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 gateway/router/marshal/json/marshaller_gojay_object.go diff --git a/gateway/router/marshal/json/marshaller_gojay_object.go b/gateway/router/marshal/json/marshaller_gojay_object.go new file mode 100644 index 000000000..af3cbbec3 --- /dev/null +++ b/gateway/router/marshal/json/marshaller_gojay_object.go @@ -0,0 +1,68 @@ +package json + +import ( + "github.com/francoispqt/gojay" + "github.com/viant/xunsafe" + "unsafe" +) + +// gojayObjectMarshaller delegates to gojay's Marshaler/UnmarshalerJSONObject when available, +// and falls back to the generic struct marshaller for the other direction. +type gojayObjectMarshaller struct { + valueType *xunsafe.Type + addrType *xunsafe.Type + fallback marshaler + useMarshal bool + useUnmarshal bool +} + +func newGojayObjectMarshaller(valueType *xunsafe.Type, addrType *xunsafe.Type, fallback marshaler, useMarshal, useUnmarshal bool) *gojayObjectMarshaller { + return &gojayObjectMarshaller{ + valueType: valueType, + addrType: addrType, + fallback: fallback, + useMarshal: useMarshal, + useUnmarshal: useUnmarshal, + } +} + +func (g *gojayObjectMarshaller) MarshallObject(ptr unsafe.Pointer, session *MarshallSession) error { + if ptr == nil { + session.Write(nullBytes) + return nil + } + + if g.useMarshal { + // Prefer pointer receiver if (*T) implements MarshalerJSONObject + if m, ok := g.addrType.Value(ptr).(gojay.MarshalerJSONObject); ok { + enc := gojay.NewEncoder(session.Buffer) + return enc.EncodeObject(m) + } + // Fallback to value receiver if (T) implements MarshalerJSONObject + if m, ok := g.valueType.Interface(ptr).(gojay.MarshalerJSONObject); ok { + enc := gojay.NewEncoder(session.Buffer) + return enc.EncodeObject(m) + } + // If neither matched at runtime, fallback to generic marshaller + } + return g.fallback.MarshallObject(ptr, session) +} + +func (g *gojayObjectMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder *gojay.Decoder, auxiliaryDecoder *gojay.Decoder, session *UnmarshalSession) error { + if !g.useUnmarshal { + return g.fallback.UnmarshallObject(pointer, decoder, auxiliaryDecoder, session) + } + + d := decoder + if auxiliaryDecoder != nil { + d = auxiliaryDecoder + } + + // Prefer pointer receiver only; value receiver cannot mutate destination reliably. + if u, ok := g.addrType.Value(pointer).(gojay.UnmarshalerJSONObject); ok { + return d.Object(u) + } + + // If neither matched at runtime, fallback to generic unmarshaller + return g.fallback.UnmarshallObject(pointer, decoder, auxiliaryDecoder, session) +} From 16e573bd58876da1df099ab70e5eaa3966ebae62 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 24 Sep 2025 14:43:06 -0700 Subject: [PATCH 035/279] enhanced marhsller --- gateway/router/marshal/json/cache.go | 31 +++++++++++++------ .../router/marshal/json/marshaller_struct.go | 4 ++- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/gateway/router/marshal/json/cache.go b/gateway/router/marshal/json/cache.go index b5c562932..4329b4905 100644 --- a/gateway/router/marshal/json/cache.go +++ b/gateway/router/marshal/json/cache.go @@ -3,13 +3,14 @@ package json import ( "bytes" "fmt" + "reflect" + "sync" + "github.com/viant/datly/gateway/router/marshal/config" "github.com/viant/tagly/format" "github.com/viant/tagly/format/text" "github.com/viant/xreflect" "github.com/viant/xunsafe" - "reflect" - "sync" ) var buffersPool *buffers @@ -215,22 +216,32 @@ func (c *pathCache) getMarshaller(rType reflect.Type, config *config.IOConfig, p return newTimeMarshaller(tag, config), nil } - // Build base struct marshaller first. + // Decide if type uses gojay; build base without init to handle self-references safely. + hasMarshal := (aConfig == nil || !aConfig.IgnoreCustomMarshaller) && (rType.Implements(marshalerJSONObjectType) || reflect.PtrTo(rType).Implements(marshalerJSONObjectType)) + hasUnmarshal := (aConfig == nil || !aConfig.IgnoreCustomMarshaller) && (rType.Implements(unmarshalerJSONObjectType) || reflect.PtrTo(rType).Implements(unmarshalerJSONObjectType)) + base, err := newStructMarshaller(config, rType, path, outputPath, tag, c.parent) if err != nil { return nil, err } - // If struct defines gojay interfaces, wrap the base. - if aConfig == nil || !aConfig.IgnoreCustomMarshaller { - hasMarshal := rType.Implements(marshalerJSONObjectType) || reflect.PtrTo(rType).Implements(marshalerJSONObjectType) - hasUnmarshal := rType.Implements(unmarshalerJSONObjectType) || reflect.PtrTo(rType).Implements(unmarshalerJSONObjectType) - if hasMarshal || hasUnmarshal { - return newGojayObjectMarshaller(getXType(rType), getXType(reflect.PtrTo(rType)), base, hasMarshal, hasUnmarshal), nil + if hasMarshal || hasUnmarshal { + // Wrap base with gojay and store wrapper first to break cycles and ensure self-references use wrapper. + wrapper := newGojayObjectMarshaller(getXType(rType), getXType(reflect.PtrTo(rType)), base, hasMarshal, hasUnmarshal) + c.storeMarshaler(rType, wrapper) + if err := base.init(); err != nil { + return nil, err } + return wrapper, nil + } + + // No gojay: store base first to break cycles, then init. + c.storeMarshaler(rType, base) + if err := base.init(); err != nil { + return nil, err } - // Otherwise, allow custom unmarshaller on structs if defined. + // Allow custom unmarshaller on structs if defined and not ignored (only if no gojay used). if (aConfig == nil || !aConfig.IgnoreCustomUnmarshaller) && rType.Implements(unmarshallerIntoType) { return newCustomUnmarshaller(rType, config, path, outputPath, tag, c.parent) } diff --git a/gateway/router/marshal/json/marshaller_struct.go b/gateway/router/marshal/json/marshaller_struct.go index a0d2d1070..bb66bd01d 100644 --- a/gateway/router/marshal/json/marshaller_struct.go +++ b/gateway/router/marshal/json/marshaller_struct.go @@ -68,7 +68,9 @@ func newStructMarshaller(config *config.IOConfig, rType reflect.Type, path strin marshallersIndex: map[string]int{}, } - return result, result.init() + // Initialization is invoked by cache after it stores the marshaller (or wrapper) + // to break cycles for self-referential types. + return result, nil } func (s *structMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder *gojay.Decoder, auxiliaryDecoder *gojay.Decoder, session *UnmarshalSession) error { From 6208351fc5d4a2673c205c82a00455b81410e0c4 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 24 Sep 2025 14:48:15 -0700 Subject: [PATCH 036/279] enhanced marhsller --- gateway/router/marshal/json/cache.go | 13 +++++--- .../marshal/json/marshaller_deferred.go | 31 +++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 gateway/router/marshal/json/marshaller_deferred.go diff --git a/gateway/router/marshal/json/cache.go b/gateway/router/marshal/json/cache.go index 4329b4905..728af816d 100644 --- a/gateway/router/marshal/json/cache.go +++ b/gateway/router/marshal/json/cache.go @@ -106,12 +106,17 @@ func (c *pathCache) loadOrGetMarshaller(rType reflect.Type, config *config.IOCon return value.(marshaler), nil } - aMarshaler, err := c.getMarshaller(rType, config, path, outputPath, tag, options...) + // Place a deferred placeholder to break recursive graphs for this path and type. + placeholder := &deferredMarshaller{} + c.storeMarshaler(rType, placeholder) + aMarshaler, err := c.getMarshaller(rType, config, path, outputPath, tag, options...) if err != nil { return nil, err } + // Swap placeholder with the real marshaller and set target for any users that captured it. + placeholder.setTarget(aMarshaler) c.storeMarshaler(rType, aMarshaler) return aMarshaler, nil } @@ -226,17 +231,15 @@ func (c *pathCache) getMarshaller(rType reflect.Type, config *config.IOConfig, p } if hasMarshal || hasUnmarshal { - // Wrap base with gojay and store wrapper first to break cycles and ensure self-references use wrapper. + // Wrap base with gojay; placeholder at loadOrGet level already breaks cycles. wrapper := newGojayObjectMarshaller(getXType(rType), getXType(reflect.PtrTo(rType)), base, hasMarshal, hasUnmarshal) - c.storeMarshaler(rType, wrapper) if err := base.init(); err != nil { return nil, err } return wrapper, nil } - // No gojay: store base first to break cycles, then init. - c.storeMarshaler(rType, base) + // No gojay: just init base and return (placeholder already in place). if err := base.init(); err != nil { return nil, err } diff --git a/gateway/router/marshal/json/marshaller_deferred.go b/gateway/router/marshal/json/marshaller_deferred.go new file mode 100644 index 000000000..fef24a83d --- /dev/null +++ b/gateway/router/marshal/json/marshaller_deferred.go @@ -0,0 +1,31 @@ +package json + +import ( + "fmt" + "github.com/francoispqt/gojay" + "unsafe" +) + +// deferredMarshaller is a placeholder used to break recursive type graphs during construction. +// It forwards calls to the actual target once it is set. +type deferredMarshaller struct { + target marshaler +} + +func (d *deferredMarshaller) setTarget(m marshaler) { + d.target = m +} + +func (d *deferredMarshaller) MarshallObject(ptr unsafe.Pointer, session *MarshallSession) error { + if d.target == nil { + return fmt.Errorf("marshaller not initialized") + } + return d.target.MarshallObject(ptr, session) +} + +func (d *deferredMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder *gojay.Decoder, auxiliaryDecoder *gojay.Decoder, session *UnmarshalSession) error { + if d.target == nil { + return fmt.Errorf("marshaller not initialized") + } + return d.target.UnmarshallObject(pointer, decoder, auxiliaryDecoder, session) +} From c5404492eaf77bf094bfbcf0824e3bfe5e25b508 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 24 Sep 2025 14:55:33 -0700 Subject: [PATCH 037/279] enhanced marhsller --- gateway/router/marshal/json/marshaller_struct.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gateway/router/marshal/json/marshaller_struct.go b/gateway/router/marshal/json/marshaller_struct.go index bb66bd01d..23e2392a3 100644 --- a/gateway/router/marshal/json/marshaller_struct.go +++ b/gateway/router/marshal/json/marshaller_struct.go @@ -256,8 +256,8 @@ func (s *structMarshaller) createStructMarshallers(fields *groupedFields, path s } elemType := field.Type - switch elemType.Kind() { - case reflect.Ptr, reflect.Slice: + // Unwrap nested pointers/slices to detect self-references like []*T or [][]*T + for elemType.Kind() == reflect.Ptr || elemType.Kind() == reflect.Slice { elemType = elemType.Elem() } if elemType == fields.owner { From a782238afa5ee83c831029c03ab46bd34c852aae Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 29 Sep 2025 13:45:57 -0700 Subject: [PATCH 038/279] enhanced marhsller --- doc/extension/EXAMPLES.md | 123 +----------------------------------- go.mod | 1 - service/executor/service.go | 32 +++++++++- 3 files changed, 31 insertions(+), 125 deletions(-) diff --git a/doc/extension/EXAMPLES.md b/doc/extension/EXAMPLES.md index 1e6d6614a..6f476a0a7 100644 --- a/doc/extension/EXAMPLES.md +++ b/doc/extension/EXAMPLES.md @@ -2200,128 +2200,7 @@ go 1.21 require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible - github.com/aws/aws-lambda-go v1.31.0 - github.com/francoispqt/gojay v1.2.13 - github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/go-playground/validator v9.31.0+incompatible - github.com/go-sql-driver/mysql v1.7.0 - github.com/goccy/go-json v0.9.11 - github.com/golang-jwt/jwt/v4 v4.4.1 - github.com/google/gops v0.3.23 - github.com/google/uuid v1.3.0 - github.com/jessevdk/go-flags v1.5.0 - github.com/leodido/go-urn v1.2.1 // indirect - github.com/lib/pq v1.10.6 - github.com/mattn/go-sqlite3 v1.14.16 - github.com/onsi/gomega v1.20.2 // indirect - github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.8.4 - github.com/viant/afs v1.24.2 - github.com/viant/afsc v1.9.0 - github.com/viant/assertly v0.9.1-0.20220620174148-bab013f93a60 - github.com/viant/bigquery v0.2.1 - github.com/viant/cloudless v1.8.1 - github.com/viant/dsc v0.16.2 // indirect - github.com/viant/dsunit v0.10.8 - github.com/viant/dyndb v0.1.4-0.20221214043424-27654ab6ed9c - github.com/viant/gmetric v0.2.7-0.20220508155136-c2e3c95db446 - github.com/viant/godiff v0.4.1 - github.com/viant/parsly v0.2.0 - github.com/viant/pgo v0.10.3 - github.com/viant/scy v0.6.0 - github.com/viant/sqlx v0.8.0 - github.com/viant/structql v0.2.2 - github.com/viant/toolbox v0.34.6-0.20221112031702-3e7cdde7f888 - github.com/viant/velty v0.2.0 - github.com/viant/xdatly/types/custom v0.0.0-20230309034540-231985618fc7 - github.com/viant/xreflect v0.0.0-20230303201326-f50afb0feb0d - github.com/viant/xunsafe v0.8.4 - golang.org/x/mod v0.9.0 - golang.org/x/oauth2 v0.7.0 - google.golang.org/api v0.114.0 - gopkg.in/go-playground/assert.v1 v1.2.1 // indirect - gopkg.in/yaml.v3 v3.0.1 -) - -require ( - github.com/viant/govalidator v0.2.1 - github.com/viant/sqlparser v0.3.1-0.20230320162628-96274e82953f - golang.org/x/crypto v0.7.0 // indirect -) - -require ( - github.com/aws/aws-sdk-go v1.44.12 - github.com/aws/aws-sdk-go-v2/config v1.18.3 - github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 - github.com/viant/structology v0.2.0 - github.com/viant/xdatly/extension v0.0.0-20230323215422-3e5c3147f0e6 - github.com/viant/xdatly/handler v0.0.0-20230619231115-e622dd6aff79 - github.com/viant/xdatly/types/core v0.0.0-20230615201419-f5e46b6b011f -) - -require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.19.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - cloud.google.com/go/secretmanager v1.10.0 // indirect - cloud.google.com/go/storage v1.29.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.18.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect - github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 // indirect - github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.8 // indirect - github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.27 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sns v1.20.11 // indirect - github.com/aws/aws-sdk-go-v2/service/sqs v1.22.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect - github.com/aws/smithy-go v1.13.5 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d // indirect - github.com/go-errors/errors v1.4.2 // indirect - github.com/go-playground/locales v0.14.0 // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.8.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/kr/pretty v0.3.0 // indirect - github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect - github.com/lestrrat-go/blackmagic v1.0.0 // indirect - github.com/lestrrat-go/httpcc v1.0.1 // indirect - github.com/lestrrat-go/iter v1.0.1 // indirect - github.com/lestrrat-go/jwx v1.2.25 // indirect - github.com/lestrrat-go/option v1.0.0 // indirect - github.com/michael/mymodule2 v0.0.0-00010101000000-000000000000 // indirect - github.com/nxadm/tail v1.4.8 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect - github.com/viant/igo v0.1.0 // indirect - github.com/yuin/gopher-lua v0.0.0-20221210110428-332342483e3f // indirect - go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.54.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + .... gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.mod b/go.mod index fa54285be..16e40247d 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,6 @@ require ( github.com/go-errors/errors v1.5.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/service/executor/service.go b/service/executor/service.go index c89e8fc41..01df6c631 100644 --- a/service/executor/service.go +++ b/service/executor/service.go @@ -4,6 +4,11 @@ import ( "context" "database/sql" "fmt" + "reflect" + "strings" + "sync/atomic" + "time" + "github.com/viant/datly/logger" expand2 "github.com/viant/datly/service/executor/expand" vsession "github.com/viant/datly/service/session" @@ -13,8 +18,6 @@ import ( "github.com/viant/sqlx/option" "github.com/viant/xdatly/handler/exec" "github.com/viant/xdatly/handler/response" - "reflect" - "time" ) type ( @@ -31,6 +34,8 @@ type ( dbSource DBSource collections map[string]*batcher.Collection logger *logger.Adapter + inserted int32 + updated int32 } DBOption func(options *DBOptions) @@ -190,6 +195,9 @@ func (e *Executor) handleUpdate(ctx context.Context, sess *dbSession, db *sql.DB options = append(options, db) updated, err := service.Exec(ctx, executable.Data, options...) + if err == nil { + atomic.AddInt32(&sess.updated, int32(updated)) + } e.logMetrics(ctx, executable.Table, "UPDATE", updated, now, err) return err } @@ -233,6 +241,9 @@ func (e *Executor) handleInsert(ctx context.Context, sess *dbSession, executable } options = append(options, tx) inserted, _, err = service.Exec(ctx, executable.Data, options...) + if err == nil { + atomic.AddInt32(&sess.inserted, int32(inserted)) + } e.logMetrics(ctx, executable.Table, "INSERT", inserted, started, err) return err } @@ -252,6 +263,23 @@ func (e *Executor) handleInsert(ctx context.Context, sess *dbSession, executable options = append(options, option.BatchSize(batchSize)) options = append(options, e.dbOptions(db, sess)) inserted, _, err = service.Exec(ctx, executable.Data, options...) + if err == nil { + atomic.AddInt32(&sess.inserted, int32(inserted)) + } + isInvalidConnection := err != nil && strings.Contains(err.Error(), "invalid connection") + if isInvalidConnection && atomic.LoadInt32(&sess.inserted) == 0 && atomic.LoadInt32(&sess.updated) == 0 { + var dErr error + db, dErr = sess.dbSource.Db(ctx) + if dErr != nil { + return fmt.Errorf("failed after retry: %w", err) + } + sess.tx.db = db + sess.tx.tx = nil + if _, err = sess.tx.Tx(); err != nil { + return err + } + inserted, _, err = service.Exec(ctx, executable.Data, options...) + } e.logMetrics(ctx, executable.Table, "INSERT", inserted, started, err) return err } From f8fcc28d4e9927dfb319c32dffd80294f86ac99b Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 7 Oct 2025 06:24:55 -0700 Subject: [PATCH 039/279] expose marshallers, move queryselector to xdatly --- cmd/datly/build.yaml | 2 +- e2e/local/build.yaml | 2 +- e2e/local/regression/regression.yaml | 2 +- gateway/router/handler.go | 21 ++-- gateway/runtime/apigw/deploy.yaml | 2 +- gateway/runtime/gcr/deploy.yaml | 2 +- gateway/runtime/lambda/deploy.yaml | 2 +- go.mod | 10 +- go.sum | 18 ++-- internal/translator/rule.go | 7 +- internal/translator/service.go | 14 ++- repository/component.go | 144 +++++++++++++++++++++++--- repository/contract/meta.go | 5 +- service.go | 36 +++++-- service/executor/extension/session.go | 8 +- service/session/selector.go | 14 ++- view/state.go | 76 ++++---------- view/state/kind/locator/options.go | 19 ++-- 18 files changed, 256 insertions(+), 128 deletions(-) diff --git a/cmd/datly/build.yaml b/cmd/datly/build.yaml index 9b2fefa03..92ae41d35 100644 --- a/cmd/datly/build.yaml +++ b/cmd/datly/build.yaml @@ -9,7 +9,7 @@ pipeline: set_sdk: action: sdk.set target: $target - sdk: go:1.23 + sdk: go:1.25.1 build: action: exec:run target: $target diff --git a/e2e/local/build.yaml b/e2e/local/build.yaml index b9b3f87b3..78fd0be42 100644 --- a/e2e/local/build.yaml +++ b/e2e/local/build.yaml @@ -14,7 +14,7 @@ pipeline: set_sdk: action: sdk.set target: $target - sdk: go:1.23 + sdk: go:1.25.1 buildValidator: action: exec:run diff --git a/e2e/local/regression/regression.yaml b/e2e/local/regression/regression.yaml index f4ce9a18c..10cbbf501 100644 --- a/e2e/local/regression/regression.yaml +++ b/e2e/local/regression/regression.yaml @@ -5,7 +5,7 @@ pipeline: set_sdk: action: sdk.set target: $target - sdk: go:1.23 + sdk: go:1.25.1 database: action: run diff --git a/gateway/router/handler.go b/gateway/router/handler.go index ce7398a41..ef3b129c3 100644 --- a/gateway/router/handler.go +++ b/gateway/router/handler.go @@ -261,11 +261,8 @@ func (r *Handler) writeErrorResponse(ctx context.Context, w http.ResponseWriter, http.Error(w, err.Error(), http.StatusInternalServerError) return } - if aComponent.Content.Marshaller.JSON.CanMarshal() { - data, err = aComponent.Marshaller.JSON.Codec.Marshal(aResponse.State()) - } else { - data, err = aComponent.Marshaller.JSON.JsonMarshaller.Marshal(aResponse.State()) - } + mf := aComponent.MarshalFunc() + data, err = mf(aResponse.State()) if err != nil { w.Write(data) if execCtx != nil { @@ -462,8 +459,10 @@ func (r *Handler) handleComponent(ctx context.Context, request *http.Request, aC options.Append(response.WithHeader("Content-Disposition", fmt.Sprintf(`attachment; filename="%s.xlsx"`, aComponent.Output.GetTitle()))) } } + // Use component-level marshaller with request-scoped options filters := aComponent.Exclusion(aSession.State()) - data, err := aComponent.Content.Marshal(format, aComponent.Output.Field(), output, filters) + mf := aComponent.MarshalFunc(repository.WithRequest(request), repository.WithFormat(format), repository.WithFilters(filters)) + data, err := mf(output) if err != nil { return nil, response.NewError(500, fmt.Sprintf("failed to marshal response: %v", err), response.WithError(err)) } @@ -501,13 +500,9 @@ func (r *Handler) marshalComponentOutput(output interface{}, aComponent *reposit case []byte: return response.NewBuffered(response.WithBytes(actual)), nil default: - var data []byte - var err error - if aComponent.Content.Marshaller.JSON.CanMarshal() { - data, err = aComponent.Content.Marshaller.JSON.Codec.Marshal(output) - } else { - data, err = aComponent.Content.Marshaller.JSON.JsonMarshaller.Marshal(output) - } + // Default to JSON marshalling using component-level marshaller + mf := aComponent.MarshalFunc() + data, err := mf(output) if err != nil { return nil, response.NewError(http.StatusInternalServerError, err.Error(), response.WithError(err)) } diff --git a/gateway/runtime/apigw/deploy.yaml b/gateway/runtime/apigw/deploy.yaml index 5a7d85dc6..0511616b0 100644 --- a/gateway/runtime/apigw/deploy.yaml +++ b/gateway/runtime/apigw/deploy.yaml @@ -25,7 +25,7 @@ pipeline: set_sdk: action: sdk.set target: $target - sdk: go:1.21 + sdk: go:1.25.1 build: package: diff --git a/gateway/runtime/gcr/deploy.yaml b/gateway/runtime/gcr/deploy.yaml index 0257ecf68..7cc921e0d 100644 --- a/gateway/runtime/gcr/deploy.yaml +++ b/gateway/runtime/gcr/deploy.yaml @@ -18,7 +18,7 @@ pipeline: setSdk: action: sdk.set target: $target - sdk: go:1.21 + sdk: go:1.25.1 deploy: buildBinary: diff --git a/gateway/runtime/lambda/deploy.yaml b/gateway/runtime/lambda/deploy.yaml index 6a7fbae63..0ea83df8d 100644 --- a/gateway/runtime/lambda/deploy.yaml +++ b/gateway/runtime/lambda/deploy.yaml @@ -27,7 +27,7 @@ pipeline: set_sdk: action: sdk.set target: $target - sdk: go:1.21 + sdk: go:1.25.1 build: package: diff --git a/go.mod b/go.mod index 16e40247d..4aa5d5121 100644 --- a/go.mod +++ b/go.mod @@ -48,14 +48,14 @@ require ( require ( github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d github.com/viant/firebase v0.1.1 - github.com/viant/jsonrpc v0.7.5 - github.com/viant/mcp v0.5.2 + github.com/viant/jsonrpc v0.9.0 + github.com/viant/mcp v0.6.0 github.com/viant/mcp-protocol v0.5.7 github.com/viant/structology v0.6.1 github.com/viant/tagly v0.2.2 - github.com/viant/xdatly v0.5.4-0.20250806192028-819cadf93282 + github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 - github.com/viant/xdatly/handler v0.0.0-20250806192028-819cadf93282 + github.com/viant/xdatly/handler v0.0.0-20251006174948-cb34263ae8aa github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a github.com/viant/xlsy v0.3.1 @@ -105,6 +105,7 @@ require ( github.com/go-errors/errors v1.5.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -127,6 +128,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect + github.com/viant/gosh v0.2.1 // indirect github.com/viant/igo v0.2.0 // indirect github.com/viant/x v0.3.0 // indirect github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca // indirect diff --git a/go.sum b/go.sum index 3914ffdf7..3912edbdf 100644 --- a/go.sum +++ b/go.sum @@ -1124,10 +1124,10 @@ github.com/viant/govalidator v0.3.1 h1:V7f/KgfzbP8fVDc+Kj+jyPvfXxMr2N1x7srOlDV6l github.com/viant/govalidator v0.3.1/go.mod h1:D35Dwx0R8rR1knRxhlseoYvOkiqo24kpMg1/o977i9Y= github.com/viant/igo v0.2.0 h1:ygWmTCinnGPaeV7omJLiyneOpzYZ5kiw7oYz7mUJZVQ= github.com/viant/igo v0.2.0/go.mod h1:7V6AWsLhKWeGzXNTNH3AZiIEKa0m33DrQbdWtapsI74= -github.com/viant/jsonrpc v0.7.5 h1:QiLEVl5nP7j5i55jDQp4HUURKJLf+ENPafTlJT13u38= -github.com/viant/jsonrpc v0.7.5/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= -github.com/viant/mcp v0.5.2 h1:m0Z7LdYOQOs7MhYg7Ql9nZlJDiedHfC86VNRzfKg3Gs= -github.com/viant/mcp v0.5.2/go.mod h1:ybylH9mD3/aVLrT8KQsakmsKxtfw7Kpo4A9hmTuetGw= +github.com/viant/jsonrpc v0.9.0 h1:vTZsApJxTd3Y50ygOBs8HKCJ24NrwgCa7lqG1oYXpdE= +github.com/viant/jsonrpc v0.9.0/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= +github.com/viant/mcp v0.6.0 h1:+BCsLSW5pux07avEhS550hZno8Y5ZKKSfdLm6NHRU+8= +github.com/viant/mcp v0.6.0/go.mod h1:fb5wpE9kc/R32pNE4Pdo1DR4ZW6+0em3rsFuBHoqmp4= github.com/viant/mcp-protocol v0.5.7 h1:3ifypMAy+oUjQEAsq+XwrAhE/B/3eIes4yXdhoRF9Eo= github.com/viant/mcp-protocol v0.5.7/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b h1:3q166tV28yFdbFV+tXXjH7ViKAmgAgGdoWzMtvhQv28= @@ -1138,8 +1138,6 @@ github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns= github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.17.7 h1:drUv3N8mOboq917gnmcT9zC4G9vj4jU11bO/SsLpmc8= -github.com/viant/sqlx v0.17.7/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/sqlx v0.17.8 h1:YxGTrXC2B1JmDz1qp8G+G9hGPk7XCRevWN1E4E+ZlCI= github.com/viant/sqlx v0.17.8/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.6.1 h1:Forza+RF/1tmlQFk9ABNhu+IQ8vMAqbYM6FOsYtGh9E= @@ -1156,12 +1154,12 @@ github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 h1:zKk+6hqUipkJXCPCH github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= github.com/viant/x v0.3.0 h1:/3A0z/uySGxMo6ixH90VAcdjI00w5e3REC1zg5hzhJA= github.com/viant/x v0.3.0/go.mod h1:54jP3qV+nnQdNDaWxEwGTAAzCu9sx9er9htiwTW/Mcw= -github.com/viant/xdatly v0.5.4-0.20250806192028-819cadf93282 h1:CqRQGsior7arN1lQA11oCoWdC/LZv1ObhCOGpdwvR3k= -github.com/viant/xdatly v0.5.4-0.20250806192028-819cadf93282/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= +github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa h1:o5o1CmraGb/LSpfrgmDoMdi9JJGjiopH8cmX98ukJS0= +github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUBDzc4rWacOYvAq/TKrTV0agvMF0gwm2gaoHI= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259/go.mod h1:fb8YgbVadk8X5ZLz49LWGzWmQlZd7Y/I5wE0ru44bIo= -github.com/viant/xdatly/handler v0.0.0-20250806192028-819cadf93282 h1:oNhkNyC6bRBifxWLyd7MTEFmCMwfg1LaAjKAmubrWCM= -github.com/viant/xdatly/handler v0.0.0-20250806192028-819cadf93282/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= +github.com/viant/xdatly/handler v0.0.0-20251006174948-cb34263ae8aa h1:UzX1wB23RMENSKF5X0fQZR/cIy7wB7z2ODWCIm358IQ= +github.com/viant/xdatly/handler v0.0.0-20251006174948-cb34263ae8aa/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 h1:G+e1MMDxQXUPPlAXVNlRqSLTLra7udGQZUu3hnr0Y8M= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52/go.mod h1:LJN2m8xJjtYNCvyvNrVanJwvzj8+hYCuPswL8H4qRG0= github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a h1:jecH7mH63gj1zJwD18SdvSHM9Ttr9FEOnhHkYfkCNkI= diff --git a/internal/translator/rule.go b/internal/translator/rule.go index 98ba973a2..ec42c5388 100644 --- a/internal/translator/rule.go +++ b/internal/translator/rule.go @@ -67,6 +67,7 @@ type ( IsGeneratation bool XMLUnmarshalType string `json:",omitempty"` JSONUnmarshalType string `json:",omitempty"` + JSONMarshalType string `json:",omitempty"` OutputParameter *inference.Parameter } @@ -132,6 +133,7 @@ func (r *Rule) DSQLSetting() interface{} { DocURLs []string `json:",omitempty"` Internal bool `json:",omitempty"` JSONUnmarshalType string `json:",omitempty"` + JSONMarshalType string `json:",omitempty"` Connector string `json:",omitempty"` contract.ModelContextProtocol contract.Meta @@ -148,6 +150,7 @@ func (r *Rule) DSQLSetting() interface{} { DocURLs: r.DocURLs, Internal: r.Internal, JSONUnmarshalType: r.JSONUnmarshalType, + JSONMarshalType: r.JSONMarshalType, Connector: r.Connector, ModelContextProtocol: r.ModelContextProtocol, Meta: r.Meta, @@ -321,7 +324,9 @@ func (r *Rule) applyDefaults() { if r.XMLUnmarshalType != "" { r.Route.Content.Marshaller.XML.TypeName = r.XMLUnmarshalType } - if r.JSONUnmarshalType != "" { + if r.JSONMarshalType != "" { + r.Route.Content.Marshaller.JSON.TypeName = r.JSONMarshalType + } else if r.JSONUnmarshalType != "" { r.Route.Content.Marshaller.JSON.TypeName = r.JSONUnmarshalType } } diff --git a/internal/translator/service.go b/internal/translator/service.go index 70e71d683..6f7cbcbc2 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -329,6 +329,15 @@ func (s *Service) persistRouterRule(ctx context.Context, resource *Resource, ser } route.Component.Meta = resource.Rule.Meta + if route.Component.Meta.DescriptionURI != "" { + URL := url.Join(baseRuleURL, route.Component.Meta.DescriptionURI) + description, err := s.fs.DownloadWithURL(ctx, URL) + if err != nil { + return fmt.Errorf("failed to download meta description: %v %w", URL, err) + } + route.Component.Meta.Description = string(description) + } + route.ModelContextProtocol = resource.Rule.ModelContextProtocol if route.Handler != nil { if route.Component.Output.Type.Schema == nil { @@ -362,7 +371,10 @@ func (s *Service) persistRouterRule(ctx context.Context, resource *Resource, ser if resource.Rule.XMLUnmarshalType != "" { route.Content.Marshaller.XML.TypeName = resource.Rule.XMLUnmarshalType } - if resource.Rule.JSONUnmarshalType != "" { + // JSON marshaller/unmarshaller customization: prefer MarshalType if provided, fallback to UnmarshalType. + if resource.Rule.JSONMarshalType != "" { + route.Content.Marshaller.JSON.TypeName = resource.Rule.JSONMarshalType + } else if resource.Rule.JSONUnmarshalType != "" { route.Content.Marshaller.JSON.TypeName = resource.Rule.JSONUnmarshalType } route.Component.Output.DataFormat = resource.Rule.DataFormat diff --git a/repository/component.go b/repository/component.go index 109ded23a..5a22bdee6 100644 --- a/repository/component.go +++ b/repository/component.go @@ -15,7 +15,7 @@ import ( "github.com/viant/datly/gateway/router/marshal/json" "github.com/viant/datly/internal/setter" "github.com/viant/datly/repository/async" - "github.com/viant/datly/repository/content" + content "github.com/viant/datly/repository/content" "github.com/viant/datly/repository/contract" "github.com/viant/datly/repository/handler" "github.com/viant/datly/repository/version" @@ -261,27 +261,143 @@ func (c *Component) IOConfig() *config.IOConfig { } func (c *Component) UnmarshalFunc(request *http.Request) shared.Unmarshal { - contentType := request.Header.Get(content.HeaderContentType) - setter.SetStringIfEmpty(&contentType, request.Header.Get(strings.ToLower(content.HeaderContentType))) + // Delegate to options-based variant for symmetry and centralization. + return c.UnmarshalFor(WithUnmarshalRequest(request)) +} + +// UnmarshalOption configures unmarshal behavior for Component.UnmarshalFor. +type UnmarshalOption func(*unmarshalOptions) + +type unmarshalOptions struct { + request *http.Request + contentType string + interceptors json.UnmarshalerInterceptors +} + +// WithUnmarshalRequest supplies an http request for content-type detection and transforms. +func WithUnmarshalRequest(r *http.Request) UnmarshalOption { + return func(o *unmarshalOptions) { o.request = r } +} + +// WithContentType overrides the detected content type. +func WithContentType(ct string) UnmarshalOption { + return func(o *unmarshalOptions) { o.contentType = ct } +} + +// WithUnmarshalInterceptors adds/overrides JSON path interceptors. +func WithUnmarshalInterceptors(m json.UnmarshalerInterceptors) UnmarshalOption { + return func(o *unmarshalOptions) { + if o.interceptors == nil { + o.interceptors = json.UnmarshalerInterceptors{} + } + for k, v := range m { + o.interceptors[k] = v + } + } +} + +// UnmarshalFor returns a request-scoped unmarshal function applying content-type detection and transforms. +func (c *Component) UnmarshalFor(opts ...UnmarshalOption) shared.Unmarshal { + options := &unmarshalOptions{} + for _, opt := range opts { + if opt != nil { + opt(options) + } + } + + // Resolve content type if request present + contentType := options.contentType + if contentType == "" && options.request != nil { + contentType = options.request.Header.Get(content.HeaderContentType) + setter.SetStringIfEmpty(&contentType, options.request.Header.Get(strings.ToLower(content.HeaderContentType))) + } + switch contentType { case content.XMLContentType: return c.Content.Marshaller.XML.Unmarshal case content.CSVContentType: return c.Content.CSV.Unmarshal - default: - switch c.Output.DataFormat { - case content.XMLFormat: - return c.Content.Marshaller.XML.Unmarshal + } + // Fallback to data format preference when no content type or not matched + if c.Output.DataFormat == content.XMLFormat { + return c.Content.Marshaller.XML.Unmarshal + } + + // Build JSON path interceptors from component transforms and any user-provided ones + interceptors := options.interceptors + if interceptors == nil { + interceptors = json.UnmarshalerInterceptors{} + } + if options.request != nil { + for _, transform := range c.UnmarshallerInterceptors() { + interceptors[transform.Path] = c.transformFn(options.request, transform) } } - jsonPathInterceptor := json.UnmarshalerInterceptors{} - unmarshallerInterceptors := c.UnmarshallerInterceptors() - for i := range unmarshallerInterceptors { - transform := unmarshallerInterceptors[i] - jsonPathInterceptor[transform.Path] = c.transformFn(request, transform) + + req := options.request // capture for closure + return func(data []byte, dest interface{}) error { + if len(interceptors) > 0 || req != nil { + return c.Content.Marshaller.JSON.JsonMarshaller.Unmarshal(data, dest, interceptors, req) + } + return c.Content.Marshaller.JSON.JsonMarshaller.Unmarshal(data, dest) } - return func(bytes []byte, i interface{}) error { - return c.Content.Marshaller.JSON.JsonMarshaller.Unmarshal(bytes, i, jsonPathInterceptor, request) +} + +// MarshalOption configures marshal behavior for Component.MarshalFunc. +type MarshalOption func(*marshalOptions) + +type marshalOptions struct { + request *http.Request + format string + field string + filters []*json.FilterEntry +} + +// WithRequest supplies an http request for deriving format and state-based exclusions. +func WithRequest(r *http.Request) MarshalOption { return func(o *marshalOptions) { o.request = r } } + +// WithFormat overrides the output format (e.g. content.JSONFormat, content.CSVFormat, etc.). +func WithFormat(format string) MarshalOption { return func(o *marshalOptions) { o.format = format } } + +// WithField overrides the field used by tabular JSON embedding. +func WithField(field string) MarshalOption { return func(o *marshalOptions) { o.field = field } } + +// WithFilters sets explicit JSON field filters (exclusion-based projection). +func WithFilters(filters []*json.FilterEntry) MarshalOption { + return func(o *marshalOptions) { o.filters = filters } +} + +// MarshalFunc returns a request-scoped marshaller closure applying options like format and exclusions. +// If no format is specified, it defaults to JSON for non-reader services and derives from request for readers. +func (c *Component) MarshalFunc(opts ...MarshalOption) shared.Marshal { + options := &marshalOptions{} + for _, opt := range opts { + if opt != nil { + opt(options) + } + } + + // Resolve format + format := options.format + if format == "" { + if options.request != nil && c.Service == service.TypeReader { + format = c.Output.Format(options.request.URL.Query()) + } else { + format = content.JSONFormat + } + } + + // Resolve field (used for tabular JSON embedding) + field := options.field + if field == "" { + field = c.Output.Field() + } + + // Resolve filters (explicit only) + filters := options.filters + + return func(src interface{}) ([]byte, error) { + return c.Content.Marshal(format, field, src, filters) } } diff --git a/repository/contract/meta.go b/repository/contract/meta.go index 878128b5e..8b0cfa71a 100644 --- a/repository/contract/meta.go +++ b/repository/contract/meta.go @@ -7,8 +7,9 @@ import ( // MCP Model Configuration Protocol path integration type Meta struct { - Name string `json:",omitempty" yaml:"Name"` // name of the MCP - Description string `json:",omitempty" yaml:"Description"` // optional description for documentation purposes + Name string `json:",omitempty" yaml:"Name"` // name of the MCP + Description string `json:",omitempty" yaml:"Description"` // optional description for documentation purposes + DescriptionURI string `json:",omitempty" yaml:"DescriptionURI"` } type ModelContextProtocol struct { diff --git a/service.go b/service.go index 68f537d55..78d5acf85 100644 --- a/service.go +++ b/service.go @@ -4,6 +4,7 @@ import ( "context" _ "embed" "fmt" + "github.com/viant/cloudless/async/mbus" "github.com/viant/datly/gateway" "github.com/viant/datly/repository" @@ -17,20 +18,22 @@ import ( "github.com/viant/datly/service/session" "github.com/viant/datly/view" "github.com/viant/datly/view/extension" + "github.com/viant/datly/view/state/kind/locator" verifier2 "github.com/viant/scy/auth/jwt/verifier" hstate "github.com/viant/xdatly/handler/state" + "net/http" + nurl "net/url" + "reflect" + "strings" + "time" + "github.com/viant/datly/view/state" "github.com/viant/scy/auth/jwt" "github.com/viant/scy/auth/jwt/signer" "github.com/viant/structology" "github.com/viant/xdatly/codec" xhandler "github.com/viant/xdatly/handler" - "net/http" - nurl "net/url" - "reflect" - "strings" - "time" ) //go:embed Version @@ -50,16 +53,18 @@ type ( } sessionOptions struct { - request *http.Request - resource state.Resource - form *hstate.Form + request *http.Request + resource state.Resource + form *hstate.Form + querySelectors []*hstate.NamedQuerySelector } SessionOption func(o *sessionOptions) operateOptions struct { - path *contract.Path - component *repository.Component - session *session.Session + path *contract.Path + component *repository.Component + session *session.Session + output interface{} input interface{} sessionOptions []SessionOption @@ -151,6 +156,12 @@ func WithForm(form *hstate.Form) SessionOption { } } +func WithQuerySelectors(selectors ...*hstate.NamedQuerySelector) SessionOption { + return func(o *sessionOptions) { + o.querySelectors = selectors + } +} + func WithStateResource(resource state.Resource) SessionOption { return func(o *sessionOptions) { o.resource = resource @@ -160,6 +171,9 @@ func WithStateResource(resource state.Resource) SessionOption { func (s *Service) NewComponentSession(aComponent *repository.Component, opts ...SessionOption) *session.Session { sessionOpt := newSessionOptions(opts) options := aComponent.LocatorOptions(sessionOpt.request, sessionOpt.form, aComponent.UnmarshalFunc(sessionOpt.request)) + if sessionOpt.querySelectors != nil { + options = append(options, locator.WithQuerySelectors(sessionOpt.querySelectors)) + } aSession := session.New(aComponent.View, session.WithLocatorOptions(options...), session.WithAuth(s.repository.Auth()), session.WithStateResource(sessionOpt.resource), session.WithOperate(s.operator.Operate)) diff --git a/service/executor/extension/session.go b/service/executor/extension/session.go index d52f72f5d..9fbfe51fc 100644 --- a/service/executor/extension/session.go +++ b/service/executor/extension/session.go @@ -19,7 +19,7 @@ import ( type ( Session struct { sqlService SqlServiceFn - stater state.Stater + injector state.Injector validator *validator.Service differ *differ.Service mbus *xmbus.Service @@ -92,7 +92,7 @@ func (s *Session) Db(opts ...sqlx.Option) (*sqlx.Service, error) { } func (s *Session) Stater() *state.Service { - return state.New(s.stater) + return state.New(s.injector) } func (s *Session) FlushTemplate(ctx context.Context) error { @@ -148,8 +148,8 @@ func WithMessageBus(messageBusses []*mbus.Resource) Option { } } -func WithStater(stater state.Stater) Option { +func WithStater(injector state.Injector) Option { return func(s *Session) { - s.stater = stater + s.injector = injector } } diff --git a/service/session/selector.go b/service/session/selector.go index 895a953d2..e4a39b8d3 100644 --- a/service/session/selector.go +++ b/service/session/selector.go @@ -3,13 +3,14 @@ package session import ( "context" "fmt" + "strconv" + "strings" + "github.com/viant/datly/service/session/criteria" "github.com/viant/datly/view" "github.com/viant/tagly/format/text" "github.com/viant/xdatly/codec" "github.com/viant/xdatly/handler/response" - "strconv" - "strings" ) func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, opts *Options) (err error) { @@ -18,6 +19,14 @@ func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, return nil } + selector := s.state.Lookup(ns.View) + + if opts != nil && opts.locatorOpt != nil && opts.locatorOpt.QuerySelectors != nil { //override selector + querySelectors := opts.locatorOpt.QuerySelectors + if namedSelector := querySelectors.Find(ns.View.Name); namedSelector != nil { + selector.QuerySelector = namedSelector.QuerySelector + } + } if err = s.populateFieldQuerySelector(ctx, ns, opts); err != nil { return response.NewParameterError(ns.View.Name, selectorParameters.FieldsParameter.Name, err) } @@ -36,7 +45,6 @@ func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, if err = s.populatePageQuerySelector(ctx, ns, opts); err != nil { return response.NewParameterError(ns.View.Name, selectorParameters.PageParameter.Name, err) } - selector := s.state.Lookup(ns.View) if selector.Limit == 0 && selector.Offset != 0 { return fmt.Errorf("can't use offset without limit - view: %v", ns.View.Name) } diff --git a/view/state.go b/view/state.go index a7ff10ece..8bfd3715b 100644 --- a/view/state.go +++ b/view/state.go @@ -1,12 +1,14 @@ package view import ( + "strings" + "sync" + "github.com/viant/datly/view/state/predicate" "github.com/viant/sqlx/io/read/cache" "github.com/viant/structology" "github.com/viant/tagly/format/text" - "strings" - "sync" + "github.com/viant/xdatly/handler/state" ) // Statelet allows customizing View fetched from Database @@ -14,9 +16,18 @@ type ( //InputType represents view state Statelet struct { - Template *structology.State - QuerySelector + //SELECTORS + DatabaseFormat text.CaseFormat + OutputFormat text.CaseFormat + Template *structology.State + state.QuerySelector QuerySettings + filtersMu sync.Mutex + initialized bool + _columnNames map[string]bool + result *cache.ParmetrizedQuery + predicate.Filters + Ignore bool } QuerySettings struct { @@ -24,42 +35,8 @@ type ( SyncFlag bool ContentFormat string } - - QuerySelector struct { - //SELECTORS - DatabaseFormat text.CaseFormat - OutputFormat text.CaseFormat - Columns []string `json:",omitempty"` - Fields []string `json:",omitempty"` - OrderBy string `json:",omitempty"` - Offset int `json:",omitempty"` - Limit int `json:",omitempty"` - - Criteria string `json:",omitempty"` - Placeholders []interface{} `json:",omitempty"` - Page int - Ignore bool - predicate.Filters - - initialized bool - _columnNames map[string]bool - result *cache.ParmetrizedQuery - filtersMu sync.Mutex - } ) -func (s *QuerySelector) CurrentLimit() int { - return s.Limit -} - -func (s *QuerySelector) CurrentOffset() int { - return s.Offset -} - -func (s *QuerySelector) CurrentPage() int { - return s.Page -} - // Init initializes Statelet func (s *Statelet) Init(aView *View) { if aView != nil && s.Template == nil && aView.Template.stateType != nil { @@ -72,12 +49,12 @@ func (s *Statelet) Init(aView *View) { } // Has checks if Field is present in Template.Columns -func (s *QuerySelector) Has(field string) bool { +func (s *Statelet) Has(field string) bool { _, ok := s._columnNames[field] return ok } -func (s *QuerySelector) Add(fieldName string, isHolder bool) { +func (s *Statelet) Add(fieldName string, isHolder bool) { toLower := strings.ToLower(fieldName) if _, ok := s._columnNames[toLower]; ok { return @@ -95,28 +72,21 @@ func (s *QuerySelector) Add(fieldName string, isHolder bool) { } } -func (s *QuerySelector) SetCriteria(expanded string, placeholders []interface{}) { - s.Criteria = expanded - s.Placeholders = placeholders -} - // AppendFilters safely appends filters to the selector's Filters to avoid data races. func (s *Statelet) AppendFilters(filters predicate.Filters) { if len(filters) == 0 { return } - s.QuerySelector.filtersMu.Lock() - s.QuerySelector.Filters = append(s.QuerySelector.Filters, filters...) - s.QuerySelector.filtersMu.Unlock() + s.filtersMu.Lock() + s.Filters = append(s.Filters, filters...) + s.filtersMu.Unlock() } // NewStatelet creates a selector func NewStatelet() *Statelet { return &Statelet{ - QuerySelector: QuerySelector{ - _columnNames: map[string]bool{}, - initialized: true, - }, + _columnNames: map[string]bool{}, + initialized: true, } } @@ -127,7 +97,7 @@ type State struct { } // QuerySelector returns query selector -func (s *State) QuerySelector(view *View) *QuerySelector { +func (s *State) QuerySelector(view *View) *state.QuerySelector { statelet := s.Lookup(view) if statelet == nil { return nil diff --git a/view/state/kind/locator/options.go b/view/state/kind/locator/options.go index d591e8666..9cf99b2eb 100644 --- a/view/state/kind/locator/options.go +++ b/view/state/kind/locator/options.go @@ -21,12 +21,13 @@ import ( // Options represents locator options type ( Options struct { - request *http.Request - Form *hstate.Form - Path map[string]string - Query url.Values - Header http.Header - Body []byte + request *http.Request + Form *hstate.Form + QuerySelectors hstate.QuerySelectors + Path map[string]string + Query url.Values + Header http.Header + Body []byte fromError error Parent *KindLocator @@ -181,6 +182,12 @@ func WithInputParameters(parameters state.NamedParameters) Option { } } +func WithQuerySelectors(selectors hstate.QuerySelectors) Option { + return func(o *Options) { + o.QuerySelectors = selectors + } +} + // WithPathParameters create with path parameters options func WithPathParameters(parameters map[string]string) Option { return func(o *Options) { From f09ab1ff9144f503e07471821ca358a0ec3f938a Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 7 Oct 2025 07:41:48 -0700 Subject: [PATCH 040/279] expose marshallers, move queryselector to xdatly --- service.go | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/service.go b/service.go index 78d5acf85..08fc5eb05 100644 --- a/service.go +++ b/service.go @@ -8,14 +8,17 @@ import ( "github.com/viant/cloudless/async/mbus" "github.com/viant/datly/gateway" "github.com/viant/datly/repository" + rcontent "github.com/viant/datly/repository/content" "github.com/viant/datly/repository/contract" "github.com/viant/datly/repository/locator/component/dispatcher" + srv "github.com/viant/datly/service" sjwt "github.com/viant/datly/service/auth/jwt" "github.com/viant/datly/service/auth/mock" "github.com/viant/datly/service/executor" "github.com/viant/datly/service/operator" "github.com/viant/datly/service/reader" "github.com/viant/datly/service/session" + "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/datly/view/extension" "github.com/viant/datly/view/state/kind/locator" @@ -282,6 +285,67 @@ func (s *Service) PopulateInput(ctx context.Context, aComponent *repository.Comp return nil } +// GetMarshaller prepares a request-scoped marshaller closure and resolved content type for the given component path. +// It preserves existing behavior for readers (format derived from query) and defaults to JSON otherwise. +func (s *Service) GetMarshaller(r *http.Request, methodAndPath string, extra ...repository.MarshalOption) (marshal shared.Marshal, contentType string, comp *repository.Component, err error) { + comp, err = s.Component(r.Context(), methodAndPath) + if err != nil || comp == nil { + if err == nil { + err = fmt.Errorf("component not found: %s", methodAndPath) + } + return nil, "", nil, err + } + + // Build component session to populate state (for exclusion filters) + sess := s.NewComponentSession(comp, WithRequest(r), WithStateResource(comp.View.Resource())) + // Compute JSON field filters from populated state + filters := comp.Exclusion(sess.State()) + + // Optional format override from query parameter `format` + override := strings.TrimSpace(r.URL.Query().Get("format")) + + var opts []repository.MarshalOption + opts = append(opts, repository.WithRequest(r), repository.WithFilters(filters)) + if override != "" { + opts = append(opts, repository.WithFormat(override)) + } + if len(extra) > 0 { + opts = append(opts, extra...) + } + + // Prepare marshaller closure + marshal = comp.MarshalFunc(opts...) + + // Resolve content type for headers + resolved := override + if resolved == "" && comp.Service == srv.TypeReader { + resolved = comp.Output.Format(r.URL.Query()) + } + if resolved == "" { + resolved = rcontent.JSONFormat + } + contentType = comp.Output.ContentType(resolved) + return marshal, contentType, comp, nil +} + +// GetUnmarshaller prepares a request-scoped unmarshaller for the given component path. +func (s *Service) GetUnmarshaller(r *http.Request, methodAndPath string, extra ...repository.UnmarshalOption) (unmarshal shared.Unmarshal, comp *repository.Component, err error) { + comp, err = s.Component(r.Context(), methodAndPath) + if err != nil || comp == nil { + if err == nil { + err = fmt.Errorf("component not found: %s", methodAndPath) + } + return nil, nil, err + } + var opts []repository.UnmarshalOption + opts = append(opts, repository.WithUnmarshalRequest(r)) + if len(extra) > 0 { + opts = append(opts, extra...) + } + unmarshal = comp.UnmarshalFor(opts...) + return unmarshal, comp, nil +} + // Read reads data from a view func (s *Service) Read(ctx context.Context, locator string, dest interface{}, option ...reader.Option) error { aView, err := s.View(ctx, wrapWithMethod(http.MethodGet, locator)) From 217831c16c85c2684e354786962b05b104308cad Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 7 Oct 2025 08:17:01 -0700 Subject: [PATCH 041/279] expose marshallers, move queryselector to xdatly --- .../router/marshal/json/marshaller_struct.go | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/gateway/router/marshal/json/marshaller_struct.go b/gateway/router/marshal/json/marshaller_struct.go index 23e2392a3..e3d962191 100644 --- a/gateway/router/marshal/json/marshaller_struct.go +++ b/gateway/router/marshal/json/marshaller_struct.go @@ -1,16 +1,18 @@ package json import ( + "reflect" + "strings" + "unicode" + "unsafe" + "github.com/francoispqt/gojay" "github.com/viant/datly/gateway/router/marshal/config" + "github.com/viant/datly/view/tags" structology "github.com/viant/structology" "github.com/viant/tagly/format" "github.com/viant/tagly/format/text" xunsafe "github.com/viant/xunsafe" - "reflect" - "strings" - "unicode" - "unsafe" ) type ( @@ -254,6 +256,15 @@ func (s *structMarshaller) createStructMarshallers(fields *groupedFields, path s if err != nil { return nil, err } + if dTag.Name == "" { //fallback to parameter + if parameterTag := field.Tag.Get("parameter"); parameterTag != "" { + if aTag, _ := tags.Parse(field.Tag, nil, tags.ParameterTag); aTag != nil && aTag.Parameter != nil { + if aTag.Parameter.Kind == "body" { + dTag.Name = aTag.Parameter.In + } + } + } + } elemType := field.Type // Unwrap nested pointers/slices to detect self-references like []*T or [][]*T @@ -315,6 +326,7 @@ func (s *structMarshaller) newFieldMarshaller(marshallers *[]*marshallerWithFiel } else if s.config.CaseFormat != "" { jsonName = formatName(jsonName, s.config.CaseFormat) } + path, outputPath = addToPath(path, field.Name), addToPath(outputPath, jsonName) xField := xunsafe.NewField(field) From 672e82b7059aa4efea643e93c275708e4ce019cf Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 8 Oct 2025 09:37:27 -0700 Subject: [PATCH 042/279] added generic router --- go.mod | 1 + router.go | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++++ service.go | 56 ++++++++++++++++++++++-- 3 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 router.go diff --git a/go.mod b/go.mod index 4aa5d5121..7af2f0c20 100644 --- a/go.mod +++ b/go.mod @@ -165,3 +165,4 @@ require ( modernc.org/strutil v1.1.3 // indirect modernc.org/token v1.0.0 // indirect ) + diff --git a/router.go b/router.go new file mode 100644 index 000000000..5e556376e --- /dev/null +++ b/router.go @@ -0,0 +1,125 @@ +package datly + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/xdatly/handler/response" + hstate "github.com/viant/xdatly/handler/state" +) + +type Handler[T any] func(ctx context.Context, service T, request *http.Request, injector hstate.Injector, extra ...OperateOption) (interface{}, error) + +type Route[T any] struct { + dao *Service + handler Handler[T] + service T + path *contract.Path + component *repository.Component +} + +func (r Route[T]) ensureComponent(ctx context.Context) (*repository.Component, error) { + if r.component == nil { + var err error + r.component, err = r.dao.repository.Registry().Lookup(ctx, r.path) + if err != nil { + return nil, err + } + } + return r.component, nil +} + +func (r Route[T]) Run(ctx context.Context, writer http.ResponseWriter, request *http.Request) error { + marshaller, contentType, _, err := r.dao.getMarshaller(request, r.component) + if err != nil { + return fmt.Errorf("failed to lookup marshaller: %w", err) + } + injector, err := r.dao.GetInjector(request, r.component) + if err != nil { + return fmt.Errorf("failed to lookup injector: %w", err) + } + selectors := []*hstate.NamedQuerySelector{} + values := request.URL.Query() + if page := values.Get("page"); page != "" { + selector := &hstate.NamedQuerySelector{Name: r.component.View.Name} + selector.Page, _ = strconv.Atoi(page) + selectors = append(selectors, selector) + } + result, err := r.handler(ctx, r.service, request, injector, WithSessionOptions(WithRequest(request), WithQuerySelectors(selectors...))) + var data []byte + if err != nil { + rErr, ok := err.(*response.Error) + if !ok { + rErr = response.NewError(http.StatusInternalServerError, err.Error()) + } + data, err = marshaller(rErr) + } else { + data, err = marshaller(result) + } + if err != nil { + http.Error(writer, err.Error(), http.StatusInternalServerError) + return nil + } + statusCode := http.StatusOK + statusCoder, ok := result.(response.StatusCoder) + if ok { + statusCode = statusCoder.StatusCode() + } + + writer.Header().Set("Content-Type", contentType) + writer.WriteHeader(statusCode) + _, err = writer.Write(data) + return err +} + +func newRoute[T any](dao *Service, path *contract.Path, component *repository.Component, service T, handler Handler[T]) *Route[T] { + return &Route[T]{path: path, handler: handler, dao: dao, component: component, service: service} +} + +type Router[T any] struct { + registry map[string]*Route[T] + dao *Service + service T +} + +type routeNotFound struct { + error +} + +// IsRouteNotFound checks if error is route not found +func IsRouteNotFound(err error) bool { + _, ok := err.(*routeNotFound) + return ok +} + +func (r *Router[T]) Run(writer http.ResponseWriter, request *http.Request) error { + aPath := contract.NewPath(request.Method, request.URL.Path) + component, err := r.dao.repository.Registry().Lookup(request.Context(), aPath) + if err != nil { + return &routeNotFound{err} + } + route, ok := r.registry[component.Path.Key()] + if !ok { + return &routeNotFound{errors.New("route not found")} + } + return route.Run(request.Context(), writer, request) +} + +func (r *Router[T]) Register(ctx context.Context, path *contract.Path, handler Handler[T]) error { + component, err := r.dao.repository.Registry().Lookup(ctx, path) + if err != nil { + return fmt.Errorf("failed to lookup component: %w for path: %+v", err, path) + } + route := newRoute[T](r.dao, path, component, r.service, handler) + r.registry[path.Key()] = route + return nil +} + +func NewRouter[T any](dao *Service, service T) *Router[T] { + return &Router[T]{registry: make(map[string]*Route[T]), dao: dao, service: service} +} diff --git a/service.go b/service.go index 08fc5eb05..1b67c0847 100644 --- a/service.go +++ b/service.go @@ -285,6 +285,15 @@ func (s *Service) PopulateInput(ctx context.Context, aComponent *repository.Comp return nil } +func (s *Service) GetInjector(r *http.Request, comp *repository.Component) (hstate.Injector, error) { + if err := s.ensureComponentInitialized(comp); err != nil { + return nil, err + } + // Build component session to populate state (for exclusion filters) + sess := s.NewComponentSession(comp, WithRequest(r), WithStateResource(comp.View.Resource())) + return sess, nil +} + // GetMarshaller prepares a request-scoped marshaller closure and resolved content type for the given component path. // It preserves existing behavior for readers (format derived from query) and defaults to JSON otherwise. func (s *Service) GetMarshaller(r *http.Request, methodAndPath string, extra ...repository.MarshalOption) (marshal shared.Marshal, contentType string, comp *repository.Component, err error) { @@ -295,6 +304,14 @@ func (s *Service) GetMarshaller(r *http.Request, methodAndPath string, extra ... } return nil, "", nil, err } + return s.getMarshaller(r, comp, extra...) +} + +func (s *Service) getMarshaller(r *http.Request, comp *repository.Component, extra ...repository.MarshalOption) (shared.Marshal, string, *repository.Component, error) { + // Ensure component content marshallers are initialized (defensive when invoked outside router lifecycle) + if err := s.ensureComponentInitialized(comp); err != nil { + return nil, "", nil, err + } // Build component session to populate state (for exclusion filters) sess := s.NewComponentSession(comp, WithRequest(r), WithStateResource(comp.View.Resource())) @@ -314,7 +331,7 @@ func (s *Service) GetMarshaller(r *http.Request, methodAndPath string, extra ... } // Prepare marshaller closure - marshal = comp.MarshalFunc(opts...) + marshal := comp.MarshalFunc(opts...) // Resolve content type for headers resolved := override @@ -324,7 +341,7 @@ func (s *Service) GetMarshaller(r *http.Request, methodAndPath string, extra ... if resolved == "" { resolved = rcontent.JSONFormat } - contentType = comp.Output.ContentType(resolved) + contentType := comp.Output.ContentType(resolved) return marshal, contentType, comp, nil } @@ -337,15 +354,46 @@ func (s *Service) GetUnmarshaller(r *http.Request, methodAndPath string, extra . } return nil, nil, err } + return s.getUnmarshaller(r, comp, extra...) +} + +func (s *Service) getUnmarshaller(r *http.Request, comp *repository.Component, extra ...repository.UnmarshalOption) (shared.Unmarshal, *repository.Component, error) { + // Ensure component content marshallers are initialized (defensive) + if err := s.ensureComponentInitialized(comp); err != nil { + return nil, nil, err + } var opts []repository.UnmarshalOption opts = append(opts, repository.WithUnmarshalRequest(r)) if len(extra) > 0 { opts = append(opts, extra...) } - unmarshal = comp.UnmarshalFor(opts...) + unmarshal := comp.UnmarshalFor(opts...) return unmarshal, comp, nil } +// ensureComponentInitialized defensively initializes component content marshallers when called from external contexts. +func (s *Service) ensureComponentInitialized(comp *repository.Component) error { + if comp == nil { + return fmt.Errorf("component was nil") + } + res := comp.View.GetResource() + if res == nil { + return nil + } + // If JSON marshaller already present, assume initialized. + if comp.Content.Marshaller.JSON.JsonMarshaller != nil { + return nil + } + // Initialize content marshallers as in Component.Init + if err := comp.Content.InitMarshaller(comp.IOConfig(), comp.Output.Exclude, comp.BodyType(), comp.OutputType()); err != nil { + return err + } + if err := comp.Content.Marshaller.Init(res.LookupType()); err != nil { + return err + } + return nil +} + // Read reads data from a view func (s *Service) Read(ctx context.Context, locator string, dest interface{}, option ...reader.Option) error { aView, err := s.View(ctx, wrapWithMethod(http.MethodGet, locator)) @@ -595,7 +643,7 @@ func (s *Service) HTTPHandler(ctx context.Context, options ...gateway.Option) (h return s.handler, nil } -// New creates a datly service, repository allows you to bootstrap empty or existing yaml repository +// New creates a dao dao, repository allows you to bootstrap empty or existing yaml repository func New(ctx context.Context, options ...repository.Option) (*Service, error) { options = append([]repository.Option{ repository.WithJWTSigner(mock.HmacJwtSigner()), From 1a97134326b303d66d19a251fba34c6acac226a6 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 8 Oct 2025 10:20:48 -0700 Subject: [PATCH 043/279] added generic router --- service.go | 1 + service/session/stater.go | 35 +++++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/service.go b/service.go index 1b67c0847..32a2cdcc4 100644 --- a/service.go +++ b/service.go @@ -179,6 +179,7 @@ func (s *Service) NewComponentSession(aComponent *repository.Component, opts ... } aSession := session.New(aComponent.View, session.WithLocatorOptions(options...), session.WithAuth(s.repository.Auth()), + session.WithComponent(aComponent), session.WithStateResource(sessionOpt.resource), session.WithOperate(s.operator.Operate)) return aSession } diff --git a/service/session/stater.go b/service/session/stater.go index d2bda7a34..a98548014 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -7,6 +7,8 @@ import ( "reflect" "runtime/debug" + "embed" + "github.com/viant/datly/utils/types" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind/locator" @@ -55,12 +57,33 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt destType := reflect.TypeOf(dest) sType := types.EnsureStruct(destType) stateType, ok := s.Types.Lookup(sType) - if !ok { - if stateType, err = state.NewType( - state.WithSchema(state.NewSchema(destType)), - state.WithResource(s.resource), - ); err != nil { - return err + + var embedFs *embed.FS + if embedder, ok := dest.(state.Embedder); ok { + embedFs = embedder.EmbedFS() + } + + if !ok && s.component != nil { + + if s.component.Input.Type.Type() != nil { + if destType == s.component.Input.Type.Type().Type() { + stateType = &s.component.Input.Type + } + } + if s.component.Output.Type.Type() != nil { + if destType == s.component.Output.Type.Type().Type() { + stateType = &s.component.Output.Type + } + } + + if stateType == nil { + if stateType, err = state.NewType( + state.WithSchema(state.NewSchema(destType)), + state.WithResource(s.resource), + state.WithFS(embedFs), + ); err != nil { + return err + } } s.Types.Put(stateType) } From a0ec4c34db4d347bee5d965e8b8acb4dc99391ad Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 8 Oct 2025 12:50:53 -0700 Subject: [PATCH 044/279] updated locator signature --- repository/locator/async/locator.go | 3 +- repository/locator/component/component.go | 2 +- repository/locator/meta/locator.go | 3 +- repository/locator/output/output.go | 3 +- router.go | 1 + service/executor/handler/locator/handler.go | 3 +- service/session/state.go | 5 +- view/state/kind/locator.go | 4 +- view/state/kind/locator/body.go | 54 +++++++++++++-------- view/state/kind/locator/constants.go | 3 +- view/state/kind/locator/context.go | 3 +- view/state/kind/locator/cookie.go | 3 +- view/state/kind/locator/data.go | 2 +- view/state/kind/locator/env.go | 3 +- view/state/kind/locator/form.go | 3 +- view/state/kind/locator/generator.go | 3 +- view/state/kind/locator/header.go | 3 +- view/state/kind/locator/http.go | 3 +- view/state/kind/locator/object.go | 3 +- view/state/kind/locator/parameter.go | 3 +- view/state/kind/locator/path.go | 3 +- view/state/kind/locator/query.go | 3 +- view/state/kind/locator/repeated.go | 2 +- view/state/kind/locator/state.go | 3 +- view/state/kind/locator/transient.go | 3 +- 25 files changed, 81 insertions(+), 43 deletions(-) diff --git a/repository/locator/async/locator.go b/repository/locator/async/locator.go index bc8141cde..7d047fe42 100644 --- a/repository/locator/async/locator.go +++ b/repository/locator/async/locator.go @@ -9,13 +9,14 @@ import ( "github.com/viant/xdatly/handler/async" "github.com/viant/xdatly/handler/exec" "github.com/viant/xdatly/handler/response" + "reflect" "strings" "time" ) type Locator struct{} -func (l *Locator) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (l *Locator) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { name = strings.ToLower(name) if name == keys.JobError { diff --git a/repository/locator/component/component.go b/repository/locator/component/component.go index 9fd8f6dea..b38907df6 100644 --- a/repository/locator/component/component.go +++ b/repository/locator/component/component.go @@ -35,7 +35,7 @@ func (l *componentLocator) Names() []string { return nil } -func (l *componentLocator) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (l *componentLocator) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { method, URI := shared.ExtractPath(name) request, err := l.getRequest() if err != nil { diff --git a/repository/locator/meta/locator.go b/repository/locator/meta/locator.go index 487a58be8..ba1b0c1af 100644 --- a/repository/locator/meta/locator.go +++ b/repository/locator/meta/locator.go @@ -7,13 +7,14 @@ import ( "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind" "github.com/viant/datly/view/state/kind/locator" + "reflect" "strings" ) type metaLocator struct { } -func (l *metaLocator) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (l *metaLocator) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { value := ctx.Value(view.ContextKey) if value == nil { return nil, false, nil diff --git a/repository/locator/output/output.go b/repository/locator/output/output.go index c9368971a..ec6b3ed5f 100644 --- a/repository/locator/output/output.go +++ b/repository/locator/output/output.go @@ -3,6 +3,7 @@ package output import ( "context" "encoding/json" + "reflect" "strings" "github.com/viant/datly/repository/locator/output/keys" @@ -25,7 +26,7 @@ func (l *Locator) Names() []string { return nil } -func (l *Locator) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (l *Locator) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { aName := strings.ToLower(name) switch aName { case keys.ViewData: diff --git a/router.go b/router.go index 5e556376e..1117ee905 100644 --- a/router.go +++ b/router.go @@ -101,6 +101,7 @@ func (r *Router[T]) Run(writer http.ResponseWriter, request *http.Request) error aPath := contract.NewPath(request.Method, request.URL.Path) component, err := r.dao.repository.Registry().Lookup(request.Context(), aPath) if err != nil { + fmt.Println(err) return &routeNotFound{err} } route, ok := r.registry[component.Path.Key()] diff --git a/service/executor/handler/locator/handler.go b/service/executor/handler/locator/handler.go index 3758b8abd..6f0c31aeb 100644 --- a/service/executor/handler/locator/handler.go +++ b/service/executor/handler/locator/handler.go @@ -11,6 +11,7 @@ import ( "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind" "github.com/viant/datly/view/state/kind/locator" + "reflect" ) type Handler struct { @@ -22,7 +23,7 @@ func (v *Handler) Names() []string { return nil } -func (v *Handler) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (v *Handler) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { resource := v.options.Resource if resource == nil { return nil, false, fmt.Errorf("failed to lookup handler resource: %v", name) diff --git a/service/session/state.go b/service/session/state.go index f7b0e8ce5..8e6eab3f3 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -581,7 +581,8 @@ func (s *Session) lookupValue(ctx context.Context, parameter *state.Parameter, o if err != nil { return nil, false, fmt.Errorf("failed to locate parameter: %v, %w", parameter.Name, err) } - if value, has, err = parameterLocator.Value(ctx, parameter.In.Name); err != nil { + + if value, has, err = parameterLocator.Value(ctx, parameter.OutputType(), parameter.In.Name); err != nil { return nil, false, err } if parameter.In.Kind == state.KindConst && !has { //if parameter is const and has no value, use default value @@ -596,7 +597,7 @@ func (s *Session) lookupValue(ctx context.Context, parameter *state.Parameter, o if err != nil { return nil, false, fmt.Errorf("failed to locate parameter: %v, %w", baseParameter.Name, err) } - if value, has, err = parameterLocator.Value(ctx, baseParameter.In.Name); err != nil { + if value, has, err = parameterLocator.Value(ctx, baseParameter.OutputType(), baseParameter.In.Name); err != nil { return nil, false, err } } diff --git a/view/state/kind/locator.go b/view/state/kind/locator.go index 57f765f0e..864d85b00 100644 --- a/view/state/kind/locator.go +++ b/view/state/kind/locator.go @@ -2,6 +2,8 @@ package kind import ( "context" + "reflect" + "github.com/viant/datly/view/state" ) @@ -9,7 +11,7 @@ import ( type Locator interface { //Value returns parameter value - Value(ctx context.Context, name string) (interface{}, bool, error) + Value(ctx context.Context, rType reflect.Type, name string) (interface{}, bool, error) //Names returns names of supported parameters Names() []string diff --git a/view/state/kind/locator/body.go b/view/state/kind/locator/body.go index fc41a49aa..b80d112dd 100644 --- a/view/state/kind/locator/body.go +++ b/view/state/kind/locator/body.go @@ -27,16 +27,32 @@ func (r *Body) Names() []string { return nil } -func (r *Body) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (r *Body) Value(ctx context.Context, rType reflect.Type, name string) (interface{}, bool, error) { var err error + r.Once.Do(func() { var request *http.Request request, r.err = shared.CloneHTTPRequest(r.request) r.body, r.err = readRequestBody(request) - if len(r.body) > 0 { - r.err = r.ensureRequest() - } + }) + + var requestState *structology.State + + if len(r.body) > 0 { + if r.requestState != nil && r.requestState.Type().Type() == rType { + requestState = r.requestState + } + if name == "" { + requestState, r.err = r.ensureRequest(rType) + } else { + requestState, r.err = r.ensureRequest(r.bodyType) + } + if r.err == nil { + r.requestState = requestState + } + } + if len(r.body) == 0 { return nil, false, nil } @@ -47,16 +63,16 @@ func (r *Body) Value(ctx context.Context, name string) (interface{}, bool, error return r.decodeBodyMap(ctx) } if name == "" { - return r.requestState.State(), true, nil + return requestState.State(), true, nil } - sel, err := r.requestState.Selector(name) + sel, err := requestState.Selector(name) if err != nil { return nil, false, err } - if !sel.Has(r.requestState.Pointer()) { + if !sel.Has(requestState.Pointer()) { return nil, false, nil } - return sel.Value(r.requestState.Pointer()), true, nil + return sel.Value(requestState.Pointer()), true, nil } func (r *Body) decodeBodyMap(ctx context.Context) (interface{}, bool, error) { @@ -87,21 +103,21 @@ func NewBody(opts ...Option) (kind.Locator, error) { return ret, nil } -func (r *Body) ensureRequest() (err error) { - if r.bodyType == nil { - return nil +func (r *Body) ensureRequest(rType reflect.Type) (*structology.State, error) { + if rType == nil { + return nil, nil } - rType := r.bodyType if rType.Kind() == reflect.Map { - return nil + return nil, nil } - bodyType := structology.NewStateType(r.bodyType) - r.requestState = bodyType.NewState() - dest := r.requestState.StatePtr() - if err = r.unmarshal(r.body, dest); err == nil { - r.requestState.Sync() + bodyType := structology.NewStateType(rType) + requestState := bodyType.NewState() + dest := requestState.StatePtr() + err := r.unmarshal(r.body, dest) + if err == nil { + requestState.Sync() } - return err + return requestState, err } func (r *Body) updateQueryString(ctx context.Context, body interface{}) { diff --git a/view/state/kind/locator/constants.go b/view/state/kind/locator/constants.go index f6cd81eeb..516e7c4a2 100644 --- a/view/state/kind/locator/constants.go +++ b/view/state/kind/locator/constants.go @@ -3,6 +3,7 @@ package locator import ( "context" "github.com/viant/datly/view/state/kind" + "reflect" "sync" ) @@ -16,7 +17,7 @@ func (r *Constants) Names() []string { return nil } -func (r *Constants) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (r *Constants) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { if len(r.constants) > 0 { if value, ok := r.constants[name]; ok { return value, true, nil diff --git a/view/state/kind/locator/context.go b/view/state/kind/locator/context.go index d5fcd827c..33d0985c4 100644 --- a/view/state/kind/locator/context.go +++ b/view/state/kind/locator/context.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/viant/datly/view/state/kind" "github.com/viant/xdatly/handler/exec" + "reflect" ) type Context struct { @@ -14,7 +15,7 @@ func (v *Context) Names() []string { return nil } -func (v *Context) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (v *Context) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { rawValue := ctx.Value(exec.ContextKey) if rawValue == nil { diff --git a/view/state/kind/locator/cookie.go b/view/state/kind/locator/cookie.go index 804949592..117550935 100644 --- a/view/state/kind/locator/cookie.go +++ b/view/state/kind/locator/cookie.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/viant/datly/view/state/kind" "net/http" + "reflect" ) type Cookie struct { @@ -19,7 +20,7 @@ func (v *Cookie) Names() []string { return result } -func (v *Cookie) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (v *Cookie) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { for _, cookie := range v.cookies { if cookie.Name == name { return cookie.Value, true, nil diff --git a/view/state/kind/locator/data.go b/view/state/kind/locator/data.go index efcabfdb5..db35876c5 100644 --- a/view/state/kind/locator/data.go +++ b/view/state/kind/locator/data.go @@ -18,7 +18,7 @@ func (p *DataView) Names() []string { return nil } -func (p *DataView) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (p *DataView) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { aView, ok := p.Views[name] if !ok { return nil, false, fmt.Errorf("failed to lookup view: %v", name) diff --git a/view/state/kind/locator/env.go b/view/state/kind/locator/env.go index 05ccc5217..28b980e5d 100644 --- a/view/state/kind/locator/env.go +++ b/view/state/kind/locator/env.go @@ -4,6 +4,7 @@ import ( "context" "github.com/viant/datly/view/state/kind" "os" + "reflect" ) type Env struct { @@ -14,7 +15,7 @@ func (v *Env) Names() []string { return os.Environ() } -func (v *Env) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (v *Env) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { ret, ok := v.env[name] return ret, ok, nil } diff --git a/view/state/kind/locator/form.go b/view/state/kind/locator/form.go index 387815e79..174f66255 100644 --- a/view/state/kind/locator/form.go +++ b/view/state/kind/locator/form.go @@ -5,6 +5,7 @@ import ( "github.com/viant/datly/view/state/kind" "github.com/viant/xdatly/handler/state" "net/http" + "reflect" ) type Form struct { @@ -16,7 +17,7 @@ func (r *Form) Names() []string { return nil } -func (r *Form) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (r *Form) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { if r.form != nil && len(r.form.Values) == 0 && r.request == nil { return nil, false, nil } diff --git a/view/state/kind/locator/generator.go b/view/state/kind/locator/generator.go index 1583357f8..f020b40ff 100644 --- a/view/state/kind/locator/generator.go +++ b/view/state/kind/locator/generator.go @@ -4,6 +4,7 @@ import ( "context" "github.com/google/uuid" "github.com/viant/datly/view/state/kind" + "reflect" "strings" "time" ) @@ -14,7 +15,7 @@ func (v *Generator) Names() []string { return nil } -func (v *Generator) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (v *Generator) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { switch strings.ToLower(name) { case "nil": return nil, true, nil diff --git a/view/state/kind/locator/header.go b/view/state/kind/locator/header.go index e6a8135e2..c2fd3962f 100644 --- a/view/state/kind/locator/header.go +++ b/view/state/kind/locator/header.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/viant/datly/view/state/kind" "net/http" + "reflect" ) type Header struct { @@ -20,7 +21,7 @@ func (q *Header) Names() []string { return result } -func (q *Header) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (q *Header) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { value, ok := q.header[name] if !ok { return nil, false, nil diff --git a/view/state/kind/locator/http.go b/view/state/kind/locator/http.go index a2becfe56..8fdd0d9bc 100644 --- a/view/state/kind/locator/http.go +++ b/view/state/kind/locator/http.go @@ -7,6 +7,7 @@ import ( "github.com/viant/datly/view/state/kind" "io" "net/http" + "reflect" "strings" ) @@ -19,7 +20,7 @@ func (p *HttpRequest) Names() []string { return nil } -func (p *HttpRequest) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (p *HttpRequest) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { request := p.request if p.request == nil { var err error diff --git a/view/state/kind/locator/object.go b/view/state/kind/locator/object.go index f1c066718..ced3fe835 100644 --- a/view/state/kind/locator/object.go +++ b/view/state/kind/locator/object.go @@ -6,6 +6,7 @@ import ( "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind" "github.com/viant/structology" + "reflect" ) type Object struct { @@ -19,7 +20,7 @@ func (p *Object) Names() []string { return nil } -func (p *Object) Value(ctx context.Context, names string) (interface{}, bool, error) { +func (p *Object) Value(ctx context.Context, _ reflect.Type, names string) (interface{}, bool, error) { parameter := p.matchByLocation(names) if parameter == nil { return nil, false, fmt.Errorf("failed to match parameter by location: %v", names) diff --git a/view/state/kind/locator/parameter.go b/view/state/kind/locator/parameter.go index b46f54ed0..35578c13c 100644 --- a/view/state/kind/locator/parameter.go +++ b/view/state/kind/locator/parameter.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind" + "reflect" ) type Parameter struct { @@ -16,7 +17,7 @@ func (p *Parameter) Names() []string { return nil } -func (p *Parameter) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (p *Parameter) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { parameter, ok := p.Parameters[name] if !ok { return nil, false, fmt.Errorf("uknonw parameter: %s", name) diff --git a/view/state/kind/locator/path.go b/view/state/kind/locator/path.go index eecabec7c..79a8af548 100644 --- a/view/state/kind/locator/path.go +++ b/view/state/kind/locator/path.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/viant/datly/view/state/kind" "github.com/viant/toolbox" + "reflect" ) type Path struct { @@ -20,7 +21,7 @@ func (v *Path) Names() []string { return result } -func (v *Path) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (v *Path) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { if name == "" { return v.path, true, nil } diff --git a/view/state/kind/locator/query.go b/view/state/kind/locator/query.go index 605f1b1a7..b532215aa 100644 --- a/view/state/kind/locator/query.go +++ b/view/state/kind/locator/query.go @@ -7,6 +7,7 @@ import ( "github.com/viant/xdatly/handler/exec" "net/http" "net/url" + "reflect" ) type Query struct { @@ -23,7 +24,7 @@ func (q *Query) Names() []string { return result } -func (q *Query) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (q *Query) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { if name == "" { return q.rawQuery, true, nil } diff --git a/view/state/kind/locator/repeated.go b/view/state/kind/locator/repeated.go index 1d07fde50..2f19e91ae 100644 --- a/view/state/kind/locator/repeated.go +++ b/view/state/kind/locator/repeated.go @@ -28,7 +28,7 @@ func (p *Repeated) Names() []string { return nil } -func (p *Repeated) Value(ctx context.Context, names string) (interface{}, bool, error) { +func (p *Repeated) Value(ctx context.Context, _ reflect.Type, names string) (interface{}, bool, error) { parameter := p.matchByLocation(names) if parameter == nil { return nil, false, fmt.Errorf("failed to match parameter by location: %v", names) diff --git a/view/state/kind/locator/state.go b/view/state/kind/locator/state.go index bbd44cd4c..503dba3f5 100644 --- a/view/state/kind/locator/state.go +++ b/view/state/kind/locator/state.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/viant/datly/view/state/kind" "github.com/viant/structology" + "reflect" ) type State struct { @@ -14,7 +15,7 @@ type State struct { func (p *State) Names() []string { return nil } -func (p *State) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (p *State) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { _, err := p.State.Selector(name) if err != nil { return nil, false, nil diff --git a/view/state/kind/locator/transient.go b/view/state/kind/locator/transient.go index 0fde22e6e..72ac34cca 100644 --- a/view/state/kind/locator/transient.go +++ b/view/state/kind/locator/transient.go @@ -3,6 +3,7 @@ package locator import ( "context" "github.com/viant/datly/view/state/kind" + "reflect" ) type Transient struct{} @@ -11,7 +12,7 @@ func (v *Transient) Names() []string { return nil } -func (v *Transient) Value(ctx context.Context, name string) (interface{}, bool, error) { +func (v *Transient) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { if name == "" { return nil, false, nil } From 9972471896cc58ee1e255ac1ef1f1bab94e846b9 Mon Sep 17 00:00:00 2001 From: vc42 Date: Thu, 9 Oct 2025 13:39:06 -0400 Subject: [PATCH 045/279] allow large json objects in LoadData --- view/extension/handler/loader.go | 81 +++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/view/extension/handler/loader.go b/view/extension/handler/loader.go index 7dd4fc96f..86a5966c2 100644 --- a/view/extension/handler/loader.go +++ b/view/extension/handler/loader.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "context" "encoding/json" + "errors" "fmt" "github.com/viant/afs" "github.com/viant/datly/utils/types" @@ -42,56 +43,92 @@ func (l *LoadData) Exec(ctx context.Context, session handler.Session) (interface if !ok || err != nil { return nil, fmt.Errorf("invalid Loader URL: %w", err) } + var URL string - switch URLValue.(type) { + switch v := URLValue.(type) { case string: - URL = URLValue.(string) + URL = v case *string: - URL = *URLValue.(*string) + URL = *v default: - return nil, fmt.Errorf("invalid Loader URL: expected %T, but had %T", URL, URLValue) + return nil, fmt.Errorf("invalid Loader URL: expected %T, but had %T", "", URLValue) } + // Prefer .gz if the plain URL doesn't exist. if ok, _ := l.fs.Exists(ctx, URL); !ok { if ok, _ := l.fs.Exists(ctx, URL+".gz"); ok { URL += ".gz" } } - isCompressed := strings.HasSuffix(URL, ".gz") + // Download compressed or plain bytes (API returns []byte). data, err := l.fs.DownloadWithURL(ctx, URL) if err != nil { return nil, fmt.Errorf("failed to load URL: %w", err) } - if isCompressed { - reader, err := gzip.NewReader(bytes.NewReader(data)) + + // Build a streaming reader chain; avoid io.ReadAll on gzip. + var r io.Reader = bytes.NewReader(data) + if strings.HasSuffix(URL, ".gz") { + gzr, err := gzip.NewReader(r) if err != nil { return nil, fmt.Errorf("failed to decompress URL: failed to create reader: %w (used URL: %s)", err, URL) } - defer reader.Close() - if data, err = io.ReadAll(reader); err != nil { - return nil, fmt.Errorf("failed to decompress URL:%w (used URL: %s)", err, URL) - } + defer gzr.Close() + r = gzr } + + br := bufio.NewReaderSize(r, 1<<20) // read-ahead; does NOT cap JSON size + dec := json.NewDecoder(br) + dec.UseNumber() + + // Output slice + appender (kept from your original design) itemType := l.Options.OutputType.Elem() xSlice := xunsafe.NewSlice(l.Options.OutputType) - scanner := bufio.NewScanner(bytes.NewReader(data)) response := reflect.New(l.Options.OutputType).Interface() appender := xSlice.Appender(xunsafe.AsPointer(response)) - scanner.Buffer(make([]byte, 1024*1024), 5*1024*1024) - for scanner.Scan() { - line := scanner.Bytes() - if len(line) == 0 { - continue + + // Reject top-level arrays to keep the code simple (no streaming array parsing). + first, err := peekFirstNonSpace(br) + if err != nil { + if errors.Is(err, io.EOF) { + return response, nil // empty file -> empty slice } - item := types.NewValue(itemType) - err := json.Unmarshal(scanner.Bytes(), item) - if err != nil { - return nil, fmt.Errorf("invalid item: %w, %s", err, line) + return nil, fmt.Errorf("read error: %w", err) + } + if first == '[' { + return nil, fmt.Errorf("top-level JSON arrays are not supported; provide NDJSON (one object per line) or a single JSON object") + } + // Put the byte back so the decoder sees it. + _ = br.UnreadByte() + + // Decode one value per call: supports single object or NDJSON. + for { + item := types.NewValue(itemType) // pointer to zero value of element type + if err := dec.Decode(item); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("invalid item: %w", err) } appender.Append(item) } - return response, scanner.Err() + + return response, nil +} + +// Reads and returns the first non-space byte without consuming input for the decoder. +func peekFirstNonSpace(br *bufio.Reader) (byte, error) { + for { + b, err := br.ReadByte() + if err != nil { + return 0, err + } + if b == ' ' || b == '\n' || b == '\r' || b == '\t' { + continue + } + return b, nil + } } func (*LoadDataProvider) New(ctx context.Context, opts ...handler.Option) (handler.Handler, error) { From ad9cc45d85d05f7157f0ff01912745a8e2c166e5 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 9 Oct 2025 13:57:41 -0700 Subject: [PATCH 046/279] updated locator signature --- router.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/router.go b/router.go index 1117ee905..03504a4eb 100644 --- a/router.go +++ b/router.go @@ -124,3 +124,7 @@ func (r *Router[T]) Register(ctx context.Context, path *contract.Path, handler H func NewRouter[T any](dao *Service, service T) *Router[T] { return &Router[T]{registry: make(map[string]*Route[T]), dao: dao, service: service} } + +type BodyEnvelope[T any] struct { + Body T `parameter:",kind=body"` +} From a5e53061cbf1ef11ee28db6c7b66b72e1b420c6d Mon Sep 17 00:00:00 2001 From: vc42 Date: Sun, 12 Oct 2025 08:21:55 -0400 Subject: [PATCH 047/279] fixed concurrency issues in deferredMarshaller --- gateway/router/marshal/json/cache.go | 20 ++++----- .../marshal/json/marshaller_deferred.go | 42 +++++++++++++++---- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/gateway/router/marshal/json/cache.go b/gateway/router/marshal/json/cache.go index 728af816d..d08a4704f 100644 --- a/gateway/router/marshal/json/cache.go +++ b/gateway/router/marshal/json/cache.go @@ -100,25 +100,23 @@ func (m *marshallersCache) loadMarshaller(rType reflect.Type, config *config.IOC return marshaller, nil } -func (c *pathCache) loadOrGetMarshaller(rType reflect.Type, config *config.IOConfig, path string, outputPath string, tag *format.Tag, options ...interface{}) (marshaler, error) { - value, ok := c.cache.Load(rType) +func (c *pathCache) loadOrGetMarshaller(rType reflect.Type, cfg *config.IOConfig, path, outPath string, tag *format.Tag, options ...interface{}) (marshaler, error) { + + placeholder := newDeferred() + value, ok := c.cache.LoadOrStore(rType, placeholder) if ok { return value.(marshaler), nil } - // Place a deferred placeholder to break recursive graphs for this path and type. - placeholder := &deferredMarshaller{} - c.storeMarshaler(rType, placeholder) - - aMarshaler, err := c.getMarshaller(rType, config, path, outputPath, tag, options...) + aMarshaller, err := c.getMarshaller(rType, cfg, path, outPath, tag, options...) if err != nil { + placeholder.fail(err) // unblock anyone holding the promise + c.cache.CompareAndDelete(rType, placeholder) // allow a clean retry later return nil, err } - // Swap placeholder with the real marshaller and set target for any users that captured it. - placeholder.setTarget(aMarshaler) - c.storeMarshaler(rType, aMarshaler) - return aMarshaler, nil + placeholder.setTarget(aMarshaller) // resolve success + return aMarshaller, nil } func (c *pathCache) getMarshaller(rType reflect.Type, config *config.IOConfig, path string, outputPath string, tag *format.Tag, options ...interface{}) (marshaler, error) { diff --git a/gateway/router/marshal/json/marshaller_deferred.go b/gateway/router/marshal/json/marshaller_deferred.go index fef24a83d..48955de80 100644 --- a/gateway/router/marshal/json/marshaller_deferred.go +++ b/gateway/router/marshal/json/marshaller_deferred.go @@ -2,30 +2,56 @@ package json import ( "fmt" - "github.com/francoispqt/gojay" "unsafe" + + "github.com/francoispqt/gojay" ) // deferredMarshaller is a placeholder used to break recursive type graphs during construction. // It forwards calls to the actual target once it is set. type deferredMarshaller struct { target marshaler + ready chan struct{} + err error +} + +func newDeferred() *deferredMarshaller { + return &deferredMarshaller{ready: make(chan struct{})} } func (d *deferredMarshaller) setTarget(m marshaler) { d.target = m + close(d.ready) +} + +func (d *deferredMarshaller) fail(e error) { + d.err = e + close(d.ready) // writes to err happen-before any receive on ready } -func (d *deferredMarshaller) MarshallObject(ptr unsafe.Pointer, session *MarshallSession) error { +func (d *deferredMarshaller) resolved() (marshaler, error) { + <-d.ready // wait for resolve/fail + if d.err != nil { + return nil, d.err + } if d.target == nil { - return fmt.Errorf("marshaller not initialized") + return nil, fmt.Errorf("marshaller not initialized") } - return d.target.MarshallObject(ptr, session) + return d.target, nil } -func (d *deferredMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder *gojay.Decoder, auxiliaryDecoder *gojay.Decoder, session *UnmarshalSession) error { - if d.target == nil { - return fmt.Errorf("marshaller not initialized") +func (d *deferredMarshaller) MarshallObject(ptr unsafe.Pointer, s *MarshallSession) error { + m, err := d.resolved() + if err != nil { + return err + } + return m.MarshallObject(ptr, s) +} + +func (d *deferredMarshaller) UnmarshallObject(p unsafe.Pointer, dec, aux *gojay.Decoder, s *UnmarshalSession) error { + m, err := d.resolved() + if err != nil { + return err } - return d.target.UnmarshallObject(pointer, decoder, auxiliaryDecoder, session) + return m.UnmarshallObject(p, dec, aux, s) } From 71810a809d693135c81fbb58ac56d5e3f1d990c2 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 15 Oct 2025 12:15:38 -0700 Subject: [PATCH 048/279] updated locator signature --- service/executor/expand/data_unit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/executor/expand/data_unit.go b/service/executor/expand/data_unit.go index 784953fc9..9d2c22dad 100644 --- a/service/executor/expand/data_unit.go +++ b/service/executor/expand/data_unit.go @@ -148,7 +148,7 @@ func (c *DataUnit) Next() (interface{}, error) { return c.ParamsGroup[index], nil } - return nil, fmt.Errorf("expected to get binding parameter, but noone was found, ParamsGroup: %v, placeholderCounter: %v", c.ParamsGroup, c.placeholderCounter) + return nil, fmt.Errorf("expected to get binding parameter, but none was found, ParamsGroup: %v, placeholderCounter: %v", c.ParamsGroup, c.placeholderCounter) } func (c *DataUnit) ensureSliceIndex() { From cca251ddc956c026cb2140655ebb24f95d4c56d6 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 20 Oct 2025 09:17:34 -0700 Subject: [PATCH 049/279] enhanced mcp integration --- gateway/mcp.go | 327 +++++++++++++----- gateway/router/marshal/json/marshaller_map.go | 9 +- .../router/marshal/json/marshaller_strings.go | 49 ++- go.mod | 5 +- go.sum | 10 +- repository/component.go | 103 ++++++ 6 files changed, 397 insertions(+), 106 deletions(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index 45310d541..0a9624afe 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -4,6 +4,12 @@ import ( "context" "encoding/json" "fmt" + "io" + "net/http" + "net/url" + "reflect" + "strings" + furl "github.com/viant/afs/url" "github.com/viant/datly/gateway/router/proxy" "github.com/viant/datly/repository" @@ -14,11 +20,6 @@ import ( "github.com/viant/mcp-protocol/schema" serverproto "github.com/viant/mcp-protocol/server" "github.com/viant/toolbox" - "io" - "net/http" - "net/url" - "reflect" - "strings" ) func (r *Router) buildToolsIntegration(item *dpath.Item, aPath *dpath.Path, aRoute *Route, provider *repository.Provider) error { @@ -53,109 +54,188 @@ func (r *Router) buildToolsIntegration(item *dpath.Item, aPath *dpath.Path, aRou } func (r *Router) mcpToolCallHandler(component *repository.Component, aRoute *Route) serverproto.ToolHandlerFunc { - handler := func(ctx context.Context, req *schema.CallToolRequest) (*schema.CallToolResult, *jsonrpc.Error) { + return func(ctx context.Context, req *schema.CallToolRequest) (*schema.CallToolResult, *jsonrpc.Error) { params := req.Params - URI := r.matchToolCallComponentURI(aRoute, component, params) - URL := fmt.Sprintf("http://localhost/%v", strings.TrimLeft(URI, "/")) // fallback to a local URL for now, this should be replaced with the actual service URL + uri := r.matchToolCallComponentURI(aRoute, component, params) + baseURL := fmt.Sprintf("http://localhost/%v", strings.TrimLeft(uri, "/")) // replace with actual service URL when available + values := url.Values{} var body io.Reader - var uniquePath = make(map[string]bool) - var uniqueQuery = make(map[string]bool) - for _, parameter := range component.Input.Type.Parameters { - paramName := strings.Title(parameter.Name) - value := params.Arguments[paramName] - paramType := parameter.Schema.Type() - if paramType.Kind() == reflect.Ptr { - paramType = paramType.Elem() + uniquePath := map[string]bool{} + uniqueQuery := map[string]bool{} + + // 1) Collect parameters (component + selector pagination) + allParams := r.collectToolParameters(component) + + // 2) Apply parameters to request URL/query/body + for _, p := range allParams { + name := strings.Title(p.Name) + value := params.Arguments[name] + pType := p.Schema.Type() + if pType.Kind() == reflect.Ptr { + pType = pType.Elem() } + value = r.coerceNumericValue(value, pType) + var rpcErr *jsonrpc.Error + baseURL, body, rpcErr = r.applyParamToRequest(baseURL, values, p, value, uniquePath, uniqueQuery, body) + if rpcErr != nil { + return nil, rpcErr + } + } - switch paramType.Kind() { - case reflect.Int, reflect.Int64, reflect.Uint, reflect.Uint64, reflect.Float64: - if value == nil { - continue - } - value = toolbox.AsInt(value) + // 3) Finalize URL with query string + finalURL := baseURL + if enc := values.Encode(); enc != "" { + if strings.Contains(finalURL, "?") { + finalURL += "&" + enc + } else { + finalURL += "?" + enc } + } - switch parameter.In.Kind { - case state.KindPath: - if uniquePath[parameter.In.Name] { - continue - } - uniquePath[parameter.In.Name] = true + // 4) Build HTTP request and route + httpReq, rpcErr := r.newToolHTTPRequest(aRoute.Path.Method, finalURL, body) + if rpcErr != nil { + return nil, rpcErr + } + r.addAuthTokenIfPresent(ctx, httpReq) + httpReq.RequestURI = httpReq.URL.RequestURI() + if uri != aRoute.URI() { + if matched, _ := r.match(component.Method, uri, httpReq); matched != nil { + aRoute = matched + } + } - if value == nil { - return nil, jsonrpc.NewInvalidRequest("missing path parameter: "+parameter.In.Name, nil) - } + rw := proxy.NewWriter() + aRoute.Handle(rw, httpReq) - URL = strings.ReplaceAll(URL, "{"+parameter.In.Name+"}", fmt.Sprintf("%v", value)) - case state.KindQuery, state.KindForm: - if uniqueQuery[parameter.In.Name] { - continue - } - uniqueQuery[parameter.In.Name] = true - if value == nil || value == "" { - continue - } - // Check if value is a slice and create a comma-separated string - if slice, ok := value.([]interface{}); ok { - var items []string - for _, item := range slice { - if f, ok := item.(float64); ok { - items = append(items, fmt.Sprintf("%v", int64(f))) - } else { - items = append(items, fmt.Sprintf("%v", item)) - } - } - values.Add(parameter.In.Name, strings.Join(items, ",")) - } else { - values.Add(parameter.In.Name, fmt.Sprintf("%v", value)) - } - case state.KindRequestBody: - if text, ok := value.(string); ok { - body = strings.NewReader(text) - } else { - data, err := json.Marshal(value) - if err != nil { - return nil, jsonrpc.NewInvalidParamsError("failed to marshal request body: %w", data) - } - body = strings.NewReader(string(data)) - } - } + // 5) Build tool result (text + structured on error) + return r.buildToolCallResult(rw, finalURL, aRoute.Path.Method), nil + } +} + +// collectToolParameters aggregates component input parameters with selector pagination (limit/offset) when available. +func (r *Router) collectToolParameters(component *repository.Component) []*state.Parameter { + var all []*state.Parameter + all = append(all, component.Input.Type.Parameters...) + if component.View != nil && component.View.Selector != nil { + if p := component.View.Selector.LimitParameter; p != nil { + all = append(all, p) + } + if p := component.View.Selector.OffsetParameter; p != nil { + all = append(all, p) + } + if p := component.View.Selector.FieldsParameter; p != nil { + all = append(all, p) } - responseWriter := proxy.NewWriter() + if p := component.View.Selector.PageParameter; p != nil { + all = append(all, p) + } + } + return all +} - // Add query parameters to URL if any exist - if len(values) > 0 { - if strings.Contains(URL, "?") { - URL += "&" + values.Encode() - } else { - URL += "?" + values.Encode() +// coerceNumericValue normalizes numeric values to integers when appropriate. +func (r *Router) coerceNumericValue(value interface{}, paramType reflect.Type) interface{} { + switch paramType.Kind() { + case reflect.Int, reflect.Int64, reflect.Uint, reflect.Uint64, reflect.Float64: + if value == nil { + return nil + } + return toolbox.AsInt(value) + } + return value +} + +// applyParamToRequest applies a single parameter into path placeholders, query/form values, or request body. +func (r *Router) applyParamToRequest(baseURL string, values url.Values, p *state.Parameter, value interface{}, uniquePath, uniqueQuery map[string]bool, body io.Reader) (string, io.Reader, *jsonrpc.Error) { + switch p.In.Kind { + case state.KindPath: + if uniquePath[p.In.Name] { + return baseURL, body, nil + } + uniquePath[p.In.Name] = true + if value == nil { + // If parameter has its own URI segment configured, treat as optional and strip the placeholder. + if p.URI != "" { + baseURL = strings.ReplaceAll(baseURL, "/{"+p.In.Name+"}", "") + baseURL = strings.ReplaceAll(baseURL, "{"+p.In.Name+"}", "") + return baseURL, body, nil } + return baseURL, body, jsonrpc.NewInvalidRequest("missing path parameter: "+p.In.Name, nil) } - httpRequest, err := http.NewRequest(aRoute.Path.Method, URL, body) - if err != nil { - return nil, jsonrpc.NewInvalidRequest(err.Error(), nil) + baseURL = strings.ReplaceAll(baseURL, "{"+p.In.Name+"}", fmt.Sprintf("%v", value)) + case state.KindQuery, state.KindForm: + if uniqueQuery[p.In.Name] { + return baseURL, body, nil + } + uniqueQuery[p.In.Name] = true + if value == nil || value == "" { + return baseURL, body, nil } - r.addAuthTokenIfPresent(ctx, httpRequest) - httpRequest.RequestURI = httpRequest.URL.RequestURI() - if URI != aRoute.URI() { - if matchedRoute, _ := r.match(component.Method, URI, httpRequest); matchedRoute != nil { - aRoute = matchedRoute + if slice, ok := value.([]interface{}); ok { + var items []string + for _, item := range slice { + if f, ok := item.(float64); ok { + items = append(items, fmt.Sprintf("%v", int64(f))) + } else { + items = append(items, fmt.Sprintf("%v", item)) + } } + values.Add(p.In.Name, strings.Join(items, ",")) + } else { + values.Add(p.In.Name, fmt.Sprintf("%v", value)) } - aRoute.Handle(responseWriter, httpRequest) // route the request to the actual handler - var result = schema.CallToolResult{} - mimeType := "application/json" - item := schema.CallToolResultContentElem{ - MimeType: mimeType, - Type: "text", // use data for some clients - Text: responseWriter.Body.String(), + case state.KindRequestBody: + if text, ok := value.(string); ok { + body = strings.NewReader(text) + } else { + data, err := json.Marshal(value) + if err != nil { + return baseURL, body, jsonrpc.NewInvalidParamsError("failed to marshal request body", nil) + } + body = strings.NewReader(string(data)) } - result.Content = append(result.Content, item) - return &result, nil } - return handler + return baseURL, body, nil +} + +// newToolHTTPRequest constructs an HTTP request for routed tool invocation. +func (r *Router) newToolHTTPRequest(method, URL string, body io.Reader) (*http.Request, *jsonrpc.Error) { + httpRequest, err := http.NewRequest(method, URL, body) + if err != nil { + return nil, jsonrpc.NewInvalidRequest(err.Error(), nil) + } + return httpRequest, nil +} + +// buildToolCallResult composes a CallToolResult with text content and structured error info if status is not OK. +func (r *Router) buildToolCallResult(responseWriter *proxy.Writer, URL, method string) *schema.CallToolResult { + var result = &schema.CallToolResult{} + mimeType := responseWriter.HeaderMap.Get("Content-Type") + if mimeType == "" { + mimeType = "application/json" + } + data := responseWriter.Body.Bytes() + result.Content = append(result.Content, schema.CallToolResultContentElem{ + MimeType: mimeType, + Type: "text", + Text: string(data), + }) + _ = json.Unmarshal(data, &result.StructuredContent) + if responseWriter.Code >= http.StatusBadRequest { + isErr := true + result.IsError = &isErr + result.StructuredContent = map[string]interface{}{ + "status": responseWriter.Code, + "error": true, + "message": responseWriter.Body.String(), + "headers": responseWriter.HeaderMap, + "uri": URL, + "method": method, + } + } + return result } func (r *Router) matchToolCallComponentURI(aRoute *Route, component *repository.Component, params schema.CallToolRequestParams) string { @@ -190,6 +270,7 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty var inputFields []reflect.StructField var uniqueQuery = make(map[string]bool) var uniquePath = make(map[string]bool) + // Include component input parameters for _, parameter := range components.Input.Type.Parameters { name := strings.Title(parameter.Name) switch parameter.In.Kind { @@ -198,20 +279,63 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty continue } uniquePath[parameter.In.Name] = true - inputFields = append(inputFields, reflect.StructField{Name: name, Type: parameter.Schema.Type()}) + // If parameter is a slice, make it optional in schema via `omitempty` and optional:"true". + var tag reflect.StructTag + if parameter.Schema != nil && parameter.Schema.Type().Kind() == reflect.Slice { + tag = `json:",omitempty" optional:"true"` + } + inputFields = append(inputFields, reflect.StructField{Name: name, Type: parameter.Schema.Type(), Tag: tag}) case state.KindQuery, state.KindForm: if uniqueQuery[parameter.In.Name] { continue } uniqueQuery[parameter.In.Name] = true + // Repeated (slice) params are optional regardless of "required" tag. + // Otherwise, respect explicit required; default to optional. tag := reflect.StructTag(parameter.Tag) - if !strings.Contains(parameter.Tag, "required") { + if parameter.Schema != nil && parameter.Schema.Type().Kind() == reflect.Slice { + tag = `json:",omitempty" optional:"true"` + } else if !strings.Contains(parameter.Tag, "required") { tag = `json:",omitempty"` } inputFields = append(inputFields, reflect.StructField{Name: name, Type: parameter.Schema.Type(), Tag: tag}) case state.KindRequestBody: - inputFields = append(inputFields, reflect.StructField{Name: name, Type: parameter.Schema.Type()}) + // If body is a slice, mark optional in schema. + var tag reflect.StructTag + if parameter.Schema != nil && parameter.Schema.Type().Kind() == reflect.Slice { + tag = `json:",omitempty" optional:"true"` + } + inputFields = append(inputFields, reflect.StructField{Name: name, Type: parameter.Schema.Type(), Tag: tag}) + } + } + + // Include selector (limit/offset/fields/page) for read components when available + if components.View != nil && components.View.Selector != nil { + if p := components.View.Selector.LimitParameter; p != nil && p.In != nil && p.In.Name != "" { + if !uniqueQuery[p.In.Name] { // avoid duplicates + uniqueQuery[p.In.Name] = true + inputFields = append(inputFields, reflect.StructField{Name: strings.Title(p.Name), Type: p.Schema.Type(), Tag: `json:",omitempty"`}) + } + } + if p := components.View.Selector.OffsetParameter; p != nil && p.In != nil && p.In.Name != "" { + if !uniqueQuery[p.In.Name] { + uniqueQuery[p.In.Name] = true + inputFields = append(inputFields, reflect.StructField{Name: strings.Title(p.Name), Type: p.Schema.Type(), Tag: `json:",omitempty"`}) + } + } + if p := components.View.Selector.FieldsParameter; p != nil && p.In != nil && p.In.Name != "" { + if !uniqueQuery[p.In.Name] { + uniqueQuery[p.In.Name] = true + // Fields is a []string – ensure optional in schema + inputFields = append(inputFields, reflect.StructField{Name: strings.Title(p.Name), Type: p.Schema.Type(), Tag: `json:",omitempty" optional:"true"`}) + } + } + if p := components.View.Selector.PageParameter; p != nil && p.In != nil && p.In.Name != "" { + if !uniqueQuery[p.In.Name] { + uniqueQuery[p.In.Name] = true + inputFields = append(inputFields, reflect.StructField{Name: strings.Title(p.Name), Type: p.Schema.Type(), Tag: `json:",omitempty"`}) + } } } @@ -229,6 +353,23 @@ func (r *Router) buildTemplateResourceIntegration(item *dpath.Item, aPath *dpath parameterNames = append(parameterNames, parameter.In.Name) } } + // Also expose view selector pagination controls in URI template if present + if provider != nil { + if comp, err := provider.Component(context.Background()); err == nil && comp.View != nil && comp.View.Selector != nil { + if p := comp.View.Selector.LimitParameter; p != nil && p.In != nil && p.In.Name != "" { + parameterNames = append(parameterNames, p.In.Name) + } + if p := comp.View.Selector.OffsetParameter; p != nil && p.In != nil && p.In.Name != "" { + parameterNames = append(parameterNames, p.In.Name) + } + if p := comp.View.Selector.FieldsParameter; p != nil && p.In != nil && p.In.Name != "" { + parameterNames = append(parameterNames, p.In.Name) + } + if p := comp.View.Selector.PageParameter; p != nil && p.In != nil && p.In.Name != "" { + parameterNames = append(parameterNames, p.In.Name) + } + } + } canBuildTemplateResource := len(parameterNames) > 0 || strings.Contains(aPath.URI, "{") if !canBuildTemplateResource { return nil diff --git a/gateway/router/marshal/json/marshaller_map.go b/gateway/router/marshal/json/marshaller_map.go index 320001bbe..427868bf1 100644 --- a/gateway/router/marshal/json/marshaller_map.go +++ b/gateway/router/marshal/json/marshaller_map.go @@ -203,6 +203,9 @@ func (m *mapMarshaller) mapStringIfaceMarshaller() func(pointer unsafe.Pointer, return nil } + // Ensure JSON special characters in keys are escaped + replacer := getReplacer() + if !m.isEmbedded { sb.WriteString("{") } @@ -214,9 +217,9 @@ func (m *mapMarshaller) mapStringIfaceMarshaller() func(pointer unsafe.Pointer, sb.WriteString(",") } counter++ - sb.WriteString(`"`) - sb.WriteString(namesIndex.formatTo(aKey, m.config.CaseFormat)) - sb.WriteString(`":`) + // Write escaped key + marshallString(namesIndex.formatTo(aKey, m.config.CaseFormat), sb, replacer) + sb.WriteString(`:`) if err := m.valueMarshaller.MarshallObject(AsPtr(aValue, m.valueType), sb); err != nil { return err diff --git a/gateway/router/marshal/json/marshaller_strings.go b/gateway/router/marshal/json/marshaller_strings.go index c044fc5b6..f19f91325 100644 --- a/gateway/router/marshal/json/marshaller_strings.go +++ b/gateway/router/marshal/json/marshaller_strings.go @@ -49,9 +49,54 @@ func (i *stringMarshaller) ensureReplacer() { } } -func marshallString(asString string, sb *MarshallSession, replacer *strings.Replacer) { +func marshallString(asString string, sb *MarshallSession, _ *strings.Replacer) { + // Fully JSON-escape the string, including control chars and JS line/paragraph separators. + const hexDigits = "0123456789abcdef" sb.WriteByte('"') - sb.WriteString(replacer.Replace(asString)) + for i := 0; i < len(asString); i++ { + c := asString[i] + switch c { + case '\\', '"': + sb.WriteByte('\\') + sb.WriteByte(c) + case '/': + sb.WriteByte('\\') + sb.WriteByte('/') + case '\b': + sb.WriteString(`\\b`) + case '\f': + sb.WriteString(`\\f`) + case '\n': + sb.WriteString(`\\n`) + case '\r': + sb.WriteString(`\\r`) + case '\t': + sb.WriteString(`\\t`) + default: + // Escape other control characters < 0x20 as \u00XX + if c < 0x20 { + sb.WriteString(`\\u00`) + sb.WriteByte(hexDigits[c>>4]) + sb.WriteByte(hexDigits[c&0x0F]) + continue + } + // Escape U+2028 and U+2029 to be safe for JS embed contexts + if c == 0xE2 && i+2 < len(asString) { + c1 := asString[i+1] + c2 := asString[i+2] + if c1 == 0x80 && (c2 == 0xA8 || c2 == 0xA9) { + if c2 == 0xA8 { + sb.WriteString(`\\u2028`) + } else { + sb.WriteString(`\\u2029`) + } + i += 2 + continue + } + } + sb.WriteByte(c) + } + } sb.WriteByte('"') } diff --git a/go.mod b/go.mod index 7af2f0c20..5ff45f3ab 100644 --- a/go.mod +++ b/go.mod @@ -48,9 +48,9 @@ require ( require ( github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d github.com/viant/firebase v0.1.1 - github.com/viant/jsonrpc v0.9.0 + github.com/viant/jsonrpc v0.11.0 github.com/viant/mcp v0.6.0 - github.com/viant/mcp-protocol v0.5.7 + github.com/viant/mcp-protocol v0.5.10 github.com/viant/structology v0.6.1 github.com/viant/tagly v0.2.2 github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa @@ -166,3 +166,4 @@ require ( modernc.org/token v1.0.0 // indirect ) +replace github.com/viant/mcp => ../mcp diff --git a/go.sum b/go.sum index 3912edbdf..2f063153d 100644 --- a/go.sum +++ b/go.sum @@ -1124,12 +1124,10 @@ github.com/viant/govalidator v0.3.1 h1:V7f/KgfzbP8fVDc+Kj+jyPvfXxMr2N1x7srOlDV6l github.com/viant/govalidator v0.3.1/go.mod h1:D35Dwx0R8rR1knRxhlseoYvOkiqo24kpMg1/o977i9Y= github.com/viant/igo v0.2.0 h1:ygWmTCinnGPaeV7omJLiyneOpzYZ5kiw7oYz7mUJZVQ= github.com/viant/igo v0.2.0/go.mod h1:7V6AWsLhKWeGzXNTNH3AZiIEKa0m33DrQbdWtapsI74= -github.com/viant/jsonrpc v0.9.0 h1:vTZsApJxTd3Y50ygOBs8HKCJ24NrwgCa7lqG1oYXpdE= -github.com/viant/jsonrpc v0.9.0/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= -github.com/viant/mcp v0.6.0 h1:+BCsLSW5pux07avEhS550hZno8Y5ZKKSfdLm6NHRU+8= -github.com/viant/mcp v0.6.0/go.mod h1:fb5wpE9kc/R32pNE4Pdo1DR4ZW6+0em3rsFuBHoqmp4= -github.com/viant/mcp-protocol v0.5.7 h1:3ifypMAy+oUjQEAsq+XwrAhE/B/3eIes4yXdhoRF9Eo= -github.com/viant/mcp-protocol v0.5.7/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= +github.com/viant/jsonrpc v0.11.0 h1:SqOztRwLWTCdK+VSU0XhZvwqeHrJ1hpQcmhPY6NXH5g= +github.com/viant/jsonrpc v0.11.0/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= +github.com/viant/mcp-protocol v0.5.10 h1:915EC1GKgBbyYF4efzRSZ/AE6f4vobkbwa2qe6OOjJ0= +github.com/viant/mcp-protocol v0.5.10/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b h1:3q166tV28yFdbFV+tXXjH7ViKAmgAgGdoWzMtvhQv28= github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b/go.mod h1:85fneXJbErKMGhSQto3A5ElTQCwl3t74U9cSV0waBHw= github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= diff --git a/repository/component.go b/repository/component.go index 5a22bdee6..ec106e47a 100644 --- a/repository/component.go +++ b/repository/component.go @@ -170,6 +170,109 @@ func (c *Component) initView(ctx context.Context, resource *view.Resource) error if err := c.View.Init(ctx, resource); err != nil { return err } + // For read components (GET), expose and enable offset/limit/fields/page/orderBy for each namespaced view. + if strings.EqualFold(c.Path.Method, http.MethodGet) { + // Helper to enable limit/offset for a view with namespace prefix (if any) + ensureSelectors := func(v *view.View, nsPrefix string) { + if v == nil { + return + } + if v.Selector == nil { + v.Selector = &view.Config{} + } + if v.Selector.Constraints == nil { + v.Selector.Constraints = &view.Constraints{} + } + // Enable constraints + v.Selector.Constraints.Limit = true + v.Selector.Constraints.Offset = true + v.Selector.Constraints.Projection = true + v.Selector.Constraints.OrderBy = true + + // Limit param + if v.Selector.LimitParameter == nil { + p := *view.QueryStateParameters.LimitParameter + p.Description = view.Description(view.LimitQuery, v.Name) + if nsPrefix != "" { + p.In = state.NewQueryLocation(nsPrefix + view.LimitQuery) + } + v.Selector.LimitParameter = &p + } else if v.Selector.LimitParameter.Description == "" { + v.Selector.LimitParameter.Description = view.Description(view.LimitQuery, v.Name) + } + + // Offset param + if v.Selector.OffsetParameter == nil { + p := *view.QueryStateParameters.OffsetParameter + p.Description = view.Description(view.OffsetQuery, v.Name) + if nsPrefix != "" { + p.In = state.NewQueryLocation(nsPrefix + view.OffsetQuery) + } + v.Selector.OffsetParameter = &p + } else if v.Selector.OffsetParameter.Description == "" { + v.Selector.OffsetParameter.Description = view.Description(view.OffsetQuery, v.Name) + } + + // Fields param (controls which fields are included) + if v.Selector.FieldsParameter == nil { + p := *view.QueryStateParameters.FieldsParameter + p.Description = view.Description(view.FieldsQuery, v.Name) + if nsPrefix != "" { + p.In = state.NewQueryLocation(nsPrefix + view.FieldsQuery) + } + v.Selector.FieldsParameter = &p + } else if v.Selector.FieldsParameter.Description == "" { + v.Selector.FieldsParameter.Description = view.Description(view.FieldsQuery, v.Name) + } + + // Page param (paging interface on top of limit/offset) + if v.Selector.PageParameter == nil { + p := *view.QueryStateParameters.PageParameter + p.Description = view.Description(view.PageQuery, v.Name) + if nsPrefix != "" { + p.In = state.NewQueryLocation(nsPrefix + view.PageQuery) + } + v.Selector.PageParameter = &p + } else if v.Selector.PageParameter.Description == "" { + v.Selector.PageParameter.Description = view.Description(view.PageQuery, v.Name) + } + + // OrderBy param + if v.Selector.OrderByParameter == nil { + p := *view.QueryStateParameters.OrderByParameter + p.Description = view.Description(view.OrderByQuery, v.Name) + if nsPrefix != "" { + p.In = state.NewQueryLocation(nsPrefix + view.OrderByQuery) + } + v.Selector.OrderByParameter = &p + } else if v.Selector.OrderByParameter.Description == "" { + v.Selector.OrderByParameter.Description = view.Description(view.OrderByQuery, v.Name) + } + } + + // Root view + nsPrefix := "" + if c.View.Selector != nil && c.View.Selector.Namespace != "" { + nsPrefix = c.View.Selector.Namespace + } + ensureSelectors(c.View, nsPrefix) + + // All related views via NamespacedView + if c.NamespacedView != nil { + for _, nsView := range c.NamespacedView.Views { + v := nsView.View + // Determine ns prefix from NamespacedView (prefer non-empty namespace if present) + pfx := "" + for _, ns := range nsView.Namespaces { + if ns != "" { + pfx = ns + break + } + } + ensureSelectors(v, pfx) + } + } + } holder := "" if c.Contract.Output.Type.Parameters != nil { if rootHolder := c.Contract.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); rootHolder != nil { From 520d4c353ba673f0711feb3a02e148d8a2c9b1e5 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 20 Oct 2025 09:17:50 -0700 Subject: [PATCH 050/279] enhanced mcp integration --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 5ff45f3ab..b9bf9e443 100644 --- a/go.mod +++ b/go.mod @@ -166,4 +166,3 @@ require ( modernc.org/token v1.0.0 // indirect ) -replace github.com/viant/mcp => ../mcp From fce13f39c3a08cda87761f4bd45886175dbc2890 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 20 Oct 2025 09:19:02 -0700 Subject: [PATCH 051/279] enhanced mcp integration --- go.mod | 3 +-- go.sum | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b9bf9e443..43b7b8438 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d github.com/viant/firebase v0.1.1 github.com/viant/jsonrpc v0.11.0 - github.com/viant/mcp v0.6.0 + github.com/viant/mcp v0.7.0 github.com/viant/mcp-protocol v0.5.10 github.com/viant/structology v0.6.1 github.com/viant/tagly v0.2.2 @@ -165,4 +165,3 @@ require ( modernc.org/strutil v1.1.3 // indirect modernc.org/token v1.0.0 // indirect ) - diff --git a/go.sum b/go.sum index 2f063153d..20c6ae58a 100644 --- a/go.sum +++ b/go.sum @@ -1126,6 +1126,8 @@ github.com/viant/igo v0.2.0 h1:ygWmTCinnGPaeV7omJLiyneOpzYZ5kiw7oYz7mUJZVQ= github.com/viant/igo v0.2.0/go.mod h1:7V6AWsLhKWeGzXNTNH3AZiIEKa0m33DrQbdWtapsI74= github.com/viant/jsonrpc v0.11.0 h1:SqOztRwLWTCdK+VSU0XhZvwqeHrJ1hpQcmhPY6NXH5g= github.com/viant/jsonrpc v0.11.0/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= +github.com/viant/mcp v0.7.0 h1:pIsT93/45pDxpphsZgS8d+0mIzNDihZH0zCDADxGqe8= +github.com/viant/mcp v0.7.0/go.mod h1:4Kk48IEvnAwkplg9sLs1lOBY3cRdsYjrn4lXXdPOPMo= github.com/viant/mcp-protocol v0.5.10 h1:915EC1GKgBbyYF4efzRSZ/AE6f4vobkbwa2qe6OOjJ0= github.com/viant/mcp-protocol v0.5.10/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b h1:3q166tV28yFdbFV+tXXjH7ViKAmgAgGdoWzMtvhQv28= From 120a696c2fa04a4794b0565e2f518f1dc3a37b05 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 20 Oct 2025 13:25:22 -0700 Subject: [PATCH 052/279] enhanced mcp integration --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 43b7b8438..eaa512277 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d github.com/viant/firebase v0.1.1 github.com/viant/jsonrpc v0.11.0 - github.com/viant/mcp v0.7.0 + github.com/viant/mcp v0.7.2 github.com/viant/mcp-protocol v0.5.10 github.com/viant/structology v0.6.1 github.com/viant/tagly v0.2.2 diff --git a/go.sum b/go.sum index 20c6ae58a..318e88f4e 100644 --- a/go.sum +++ b/go.sum @@ -1128,6 +1128,8 @@ github.com/viant/jsonrpc v0.11.0 h1:SqOztRwLWTCdK+VSU0XhZvwqeHrJ1hpQcmhPY6NXH5g= github.com/viant/jsonrpc v0.11.0/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= github.com/viant/mcp v0.7.0 h1:pIsT93/45pDxpphsZgS8d+0mIzNDihZH0zCDADxGqe8= github.com/viant/mcp v0.7.0/go.mod h1:4Kk48IEvnAwkplg9sLs1lOBY3cRdsYjrn4lXXdPOPMo= +github.com/viant/mcp v0.7.2 h1:+vkzxFIlKWsjTY/56oBcHuPBlO+9lNb1cwwm2FXA/cA= +github.com/viant/mcp v0.7.2/go.mod h1:4Kk48IEvnAwkplg9sLs1lOBY3cRdsYjrn4lXXdPOPMo= github.com/viant/mcp-protocol v0.5.10 h1:915EC1GKgBbyYF4efzRSZ/AE6f4vobkbwa2qe6OOjJ0= github.com/viant/mcp-protocol v0.5.10/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b h1:3q166tV28yFdbFV+tXXjH7ViKAmgAgGdoWzMtvhQv28= From bcb769bc1726954f0b61b89ceb2df7d3aa747cf0 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sun, 26 Oct 2025 20:51:37 -0700 Subject: [PATCH 053/279] enhanced mcp integration --- go.mod | 4 ++-- go.sum | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index eaa512277..1ec14f461 100644 --- a/go.mod +++ b/go.mod @@ -48,8 +48,8 @@ require ( require ( github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d github.com/viant/firebase v0.1.1 - github.com/viant/jsonrpc v0.11.0 - github.com/viant/mcp v0.7.2 + github.com/viant/jsonrpc v0.15.0 + github.com/viant/mcp v0.8.0 github.com/viant/mcp-protocol v0.5.10 github.com/viant/structology v0.6.1 github.com/viant/tagly v0.2.2 diff --git a/go.sum b/go.sum index 318e88f4e..d80772bcb 100644 --- a/go.sum +++ b/go.sum @@ -1126,10 +1126,18 @@ github.com/viant/igo v0.2.0 h1:ygWmTCinnGPaeV7omJLiyneOpzYZ5kiw7oYz7mUJZVQ= github.com/viant/igo v0.2.0/go.mod h1:7V6AWsLhKWeGzXNTNH3AZiIEKa0m33DrQbdWtapsI74= github.com/viant/jsonrpc v0.11.0 h1:SqOztRwLWTCdK+VSU0XhZvwqeHrJ1hpQcmhPY6NXH5g= github.com/viant/jsonrpc v0.11.0/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= +github.com/viant/jsonrpc v0.14.0 h1:YppPzIidbd9bgjKHCREXkvjkJXf8AaFGGWfk1r+nCJE= +github.com/viant/jsonrpc v0.14.0/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= +github.com/viant/jsonrpc v0.15.0 h1:0qy9vzgNwR9Gj1C+ouSrzNUtNDzKGogO+7TZR+cFrA4= +github.com/viant/jsonrpc v0.15.0/go.mod h1:b214Lo4zBwLqbu6Tf2bRlgQkFfPMBW5ap4qS+I3zcJ8= github.com/viant/mcp v0.7.0 h1:pIsT93/45pDxpphsZgS8d+0mIzNDihZH0zCDADxGqe8= github.com/viant/mcp v0.7.0/go.mod h1:4Kk48IEvnAwkplg9sLs1lOBY3cRdsYjrn4lXXdPOPMo= github.com/viant/mcp v0.7.2 h1:+vkzxFIlKWsjTY/56oBcHuPBlO+9lNb1cwwm2FXA/cA= github.com/viant/mcp v0.7.2/go.mod h1:4Kk48IEvnAwkplg9sLs1lOBY3cRdsYjrn4lXXdPOPMo= +github.com/viant/mcp v0.7.5 h1:8Gmdz4LiZ1Ot8eUaLCufqWtivrkGGjBA1ra41cdmgmo= +github.com/viant/mcp v0.7.5/go.mod h1:3eBNG5U/CCOPbLdBpF3clwS11WfxEYB6MdzTL2s4jLo= +github.com/viant/mcp v0.8.0 h1:n4tnLXpOtpnrLZtHyNG2mmZ9SUbGWKsWGla10iMfuDg= +github.com/viant/mcp v0.8.0/go.mod h1:fyuB1TSQYbbGNn7U6rLmlr9gD+Yg5+Na32D34Uvm0sk= github.com/viant/mcp-protocol v0.5.10 h1:915EC1GKgBbyYF4efzRSZ/AE6f4vobkbwa2qe6OOjJ0= github.com/viant/mcp-protocol v0.5.10/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b h1:3q166tV28yFdbFV+tXXjH7ViKAmgAgGdoWzMtvhQv28= From d847cf1fd63a7689b040cebd3b92e372a3939987 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 27 Oct 2025 05:51:57 -0700 Subject: [PATCH 054/279] patched parser --- internal/translator/parser/statement.go | 43 ++++++++++++++----------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/internal/translator/parser/statement.go b/internal/translator/parser/statement.go index 3874f4c48..39633964f 100644 --- a/internal/translator/parser/statement.go +++ b/internal/translator/parser/statement.go @@ -38,42 +38,49 @@ func (s Statements) DMLTables(rawSQL string) []string { var tables = make(map[string]bool) var result []string for _, statement := range s { + // Only consider exec statements for DML table extraction. + if !statement.IsExec { + continue + } SQL := rawSQL[statement.Start:statement.End] - usesService := strings.Contains(SQL, "$sql.") - lowerCasedDML := strings.ToLower(SQL) - quoted := "" - - if index := strings.Index(SQL, `"`); index != -1 { - quoted = SQL[index+1:] - if index = strings.Index(quoted, `"`); index != -1 { - quoted = quoted[:index] + // Handle service-based exec ($sql.Insert/$sql.Update) only when explicitly detected as service. + if statement.Kind == shared.ExecKindService { + quoted := "" + if index := strings.Index(SQL, `"`); index != -1 { + quoted = SQL[index+1:] + if index = strings.Index(quoted, `"`); index != -1 { + quoted = quoted[:index] + } } - } - if usesService && quoted != "" { - statement.Table = quoted - if _, ok := tables[statement.Table]; ok { + if quoted != "" { + statement.Table = quoted + if _, ok := tables[statement.Table]; ok { + continue + } + result = append(result, statement.Table) + tables[statement.Table] = true continue } - result = append(result, statement.Table) - tables[statement.Table] = true - continue } + + lowerCasedDML := strings.ToLower(SQL) + if strings.Contains(lowerCasedDML, "insert") { - if stmt, _ := sqlparser.ParseInsert(SQL); stmt != nil { + if stmt, _ := sqlparser.ParseInsert(SQL); stmt != nil && stmt.Target.X != nil { if table := sqlparser.Stringify(stmt.Target.X); table != "" { statement.Table = table } } } else if strings.Contains(lowerCasedDML, "update") { - if stmt, _ := sqlparser.ParseUpdate(SQL); stmt != nil { + if stmt, _ := sqlparser.ParseUpdate(SQL); stmt != nil && stmt.Target.X != nil { if table := sqlparser.Stringify(stmt.Target.X); table != "" { statement.Table = table } } } else if strings.Contains(lowerCasedDML, "delete") { - if stmt, _ := sqlparser.ParseDelete(SQL); stmt != nil { + if stmt, _ := sqlparser.ParseDelete(SQL); stmt != nil && stmt.Target.X != nil { if table := sqlparser.Stringify(stmt.Target.X); table != "" { statement.Table = table } From d06c78404fd4a425fb64441832ced78b727b121b Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 28 Oct 2025 12:20:49 -0700 Subject: [PATCH 055/279] patched parser --- service/operator/reader.go | 2 +- service/operator/service.go | 15 ++++++++++----- view/state/hook.go | 10 +++++++++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/service/operator/reader.go b/service/operator/reader.go index e882d0643..1e1ae41f4 100644 --- a/service/operator/reader.go +++ b/service/operator/reader.go @@ -44,7 +44,7 @@ func (s *Service) runQuery(ctx context.Context, component *repository.Component, if err := s.updateJobStatusDone(ctx, component, handlerResponse, setting.SyncFlag, startTime); err != nil { return nil, err } - if output, err = s.finalize(ctx, handlerResponse.Output, handlerResponse.Error); err != nil { + if output, err = s.finalize(ctx, handlerResponse.Output, handlerResponse.Error, nil); err != nil { aSession.ClearCache(component.Output.Type.Parameters) return s.HandleError(ctx, aSession, component, err) } diff --git a/service/operator/service.go b/service/operator/service.go index bf50ece0b..960105e6a 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -6,6 +6,10 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "reflect" + "time" + "github.com/viant/afs" "github.com/viant/afs/file" "github.com/viant/datly/repository" @@ -29,9 +33,6 @@ import ( "github.com/viant/xdatly/handler/response" hstate "github.com/viant/xdatly/handler/state" "google.golang.org/api/googleapi" - "net/http" - "reflect" - "time" ) type Service struct { @@ -118,13 +119,17 @@ func (s *Service) operate(ctx context.Context, aComponent *repository.Component, } } - return s.finalize(ctx, ret, err) + return s.finalize(ctx, ret, err, aSession) } return nil, response.NewError(500, fmt.Sprintf("unsupported Type %v", aComponent.Service)) } -func (s *Service) finalize(ctx context.Context, ret interface{}, err error) (interface{}, error) { +func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSession *session.Session) (interface{}, error) { + if injectorFinalizer, ok := ret.(state.InjectorFinalizer); ok { + err = injectorFinalizer.Finalize(ctx, aSession) + return ret, err + } if err != nil { return ret, err } diff --git a/view/state/hook.go b/view/state/hook.go index 9ae5b49c4..a4125cb44 100644 --- a/view/state/hook.go +++ b/view/state/hook.go @@ -1,6 +1,10 @@ package state -import "context" +import ( + "context" + + "github.com/viant/xdatly/handler/state" +) // Initializer is an interface that should be implemented by any type that needs to be initialized type Initializer interface { @@ -11,3 +15,7 @@ type Initializer interface { type Finalizer interface { Finalize(ctx context.Context) error } + +type InjectorFinalizer interface { + Finalize(ctx context.Context, injector state.Injector) error +} From b88cbd582b0a962218a01ec5556298480256b8c5 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 28 Oct 2025 12:40:49 -0700 Subject: [PATCH 056/279] patched parser --- view/state/types.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/view/state/types.go b/view/state/types.go index 292ad4147..f301290ea 100644 --- a/view/state/types.go +++ b/view/state/types.go @@ -11,6 +11,9 @@ type Types struct { } func (c *Types) Lookup(p reflect.Type) (*Type, bool) { + if len(c.types) == 0 { + return nil, false + } c.RWMutex.RLock() ret, ok := c.types[p] c.RWMutex.RUnlock() From 1b5e79ff29bc13859ab3aacea5d408f487e62505 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 28 Oct 2025 12:47:53 -0700 Subject: [PATCH 057/279] patched parser --- service/operator/reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/operator/reader.go b/service/operator/reader.go index 1e1ae41f4..801638e12 100644 --- a/service/operator/reader.go +++ b/service/operator/reader.go @@ -44,7 +44,7 @@ func (s *Service) runQuery(ctx context.Context, component *repository.Component, if err := s.updateJobStatusDone(ctx, component, handlerResponse, setting.SyncFlag, startTime); err != nil { return nil, err } - if output, err = s.finalize(ctx, handlerResponse.Output, handlerResponse.Error, nil); err != nil { + if output, err = s.finalize(ctx, handlerResponse.Output, handlerResponse.Error, aSession); err != nil { aSession.ClearCache(component.Output.Type.Parameters) return s.HandleError(ctx, aSession, component, err) } From c26fd5e5fd7edf5a275ce1dcfd882882ced651f8 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 28 Oct 2025 13:13:44 -0700 Subject: [PATCH 058/279] patched parser --- service/operator/service.go | 15 ++++++++++++++- service/session/state.go | 8 ++++++++ view/state/hook.go | 3 ++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/service/operator/service.go b/service/operator/service.go index 960105e6a..a5a61ecb1 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -15,6 +15,7 @@ import ( "github.com/viant/datly/repository" rasync "github.com/viant/datly/repository/async" "github.com/viant/datly/repository/content" + "github.com/viant/datly/repository/contract" "github.com/viant/datly/service" "github.com/viant/datly/service/reader" "github.com/viant/datly/service/session" @@ -29,9 +30,11 @@ import ( xhandler "github.com/viant/xdatly/handler" "github.com/viant/xdatly/handler/async" "github.com/viant/xdatly/handler/exec" + xhttp "github.com/viant/xdatly/handler/http" "github.com/viant/xdatly/handler/logger" "github.com/viant/xdatly/handler/response" hstate "github.com/viant/xdatly/handler/state" + xstate "github.com/viant/xdatly/handler/state" "google.golang.org/api/googleapi" ) @@ -85,6 +88,7 @@ func (s *Service) HandleError(ctx context.Context, aSession *session.Session, aC func (s *Service) operate(ctx context.Context, aComponent *repository.Component, aSession *session.Session) (interface{}, error) { var err error + ctx, err = s.EnsureContext(ctx, aSession, aComponent) if err != nil { return nil, err @@ -127,7 +131,16 @@ func (s *Service) operate(ctx context.Context, aComponent *repository.Component, func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSession *session.Session) (interface{}, error) { if injectorFinalizer, ok := ret.(state.InjectorFinalizer); ok { - err = injectorFinalizer.Finalize(ctx, aSession) + + lookup := func(ctx context.Context, route xhttp.Route) (xstate.Injector, error) { + aComponent, err := aSession.Registry().Lookup(ctx, contract.NewPath(route.Method, route.URL)) + if err != nil { + return nil, err + } + return aSession.NewSession(aComponent), nil + } + + err = injectorFinalizer.Finalize(ctx, lookup) return ret, err } if err != nil { diff --git a/service/session/state.go b/service/session/state.go index 8e6eab3f3..63904496e 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/viant/datly/internal/converter" + "github.com/viant/datly/repository" "github.com/viant/datly/service/auth" "github.com/viant/datly/utils/types" "github.com/viant/datly/view" @@ -42,6 +43,13 @@ type ( } ) +func (s *Session) NewSession(component *repository.Component) *Session { + ret := *s + s.component = component + s.view = component.View + return &ret +} + func (s *Session) SetView(view *view.View) { s.view = view } diff --git a/view/state/hook.go b/view/state/hook.go index a4125cb44..e871f4d6d 100644 --- a/view/state/hook.go +++ b/view/state/hook.go @@ -3,6 +3,7 @@ package state import ( "context" + "github.com/viant/xdatly/handler/http" "github.com/viant/xdatly/handler/state" ) @@ -17,5 +18,5 @@ type Finalizer interface { } type InjectorFinalizer interface { - Finalize(ctx context.Context, injector state.Injector) error + Finalize(ctx context.Context, getInjector func(ctx context.Context, path http.Route) (state.Injector, error)) error } From 57d9ca6a83439d62fbe672b690be8b0e82ae9936 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 28 Oct 2025 13:45:22 -0700 Subject: [PATCH 059/279] patched parser --- service/session/state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/session/state.go b/service/session/state.go index 63904496e..68fef2ae7 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -45,8 +45,8 @@ type ( func (s *Session) NewSession(component *repository.Component) *Session { ret := *s - s.component = component - s.view = component.View + ret.component = component + ret.view = component.View return &ret } From 452af4fa118ac00a3503b6616d2d53e305364b67 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 28 Oct 2025 14:02:27 -0700 Subject: [PATCH 060/279] patched parser --- service/session/state.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/service/session/state.go b/service/session/state.go index 68fef2ae7..005425dfc 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -45,8 +45,9 @@ type ( func (s *Session) NewSession(component *repository.Component) *Session { ret := *s - ret.component = component - ret.view = component.View + s.component = component + s.locatorOpt.Views.Register(component.View) + s.view = component.View return &ret } From 82d6bb5c6914c61486a9cbbd8bc9b272476e1aff Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 29 Oct 2025 07:58:29 -0700 Subject: [PATCH 061/279] patched parser --- service/executor/handler/executor.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/service/executor/handler/executor.go b/service/executor/handler/executor.go index ce20877d0..3e21d27f1 100644 --- a/service/executor/handler/executor.go +++ b/service/executor/handler/executor.go @@ -173,6 +173,7 @@ func (e *Executor) newSqlService(options *sqlx.Options) (sqlx.Sqlx, error) { } func (e *Executor) getDataUnit(options *sqlx.Options) (*expand.DataUnit, error) { + e.ensureConnectors() if (options.WithDb == nil && options.WithTx == nil) && options.WithConnector == e.view.Connector.Name { return e.dataUnit, nil } @@ -197,6 +198,11 @@ func (e *Executor) getDataUnit(options *sqlx.Options) (*expand.DataUnit, error) if connector == nil { return nil, fmt.Errorf("failed to lookup connector %v", options.WithConnector) } + + if _, ok := e.connectors[options.WithConnector]; !ok { + e.connectors[options.WithConnector] = connector + } + db, err := connector.DB() if err != nil { return nil, err @@ -211,6 +217,17 @@ func (e *Executor) getDataUnit(options *sqlx.Options) (*expand.DataUnit, error) return e.dataUnit, nil } +func (e *Executor) ensureConnectors() { + if len(e.connectors) == 0 { + e.connectors = make(view.Connectors) + if res := e.view.GetResource(); res != nil { + for _, connector := range res.Connectors { + e.connectors[connector.Name] = connector + } + } + } +} + func (e *Executor) Execute(ctx context.Context) error { if e.executed { return nil @@ -222,6 +239,10 @@ func (e *Executor) Execute(ctx context.Context) error { dbOptions = append(dbOptions, executor.WithTx(e.tx)) } + err := service.ExecuteStmts(ctx, executor.NewViewDBSource(e.view), newSqlxIterator(e.dataUnit.Statements.Executable), dbOptions...) + if err != nil { + return err + } for _, unit := range e.dataUnits { dbSource := &DbSource{} dbSource.db, _ = unit.MetaSource.Db() @@ -230,7 +251,7 @@ func (e *Executor) Execute(ctx context.Context) error { } } - return service.ExecuteStmts(ctx, executor.NewViewDBSource(e.view), newSqlxIterator(e.dataUnit.Statements.Executable), dbOptions...) + return err } func (e *Executor) ExpandAndExecute(ctx context.Context) (*executor.Session, error) { From d002e57dc294d5598b81b450398b82f87a60c972 Mon Sep 17 00:00:00 2001 From: arao Date: Thu, 30 Oct 2025 15:32:20 -0700 Subject: [PATCH 062/279] ENG-00000 structql update --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index fa54285be..df198a925 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 github.com/viant/sqlx v0.17.8 - github.com/viant/structql v0.5.2 + github.com/viant/structql v0.5.3 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 github.com/viant/xreflect v0.7.3 diff --git a/go.sum b/go.sum index 3914ffdf7..a3e9e8c48 100644 --- a/go.sum +++ b/go.sum @@ -1138,14 +1138,12 @@ github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns= github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.17.7 h1:drUv3N8mOboq917gnmcT9zC4G9vj4jU11bO/SsLpmc8= -github.com/viant/sqlx v0.17.7/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/sqlx v0.17.8 h1:YxGTrXC2B1JmDz1qp8G+G9hGPk7XCRevWN1E4E+ZlCI= github.com/viant/sqlx v0.17.8/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.6.1 h1:Forza+RF/1tmlQFk9ABNhu+IQ8vMAqbYM6FOsYtGh9E= github.com/viant/structology v0.6.1/go.mod h1:63XfkzUyNw7wdi99HJIsH2Rg3d5AOumqbWLUYytOkxU= -github.com/viant/structql v0.5.2 h1:0dAratszxC6AD/TNaV8BnLQQprNO5GJHaKjmszrIoeY= -github.com/viant/structql v0.5.2/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= +github.com/viant/structql v0.5.3 h1:QeOxvF0so8VFGt5bm+Jr6FL8uRnXkvwWY+ZCkbe3zsI= +github.com/viant/structql v0.5.3/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= github.com/viant/tagly v0.2.2 h1:qqb4Dov83i7nl7Gewph/lLaYAF8MKv0N7y34scgRNmE= github.com/viant/tagly v0.2.2/go.mod h1:vV8QgJkhug+X+qyKds8av0fhjD+4u7IhNtowL1KGQ5A= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= From 37994e2c3cb56071a7689b3fd1b23aead6982581 Mon Sep 17 00:00:00 2001 From: arao Date: Fri, 31 Oct 2025 10:35:34 -0700 Subject: [PATCH 063/279] ENG-0000 datly tarslate for Patch --- internal/translator/resource.go | 8 +++++--- internal/translator/service.go | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/translator/resource.go b/internal/translator/resource.go index 163684277..db7e97121 100644 --- a/internal/translator/resource.go +++ b/internal/translator/resource.go @@ -474,9 +474,11 @@ func (r *Resource) expandSQL(viewlet *Viewlet) (*sqlx.SQL, error) { func (r *Resource) ensureViewParametersSchema(ctx context.Context, setType func(ctx context.Context, setType *Viewlet) error) error { viewParameters := r.State.FilterByKind(state.KindView) for _, viewParameter := range viewParameters { - if viewParameter.Schema != nil && viewParameter.Schema.Type() != nil { - continue - } + //WE DO NOT NEEDED IT + //if viewParameter.Schema != nil && viewParameter.Schema.Type() != nil { + // fmt.Printf("skipping view %v %v\n", viewParameter.Name, viewParameter.Schema) + // //continue + //} if viewParameter.In.Name == "" { //default root schema continue } diff --git a/internal/translator/service.go b/internal/translator/service.go index 6f7cbcbc2..539ceade9 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -577,7 +577,8 @@ func (s *Service) buildQueryViewletType(ctx context.Context, viewlet *Viewlet) e func (s *Service) buildViewletType(ctx context.Context, db *sql.DB, viewlet *Viewlet) (err error) { shared.EnsureArgs(viewlet.Expanded.Query, &viewlet.Expanded.Args) - if viewlet.Spec, err = inference.NewSpec(ctx, db, &s.Repository.Messages, viewlet.Table.Name, viewlet.ColumnConfig, viewlet.Expanded.Query, viewlet.Expanded.Args...); err != nil { + viewlet.Spec, err = inference.NewSpec(ctx, db, &s.Repository.Messages, viewlet.Table.Name, viewlet.ColumnConfig, viewlet.Expanded.Query, viewlet.Expanded.Args...) + if err != nil { return fmt.Errorf("failed to create spec for %v, %w", viewlet.Name, err) } From 6437e5db561d4a29442d020cba080778257a24fc Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 31 Oct 2025 14:20:40 -0700 Subject: [PATCH 064/279] patched parser --- service/session/option.go | 8 ++++++++ service/session/state.go | 34 +++++++++++++++++++++++++++++++--- service/session/stater.go | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/service/session/option.go b/service/session/option.go index 148242cb4..099835813 100644 --- a/service/session/option.go +++ b/service/session/option.go @@ -33,6 +33,7 @@ type ( scope string embeddedFS *embed.FS auth *auth.Service + preseedCache bool } Option func(o *Options) @@ -155,6 +156,13 @@ func WithAuth(auth *auth.Service) Option { } } +// WithPreseedCache controls whether NewSession should pre-seed child cache from parent (default false) +func WithPreseedCache(flag bool) Option { + return func(s *Options) { + s.preseedCache = flag + } +} + func WithComponent(component *repository.Component) Option { return func(s *Options) { s.component = component diff --git a/service/session/state.go b/service/session/state.go index 005425dfc..3f28d36b5 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -45,9 +45,37 @@ type ( func (s *Session) NewSession(component *repository.Component) *Session { ret := *s - s.component = component - s.locatorOpt.Views.Register(component.View) - s.view = component.View + // set component and view on the child session (do not mutate receiver) + ret.component = component + ret.Options.component = component + ret.view = component.View + if ret.locatorOpt != nil { + if _, ok := ret.locatorOpt.Views[component.View.Name]; !ok { + ret.locatorOpt.Views.Register(component.View) + } + } + + // create a fresh cache and optionally pre-populate from parent cache values + parent := s.cache + ret.cache = newCache() + if ret.Options.preseedCache && parent != nil { + parent.RWMutex.RLock() + for k, v := range parent.values { + ret.cache.values[k] = v + } + parent.RWMutex.RUnlock() + } + + // reset predicates (filters) on the child session state + if ret.Options.state != nil { + ret.Options.state.RWMutex.Lock() + for _, st := range ret.Options.state.Views { + if st != nil { + st.Filters = nil + } + } + ret.Options.state.RWMutex.Unlock() + } return &ret } diff --git a/service/session/stater.go b/service/session/stater.go index a98548014..abda758bd 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -92,6 +92,40 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt } hOptions := hstate.NewOptions(opts...) + + // Handle WithInput: preload cache from provided input data + if input := hOptions.Input(); input != nil { + var parameters state.Parameters + // If input type matches component input type, reuse component parameters + if s.component != nil && s.component.Input.Type.Type() != nil && s.component.Input.Type.Type().Type() != nil { + compInType := s.component.Input.Type.Type().Type() + inType := reflect.TypeOf(input) + if inType != nil && inType.Kind() != reflect.Ptr { + inType = reflect.PtrTo(inType) + } + if inType == compInType { + parameters = s.component.Input.Type.Parameters + } + } + // Otherwise, derive parameters from input type + if len(parameters) == 0 { + inType := reflect.TypeOf(input) + aType, e := state.NewType( + state.WithSchema(state.NewSchema(inType)), + state.WithResource(s.resource), + ) + if e != nil { + return e + } + if e = aType.Init(); e != nil { + return e + } + parameters = aType.Parameters + } + if e := s.LoadState(parameters, input); e != nil { + return e + } + } aState := stateType.Type().WithValue(dest) var stateOptions = []locator.Option{ locator.WithLogger(s.logger), From 8bb9c6a124074712514c79927e64a7fea024a980 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 1 Nov 2025 09:54:17 -0700 Subject: [PATCH 065/279] patched parser --- service/executor/expand/evaluator.go | 3 +-- service/session/stater.go | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/service/executor/expand/evaluator.go b/service/executor/expand/evaluator.go index f898b4ffa..c733aca79 100644 --- a/service/executor/expand/evaluator.go +++ b/service/executor/expand/evaluator.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - "reflect" - "github.com/viant/datly/view/keywords" "github.com/viant/datly/view/state/predicate" "github.com/viant/godiff" @@ -14,6 +12,7 @@ import ( "github.com/viant/velty/est" "github.com/viant/velty/est/op" "github.com/viant/xreflect" + "reflect" ) type ( diff --git a/service/session/stater.go b/service/session/stater.go index abda758bd..c87a6966f 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -10,6 +10,7 @@ import ( "embed" "github.com/viant/datly/utils/types" + "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind/locator" "github.com/viant/xdatly/handler/response" @@ -125,6 +126,9 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt if e := s.LoadState(parameters, input); e != nil { return e } + if s.view.Mode == view.ModeQuery { + s.SetViewState(ctx, s.view) + } } aState := stateType.Type().WithValue(dest) var stateOptions = []locator.Option{ From 14be98fdeb238b8389cc995f934aa48da725762a Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 5 Nov 2025 05:08:10 -0800 Subject: [PATCH 066/279] added tx option and added support for multi part --- go.mod | 2 +- go.sum | 4 + service/executor/handler/executor.go | 18 +++++ service/session/option.go | 14 ++++ view/extension/init.go | 3 +- view/state/kind/locator/body.go | 116 ++++++++++++++++++++++++--- 6 files changed, 145 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 60a5356a0..6ae51f459 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/viant/tagly v0.2.2 github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 - github.com/viant/xdatly/handler v0.0.0-20251006174948-cb34263ae8aa + github.com/viant/xdatly/handler v0.0.0-20251101182009-653093ed869e github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a github.com/viant/xlsy v0.3.1 diff --git a/go.sum b/go.sum index b5a5dd475..f427f0da7 100644 --- a/go.sum +++ b/go.sum @@ -1170,6 +1170,10 @@ github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUB github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259/go.mod h1:fb8YgbVadk8X5ZLz49LWGzWmQlZd7Y/I5wE0ru44bIo= github.com/viant/xdatly/handler v0.0.0-20251006174948-cb34263ae8aa h1:UzX1wB23RMENSKF5X0fQZR/cIy7wB7z2ODWCIm358IQ= github.com/viant/xdatly/handler v0.0.0-20251006174948-cb34263ae8aa/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= +github.com/viant/xdatly/handler v0.0.0-20251101181445-c75586bb6ea4 h1:qlYPNwGIfejalMBSoLFhpgNk0LZMRFL6NRfQnpetjPc= +github.com/viant/xdatly/handler v0.0.0-20251101181445-c75586bb6ea4/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= +github.com/viant/xdatly/handler v0.0.0-20251101182009-653093ed869e h1:WJb6NjQP/84Fqovpesy6TqST2ukHLFF+lTI9Vz43O5I= +github.com/viant/xdatly/handler v0.0.0-20251101182009-653093ed869e/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 h1:G+e1MMDxQXUPPlAXVNlRqSLTLra7udGQZUu3hnr0Y8M= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52/go.mod h1:LJN2m8xJjtYNCvyvNrVanJwvzj8+hYCuPswL8H4qRG0= github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a h1:jecH7mH63gj1zJwD18SdvSHM9Ttr9FEOnhHkYfkCNkI= diff --git a/service/executor/handler/executor.go b/service/executor/handler/executor.go index 3e21d27f1..70f1746ba 100644 --- a/service/executor/handler/executor.go +++ b/service/executor/handler/executor.go @@ -95,6 +95,12 @@ func (e *Executor) Session(ctx context.Context) (*executor.Session, error) { e.executorSession = sess sess.SessionHandler = sessionHandler + // inherit tx from session options if available + if e.tx == nil { + if tx := e.session.Options.SqlTx(); tx != nil { + e.tx = tx + } + } return e.executorSession, err } @@ -162,6 +168,10 @@ func (e *Executor) newSqlService(options *sqlx.Options) (sqlx.Sqlx, error) { if unit == e.dataUnit { //we are using View that can contain SQL Statements in Velty txStartedNotifier = e.txStarted } + // default SQLx tx to executor tx to avoid internal Begin/Commit if caller provided one + if options.WithTx == nil && e.tx != nil { + options.WithTx = e.tx + } return &Service{ txNotifier: txStartedNotifier, dataUnit: unit, @@ -314,6 +324,10 @@ func (e *Executor) redirect(ctx context.Context, route *http2.Route, opts ...hst session.WithLogger(e.logger), session.WithRegistry(registry), ) + if tx := stateOptions.SqlTx(); tx != nil { + // associate tx with session; child executor will reuse it + aSession.Apply(session.WithSQLTx(tx)) + } err = aSession.InitKinds(state.KindComponent, state.KindHeader, state.KindRequestBody, state.KindForm, state.KindQuery) if err != nil { @@ -321,6 +335,10 @@ func (e *Executor) redirect(ctx context.Context, route *http2.Route, opts ...hst } ctx = aSession.Context(ctx, true) anExecutor := NewExecutor(aComponent.View, aSession) + // ensure Execute(ctx) uses the provided tx (avoid autocommit) + if tx := stateOptions.SqlTx(); tx != nil { + anExecutor.tx = tx + } return anExecutor.NewHandlerSession(ctx, WithLogger(aSession.Logger())) } diff --git a/service/session/option.go b/service/session/option.go index 099835813..3568b7b35 100644 --- a/service/session/option.go +++ b/service/session/option.go @@ -2,6 +2,7 @@ package session import ( "context" + "database/sql" "embed" "github.com/viant/datly/repository" @@ -34,6 +35,7 @@ type ( embeddedFS *embed.FS auth *auth.Service preseedCache bool + sqlTx *sql.Tx } Option func(o *Options) @@ -47,6 +49,11 @@ func (o *Options) Registry() *repository.Registry { return o.registry } +// SqlTx returns associated SQL transaction (if any) +func (o *Options) SqlTx() *sql.Tx { + return o.sqlTx +} + func (o *Options) HasInputParameters() bool { if o.locatorOpt == nil { return false @@ -156,6 +163,13 @@ func WithAuth(auth *auth.Service) Option { } } +// WithSQLTx associates an existing SQL transaction with the session +func WithSQLTx(tx *sql.Tx) Option { + return func(s *Options) { + s.sqlTx = tx + } +} + // WithPreseedCache controls whether NewSession should pre-seed child cache from parent (default false) func WithPreseedCache(flag bool) Option { return func(s *Options) { diff --git a/view/extension/init.go b/view/extension/init.go index f05ff7140..a4cd462cd 100644 --- a/view/extension/init.go +++ b/view/extension/init.go @@ -3,6 +3,7 @@ package extension import ( "encoding/json" "fmt" + "mime/multipart" "net/http" dcodec "github.com/viant/datly/view/extension/codec" @@ -52,7 +53,7 @@ func InitRegistry() { xreflect.NewType("validator.Violation", xreflect.WithReflectType(reflect.TypeOf(validator.Violation{}))), xreflect.NewType("RawMessage", xreflect.WithReflectType(reflect.TypeOf(json.RawMessage{}))), xreflect.NewType("json.RawMessage", xreflect.WithReflectType(reflect.TypeOf(json.RawMessage{}))), - xreflect.NewType("json.RawMessage", xreflect.WithReflectType(reflect.TypeOf(json.RawMessage{}))), + xreflect.NewType("multipart.FileHeader", xreflect.WithReflectType(reflect.TypeOf(multipart.FileHeader{}))), xreflect.NewType("types.BitBool", xreflect.WithReflectType(reflect.TypeOf(types.BitBool(true)))), xreflect.NewType("time.Time", xreflect.WithReflectType(xreflect.TimeType)), xreflect.NewType("response.Status", xreflect.WithReflectType(reflect.TypeOf(response.Status{}))), diff --git a/view/state/kind/locator/body.go b/view/state/kind/locator/body.go index b80d112dd..aeb5ab428 100644 --- a/view/state/kind/locator/body.go +++ b/view/state/kind/locator/body.go @@ -3,13 +3,17 @@ package locator import ( "context" "fmt" + "mime" + "mime/multipart" + "net/http" + "reflect" + "strings" + "sync" + "github.com/viant/datly/shared" "github.com/viant/datly/view/state/kind" "github.com/viant/structology" hstate "github.com/viant/xdatly/handler/state" - "net/http" - "reflect" - "sync" ) type Body struct { @@ -21,24 +25,26 @@ type Body struct { request *http.Request err error sync.Once + isMultipart bool } +const maxMultipartMemory = 32 << 20 // 32 MiB + func (r *Body) Names() []string { return nil } func (r *Body) Value(ctx context.Context, rType reflect.Type, name string) (interface{}, bool, error) { var err error - - r.Once.Do(func() { - var request *http.Request - request, r.err = shared.CloneHTTPRequest(r.request) - r.body, r.err = readRequestBody(request) - - }) + r.initOnce() var requestState *structology.State + // Multipart handling + if r.isMultipart { + return r.handleMultipartValue(rType, name) + } + if len(r.body) > 0 { if r.requestState != nil && r.requestState.Type().Type() == rType { requestState = r.requestState @@ -75,6 +81,67 @@ func (r *Body) Value(ctx context.Context, rType reflect.Type, name string) (inte return sel.Value(requestState.Pointer()), true, nil } +// initOnce initializes body locator state based on content type (multipart vs non-multipart) +func (r *Body) initOnce() { + r.Once.Do(func() { + // Multipart branch + if r.request != nil && r.isMultipartRequest() { + r.isMultipart = true + r.err = r.request.ParseMultipartForm(maxMultipartMemory) + if r.err == nil { + r.seedFormFromMultipart() + } + return + } + // Non-multipart: clone and read body safely + var request *http.Request + request, r.err = shared.CloneHTTPRequest(r.request) + r.body, r.err = readRequestBody(request) + }) +} + +// handleMultipartValue returns value for multipart/form-data content +func (r *Body) handleMultipartValue(rType reflect.Type, name string) (interface{}, bool, error) { + if r.err != nil { + return nil, false, r.err + } + if r.request == nil || r.request.MultipartForm == nil { + return nil, false, nil + } + if name == "" { + return nil, false, nil + } + // File destinations + if rType != nil { + // []*multipart.FileHeader + if rType.Kind() == reflect.Slice && rType.Elem() == reflect.TypeOf((*multipart.FileHeader)(nil)) { + files := r.request.MultipartForm.File[name] + if len(files) == 0 { + return nil, false, nil + } + return files, true, nil + } + // *multipart.FileHeader + if rType == reflect.TypeOf((*multipart.FileHeader)(nil)) { + files := r.request.MultipartForm.File[name] + if len(files) == 0 { + return nil, false, nil + } + return files[0], true, nil + } + } + // Textual parts + if r.request.MultipartForm.Value != nil { + if vs, ok := r.request.MultipartForm.Value[name]; ok && len(vs) > 0 { + if rType != nil && rType.Kind() == reflect.Slice && rType.Elem().Kind() == reflect.String { + return vs, true, nil + } + return vs[0], true, nil + } + } + return nil, false, nil +} + func (r *Body) decodeBodyMap(ctx context.Context) (interface{}, bool, error) { aMapPtr := reflect.New(r.bodyType) aMap := reflect.MakeMap(r.bodyType) @@ -152,3 +219,32 @@ func (r *Body) updateQueryString(ctx context.Context, body interface{}) { // Encode the query string and assign it back to the request's URL req.URL.RawQuery = q.Encode() } + +// isMultipartRequest checks content type for multipart/form-data +func (r *Body) isMultipartRequest() bool { + if r.request == nil { + return false + } + ct := r.request.Header.Get("Content-Type") + if ct == "" { + return false + } + mediaType, _, err := mime.ParseMediaType(ct) + if err != nil { + return strings.Contains(strings.ToLower(ct), "multipart/form-data") + } + return strings.EqualFold(mediaType, "multipart/form-data") +} + +// seedFormFromMultipart copies textual multipart values into shared form to avoid re-parsing later +func (r *Body) seedFormFromMultipart() { + if r.request == nil || r.request.MultipartForm == nil || r.form == nil { + return + } + for k, vs := range r.request.MultipartForm.Value { + if len(vs) == 0 { + continue + } + r.form.Set(k, vs...) + } +} From 5811a0a6ae4644cd79695a850411e166c73583c9 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 5 Nov 2025 05:10:41 -0800 Subject: [PATCH 067/279] added tx option and added support for multi part --- shared/http.go | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/shared/http.go b/shared/http.go index 5b84f39f5..3504c2546 100644 --- a/shared/http.go +++ b/shared/http.go @@ -3,22 +3,35 @@ package shared import ( "bytes" "io" + "mime" "net/http" + "strings" ) // CloneHTTPRequest clones http request func CloneHTTPRequest(request *http.Request) (*http.Request, error) { - var data []byte - var err error + // Shallow clone; special-case multipart to avoid buffering entire body ret := *request ret.URL = request.URL - if request.Body != nil { - if data, err = readRequestBody(request); err != nil { - return nil, err - } - ret.Body = io.NopCloser(bytes.NewReader(data)) + + if request.Body == nil { + return &ret, nil + } + + // Detect multipart/form-data; avoid reading/consuming body + if isMultipartRequest(request) { + // share the same Body; caller must ensure only one reader consumes it + ret.Body = request.Body + return &ret, nil + } + + // Non-multipart: safe full read, reset both original and clone bodies + data, err := readRequestBody(request) + if err != nil { + return nil, err } - return &ret, err + ret.Body = io.NopCloser(bytes.NewReader(data)) + return &ret, nil } func readRequestBody(request *http.Request) ([]byte, error) { @@ -30,3 +43,18 @@ func readRequestBody(request *http.Request) ([]byte, error) { request.Body = io.NopCloser(bytes.NewReader(data)) return data, err } + +func isMultipartRequest(r *http.Request) bool { + if r == nil || r.Header == nil { + return false + } + ct := r.Header.Get("Content-Type") + if ct == "" { + return false + } + mediaType, _, err := mime.ParseMediaType(ct) + if err != nil { + return strings.Contains(strings.ToLower(ct), "multipart/form-data") + } + return strings.EqualFold(mediaType, "multipart/form-data") +} From d7ce972f738300b65388841d68a41f3567d095e8 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 5 Nov 2025 18:20:31 -0800 Subject: [PATCH 068/279] added tx option and added support for multi part --- gateway/route.go | 43 ++++- .../marshal/json/marshaller_bool_ptr.go | 2 +- .../marshal/json/marshaller_raw_message.go | 13 +- .../marshal/json/marshaller_struct_test.go | 169 ++++++++++++++++++ 4 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 gateway/router/marshal/json/marshaller_struct_test.go diff --git a/gateway/route.go b/gateway/route.go index 4bec8528a..0c15f9bdb 100644 --- a/gateway/route.go +++ b/gateway/route.go @@ -13,6 +13,9 @@ import ( "github.com/viant/xdatly/handler/exec" "net/http" "strings" + "time" + + dlogger "github.com/viant/datly/logger" ) const ( @@ -33,6 +36,9 @@ type ( Handler func(ctx context.Context, response http.ResponseWriter, req *http.Request) `json:"-"` logging.Config Version string + + // Counter is an optional per-route metrics counter + Counter dlogger.Counter `json:"-"` } ) @@ -43,7 +49,39 @@ func (r *Route) Handle(res http.ResponseWriter, req *http.Request) int { ctx := context.Background() execContext := exec.NewContext(req.Method, req.RequestURI, req.Header, r.Version) ctx = vcontext.WithValue(ctx, exec.ContextKey, execContext) + var onDone func(time.Time, ...interface{}) int64 = nil + var start time.Time + if r.Counter != nil { + start = time.Now() + onDone = r.Counter.Begin(start) + } + r.Handler(ctx, res, req) + + // finalize metrics + if onDone != nil { + end := time.Now() + onDone(end) + // Determine final status code + statusCode := execContext.StatusCode + if statusCode == 0 { + statusCode = http.StatusOK + } + // Increment error/success buckets + if statusCode >= 200 && statusCode < 300 { + r.Counter.IncrementValue("Success") + r.Counter.IncrementValue("status:2xx") + } else if statusCode >= 400 && statusCode < 500 { + r.Counter.IncrementValue("Error") + r.Counter.IncrementValue("status:4xx") + } else if statusCode >= 500 { + r.Counter.IncrementValue("Error") + r.Counter.IncrementValue("status:5xx") + } else { + // Treat other codes as success by default + r.Counter.IncrementValue("Success") + } + } if execContext.StatusCode == 0 { execContext.StatusCode = http.StatusOK } @@ -66,7 +104,7 @@ func (r *Router) NewRouteHandler(handler *router.Handler) *Route { if !strings.HasPrefix(URI, "/") { URI = "/" + URI } - return &Route{ + route := &Route{ Path: &handler.Path.Path, MCP: &handler.Path.ModelContextProtocol, Meta: &handler.Path.Meta, @@ -75,6 +113,9 @@ func (r *Router) NewRouteHandler(handler *router.Handler) *Route { Config: r.config.Logging, Version: r.config.Version, } + // Pre-register and attach per-route counter if metrics are enabled + route.Counter = r.ensureRouteCounter(context.Background(), handler.Provider) + return route } func (r *Route) URI() string { diff --git a/gateway/router/marshal/json/marshaller_bool_ptr.go b/gateway/router/marshal/json/marshaller_bool_ptr.go index 86a562a9d..9a54f47f6 100644 --- a/gateway/router/marshal/json/marshaller_bool_ptr.go +++ b/gateway/router/marshal/json/marshaller_bool_ptr.go @@ -35,5 +35,5 @@ func (i *boolPtrMarshaller) MarshallObject(ptr unsafe.Pointer, sb *MarshallSessi } func (i *boolPtrMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder *gojay.Decoder, auxiliaryDecoder *gojay.Decoder, session *UnmarshalSession) error { - return decoder.AddBool(xunsafe.AsBoolPtr(pointer)) + return decoder.AddBoolNull(xunsafe.AsBoolAddrPtr(pointer)) } diff --git a/gateway/router/marshal/json/marshaller_raw_message.go b/gateway/router/marshal/json/marshaller_raw_message.go index 5ed57e11f..75686d0c2 100644 --- a/gateway/router/marshal/json/marshaller_raw_message.go +++ b/gateway/router/marshal/json/marshaller_raw_message.go @@ -1,6 +1,7 @@ package json import ( + stdjson "encoding/json" "github.com/francoispqt/gojay" "github.com/viant/xunsafe" "unsafe" @@ -14,12 +15,16 @@ func newRawMessageMarshaller() *rawMessageMarshaller { func (r *rawMessageMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder *gojay.Decoder, auxiliaryDecoder *gojay.Decoder, session *UnmarshalSession) error { bytesPtr := xunsafe.AsBytesPtr(pointer) - dst := "" - if err := decoder.DecodeString(&dst); err != nil { + // Decode arbitrary JSON value into interface{}, then re-marshal to raw bytes. + var val interface{} + if err := decoder.AddInterface(&val); err != nil { return err } - - *bytesPtr = []byte(dst) + data, err := stdjson.Marshal(val) + if err != nil { + return err + } + *bytesPtr = data return nil } diff --git a/gateway/router/marshal/json/marshaller_struct_test.go b/gateway/router/marshal/json/marshaller_struct_test.go new file mode 100644 index 000000000..845b1e433 --- /dev/null +++ b/gateway/router/marshal/json/marshaller_struct_test.go @@ -0,0 +1,169 @@ +package json + +import ( + stdjson "encoding/json" + "reflect" + "testing" + "time" + + "github.com/viant/datly/gateway/router/marshal/config" + "github.com/viant/tagly/format/text" +) + +// Session represents a user session document. +type Session struct { + // UserID is the PK of the session set. + UserID int `aerospike:"user_id,pk"` + // LastSeen is the last activity timestamp. Stored as unix seconds. + LastSeen *time.Time `aerospike:"last_seen,unixsec"` + // Disabled marks the session as inactive. + Disabled *bool `aerospike:"disabled"` + // Attribute holds session attributes entries. + Attribute []Attribute +} + +// Attribute represents a single attribute entry stored within the session's attributes map bin. +// The PK is still `user_id`, and attribute entries are keyed by `name`. +type Attribute struct { + // UserID is the session owner and record key. + UserID int `aerospike:"user_id,pk"` + // Name is the attribute key (map key). + Name *string `aerospike:"name,mapKey"` + // Value is the attribute payload; supports native Aerospike types. + Value stdjson.RawMessage `aerospike:"value"` +} + +func newMarshaller() *Marshaller { + // We force lowerCamel JSON keys and a time layout that matches the sample payload offset (e.g. "-08"). + cfg := &config.IOConfig{ + CaseFormat: text.CaseFormatLowerCamel, + TimeLayout: "2006-01-02T15:04:05-07", + } + return New(cfg) +} + +func TestUnmarshal_SessionWithAttributes(t *testing.T) { + payload := `[{"attribute":[{"name":"theme","userId":252,"value":{"color":"dark"}}],"disabled":false,"lastSeen":"2025-11-05T17:00:07-08","userId":252}]` + + var got []Session + err := newMarshaller().Unmarshal([]byte(payload), &got) + if err != nil { + t.Fatalf("unexpected unmarshal error: %v", err) + } + if len(got) != 1 { + t.Fatalf("expected 1 session, got %d", len(got)) + } + + s := got[0] + if s.UserID != 252 { + t.Fatalf("expected userId=252, got %d", s.UserID) + } + if s.Disabled == nil || *s.Disabled != false { + t.Fatalf("expected disabled=false, got %v", s.Disabled) + } + if s.LastSeen == nil { + t.Fatalf("expected lastSeen to be set") + } + // Verify attributes + if len(s.Attribute) != 1 { + t.Fatalf("expected 1 attribute, got %d", len(s.Attribute)) + } + a := s.Attribute[0] + if a.UserID != 252 { + t.Fatalf("expected attribute.userId=252, got %d", a.UserID) + } + if a.Name == nil || *a.Name != "theme" { + if a.Name == nil { + t.Fatalf("expected attribute.name=theme, got ") + } + t.Fatalf("expected attribute.name=theme, got %s", *a.Name) + } + // Ensure raw value round-trips as expected JSON + var valueObj map[string]string + if err := stdjson.Unmarshal(a.Value, &valueObj); err != nil { + t.Fatalf("unexpected attribute.value unmarshal error: %v", err) + } + expected := map[string]string{"color": "dark"} + if !reflect.DeepEqual(valueObj, expected) { + t.Fatalf("unexpected attribute.value: got %+v want %+v", valueObj, expected) + } +} + +func TestMarshal_SessionWithAttributes(t *testing.T) { + name := "theme" + disabled := false + ts, err := time.Parse("2006-01-02T15:04:05-07", "2025-11-05T17:00:07-08") + if err != nil { + t.Fatalf("invalid test time: %v", err) + } + raw := stdjson.RawMessage(`{"color":"dark"}`) + data := []Session{ + { + UserID: 252, + LastSeen: &ts, + Disabled: &disabled, + Attribute: []Attribute{ + {UserID: 252, Name: &name, Value: raw}, + }, + }, + } + + out, err := newMarshaller().Marshal(data) + if err != nil { + t.Fatalf("unexpected marshal error: %v", err) + } + + // Compare semantically by decoding both expected and actual into generic values. + expected := `[{"attribute":[{"name":"theme","userId":252,"value":{"color":"dark"}}],"disabled":false,"lastSeen":"2025-11-05T17:00:07-08","userId":252}]` + + var gotVal, wantVal interface{} + if err := stdjson.Unmarshal(out, &gotVal); err != nil { + t.Fatalf("unexpected result json: %v, body=%s", err, string(out)) + } + if err := stdjson.Unmarshal([]byte(expected), &wantVal); err != nil { + t.Fatalf("invalid expected json: %v", err) + } + if !reflect.DeepEqual(gotVal, wantVal) { + t.Fatalf("mismatch json:\n got: %s\nwant: %s", string(out), expected) + } +} + +func TestBoolPointer_NullAndPresent(t *testing.T) { + // Case 1: disabled is null -> Disabled == nil + payloadNull := `[{"userId":1,"disabled":null}]` + var s1 []Session + if err := newMarshaller().Unmarshal([]byte(payloadNull), &s1); err != nil { + t.Fatalf("unmarshal null disabled: %v", err) + } + if len(s1) != 1 || s1[0].Disabled != nil { + t.Fatalf("expected Disabled=nil, got %+v", s1) + } + + // Case 2: disabled false -> Disabled != nil and false + payloadFalse := `[{"userId":1,"disabled":false}]` + var s2 []Session + if err := newMarshaller().Unmarshal([]byte(payloadFalse), &s2); err != nil { + t.Fatalf("unmarshal false disabled: %v", err) + } + if len(s2) != 1 || s2[0].Disabled == nil || *s2[0].Disabled != false { + t.Fatalf("expected Disabled=false pointer, got %+v", s2) + } + + // Case 3: marshal with Disabled=nil -> emits null + data := []Session{{UserID: 3}} + out, err := newMarshaller().Marshal(data) + if err != nil { + t.Fatalf("marshal nil disabled: %v", err) + } + // verify null present for disabled if not omitted by config + var v []map[string]interface{} + if err := stdjson.Unmarshal(out, &v); err != nil { + t.Fatalf("decode marshalled: %v", err) + } + if _, ok := v[0]["disabled"]; !ok { + t.Fatalf("expected disabled key present; got %s", string(out)) + } + if v[0]["disabled"] != nil { + t.Fatalf("expected disabled=null, got %v", v[0]["disabled"]) + } +} From c15bb61d7c608e7895eb3162c864ceb8fd8300af Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 6 Nov 2025 05:23:33 -0800 Subject: [PATCH 069/279] added tx option and added support for multi part --- gateway/route_metrics.go | 73 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 gateway/route_metrics.go diff --git a/gateway/route_metrics.go b/gateway/route_metrics.go new file mode 100644 index 000000000..213f6c4a3 --- /dev/null +++ b/gateway/route_metrics.go @@ -0,0 +1,73 @@ +package gateway + +import ( + "context" + "path" + "strings" + "time" + + dlogger "github.com/viant/datly/logger" + "github.com/viant/datly/repository" + gprovider "github.com/viant/gmetric/provider" +) + +// ensureRouteCounter pre-registers a per-route counter and returns a logger-compatible adapter. +func (r *Router) ensureRouteCounter(ctx context.Context, prov *repository.Provider) dlogger.Counter { + if r.metrics == nil || prov == nil { + return nil + } + component, err := prov.Component(ctx) + if err != nil || component == nil || component.View == nil { + return nil + } + + v := component.View + + // Derive a stable package from resource URL similar to view.discoverPackage + pkg := "datly" + if res := v.GetResource(); res != nil { + src := res.SourceURL + // Extract the dir and find the segment after "/routes/" + parent, _ := path.Split(src) + if idx := strings.Index(parent, "/routes/"); idx != -1 { + pkg = strings.Trim(parent[idx+len("/routes/"):], "/") + } + } + + // Build a metric operation name aligned with view metrics namespace, but scoped to component URI (.request) + method := component.Path.Method + normURI := normalizeURI(component.URI) + name := strings.Trim(normURI, "/") + ".request" + name = strings.ReplaceAll(name, "/", ".") + metricName := pkg + "." + name + if method != "" && !strings.EqualFold(method, "GET") { + metricName = method + ":" + metricName + } + metricName = strings.ReplaceAll(metricName, "/", ".") + + cnt := r.metrics.LookupOperation(metricName) + if cnt == nil { + // Title: human-friendly + title := v.Name + " request" + cnt = r.metrics.MultiOperationCounter(pkg, metricName, title, time.Millisecond, time.Minute, 2, gprovider.NewBasic()) + } + return dlogger.NewCounter(cnt) +} + +// normalizeURI replaces path parameters like {id} with a constant token to limit cardinality. +func normalizeURI(uri string) string { + res := uri + for { + i := strings.Index(res, "{") + if i == -1 { + break + } + j := strings.Index(res[i:], "}") + if j == -1 { + break + } + j = i + j + 1 + res = res[:i] + "T" + res[j:] + } + return res +} From 9ef102c0b34bea2f79c6e96a2e552c7d293e918c Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 6 Nov 2025 11:55:13 -0800 Subject: [PATCH 070/279] patched multi content upload --- gateway/route.go | 8 ++++---- view/extension/init.go | 1 + view/state/kind/locator/body.go | 22 ++++++++++++++++++---- view/state/kind/locator/http.go | 3 ++- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/gateway/route.go b/gateway/route.go index 0c15f9bdb..9f5cf6eed 100644 --- a/gateway/route.go +++ b/gateway/route.go @@ -3,6 +3,10 @@ package gateway import ( "context" "encoding/json" + "net/http" + "strings" + "time" + "github.com/viant/afs/url" "github.com/viant/datly/gateway/router" "github.com/viant/datly/repository" @@ -11,9 +15,6 @@ import ( "github.com/viant/datly/repository/path" vcontext "github.com/viant/datly/view/context" "github.com/viant/xdatly/handler/exec" - "net/http" - "strings" - "time" dlogger "github.com/viant/datly/logger" ) @@ -55,7 +56,6 @@ func (r *Route) Handle(res http.ResponseWriter, req *http.Request) int { start = time.Now() onDone = r.Counter.Begin(start) } - r.Handler(ctx, res, req) // finalize metrics diff --git a/view/extension/init.go b/view/extension/init.go index a4cd462cd..ea5ff3730 100644 --- a/view/extension/init.go +++ b/view/extension/init.go @@ -53,6 +53,7 @@ func InitRegistry() { xreflect.NewType("validator.Violation", xreflect.WithReflectType(reflect.TypeOf(validator.Violation{}))), xreflect.NewType("RawMessage", xreflect.WithReflectType(reflect.TypeOf(json.RawMessage{}))), xreflect.NewType("json.RawMessage", xreflect.WithReflectType(reflect.TypeOf(json.RawMessage{}))), + xreflect.NewType("FileHeader", xreflect.WithReflectType(reflect.TypeOf(multipart.FileHeader{}))), xreflect.NewType("multipart.FileHeader", xreflect.WithReflectType(reflect.TypeOf(multipart.FileHeader{}))), xreflect.NewType("types.BitBool", xreflect.WithReflectType(reflect.TypeOf(types.BitBool(true)))), xreflect.NewType("time.Time", xreflect.WithReflectType(xreflect.TimeType)), diff --git a/view/state/kind/locator/body.go b/view/state/kind/locator/body.go index aeb5ab428..d0c6c2150 100644 --- a/view/state/kind/locator/body.go +++ b/view/state/kind/locator/body.go @@ -37,7 +37,6 @@ func (r *Body) Names() []string { func (r *Body) Value(ctx context.Context, rType reflect.Type, name string) (interface{}, bool, error) { var err error r.initOnce() - var requestState *structology.State // Multipart handling @@ -157,15 +156,30 @@ func (r *Body) decodeBodyMap(ctx context.Context) (interface{}, bool, error) { // NewBody returns body locator func NewBody(opts ...Option) (kind.Locator, error) { options := NewOptions(opts) - if options.BodyType == nil { - return nil, fmt.Errorf("body type was empty") - } if options.request == nil { return nil, fmt.Errorf("request was empty") } if options.Unmarshal == nil { return nil, fmt.Errorf("unmarshal was empty") } + // Allow missing BodyType only for multipart/form-data requests; otherwise keep existing requirement. + if options.BodyType == nil { + ct := "" + if options.request != nil && options.request.Header != nil { + ct = options.request.Header.Get("Content-Type") + } + isMultipart := false + if ct != "" { + if mediaType, _, err := mime.ParseMediaType(ct); err == nil { + isMultipart = strings.EqualFold(mediaType, "multipart/form-data") + } else { + isMultipart = strings.Contains(strings.ToLower(ct), "multipart/form-data") + } + } + if !isMultipart { + return nil, fmt.Errorf("body type was empty") + } + } var ret = &Body{request: options.request, bodyType: options.BodyType, unmarshal: options.UnmarshalFunc(), form: options.Form} return ret, nil } diff --git a/view/state/kind/locator/http.go b/view/state/kind/locator/http.go index 8fdd0d9bc..62c9ed697 100644 --- a/view/state/kind/locator/http.go +++ b/view/state/kind/locator/http.go @@ -4,11 +4,12 @@ import ( "bytes" "context" "fmt" - "github.com/viant/datly/view/state/kind" "io" "net/http" "reflect" "strings" + + "github.com/viant/datly/view/state/kind" ) type HttpRequest struct { From 66a431862c30d80663d2a3b28a01768714c2d0a4 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 6 Nov 2025 19:49:52 -0800 Subject: [PATCH 071/279] patched multi content upload --- service/session/stater.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/service/session/stater.go b/service/session/stater.go index c87a6966f..2481d8077 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -101,10 +101,8 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt if s.component != nil && s.component.Input.Type.Type() != nil && s.component.Input.Type.Type().Type() != nil { compInType := s.component.Input.Type.Type().Type() inType := reflect.TypeOf(input) - if inType != nil && inType.Kind() != reflect.Ptr { - inType = reflect.PtrTo(inType) - } - if inType == compInType { + + if inType != nil && compInType != nil && types.EnsureStruct(inType) == types.EnsureStruct(compInType) { parameters = s.component.Input.Type.Parameters } } @@ -112,6 +110,7 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt if len(parameters) == 0 { inType := reflect.TypeOf(input) aType, e := state.NewType( + state.WithFS(embedFs), state.WithSchema(state.NewSchema(inType)), state.WithResource(s.resource), ) From 9ff2695063609beb0eefe9be3230cfa5c21b0c7b Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 7 Nov 2025 08:18:57 -0800 Subject: [PATCH 072/279] patched multi content upload --- internal/inference/parameter.go | 4 +++ service/session/state.go | 4 +++ shared/http.go | 20 +++++++++---- view/state/kind/locator/body.go | 41 +++++++++------------------ view/state/kind/locator/form.go | 50 +++++++++++++++++++++++++++++++-- 5 files changed, 85 insertions(+), 34 deletions(-) diff --git a/internal/inference/parameter.go b/internal/inference/parameter.go index eddb8efe5..ff5e5e68f 100644 --- a/internal/inference/parameter.go +++ b/internal/inference/parameter.go @@ -128,6 +128,10 @@ func (p *Parameter) veltyDeclaration(builder *strings.Builder) { builder.WriteString(".Required()") } } + + if p.Cacheable != nil { + builder.WriteString(".Cacheable('" + strconv.FormatBool(*p.Cacheable) + "')") + } if p.Connector != "" { builder.WriteString(".WithConnector('" + p.Connector + "')") } diff --git a/service/session/state.go b/service/session/state.go index 3f28d36b5..851f274af 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -761,6 +761,10 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}) err if parameter.Scope != "" { continue } + // Only warm cache for cacheable parameters; LookupValue only reads cache when cacheable + if !parameter.IsCacheable() { + continue + } selector, _ := inputState.Selector(parameter.Name) if selector == nil { continue diff --git a/shared/http.go b/shared/http.go index 3504c2546..24703bf2b 100644 --- a/shared/http.go +++ b/shared/http.go @@ -18,8 +18,8 @@ func CloneHTTPRequest(request *http.Request) (*http.Request, error) { return &ret, nil } - // Detect multipart/form-data; avoid reading/consuming body - if isMultipartRequest(request) { + // Detect multipart/*; avoid reading/consuming body + if IsMultipartRequest(request) { // share the same Body; caller must ensure only one reader consumes it ret.Body = request.Body return &ret, nil @@ -44,17 +44,27 @@ func readRequestBody(request *http.Request) ([]byte, error) { return data, err } -func isMultipartRequest(r *http.Request) bool { +// IsMultipartRequest returns true if request Content-Type is multipart/* +func IsMultipartRequest(r *http.Request) bool { if r == nil || r.Header == nil { return false } - ct := r.Header.Get("Content-Type") + return IsMultipartContentType(r.Header.Get("Content-Type")) +} + +// IsMultipartContentType returns true when the Content-Type header indicates any multipart/* +func IsMultipartContentType(ct string) bool { if ct == "" { return false } mediaType, _, err := mime.ParseMediaType(ct) if err != nil { - return strings.Contains(strings.ToLower(ct), "multipart/form-data") + return strings.Contains(strings.ToLower(ct), "multipart/") } + return strings.HasPrefix(strings.ToLower(mediaType), "multipart/") +} + +// IsFormData returns true when mediaType equals multipart/form-data +func IsFormData(mediaType string) bool { return strings.EqualFold(mediaType, "multipart/form-data") } diff --git a/view/state/kind/locator/body.go b/view/state/kind/locator/body.go index d0c6c2150..e17af401b 100644 --- a/view/state/kind/locator/body.go +++ b/view/state/kind/locator/body.go @@ -7,7 +7,6 @@ import ( "mime/multipart" "net/http" "reflect" - "strings" "sync" "github.com/viant/datly/shared" @@ -84,13 +83,18 @@ func (r *Body) Value(ctx context.Context, rType reflect.Type, name string) (inte func (r *Body) initOnce() { r.Once.Do(func() { // Multipart branch - if r.request != nil && r.isMultipartRequest() { - r.isMultipart = true - r.err = r.request.ParseMultipartForm(maxMultipartMemory) - if r.err == nil { - r.seedFormFromMultipart() + if r.request != nil { + ct := r.request.Header.Get("Content-Type") + if shared.IsMultipartContentType(ct) { + r.isMultipart = true + if mediaType, _, err := mime.ParseMediaType(ct); err == nil && shared.IsFormData(mediaType) { + r.err = r.request.ParseMultipartForm(maxMultipartMemory) + if r.err == nil { + r.seedFormFromMultipart() + } + } + return } - return } // Non-multipart: clone and read body safely var request *http.Request @@ -162,7 +166,7 @@ func NewBody(opts ...Option) (kind.Locator, error) { if options.Unmarshal == nil { return nil, fmt.Errorf("unmarshal was empty") } - // Allow missing BodyType only for multipart/form-data requests; otherwise keep existing requirement. + // Allow missing BodyType only for multipart/* requests; otherwise keep existing requirement. if options.BodyType == nil { ct := "" if options.request != nil && options.request.Header != nil { @@ -170,11 +174,7 @@ func NewBody(opts ...Option) (kind.Locator, error) { } isMultipart := false if ct != "" { - if mediaType, _, err := mime.ParseMediaType(ct); err == nil { - isMultipart = strings.EqualFold(mediaType, "multipart/form-data") - } else { - isMultipart = strings.Contains(strings.ToLower(ct), "multipart/form-data") - } + isMultipart = shared.IsMultipartContentType(ct) } if !isMultipart { return nil, fmt.Errorf("body type was empty") @@ -235,20 +235,7 @@ func (r *Body) updateQueryString(ctx context.Context, body interface{}) { } // isMultipartRequest checks content type for multipart/form-data -func (r *Body) isMultipartRequest() bool { - if r.request == nil { - return false - } - ct := r.request.Header.Get("Content-Type") - if ct == "" { - return false - } - mediaType, _, err := mime.ParseMediaType(ct) - if err != nil { - return strings.Contains(strings.ToLower(ct), "multipart/form-data") - } - return strings.EqualFold(mediaType, "multipart/form-data") -} +// removed: local isMultipartRequest; use shared.IsMultipartContentType instead // seedFormFromMultipart copies textual multipart values into shared form to avoid re-parsing later func (r *Body) seedFormFromMultipart() { diff --git a/view/state/kind/locator/form.go b/view/state/kind/locator/form.go index 174f66255..ba14605fb 100644 --- a/view/state/kind/locator/form.go +++ b/view/state/kind/locator/form.go @@ -2,15 +2,20 @@ package locator import ( "context" - "github.com/viant/datly/view/state/kind" - "github.com/viant/xdatly/handler/state" + "mime" "net/http" "reflect" + "sync" + + "github.com/viant/datly/shared" + "github.com/viant/datly/view/state/kind" + "github.com/viant/xdatly/handler/state" ) type Form struct { form *state.Form request *http.Request + once sync.Once } func (r *Form) Names() []string { @@ -26,6 +31,18 @@ func (r *Form) Value(ctx context.Context, _ reflect.Type, name string) (interfac if r.request == nil { return nil, false, nil } + // If multipart, seed from multipart and avoid FormValue fallback + if shared.IsMultipartContentType(r.request.Header.Get("Content-Type")) { + r.once.Do(func() { r.seedFormFromMultipart() }) + if values, ok = r.form.Lookup(name); ok { + if len(values) > 1 { + return values, true, nil + } + return r.form.Get(name), true, nil + } + return nil, false, nil + } + // Non-multipart: use standard FormValue fallback r.form.Mutex().Lock() defer r.form.Mutex().Unlock() value := r.request.FormValue(name) @@ -50,3 +67,32 @@ func NewForm(opts ...Option) (kind.Locator, error) { var ret = &Form{form: options.Form, request: options.request} return ret, nil } + +// seedFormFromMultipart parses multipart/form-data (if needed) and copies textual values to the shared form +func (r *Form) seedFormFromMultipart() { + if r.request == nil || r.form == nil { + return + } + if r.request.MultipartForm == nil { + // Only ParseMultipartForm for form-data; other multipart types aren't supported by ParseMultipartForm + ct := r.request.Header.Get("Content-Type") + if ct != "" { + if mediaType, _, err := mime.ParseMediaType(ct); err == nil && shared.IsFormData(mediaType) { + // Use the same default memory threshold as Body locator + const maxMultipartMemory = 32 << 20 // 32 MiB + _ = r.request.ParseMultipartForm(maxMultipartMemory) + } + } + } + if r.request.MultipartForm == nil { + return + } + r.form.Mutex().Lock() + defer r.form.Mutex().Unlock() + for k, vs := range r.request.MultipartForm.Value { + if len(vs) == 0 { + continue + } + r.form.Set(k, vs...) + } +} From f644cf7635d9ab72c15c5bce721a7b0f4c4b591a Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 7 Nov 2025 08:32:43 -0800 Subject: [PATCH 073/279] patched multi content upload --- service/session/state.go | 25 ++++++++++++++++++++++++- service/session/stater.go | 8 +++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/service/session/state.go b/service/session/state.go index 851f274af..68a4ca8e1 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -746,7 +746,27 @@ func New(aView *view.View, opts ...Option) *Session { return ret } -func (s *Session) LoadState(parameters state.Parameters, aState interface{}) error { +type loadStateOptions struct { + skipKind map[state.Kind]bool + hasSkipKind bool +} + +type LoadStateOption func(o *loadStateOptions) + +func WithLoadStateSkipKind(kinds ...state.Kind) LoadStateOption { + return func(o *loadStateOptions) { + for _, kind := range kinds { + o.skipKind[kind] = true + } + } +} + +func (s *Session) LoadState(parameters state.Parameters, aState interface{}, opts ...LoadStateOption) error { + options := &loadStateOptions{skipKind: map[state.Kind]bool{}} + for _, opt := range opts { + opt(options) + } + options.hasSkipKind = len(options.skipKind) > 0 rType := reflect.TypeOf(aState) sType := structology.NewStateType(rType, structology.WithCustomizedNames(func(name string, tag reflect.StructTag) []string { stateTag, _ := tags.ParseStateTags(tag, nil) @@ -761,6 +781,9 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}) err if parameter.Scope != "" { continue } + if options.hasSkipKind && options.skipKind[parameter.In.Kind] { + continue + } // Only warm cache for cacheable parameters; LookupValue only reads cache when cacheable if !parameter.IsCacheable() { continue diff --git a/service/session/stater.go b/service/session/stater.go index 2481d8077..cd471fd71 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -122,7 +122,13 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt } parameters = aType.Parameters } - if e := s.LoadState(parameters, input); e != nil { + + var skipOption []LoadStateOption + if s.view.Mode != view.ModeQuery { + skipOption = append(skipOption, WithLoadStateSkipKind(state.KindView, state.KindParam)) + } + + if e := s.LoadState(parameters, input, skipOption...); e != nil { return e } if s.view.Mode == view.ModeQuery { From 115062e8bfd96e5d1addcbae24c8d5d92c395829 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 7 Nov 2025 08:35:44 -0800 Subject: [PATCH 074/279] patched multi content upload --- service/session/stater.go | 1 + 1 file changed, 1 insertion(+) diff --git a/service/session/stater.go b/service/session/stater.go index cd471fd71..ab6889cd5 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -125,6 +125,7 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt var skipOption []LoadStateOption if s.view.Mode != view.ModeQuery { + //this is for patch component only (in the future we may pass it to caller when call Bind skipOption = append(skipOption, WithLoadStateSkipKind(state.KindView, state.KindParam)) } From ecf638371cce89f0efdcab88acf01e8b3d8f0b10 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 7 Nov 2025 16:25:22 -0800 Subject: [PATCH 075/279] patched multi content upload --- service/session/stater.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/session/stater.go b/service/session/stater.go index ab6889cd5..f29cabe0c 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -126,7 +126,7 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt var skipOption []LoadStateOption if s.view.Mode != view.ModeQuery { //this is for patch component only (in the future we may pass it to caller when call Bind - skipOption = append(skipOption, WithLoadStateSkipKind(state.KindView, state.KindParam)) + skipOption = append(skipOption, WithLoadStateSkipKind(state.KindComponent, state.KindView, state.KindParam)) } if e := s.LoadState(parameters, input, skipOption...); e != nil { From d6e71464435c5c4fcbfeefc6c793feb15dec5b50 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 7 Nov 2025 16:25:57 -0800 Subject: [PATCH 076/279] patched multi content upload --- service/session/stater.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/session/stater.go b/service/session/stater.go index f29cabe0c..635dff901 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -126,7 +126,7 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt var skipOption []LoadStateOption if s.view.Mode != view.ModeQuery { //this is for patch component only (in the future we may pass it to caller when call Bind - skipOption = append(skipOption, WithLoadStateSkipKind(state.KindComponent, state.KindView, state.KindParam)) + skipOption = append(skipOption, WithLoadStateSkipKind(state.KindHeader, state.KindComponent, state.KindView, state.KindParam)) } if e := s.LoadState(parameters, input, skipOption...); e != nil { From d3359f93bed8d9301971b97c116e328a8d7e4699 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 8 Nov 2025 05:53:02 -0800 Subject: [PATCH 077/279] patched multi content upload --- service/session/state.go | 48 +++++++++++++--- service/session/stater.go | 112 +++++++++++++++++++++++--------------- view/state/type.go | 7 +++ 3 files changed, 113 insertions(+), 54 deletions(-) diff --git a/service/session/state.go b/service/session/state.go index 68a4ca8e1..a85680740 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -299,13 +299,21 @@ func (s *Session) populateParameter(ctx context.Context, parameter *state.Parame parameterSelector := parameter.Selector() if options.indirectState || parameterSelector == nil { //p parameterSelector, err = aState.Selector(parameter.Name) - if parameterSelector == nil && parameter.In.Kind == state.KindConst { // TODO do we really need it? - return nil + if parameterSelector == nil { + switch parameter.In.Kind { + case state.KindConst: + return nil + case state.KindRequestBody: + if parameter.In.Name == "" { //auxiliary body wrapper + return nil + } + } } if err != nil { return err } } + if value, err = s.ensureValidValue(value, parameter, parameterSelector, options); err != nil { return err } @@ -564,8 +572,8 @@ func (s *Session) lookupFirstValue(ctx context.Context, parameters []*state.Para } func (s *Session) LookupValue(ctx context.Context, parameter *state.Parameter, opts *Options) (value interface{}, has bool, err error) { - - if value, has, err = s.lookupValue(ctx, parameter, opts); err != nil { + value, has, err = s.lookupValue(ctx, parameter, opts) + if err != nil { err = response.NewParameterError("", parameter.Name, err, response.WithObject(value), response.WithErrorStatusCode(parameter.ErrorStatusCode)) } return value, has, err @@ -656,6 +664,13 @@ func (s *Session) adjustAndCache(ctx context.Context, parameter *state.Parameter return nil, false, err } if parameter.Output != nil { + // Defensive: ensure codec is initialized before Transform. + if !parameter.Output.Initialized() { + // Initialize using session resource and current parameter input type. + if initErr := parameter.Output.Init(s.resource, parameter.Schema.Type()); initErr != nil { + return nil, false, initErr + } + } transformed, err := parameter.Output.Transform(ctx, value, opts.codecOptions...) if err != nil { return nil, false, fmt.Errorf("failed to transform %s with %s: %v, %w", parameter.Name, parameter.Output.Name, value, err) @@ -747,12 +762,18 @@ func New(aView *view.View, opts ...Option) *Session { } type loadStateOptions struct { - skipKind map[state.Kind]bool - hasSkipKind bool + skipKind map[state.Kind]bool + hasSkipKind bool + useHasMarker bool } type LoadStateOption func(o *loadStateOptions) +func WithHasMarker() LoadStateOption { + return func(o *loadStateOptions) { + o.useHasMarker = true + } +} func WithLoadStateSkipKind(kinds ...state.Kind) LoadStateOption { return func(o *loadStateOptions) { for _, kind := range kinds { @@ -777,6 +798,10 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}, opt })) inputState := sType.WithValue(aState) ptr := xunsafe.AsPointer(aState) + // Use presence markers only if enabled and supported by the input state + hasMarker := options.useHasMarker && inputState.HasMarker() + bodyParam := parameters.LookupByLocation(state.KindRequestBody, "") + for _, parameter := range parameters { if parameter.Scope != "" { continue @@ -792,23 +817,28 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}, opt if selector == nil { continue } - if !selector.Has(ptr) { + // Only use selector.Has when input supports presence markers + if hasMarker && !selector.Has(ptr) { continue } value := selector.Value(ptr) switch parameter.In.Kind { case state.KindView, state.KindParam, state.KindState: if value == nil { - return nil + continue } rType := parameter.OutputType() if rType.Kind() == reflect.Ptr { ptr := (*unsafe.Pointer)(xunsafe.AsPointer(value)) if ptr == nil || *ptr == nil { - return nil + continue } } + case state.KindRequestBody: + if bodyParam != nil { + s.setValue(bodyParam, value) + } } s.setValue(parameter, value) } diff --git a/service/session/stater.go b/service/session/stater.go index 635dff901..bbb763006 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -94,48 +94,6 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt hOptions := hstate.NewOptions(opts...) - // Handle WithInput: preload cache from provided input data - if input := hOptions.Input(); input != nil { - var parameters state.Parameters - // If input type matches component input type, reuse component parameters - if s.component != nil && s.component.Input.Type.Type() != nil && s.component.Input.Type.Type().Type() != nil { - compInType := s.component.Input.Type.Type().Type() - inType := reflect.TypeOf(input) - - if inType != nil && compInType != nil && types.EnsureStruct(inType) == types.EnsureStruct(compInType) { - parameters = s.component.Input.Type.Parameters - } - } - // Otherwise, derive parameters from input type - if len(parameters) == 0 { - inType := reflect.TypeOf(input) - aType, e := state.NewType( - state.WithFS(embedFs), - state.WithSchema(state.NewSchema(inType)), - state.WithResource(s.resource), - ) - if e != nil { - return e - } - if e = aType.Init(); e != nil { - return e - } - parameters = aType.Parameters - } - - var skipOption []LoadStateOption - if s.view.Mode != view.ModeQuery { - //this is for patch component only (in the future we may pass it to caller when call Bind - skipOption = append(skipOption, WithLoadStateSkipKind(state.KindHeader, state.KindComponent, state.KindView, state.KindParam)) - } - - if e := s.LoadState(parameters, input, skipOption...); e != nil { - return e - } - if s.view.Mode == view.ModeQuery { - s.SetViewState(ctx, s.view) - } - } aState := stateType.Type().WithValue(dest) var stateOptions = []locator.Option{ locator.WithLogger(s.logger), @@ -177,23 +135,87 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt viewOptions := s.ViewOptions(s.view, WithLocatorOptions()) stateOptions = append(viewOptions.kindLocator.Options(), stateOptions...) } + if err = s.handleInputState(ctx, hOptions, embedFs); err != nil { + return err + } - if s.component != nil && s.component.Contract.Output.Type.Type().Type() == destType { - return s.handleComponentpOutputType(ctx, dest, stateOptions) + if s.component != nil { + componentOutputType := types.EnsureStruct(s.component.Contract.Output.Type.Type().Type()) + if componentOutputType == types.EnsureStruct(destType) { + return s.handleComponentOutputType(ctx, dest, stateOptions) + } } options := s.Indirect(true, stateOptions...) options.scope = hOptions.Scope() + if err = s.SetState(ctx, stateType.Parameters, aState, options); err != nil { return err } + if initializer, ok := dest.(state.Initializer); ok { err = initializer.Init(ctx) } return err } -func (s *Session) handleComponentpOutputType(ctx context.Context, dest interface{}, stateOptions []locator.Option) error { +func (s *Session) handleInputState(ctx context.Context, hOptions *hstate.Options, embedFs *embed.FS) error { + // Handle WithInput: preload cache from provided input data + if input := hOptions.Input(); input != nil { + var parameters state.Parameters + var inputType *state.Type + // If input type matches component input type, reuse component parameters + if s.component != nil && s.component.Input.Type.Type() != nil && s.component.Input.Type.Type().Type() != nil { + compInType := s.component.Input.Type.Type().Type() + inType := reflect.TypeOf(input) + if inType != nil && compInType != nil && types.EnsureStruct(inType) == types.EnsureStruct(compInType) { + parameters = s.component.Input.Type.Parameters + inputType = &s.component.Input.Type + } + } + // Otherwise, derive parameters from input type + if len(parameters) == 0 { + inType := reflect.TypeOf(input) + aType, e := state.NewType( + state.WithFS(embedFs), + state.WithSchema(state.NewSchema(inType)), + state.WithResource(s.resource), + ) + if e != nil { + return e + } + if e = aType.Init(); e != nil { + return e + } + inputType = aType + for _, p := range aType.Parameters { + p.Init(ctx, s.view.Resource()) + } + parameters = aType.Parameters + } + + var skipOption []LoadStateOption + skipOption = append(skipOption, WithHasMarker()) + if s.view.Mode != view.ModeQuery { + //this is for patch component only (in the future we may pass it to caller when call Bind + skipOption = append(skipOption, WithLoadStateSkipKind(state.KindView, state.KindParam)) + } + if e := s.LoadState(parameters, input, skipOption...); e != nil { + return e + } + if s.view.Mode == view.ModeQuery { + inputState := inputType.Type().WithValue(input) + options := s.Options.Indirect(true) + if err := s.SetState(ctx, parameters, inputState, options); err != nil { + return err + } + _ = s.SetViewState(ctx, s.view) + } + } + return nil +} + +func (s *Session) handleComponentOutputType(ctx context.Context, dest interface{}, stateOptions []locator.Option) error { sessionOpt := s.Options s.Options = *s.Indirect(true, stateOptions...) destValue, err := s.operate(ctx, s, s.component) diff --git a/view/state/type.go b/view/state/type.go index d2ea7c0c2..523e4048e 100644 --- a/view/state/type.go +++ b/view/state/type.go @@ -82,6 +82,13 @@ func (t *Type) Init(options ...Option) (err error) { if err := t.buildParameters(); err != nil { return err } + // Ensure all derived parameters are fully initialized (schema, codecs, etc.). + for _, parameter := range t.Parameters { + t.resource.AppendParameter(parameter) + if err := parameter.Init(context.Background(), t.resource); err != nil { + return err + } + } } else if hasParameters && t.Schema.Type() == nil { if err := t.buildSchema(context.Background(), t.withMarker); err != nil { return err From e9e5ca7e5ba6ef5c15887174c5f883a2c4904d99 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 8 Nov 2025 07:00:00 -0800 Subject: [PATCH 078/279] patched multi content upload --- service/session/state.go | 12 +---- service/session/stater.go | 93 ++++++++++++++++++++------------------- view/state/type.go | 7 --- 3 files changed, 50 insertions(+), 62 deletions(-) diff --git a/service/session/state.go b/service/session/state.go index a85680740..2808c04e1 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -303,10 +303,6 @@ func (s *Session) populateParameter(ctx context.Context, parameter *state.Parame switch parameter.In.Kind { case state.KindConst: return nil - case state.KindRequestBody: - if parameter.In.Name == "" { //auxiliary body wrapper - return nil - } } } if err != nil { @@ -800,15 +796,15 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}, opt ptr := xunsafe.AsPointer(aState) // Use presence markers only if enabled and supported by the input state hasMarker := options.useHasMarker && inputState.HasMarker() - bodyParam := parameters.LookupByLocation(state.KindRequestBody, "") - for _, parameter := range parameters { if parameter.Scope != "" { continue } + if options.hasSkipKind && options.skipKind[parameter.In.Kind] { continue } + // Only warm cache for cacheable parameters; LookupValue only reads cache when cacheable if !parameter.IsCacheable() { continue @@ -835,10 +831,6 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}, opt continue } } - case state.KindRequestBody: - if bodyParam != nil { - s.setValue(bodyParam, value) - } } s.setValue(parameter, value) } diff --git a/service/session/stater.go b/service/session/stater.go index bbb763006..332c0206c 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -135,6 +135,7 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt viewOptions := s.ViewOptions(s.view, WithLocatorOptions()) stateOptions = append(viewOptions.kindLocator.Options(), stateOptions...) } + if err = s.handleInputState(ctx, hOptions, embedFs); err != nil { return err } @@ -161,56 +162,58 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt func (s *Session) handleInputState(ctx context.Context, hOptions *hstate.Options, embedFs *embed.FS) error { // Handle WithInput: preload cache from provided input data - if input := hOptions.Input(); input != nil { - var parameters state.Parameters - var inputType *state.Type - // If input type matches component input type, reuse component parameters - if s.component != nil && s.component.Input.Type.Type() != nil && s.component.Input.Type.Type().Type() != nil { - compInType := s.component.Input.Type.Type().Type() - inType := reflect.TypeOf(input) - if inType != nil && compInType != nil && types.EnsureStruct(inType) == types.EnsureStruct(compInType) { - parameters = s.component.Input.Type.Parameters - inputType = &s.component.Input.Type - } + input := hOptions.Input() + if input == nil { + return nil + } + var parameters state.Parameters + var inputType *state.Type + // If input type matches component input type, reuse component parameters + if s.component != nil && s.component.Input.Type.Type() != nil && s.component.Input.Type.Type().Type() != nil { + compInType := s.component.Input.Type.Type().Type() + inType := reflect.TypeOf(input) + if inType != nil && compInType != nil && types.EnsureStruct(inType) == types.EnsureStruct(compInType) { + parameters = s.component.Input.Type.Parameters + inputType = &s.component.Input.Type } - // Otherwise, derive parameters from input type - if len(parameters) == 0 { - inType := reflect.TypeOf(input) - aType, e := state.NewType( - state.WithFS(embedFs), - state.WithSchema(state.NewSchema(inType)), - state.WithResource(s.resource), - ) - if e != nil { - return e - } - if e = aType.Init(); e != nil { - return e - } - inputType = aType - for _, p := range aType.Parameters { - p.Init(ctx, s.view.Resource()) - } - parameters = aType.Parameters - } - - var skipOption []LoadStateOption - skipOption = append(skipOption, WithHasMarker()) - if s.view.Mode != view.ModeQuery { - //this is for patch component only (in the future we may pass it to caller when call Bind - skipOption = append(skipOption, WithLoadStateSkipKind(state.KindView, state.KindParam)) + } + // Otherwise, derive parameters from input type + if len(parameters) == 0 { + inType := reflect.TypeOf(input) + aType, e := state.NewType( + state.WithFS(embedFs), + state.WithSchema(state.NewSchema(inType)), + state.WithResource(s.resource), + ) + if e != nil { + return e } - if e := s.LoadState(parameters, input, skipOption...); e != nil { + if e = aType.Init(); e != nil { return e } - if s.view.Mode == view.ModeQuery { - inputState := inputType.Type().WithValue(input) - options := s.Options.Indirect(true) - if err := s.SetState(ctx, parameters, inputState, options); err != nil { - return err - } - _ = s.SetViewState(ctx, s.view) + inputType = aType + for _, p := range aType.Parameters { + p.Init(ctx, s.view.Resource()) + } + parameters = aType.Parameters + } + + var skipOption []LoadStateOption + skipOption = append(skipOption, WithHasMarker()) + if s.view.Mode != view.ModeQuery { + //this is for patch component only (in the future we may pass it to caller when call Bind + skipOption = append(skipOption, WithLoadStateSkipKind(state.KindView, state.KindParam)) + } + if e := s.LoadState(parameters, input, skipOption...); e != nil { + return e + } + if s.view.Mode == view.ModeQuery { + inputState := inputType.Type().WithValue(input) + options := s.Options.Indirect(true) + if err := s.SetState(ctx, parameters, inputState, options); err != nil { + return err } + _ = s.SetViewState(ctx, s.view) } return nil } diff --git a/view/state/type.go b/view/state/type.go index 523e4048e..d2ea7c0c2 100644 --- a/view/state/type.go +++ b/view/state/type.go @@ -82,13 +82,6 @@ func (t *Type) Init(options ...Option) (err error) { if err := t.buildParameters(); err != nil { return err } - // Ensure all derived parameters are fully initialized (schema, codecs, etc.). - for _, parameter := range t.Parameters { - t.resource.AppendParameter(parameter) - if err := parameter.Init(context.Background(), t.resource); err != nil { - return err - } - } } else if hasParameters && t.Schema.Type() == nil { if err := t.buildSchema(context.Background(), t.withMarker); err != nil { return err From 889497e415ba029fb0d93e328fbc40ab16867ffa Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 13 Nov 2025 10:13:37 -0800 Subject: [PATCH 079/279] patched multi content upload --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 6ae51f459..5a044b2b5 100644 --- a/go.mod +++ b/go.mod @@ -53,9 +53,9 @@ require ( github.com/viant/mcp-protocol v0.5.10 github.com/viant/structology v0.6.1 github.com/viant/tagly v0.2.2 - github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa + github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 - github.com/viant/xdatly/handler v0.0.0-20251101182009-653093ed869e + github.com/viant/xdatly/handler v0.0.0-20251113181159-0ac8b8b0ff3a github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a github.com/viant/xlsy v0.3.1 diff --git a/go.sum b/go.sum index f427f0da7..22401582d 100644 --- a/go.sum +++ b/go.sum @@ -1166,6 +1166,8 @@ github.com/viant/x v0.3.0 h1:/3A0z/uySGxMo6ixH90VAcdjI00w5e3REC1zg5hzhJA= github.com/viant/x v0.3.0/go.mod h1:54jP3qV+nnQdNDaWxEwGTAAzCu9sx9er9htiwTW/Mcw= github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa h1:o5o1CmraGb/LSpfrgmDoMdi9JJGjiopH8cmX98ukJS0= github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= +github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a h1:7CLO2LjVnFgOwN0FL3Q4y5NrD7DpclS21AiW6tDLIc8= +github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUBDzc4rWacOYvAq/TKrTV0agvMF0gwm2gaoHI= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259/go.mod h1:fb8YgbVadk8X5ZLz49LWGzWmQlZd7Y/I5wE0ru44bIo= github.com/viant/xdatly/handler v0.0.0-20251006174948-cb34263ae8aa h1:UzX1wB23RMENSKF5X0fQZR/cIy7wB7z2ODWCIm358IQ= @@ -1174,6 +1176,8 @@ github.com/viant/xdatly/handler v0.0.0-20251101181445-c75586bb6ea4 h1:qlYPNwGIfe github.com/viant/xdatly/handler v0.0.0-20251101181445-c75586bb6ea4/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= github.com/viant/xdatly/handler v0.0.0-20251101182009-653093ed869e h1:WJb6NjQP/84Fqovpesy6TqST2ukHLFF+lTI9Vz43O5I= github.com/viant/xdatly/handler v0.0.0-20251101182009-653093ed869e/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= +github.com/viant/xdatly/handler v0.0.0-20251113181159-0ac8b8b0ff3a h1:ofcLA78XVzYW0lKkmVxix00JFANZXh2JTvrzFlM/BS8= +github.com/viant/xdatly/handler v0.0.0-20251113181159-0ac8b8b0ff3a/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 h1:G+e1MMDxQXUPPlAXVNlRqSLTLra7udGQZUu3hnr0Y8M= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52/go.mod h1:LJN2m8xJjtYNCvyvNrVanJwvzj8+hYCuPswL8H4qRG0= github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a h1:jecH7mH63gj1zJwD18SdvSHM9Ttr9FEOnhHkYfkCNkI= From 8654c5cf10158c8a3220ca2ba72d1c61b01ef9c0 Mon Sep 17 00:00:00 2001 From: ppoudyal Date: Fri, 21 Nov 2025 16:08:52 -0500 Subject: [PATCH 080/279] slice interface type cast fixed --- service/session/selector.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/service/session/selector.go b/service/session/selector.go index e4a39b8d3..11c700c16 100644 --- a/service/session/selector.go +++ b/service/session/selector.go @@ -231,7 +231,19 @@ func (s *Session) setFieldsQuerySelector(value interface{}, ns *view.NamespaceVi return fmt.Errorf("can't use projection on view %v", ns.View.Name) } selector := s.state.Lookup(ns.View) - fields := value.([]string) + var fields []string + switch v := value.(type) { + case []string: + fields = v + case []interface{}: + for _, elem := range v { + text, ok := elem.(string) + if !ok { + continue + } + fields = append(fields, text) + } + } for _, field := range fields { fieldName := ns.View.CaseFormat.Format(field, text.CaseFormatUpperCamel) if err = canUseColumn(ns.View, fieldName); err != nil { From d7474f76cae3c58591fb17155bbbee9a37ae8467 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 22 Nov 2025 05:59:48 -0800 Subject: [PATCH 081/279] patched multi content upload --- go.mod | 6 ++--- go.sum | 28 +++++---------------- shared/http.go | 23 +++++++++++++++++- view/state/kind/locator/form.go | 43 +++++++++++++++++++++++++++++---- 4 files changed, 69 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 5a044b2b5..b34d19359 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/viant/dyndb v0.1.4-0.20221214043424-27654ab6ed9c github.com/viant/gmetric v0.3.2 github.com/viant/godiff v0.4.1 - github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b + github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 github.com/viant/sqlx v0.17.8 @@ -51,8 +51,8 @@ require ( github.com/viant/jsonrpc v0.15.0 github.com/viant/mcp v0.8.0 github.com/viant/mcp-protocol v0.5.10 - github.com/viant/structology v0.6.1 - github.com/viant/tagly v0.2.2 + github.com/viant/structology v0.8.0 + github.com/viant/tagly v0.3.0 github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 github.com/viant/xdatly/handler v0.0.0-20251113181159-0ac8b8b0ff3a diff --git a/go.sum b/go.sum index 22401582d..ea4bdaa33 100644 --- a/go.sum +++ b/go.sum @@ -1124,24 +1124,14 @@ github.com/viant/govalidator v0.3.1 h1:V7f/KgfzbP8fVDc+Kj+jyPvfXxMr2N1x7srOlDV6l github.com/viant/govalidator v0.3.1/go.mod h1:D35Dwx0R8rR1knRxhlseoYvOkiqo24kpMg1/o977i9Y= github.com/viant/igo v0.2.0 h1:ygWmTCinnGPaeV7omJLiyneOpzYZ5kiw7oYz7mUJZVQ= github.com/viant/igo v0.2.0/go.mod h1:7V6AWsLhKWeGzXNTNH3AZiIEKa0m33DrQbdWtapsI74= -github.com/viant/jsonrpc v0.11.0 h1:SqOztRwLWTCdK+VSU0XhZvwqeHrJ1hpQcmhPY6NXH5g= -github.com/viant/jsonrpc v0.11.0/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= -github.com/viant/jsonrpc v0.14.0 h1:YppPzIidbd9bgjKHCREXkvjkJXf8AaFGGWfk1r+nCJE= -github.com/viant/jsonrpc v0.14.0/go.mod h1:LW2l5/H4KkGCsx2ktPX59iUlycw85ZlBcRuK/WYWBX8= github.com/viant/jsonrpc v0.15.0 h1:0qy9vzgNwR9Gj1C+ouSrzNUtNDzKGogO+7TZR+cFrA4= github.com/viant/jsonrpc v0.15.0/go.mod h1:b214Lo4zBwLqbu6Tf2bRlgQkFfPMBW5ap4qS+I3zcJ8= -github.com/viant/mcp v0.7.0 h1:pIsT93/45pDxpphsZgS8d+0mIzNDihZH0zCDADxGqe8= -github.com/viant/mcp v0.7.0/go.mod h1:4Kk48IEvnAwkplg9sLs1lOBY3cRdsYjrn4lXXdPOPMo= -github.com/viant/mcp v0.7.2 h1:+vkzxFIlKWsjTY/56oBcHuPBlO+9lNb1cwwm2FXA/cA= -github.com/viant/mcp v0.7.2/go.mod h1:4Kk48IEvnAwkplg9sLs1lOBY3cRdsYjrn4lXXdPOPMo= -github.com/viant/mcp v0.7.5 h1:8Gmdz4LiZ1Ot8eUaLCufqWtivrkGGjBA1ra41cdmgmo= -github.com/viant/mcp v0.7.5/go.mod h1:3eBNG5U/CCOPbLdBpF3clwS11WfxEYB6MdzTL2s4jLo= github.com/viant/mcp v0.8.0 h1:n4tnLXpOtpnrLZtHyNG2mmZ9SUbGWKsWGla10iMfuDg= github.com/viant/mcp v0.8.0/go.mod h1:fyuB1TSQYbbGNn7U6rLmlr9gD+Yg5+Na32D34Uvm0sk= github.com/viant/mcp-protocol v0.5.10 h1:915EC1GKgBbyYF4efzRSZ/AE6f4vobkbwa2qe6OOjJ0= github.com/viant/mcp-protocol v0.5.10/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= -github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b h1:3q166tV28yFdbFV+tXXjH7ViKAmgAgGdoWzMtvhQv28= -github.com/viant/parsly v0.3.3-0.20240717150634-e1afaedb691b/go.mod h1:85fneXJbErKMGhSQto3A5ElTQCwl3t74U9cSV0waBHw= +github.com/viant/parsly v0.3.3 h1:7ytgfLOG4Ils+wviGacWxRD0gAUvVEH/iGsSE+UI8YM= +github.com/viant/parsly v0.3.3/go.mod h1:85fneXJbErKMGhSQto3A5ElTQCwl3t74U9cSV0waBHw= github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= github.com/viant/pgo v0.11.0/go.mod h1:MFzHmkRFZlciugEgUvpl/3grK789PBSH4dUVSLOSo+Q= github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= @@ -1152,10 +1142,12 @@ github.com/viant/sqlx v0.17.8 h1:YxGTrXC2B1JmDz1qp8G+G9hGPk7XCRevWN1E4E+ZlCI= github.com/viant/sqlx v0.17.8/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.6.1 h1:Forza+RF/1tmlQFk9ABNhu+IQ8vMAqbYM6FOsYtGh9E= github.com/viant/structology v0.6.1/go.mod h1:63XfkzUyNw7wdi99HJIsH2Rg3d5AOumqbWLUYytOkxU= +github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= +github.com/viant/structology v0.8.0/go.mod h1:Fnm1DyR4gfyPbnhBMkQB5lR6/isYDnncBFO1nCxxmqs= github.com/viant/structql v0.5.3 h1:QeOxvF0so8VFGt5bm+Jr6FL8uRnXkvwWY+ZCkbe3zsI= github.com/viant/structql v0.5.3/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= -github.com/viant/tagly v0.2.2 h1:qqb4Dov83i7nl7Gewph/lLaYAF8MKv0N7y34scgRNmE= -github.com/viant/tagly v0.2.2/go.mod h1:vV8QgJkhug+X+qyKds8av0fhjD+4u7IhNtowL1KGQ5A= +github.com/viant/tagly v0.3.0 h1:Y8IckveeSrroR8yisq4MBdxhcNqf4v8II01uCpamh4E= +github.com/viant/tagly v0.3.0/go.mod h1:PauQQkHmAvL5lFGr4gIgi+PE0aUPggBIBYN34sX2Oes= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/viant/toolbox v0.34.5/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/viant/toolbox v0.37.0 h1:+zwSdbQh6I6ZEyxokQJr+1gQKbLEw6erc+Av5dwKtLU= @@ -1164,18 +1156,10 @@ github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 h1:zKk+6hqUipkJXCPCH github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= github.com/viant/x v0.3.0 h1:/3A0z/uySGxMo6ixH90VAcdjI00w5e3REC1zg5hzhJA= github.com/viant/x v0.3.0/go.mod h1:54jP3qV+nnQdNDaWxEwGTAAzCu9sx9er9htiwTW/Mcw= -github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa h1:o5o1CmraGb/LSpfrgmDoMdi9JJGjiopH8cmX98ukJS0= -github.com/viant/xdatly v0.5.4-0.20251006174948-cb34263ae8aa/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a h1:7CLO2LjVnFgOwN0FL3Q4y5NrD7DpclS21AiW6tDLIc8= github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUBDzc4rWacOYvAq/TKrTV0agvMF0gwm2gaoHI= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259/go.mod h1:fb8YgbVadk8X5ZLz49LWGzWmQlZd7Y/I5wE0ru44bIo= -github.com/viant/xdatly/handler v0.0.0-20251006174948-cb34263ae8aa h1:UzX1wB23RMENSKF5X0fQZR/cIy7wB7z2ODWCIm358IQ= -github.com/viant/xdatly/handler v0.0.0-20251006174948-cb34263ae8aa/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= -github.com/viant/xdatly/handler v0.0.0-20251101181445-c75586bb6ea4 h1:qlYPNwGIfejalMBSoLFhpgNk0LZMRFL6NRfQnpetjPc= -github.com/viant/xdatly/handler v0.0.0-20251101181445-c75586bb6ea4/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= -github.com/viant/xdatly/handler v0.0.0-20251101182009-653093ed869e h1:WJb6NjQP/84Fqovpesy6TqST2ukHLFF+lTI9Vz43O5I= -github.com/viant/xdatly/handler v0.0.0-20251101182009-653093ed869e/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= github.com/viant/xdatly/handler v0.0.0-20251113181159-0ac8b8b0ff3a h1:ofcLA78XVzYW0lKkmVxix00JFANZXh2JTvrzFlM/BS8= github.com/viant/xdatly/handler v0.0.0-20251113181159-0ac8b8b0ff3a/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 h1:G+e1MMDxQXUPPlAXVNlRqSLTLra7udGQZUu3hnr0Y8M= diff --git a/shared/http.go b/shared/http.go index 24703bf2b..6be311b52 100644 --- a/shared/http.go +++ b/shared/http.go @@ -20,7 +20,28 @@ func CloneHTTPRequest(request *http.Request) (*http.Request, error) { // Detect multipart/*; avoid reading/consuming body if IsMultipartRequest(request) { - // share the same Body; caller must ensure only one reader consumes it + // If multipart form has already been parsed, we don't need to + // share or re-read the body. Instead, reuse the parsed form and + // multipart data on the clone so that downstream logic can access + // form values without touching the body again. + if request.MultipartForm != nil { + // Body is no longer needed for form access. + ret.Body = http.NoBody + // Reuse parsed forms and multipart metadata. + ret.MultipartForm = request.MultipartForm + if request.Form != nil { + ret.Form = request.Form + } + if request.PostForm != nil { + ret.PostForm = request.PostForm + } + + return &ret, nil + } + + // Backwards compatibility: if the multipart form hasn't been + // parsed yet, fall back to sharing the body. Callers must + // still ensure only one reader consumes it. ret.Body = request.Body return &ret, nil } diff --git a/view/state/kind/locator/form.go b/view/state/kind/locator/form.go index ba14605fb..2b9190533 100644 --- a/view/state/kind/locator/form.go +++ b/view/state/kind/locator/form.go @@ -3,7 +3,9 @@ package locator import ( "context" "mime" + "mime/multipart" "net/http" + "net/url" "reflect" "sync" @@ -22,10 +24,37 @@ func (r *Form) Names() []string { return nil } -func (r *Form) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { +func (r *Form) Value(ctx context.Context, rType reflect.Type, name string) (interface{}, bool, error) { if r.form != nil && len(r.form.Values) == 0 && r.request == nil { return nil, false, nil } + + // Support file uploads when parameters are declared with kind=form + // and types *multipart.FileHeader or []*multipart.FileHeader. This + // aligns multipart file fields with form semantics instead of body. + if r.request != nil && shared.IsMultipartContentType(r.request.Header.Get("Content-Type")) && rType != nil { + // Parse/seed multipart values only once + r.once.Do(func() { r.seedFormFromMultipart() }) + if r.request.MultipartForm != nil { + // []*multipart.FileHeader + if rType.Kind() == reflect.Slice && rType.Elem() == reflect.TypeOf((*multipart.FileHeader)(nil)) { + files := r.request.MultipartForm.File[name] + if len(files) == 0 { + return nil, false, nil + } + return files, true, nil + } + // *multipart.FileHeader + if rType == reflect.TypeOf((*multipart.FileHeader)(nil)) { + files := r.request.MultipartForm.File[name] + if len(files) == 0 { + return nil, false, nil + } + return files[0], true, nil + } + } + } + values, ok := r.form.Lookup(name) if !ok { if r.request == nil { @@ -73,8 +102,10 @@ func (r *Form) seedFormFromMultipart() { if r.request == nil || r.form == nil { return } - if r.request.MultipartForm == nil { - // Only ParseMultipartForm for form-data; other multipart types aren't supported by ParseMultipartForm + if r.request.MultipartForm == nil && len(r.form.Values) == 0 { + // Only ParseMultipartForm for form-data; other multipart types aren't + // supported by ParseMultipartForm. If the shared form already has + // values, treat it as authoritative and avoid parsing. ct := r.request.Header.Get("Content-Type") if ct != "" { if mediaType, _, err := mime.ParseMediaType(ct); err == nil && shared.IsFormData(mediaType) { @@ -87,12 +118,14 @@ func (r *Form) seedFormFromMultipart() { if r.request.MultipartForm == nil { return } - r.form.Mutex().Lock() - defer r.form.Mutex().Unlock() + if len(r.request.Form) == 0 { + r.request.Form = url.Values{} + } for k, vs := range r.request.MultipartForm.Value { if len(vs) == 0 { continue } r.form.Set(k, vs...) + r.request.Form[k] = vs } } From 6e2e58e57a9bb8fee94a05f6d4bc6a9b15f1cf38 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 22 Nov 2025 14:08:11 -0800 Subject: [PATCH 082/279] patched type pointers --- internal/inference/parameter.go | 7 ++++--- internal/inference/state.go | 21 +++++++++++++++------ internal/inference/type.go | 10 ++++++---- view/state/parameters.go | 7 ++++--- view/state/type.go | 12 +++++++++--- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/internal/inference/parameter.go b/internal/inference/parameter.go index ff5e5e68f..9ab895711 100644 --- a/internal/inference/parameter.go +++ b/internal/inference/parameter.go @@ -80,18 +80,19 @@ func (p *Parameter) veltyDeclaration(builder *strings.Builder) { case state.KindParam: builder.WriteString("?") default: + isPtr := strings.HasPrefix(p.Schema.DataType, "*") if p.Schema.Cardinality == state.Many { builder.WriteString("[]") - switch p.In.Kind { case "query", "form", "header": default: - if !p.IsRequired() { + if !p.IsRequired() && !isPtr { + isPtr = true builder.WriteString("*") } } - } else if !p.IsRequired() { + } else if !p.IsRequired() && !isPtr { builder.WriteString("*") } builder.WriteString(p.Schema.DataType) diff --git a/internal/inference/state.go b/internal/inference/state.go index e00c2b3b0..2783c2767 100644 --- a/internal/inference/state.go +++ b/internal/inference/state.go @@ -3,6 +3,12 @@ package inference import ( "context" "fmt" + "go/ast" + "go/parser" + "path" + "reflect" + "strings" + "github.com/viant/afs" "github.com/viant/afs/embed" "github.com/viant/afs/file" @@ -19,11 +25,6 @@ import ( "github.com/viant/toolbox/data" "github.com/viant/xreflect" "github.com/viant/xunsafe" - "go/ast" - "go/parser" - "path" - "reflect" - "strings" ) // State defines datly view/resource parameters @@ -691,12 +692,20 @@ func NewState(packageLocation, dataType string, types *xreflect.Types) (State, e } state.BuildPredicate(aTag, ¶m.Parameter) state.BuildCodec(aTag, ¶m.Parameter) + if param.Schema.DataType == "" { compType := param.Schema.CompType() + paramTypeName := compType.String() + if compType.Kind() == reflect.Pointer { compType = compType.Elem() + paramTypeName := compType.String() + + if compType.Kind() == reflect.Struct { + paramTypeName = "*" + paramTypeName + } } - param.Schema.DataType = compType.String() + param.Schema.DataType = paramTypeName param.Schema.PackagePath = compType.PkgPath() } //} diff --git a/internal/inference/type.go b/internal/inference/type.go index 3694aea03..3982045b4 100644 --- a/internal/inference/type.go +++ b/internal/inference/type.go @@ -229,11 +229,13 @@ func NewType(packageName string, name string, rType reflect.Type) (*Type, error) rType = types.EnsureStruct(rType) if rType.NumField() == 1 { wrapperField := rType.Field(0) - if canidateType, _ := wrapperField.Tag.Lookup("typeName"); canidateType != "" { - name = canidateType + if types.EnsureStruct(wrapperField.Type) != nil { + if canidateType, _ := wrapperField.Tag.Lookup("typeName"); canidateType != "" { + name = canidateType + } + structType := types.EnsureStruct(wrapperField.Type) + return NewType(packageName, name, structType) } - structType := types.EnsureStruct(wrapperField.Type) - return NewType(packageName, name, structType) } for i := 0; i < rType.NumField(); i++ { diff --git a/view/state/parameters.go b/view/state/parameters.go index 0b503b84a..ff409d1e4 100644 --- a/view/state/parameters.go +++ b/view/state/parameters.go @@ -2,6 +2,10 @@ package state import ( "fmt" + "net/http" + "reflect" + "strings" + "github.com/viant/datly/internal/setter" "github.com/viant/datly/shared" "github.com/viant/datly/utils/types" @@ -13,9 +17,6 @@ import ( "github.com/viant/velty" "github.com/viant/xreflect" "github.com/viant/xunsafe" - "net/http" - "reflect" - "strings" ) const ( diff --git a/view/state/type.go b/view/state/type.go index d2ea7c0c2..262a83959 100644 --- a/view/state/type.go +++ b/view/state/type.go @@ -4,6 +4,10 @@ import ( "context" "embed" "fmt" + "reflect" + "strings" + "unicode" + "github.com/viant/datly/internal/setter" "github.com/viant/datly/utils/types" "github.com/viant/datly/view/extension" @@ -11,9 +15,6 @@ import ( "github.com/viant/structology" "github.com/viant/tagly/format/text" "github.com/viant/xreflect" - "reflect" - "strings" - "unicode" ) type ( @@ -265,11 +266,16 @@ func BuildSchema(field *reflect.StructField, pTag *tags.Parameter, result *Param isSlice = true rawType = rawType.Elem() } + isPtr := false if rawType.Kind() == reflect.Ptr { rawType = rawType.Elem() + isPtr = true } rawName := rawType.Name() + if isPtr { + rawName = "*" + rawName + } if pTag.Cardinality != "" { result.ensureSchema() result.Schema.Cardinality = Cardinality(pTag.Cardinality) From dde0e0e7659036274b60ece530e9461551f101be Mon Sep 17 00:00:00 2001 From: ppoudyal Date: Mon, 24 Nov 2025 11:13:22 -0500 Subject: [PATCH 083/279] limit type cast modified to include other primitives than int. --- service/session/selector.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/service/session/selector.go b/service/session/selector.go index 11c700c16..467f26003 100644 --- a/service/session/selector.go +++ b/service/session/selector.go @@ -209,7 +209,10 @@ func (s *Session) setLimitQuerySelector(value interface{}, ns *view.NamespaceVie return fmt.Errorf("can't use Limit on view %v", ns.View.Name) } selector := s.state.Lookup(ns.View) - limit := value.(int) + limit, err := toInt(value) + if err != nil { + return fmt.Errorf("invalid limit value: %v", err) + } if limit <= ns.View.Selector.Limit || ns.View.Selector.Limit == 0 { selector.Limit = limit } @@ -290,3 +293,20 @@ func canUseColumn(aView *view.View, columnName string) error { } return nil } + +func toInt(v interface{}) (int, error) { + switch val := v.(type) { + case int: + return val, nil + case int32: + return int(val), nil + case int64: + return int(val), nil + case float64: + return int(val), nil + case float32: + return int(val), nil + default: + return 0, fmt.Errorf("unsupported type: %T", v) + } +} From 1e7d51171482a1638e35ab7c8e5ea2310ced2e71 Mon Sep 17 00:00:00 2001 From: ppoudyal Date: Mon, 24 Nov 2025 11:14:22 -0500 Subject: [PATCH 084/279] limit type cast modified to include other primitives than int. --- cmd/cli.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cli.go b/cmd/cli.go index 166f1d8f7..c2455c126 100644 --- a/cmd/cli.go +++ b/cmd/cli.go @@ -3,13 +3,13 @@ package cmd import ( "context" "fmt" + "github.com/jessevdk/go-flags" "github.com/viant/datly/cmd/command" soptions "github.com/viant/datly/cmd/options" ) func RunApp(version string, args soptions.Arguments) error { - options, err := buildOptions(args) if err != nil { return err From 06dba531207d0f81152f4d3df99c28b4e2dcfcf7 Mon Sep 17 00:00:00 2001 From: vagarwal-viant Date: Tue, 25 Nov 2025 11:14:31 -0800 Subject: [PATCH 085/279] ENG-51724 add safe guards across marshalling for deeper dive into panic --- repository/logging/logging.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/repository/logging/logging.go b/repository/logging/logging.go index 00440ac60..8bea12fa4 100644 --- a/repository/logging/logging.go +++ b/repository/logging/logging.go @@ -15,8 +15,8 @@ func Log(config *Config, execContext *exec.Context) { execContext.Metrics = execContext.Metrics.HideMetrics() } if config.IsAuditEnabled() { - data, _ := json.Marshal(execContext) - fmt.Println("[AUDIT] " + string(data)) + data := safeMarshal("EXECCONTEXT", execContext) + fmt.Println("[AUDIT]", string(data)) } if config.IsTracingEnabled() { trace := execContext.Trace @@ -42,7 +42,21 @@ func Log(config *Config, execContext *exec.Context) { } else { trace.Spans[0].SetStatusFromHTTPCode(execContext.StatusCode) } - traceData, _ := json.Marshal(trace) - fmt.Println("[TRACE] " + string(traceData)) + traceData := safeMarshal("TRACE", trace) + fmt.Println("[TRACE]", string(traceData)) } } + +func safeMarshal(label string, v any) []byte { + defer func() { + if r := recover(); r != nil { + fmt.Printf("[LOG-MARSHAL-PANIC] label=%s type=%T panic=%v\n", label, v, r) + } + }() + data, err := json.Marshal(v) + if err != nil { + fmt.Printf("[LOG-MARSHAL-ERROR] label=%s type=%T err=%v\n", label, v, err) + return nil + } + return data +} From 728c9e43d0ec7dace385761e6888d62a11f7481a Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Tue, 25 Nov 2025 11:57:19 -0800 Subject: [PATCH 086/279] add safeMarshal in the logging --- repository/logging/logging.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/repository/logging/logging.go b/repository/logging/logging.go index 00440ac60..5dc76e96b 100644 --- a/repository/logging/logging.go +++ b/repository/logging/logging.go @@ -3,9 +3,10 @@ package logging import ( "encoding/json" "fmt" - "github.com/viant/xdatly/handler/exec" "strconv" "time" + + "github.com/viant/xdatly/handler/exec" ) func Log(config *Config, execContext *exec.Context) { @@ -15,8 +16,8 @@ func Log(config *Config, execContext *exec.Context) { execContext.Metrics = execContext.Metrics.HideMetrics() } if config.IsAuditEnabled() { - data, _ := json.Marshal(execContext) - fmt.Println("[AUDIT] " + string(data)) + data := safeMarshal("EXECCONTEXT", execContext) + fmt.Println("[AUDIT]", string(data)) } if config.IsTracingEnabled() { trace := execContext.Trace @@ -42,7 +43,21 @@ func Log(config *Config, execContext *exec.Context) { } else { trace.Spans[0].SetStatusFromHTTPCode(execContext.StatusCode) } - traceData, _ := json.Marshal(trace) - fmt.Println("[TRACE] " + string(traceData)) + traceData := safeMarshal("TRACE", trace) + fmt.Println("[TRACE]", string(traceData)) + } +} + +func safeMarshal(label string, v any) []byte { + defer func() { + if r := recover(); r != nil { + fmt.Printf("[LOG-MARSHAL-PANIC] label=%s type=%T panic=%v\n", label, v, r) + } + }() + data, err := json.Marshal(v) + if err != nil { + fmt.Printf("[LOG-MARSHAL-ERROR] label=%s type=%T err=%v\n", label, v, err) + return nil } + return data } From 3d6b55bc5652330e5043bfb54025900dfa41a234 Mon Sep 17 00:00:00 2001 From: Himanshu Shishir Shah Date: Tue, 2 Dec 2025 13:38:53 -0800 Subject: [PATCH 087/279] ENG-52439: using AppendMetrics with mutex lock; updating logging with more info --- go.mod | 2 +- go.sum | 6 ++---- repository/logging/logging.go | 33 ++++++++++++++++++++++++++++++++- service/executor/service.go | 2 +- service/reader/service.go | 2 +- 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index b34d19359..82524a2fb 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/viant/tagly v0.3.0 github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 - github.com/viant/xdatly/handler v0.0.0-20251113181159-0ac8b8b0ff3a + github.com/viant/xdatly/handler v0.0.0-20251202205015-5f121a805ed3 github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a github.com/viant/xlsy v0.3.1 diff --git a/go.sum b/go.sum index ea4bdaa33..c9b72c11c 100644 --- a/go.sum +++ b/go.sum @@ -1140,8 +1140,6 @@ github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.17.8 h1:YxGTrXC2B1JmDz1qp8G+G9hGPk7XCRevWN1E4E+ZlCI= github.com/viant/sqlx v0.17.8/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= -github.com/viant/structology v0.6.1 h1:Forza+RF/1tmlQFk9ABNhu+IQ8vMAqbYM6FOsYtGh9E= -github.com/viant/structology v0.6.1/go.mod h1:63XfkzUyNw7wdi99HJIsH2Rg3d5AOumqbWLUYytOkxU= github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= github.com/viant/structology v0.8.0/go.mod h1:Fnm1DyR4gfyPbnhBMkQB5lR6/isYDnncBFO1nCxxmqs= github.com/viant/structql v0.5.3 h1:QeOxvF0so8VFGt5bm+Jr6FL8uRnXkvwWY+ZCkbe3zsI= @@ -1160,8 +1158,8 @@ github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a h1:7CLO2LjVnFgOwN0F github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUBDzc4rWacOYvAq/TKrTV0agvMF0gwm2gaoHI= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259/go.mod h1:fb8YgbVadk8X5ZLz49LWGzWmQlZd7Y/I5wE0ru44bIo= -github.com/viant/xdatly/handler v0.0.0-20251113181159-0ac8b8b0ff3a h1:ofcLA78XVzYW0lKkmVxix00JFANZXh2JTvrzFlM/BS8= -github.com/viant/xdatly/handler v0.0.0-20251113181159-0ac8b8b0ff3a/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= +github.com/viant/xdatly/handler v0.0.0-20251202205015-5f121a805ed3 h1:Xw0xbkb3lAu6k+p+XLlSyxPjl3XXda2qp74+itSbkKU= +github.com/viant/xdatly/handler v0.0.0-20251202205015-5f121a805ed3/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 h1:G+e1MMDxQXUPPlAXVNlRqSLTLra7udGQZUu3hnr0Y8M= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52/go.mod h1:LJN2m8xJjtYNCvyvNrVanJwvzj8+hYCuPswL8H4qRG0= github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a h1:jecH7mH63gj1zJwD18SdvSHM9Ttr9FEOnhHkYfkCNkI= diff --git a/repository/logging/logging.go b/repository/logging/logging.go index 5dc76e96b..0f28a6284 100644 --- a/repository/logging/logging.go +++ b/repository/logging/logging.go @@ -3,6 +3,8 @@ package logging import ( "encoding/json" "fmt" + "reflect" + "runtime/debug" "strconv" "time" @@ -51,7 +53,10 @@ func Log(config *Config, execContext *exec.Context) { func safeMarshal(label string, v any) []byte { defer func() { if r := recover(); r != nil { - fmt.Printf("[LOG-MARSHAL-PANIC] label=%s type=%T panic=%v\n", label, v, r) + fmt.Printf("[LOG-MARSHAL-PANIC] label=%s type=%T panic=%v\nSTACK:\n%s\n", label, v, r, debug.Stack()) + if execCtx, ok := v.(*exec.Context); ok { + findBadField(execCtx) + } } }() data, err := json.Marshal(v) @@ -61,3 +66,29 @@ func safeMarshal(label string, v any) []byte { } return data } + +func findBadField(execCtx *exec.Context) { + val := reflect.ValueOf(execCtx).Elem() + typ := val.Type() + for i := 0; i < val.NumField(); i++ { + field := val.Field(i) + fieldType := typ.Field(i) + fieldName := fieldType.Name + + // Skip unexported fields + if !field.CanInterface() { + continue + } + + func() { + defer func() { + if r := recover(); r != nil { + fmt.Printf("[BAD-FIELD-PANIC] %s (%s): %v\n", fieldName, field.Type(), r) + } + }() + if _, err := json.Marshal(field.Interface()); err != nil { + fmt.Printf("[BAD-FIELD-ERROR] %s (%s): %v\n", fieldName, field.Type(), err) + } + }() + } +} diff --git a/service/executor/service.go b/service/executor/service.go index 01df6c631..086a40695 100644 --- a/service/executor/service.go +++ b/service/executor/service.go @@ -220,7 +220,7 @@ func (e *Executor) logMetrics(ctx context.Context, table string, operation strin if err != nil { metric.Error = err.Error() } - value.(*exec.Context).Metrics.Append(&metric) + value.(*exec.Context).AppendMetrics(&metric) } func (e *Executor) handleInsert(ctx context.Context, sess *dbSession, executable *expand2.Executable, db *sql.DB) error { diff --git a/service/reader/service.go b/service/reader/service.go index d3701f9bc..1fda9fc9e 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -104,7 +104,7 @@ func (s *Service) afterRead(ctx context.Context, aSession *Session, collector *v onFinish(end) if value := ctx.Value(exec.ContextKey); value != nil { if exeCtx := value.(*exec.Context); exeCtx != nil { - exeCtx.Metrics.Append(metrics) + exeCtx.AppendMetrics(metrics) } } } From 5236e65df147a5117615f7d5e3be37d950d6912c Mon Sep 17 00:00:00 2001 From: Himanshu Shishir Shah Date: Tue, 2 Dec 2025 16:26:20 -0800 Subject: [PATCH 088/279] ENG-52439: adding unit test cases --- repository/logging/logging_test.go | 194 +++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 repository/logging/logging_test.go diff --git a/repository/logging/logging_test.go b/repository/logging/logging_test.go new file mode 100644 index 000000000..a9beb4293 --- /dev/null +++ b/repository/logging/logging_test.go @@ -0,0 +1,194 @@ +package logging + +import ( + "bytes" + "encoding/json" + "io" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/viant/xdatly/handler/exec" +) + +// TestSafeMarshal_Success tests successful JSON marshaling +func TestSafeMarshal_Success(t *testing.T) { + type TestStruct struct { + Name string `json:"name"` + Value int `json:"value"` + } + + testData := TestStruct{ + Name: "test", + Value: 42, + } + + result := safeMarshal("TEST", testData) + assert.NotNil(t, result, "safeMarshal should return non-nil for valid data") + + var unmarshaled TestStruct + err := json.Unmarshal(result, &unmarshaled) + assert.NoError(t, err) + assert.Equal(t, testData, unmarshaled) +} + +// TestSafeMarshal_Error tests safeMarshal with a value that causes a marshaling error +func TestSafeMarshal_Error(t *testing.T) { + // Channel cannot be marshaled to JSON + ch := make(chan int) + result := safeMarshal("TEST", ch) + assert.Nil(t, result, "safeMarshal should return nil when marshaling fails") +} + +// TestSafeMarshal_Panic tests safeMarshal with a value that causes a panic +func TestSafeMarshal_Panic(t *testing.T) { + // Function cannot be marshaled and may cause panic + fn := func() {} + result := safeMarshal("TEST", fn) + assert.Nil(t, result, "safeMarshal should return nil when marshaling panics") +} + +// TestSafeMarshal_ExecContext tests safeMarshal with exec.Context +func TestSafeMarshal_ExecContext(t *testing.T) { + execCtx := exec.NewContext("GET", "/test", nil, "") + result := safeMarshal("EXECCONTEXT", execCtx) + + // Should either succeed (return non-nil) or fail gracefully (return nil) + // The important thing is it doesn't panic + if result != nil { + assert.NotEmpty(t, result) + } +} + +// TestSafeMarshal_NilValue tests safeMarshal with nil value +func TestSafeMarshal_NilValue(t *testing.T) { + result := safeMarshal("TEST", nil) + assert.NotNil(t, result) + assert.Equal(t, []byte("null"), result) +} + +// TestFindBadField_ValidExecContext tests findBadField with a valid exec.Context +func TestFindBadField_ValidExecContext(t *testing.T) { + // Capture stdout to check output + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + execCtx := exec.NewContext("GET", "/test", nil, "") + findBadField(execCtx) + + // Close write pipe and restore stdout + w.Close() + os.Stdout = oldStdout + + // Read captured output + var buf bytes.Buffer + io.Copy(&buf, r) + output := buf.String() + + // With a valid exec.Context, there should be no bad field errors + assert.NotContains(t, output, "[BAD-FIELD-ERROR]", "valid exec.Context should not have bad fields") + assert.NotContains(t, output, "[BAD-FIELD-PANIC]", "valid exec.Context should not panic on field marshaling") +} + +// TestFindBadField_CompletesWithoutPanic tests that findBadField completes without panicking +func TestFindBadField_CompletesWithoutPanic(t *testing.T) { + execCtx := exec.NewContext("GET", "/test", nil, "") + + // Should complete without panicking + assert.NotPanics(t, func() { + findBadField(execCtx) + }) +} + +// TestSafeMarshal_WithLabel tests that safeMarshal uses the label parameter in error messages +func TestSafeMarshal_WithLabel(t *testing.T) { + // Capture stdout to verify label is used in error messages + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + // Use a value that will cause an error + ch := make(chan int) + result := safeMarshal("CUSTOM_LABEL", ch) + + // Close write pipe and restore stdout + w.Close() + os.Stdout = oldStdout + + // Read captured output + var buf bytes.Buffer + io.Copy(&buf, r) + output := buf.String() + + assert.Nil(t, result, "should return nil on error") + if strings.Contains(output, "[LOG-MARSHAL-ERROR]") { + assert.Contains(t, output, "CUSTOM_LABEL", "error message should include the label") + } +} + +// TestSafeMarshal_RecoversFromPanic tests that safeMarshal properly recovers from panics +func TestSafeMarshal_RecoversFromPanic(t *testing.T) { + // Capture stdout + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + // Create a type that will panic during JSON marshaling + type PanicType struct { + Value func() // Functions cannot be marshaled + } + + panicValue := PanicType{ + Value: func() {}, + } + + // This should not cause the test to panic + result := safeMarshal("PANIC_TEST", panicValue) + + // Close write pipe and restore stdout + w.Close() + os.Stdout = oldStdout + + // Read captured output + var buf bytes.Buffer + io.Copy(&buf, r) + output := buf.String() + + // Function should recover and return nil + assert.Nil(t, result, "should return nil after recovering from panic") + // Should log the panic + if strings.Contains(output, "[LOG-MARSHAL-PANIC]") { + assert.Contains(t, output, "PANIC_TEST", "panic log should include label") + } +} + +// TestSafeMarshal_ExecContextPanicCallsFindBadField tests that safeMarshal calls findBadField when exec.Context panics +func TestSafeMarshal_ExecContextPanicCallsFindBadField(t *testing.T) { + // Capture stdout + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + execCtx := exec.NewContext("GET", "/test", nil, "") + + // Try to marshal - if it panics, findBadField should be called + result := safeMarshal("EXECCONTEXT", execCtx) + + // Close write pipe and restore stdout + w.Close() + os.Stdout = oldStdout + + // Read captured output + var buf bytes.Buffer + io.Copy(&buf, r) + output := buf.String() + + // If marshaling panicked, findBadField should have been called + if result == nil && strings.Contains(output, "[LOG-MARSHAL-PANIC]") { + // findBadField should have been called (though output may be empty if no bad fields found) + // The important thing is that the function didn't crash + assert.True(t, true, "findBadField should be called when exec.Context panics") + } +} From ee5d315ca6c48e0b27a712fde94f3f17a7909a01 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 3 Dec 2025 16:51:12 -0800 Subject: [PATCH 089/279] patched type pointers --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b34d19359..c3df8efd4 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 github.com/viant/sqlx v0.17.8 - github.com/viant/structql v0.5.3 + github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 github.com/viant/xreflect v0.7.3 diff --git a/go.sum b/go.sum index ea4bdaa33..17a444947 100644 --- a/go.sum +++ b/go.sum @@ -1146,6 +1146,8 @@ github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRg github.com/viant/structology v0.8.0/go.mod h1:Fnm1DyR4gfyPbnhBMkQB5lR6/isYDnncBFO1nCxxmqs= github.com/viant/structql v0.5.3 h1:QeOxvF0so8VFGt5bm+Jr6FL8uRnXkvwWY+ZCkbe3zsI= github.com/viant/structql v0.5.3/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= +github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= +github.com/viant/structql v0.5.4/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= github.com/viant/tagly v0.3.0 h1:Y8IckveeSrroR8yisq4MBdxhcNqf4v8II01uCpamh4E= github.com/viant/tagly v0.3.0/go.mod h1:PauQQkHmAvL5lFGr4gIgi+PE0aUPggBIBYN34sX2Oes= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= From 010a92a96e0452b4e3d339a829feaafc64259afd Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 4 Dec 2025 15:14:52 -0800 Subject: [PATCH 090/279] patched type pointers --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bbb047798..4e6b4afed 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 - github.com/viant/sqlx v0.17.8 + github.com/viant/sqlx v0.21.0 github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 diff --git a/go.sum b/go.sum index eaf67828d..7a75b3fc5 100644 --- a/go.sum +++ b/go.sum @@ -1140,6 +1140,8 @@ github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.17.8 h1:YxGTrXC2B1JmDz1qp8G+G9hGPk7XCRevWN1E4E+ZlCI= github.com/viant/sqlx v0.17.8/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= +github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= +github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= github.com/viant/structology v0.8.0/go.mod h1:Fnm1DyR4gfyPbnhBMkQB5lR6/isYDnncBFO1nCxxmqs= github.com/viant/structql v0.5.3 h1:QeOxvF0so8VFGt5bm+Jr6FL8uRnXkvwWY+ZCkbe3zsI= From 0b0e8b63619f706c726e7f81b59d8548cb75bdf9 Mon Sep 17 00:00:00 2001 From: Himanshu Shishir Shah Date: Thu, 4 Dec 2025 17:46:37 -0800 Subject: [PATCH 091/279] ENG-52379: adding trace id to info logs --- shared/logging/logger.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/shared/logging/logger.go b/shared/logging/logger.go index aea30470f..9c208eb42 100644 --- a/shared/logging/logger.go +++ b/shared/logging/logger.go @@ -4,14 +4,16 @@ import ( "context" "encoding/json" "fmt" - "github.com/aws/aws-lambda-go/events" - "github.com/viant/xdatly/handler/logger" "io" "log/slog" "os" regexp "regexp" "runtime" strings "strings" + + "github.com/aws/aws-lambda-go/events" + "github.com/viant/xdatly/handler/exec" + "github.com/viant/xdatly/handler/logger" ) const ( @@ -122,6 +124,15 @@ func (s *slogger) getContextValues(ctx context.Context) []any { if openTelemetryTraceId != nil { values = append(values, "OpenTelemetryTraceId", openTelemetryTraceId) } + + execContext := ctx.Value(exec.ContextKey) + if execContext != nil { + c, ok := execContext.(*exec.Context) + if ok && c != nil && c.Trace != nil { + values = append(values, "reqTraceId", c.Trace.TraceID) + } + } + return values } From 7f60767d0ee9f0acbfbc7879c2b1f60cf308d190 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 8 Dec 2025 13:32:13 -0800 Subject: [PATCH 092/279] patched type pointers --- .../function/allowedorderbycolumn.go | 56 +++++++++++++++++++ internal/translator/function/init.go | 1 + internal/translator/function/orderby.go | 1 + internal/translator/view.go | 4 +- service/session/selector.go | 3 + view/config.go | 14 ++++- view/view.go | 19 ++++--- 7 files changed, 86 insertions(+), 12 deletions(-) create mode 100644 internal/translator/function/allowedorderbycolumn.go diff --git a/internal/translator/function/allowedorderbycolumn.go b/internal/translator/function/allowedorderbycolumn.go new file mode 100644 index 000000000..40de57f5b --- /dev/null +++ b/internal/translator/function/allowedorderbycolumn.go @@ -0,0 +1,56 @@ +package function + +import ( + "fmt" + "strings" + + "github.com/viant/datly/view" + "github.com/viant/sqlparser" +) + +type allowedOrderByColumns struct{} + +func (c *allowedOrderByColumns) Apply(args []string, column *sqlparser.Column, resource *view.Resource, aView *view.View) error { + if aView.Selector == nil { + aView.Selector = &view.Config{} + } + values, err := convertArguments(c, args) + if err != nil { + return err + } + if aView.Selector.Constraints == nil { + aView.Selector.Constraints = &view.Constraints{} + } + aView.Selector.Constraints.OrderBy = true + if len(values) == 0 { + return fmt.Errorf("failed to discover column in allowedOrderByColumns") + } + columns, ok := values[0].(string) + if !ok { + return fmt.Errorf("invalid columns type: %T, expected: %T in allowedOrderByColumns", values[0], columns) + } + for _, column := range strings.Split(columns, ",") { + column = strings.TrimSpace(column) + aView.Selector.Constraints.OrderByColumn = append(aView.Selector.Constraints.OrderByColumn, column) + } + return nil +} + +func (c *allowedOrderByColumns) Name() string { + return "allowed_order_by_columns" +} + +func (c *allowedOrderByColumns) Description() string { + return "set view.Selector.OrderBy and enables corresponding view.Selector.Constraints.OrderBy" +} + +func (c *allowedOrderByColumns) Arguments() []*Argument { + return []*Argument{ + { + Name: "allowedOrderByColumns", + Description: "query selector allowedOrderByColumns", + Required: true, + DataType: "string", + }, + } +} diff --git a/internal/translator/function/init.go b/internal/translator/function/init.go index b67d258f0..b0141c3d8 100644 --- a/internal/translator/function/init.go +++ b/internal/translator/function/init.go @@ -5,6 +5,7 @@ func init() { _registry.Register(&cache{}) _registry.Register(&limit{}) _registry.Register(&orderBy{}) + _registry.Register(&allowedOrderByColumns{}) _registry.Register(&cardinality{}) _registry.Register(&allownulls{}) _registry.Register(&matchStrategy{}) diff --git a/internal/translator/function/orderby.go b/internal/translator/function/orderby.go index 645be6058..9dc22f5ca 100644 --- a/internal/translator/function/orderby.go +++ b/internal/translator/function/orderby.go @@ -20,6 +20,7 @@ func (c *orderBy) Apply(args []string, column *sqlparser.Column, resource *view. } aView.Selector.Constraints.OrderBy = true aView.Selector.OrderBy = values[0].(string) + return nil } diff --git a/internal/translator/view.go b/internal/translator/view.go index ab94c500e..f8caa7157 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -2,15 +2,17 @@ package translator import ( "fmt" + "github.com/viant/datly/internal/asset" "github.com/viant/datly/internal/inference" "github.com/viant/datly/internal/setter" "github.com/viant/datly/internal/translator/parser" + "path" + "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/tagly/format/text" - "path" ) type ( diff --git a/service/session/selector.go b/service/session/selector.go index 467f26003..b6ff7c140 100644 --- a/service/session/selector.go +++ b/service/session/selector.go @@ -162,6 +162,9 @@ func (s *Session) setOrderByQuerySelector(value interface{}, ns *view.NamespaceV continue //position based, not need to validate } + if ns.View.Selector.Constraints.HasOrderByColumn(column) { + continue + } _, ok := ns.View.ColumnByName(column) if !ok { return fmt.Errorf("not found column %v at view %v", items, ns.View.Name) diff --git a/view/config.go b/view/config.go index 485fe5f45..8605e86f3 100644 --- a/view/config.go +++ b/view/config.go @@ -3,12 +3,13 @@ package view import ( "context" "fmt" + "reflect" + "strings" + "github.com/viant/datly/shared" "github.com/viant/datly/view/state" "github.com/viant/xdatly/codec" "github.com/viant/xreflect" - "reflect" - "strings" ) const ( @@ -90,6 +91,15 @@ func (c *Config) GetContentFormatParameter() *state.Parameter { return QueryStateParameters.ContentFormatParameter } +func (c *Constraints) HasOrderByColumn(name string) bool { + for _, candidate := range c.OrderByColumn { + if candidate == name { + return true + } + } + return false +} + func (c *Config) Init(ctx context.Context, resource *Resource, parent *View) error { if err := c.ensureConstraints(resource); err != nil { return err diff --git a/view/view.go b/view/view.go index d2f3bf567..a482cfa12 100644 --- a/view/view.go +++ b/view/view.go @@ -155,15 +155,16 @@ func (v *View) Context(ctx context.Context) context.Context { // Constraints configure what can be selected by Statelet // For each _field, default value is `false` type Constraints struct { - Criteria bool - OrderBy bool - Limit bool - Offset bool - Projection bool //enables columns projection from client (default ${NS}_fields= query param) - Filterable []string - SQLMethods []*Method `json:",omitempty"` - _sqlMethods map[string]*Method - Page *bool + Criteria bool + OrderBy bool + OrderByColumn []string + Limit bool + Offset bool + Projection bool //enables columns projection from client (default ${NS}_fields= query param) + Filterable []string + SQLMethods []*Method `json:",omitempty"` + _sqlMethods map[string]*Method + Page *bool } func (v *View) Resource() state.Resource { From 72023c4ad6f5e0f832118a18d2f13e55acb87884 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 8 Dec 2025 13:43:26 -0800 Subject: [PATCH 093/279] patched type pointers --- service/reader/sql.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/service/reader/sql.go b/service/reader/sql.go index bcb7aff93..d1f4e54b9 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -341,7 +341,7 @@ func (b *Builder) updateColumnsIn(params *view.CriteriaParam, batchData *view.Ba params.ColumnsIn = sb.String() } -func (b *Builder) appendOrderBy(sb *strings.Builder, view *view.View, selector *view.Statelet) error { +func (b *Builder) appendOrderBy(sb *strings.Builder, aView *view.View, selector *view.Statelet) error { if selector.OrderBy != "" { fragment := strings.Builder{} items := strings.Split(strings.ReplaceAll(selector.OrderBy, ":", " "), ",") @@ -362,12 +362,19 @@ func (b *Builder) appendOrderBy(sb *strings.Builder, view *view.View, selector * switch strings.ToLower(sortDirection) { case "asc", "desc", "": default: - return fmt.Errorf("invalid sort direction %v for column %v at view %v", sortDirection, column, view.Name) + return fmt.Errorf("invalid sort direction %v for column %v at aView %v", sortDirection, column, aView.Name) } - col, ok := view.ColumnByName(column) + col, ok := aView.ColumnByName(column) if !ok { - return fmt.Errorf("not found column %v at view %v", column, view.Name) + if aView.Selector.Constraints.HasOrderByColumn(column) { + col = &view.Column{ + Name: column, + } + } + } + if !ok { + return fmt.Errorf("not found column %v at aView %v", column, aView.Name) } fragment.WriteString(col.Name) if sortDirection != "" { @@ -380,9 +387,9 @@ func (b *Builder) appendOrderBy(sb *strings.Builder, view *view.View, selector * return nil } - if view.Selector.OrderBy != "" { + if aView.Selector.OrderBy != "" { sb.WriteString(orderByFragment) - sb.WriteString(strings.ReplaceAll(view.Selector.OrderBy, ":", " ")) + sb.WriteString(strings.ReplaceAll(aView.Selector.OrderBy, ":", " ")) return nil } From 93a78c4bae6ec91dc7c587bb469af5bf901be332 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 8 Dec 2025 13:43:53 -0800 Subject: [PATCH 094/279] patched type pointers --- service/reader/sql.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/reader/sql.go b/service/reader/sql.go index d1f4e54b9..82f3821e2 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -371,7 +371,9 @@ func (b *Builder) appendOrderBy(sb *strings.Builder, aView *view.View, selector col = &view.Column{ Name: column, } + ok = true } + } if !ok { return fmt.Errorf("not found column %v at aView %v", column, aView.Name) From 80bd6bfa6b7d54bc546f1d997f50aa0120e0efe9 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 8 Dec 2025 13:50:34 -0800 Subject: [PATCH 095/279] patched type pointers --- service/reader/sql.go | 1 + view/config.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/service/reader/sql.go b/service/reader/sql.go index 82f3821e2..3a34827e9 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -367,6 +367,7 @@ func (b *Builder) appendOrderBy(sb *strings.Builder, aView *view.View, selector col, ok := aView.ColumnByName(column) if !ok { + if aView.Selector.Constraints.HasOrderByColumn(column) { col = &view.Column{ Name: column, diff --git a/view/config.go b/view/config.go index 8605e86f3..dad89cb15 100644 --- a/view/config.go +++ b/view/config.go @@ -92,8 +92,9 @@ func (c *Config) GetContentFormatParameter() *state.Parameter { } func (c *Constraints) HasOrderByColumn(name string) bool { + dotedName := "." + name for _, candidate := range c.OrderByColumn { - if candidate == name { + if candidate == name || strings.HasSuffix(candidate, dotedName) { return true } } From d7ee4a3cb45ff279cc3b39eb6b4dc946a93027dd Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 8 Dec 2025 13:59:31 -0800 Subject: [PATCH 096/279] patched type pointers --- .../function/allowedorderbycolumn.go | 28 ++++++++++++++++--- service/reader/sql.go | 3 +- view/config.go | 9 ++---- view/view.go | 2 +- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/internal/translator/function/allowedorderbycolumn.go b/internal/translator/function/allowedorderbycolumn.go index 40de57f5b..cfad0bbf7 100644 --- a/internal/translator/function/allowedorderbycolumn.go +++ b/internal/translator/function/allowedorderbycolumn.go @@ -23,15 +23,35 @@ func (c *allowedOrderByColumns) Apply(args []string, column *sqlparser.Column, r } aView.Selector.Constraints.OrderBy = true if len(values) == 0 { - return fmt.Errorf("failed to discover column in allowedOrderByColumns") + return fmt.Errorf("failed to discover expression in allowedOrderByColumns") } columns, ok := values[0].(string) if !ok { return fmt.Errorf("invalid columns type: %T, expected: %T in allowedOrderByColumns", values[0], columns) } - for _, column := range strings.Split(columns, ",") { - column = strings.TrimSpace(column) - aView.Selector.Constraints.OrderByColumn = append(aView.Selector.Constraints.OrderByColumn, column) + if len(aView.Selector.Constraints.OrderByColumn) == 0 { + aView.Selector.Constraints.OrderByColumn = map[string]string{} + } + for _, expression := range strings.Split(columns, ",") { + expression = strings.TrimSpace(expression) + + key := expression + value := expression + if strings.Contains(expression, ":") { + parts := strings.SplitN(expression, ":", 2) + key = parts[0] + value = parts[1] + } + + aView.Selector.Constraints.OrderByColumn[key] = value + lcKey := strings.ToLower(key) + if lcKey != key { + aView.Selector.Constraints.OrderByColumn[lcKey] = value + } + + if index := strings.Index(key, "."); index != -1 { + aView.Selector.Constraints.OrderByColumn[key[index+1:]] = value + } } return nil } diff --git a/service/reader/sql.go b/service/reader/sql.go index 3a34827e9..256cd26f3 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -369,8 +369,9 @@ func (b *Builder) appendOrderBy(sb *strings.Builder, aView *view.View, selector if !ok { if aView.Selector.Constraints.HasOrderByColumn(column) { + mapped := aView.Selector.Constraints.OrderByColumn[column] col = &view.Column{ - Name: column, + Name: mapped, } ok = true } diff --git a/view/config.go b/view/config.go index dad89cb15..5dd70f6de 100644 --- a/view/config.go +++ b/view/config.go @@ -92,13 +92,8 @@ func (c *Config) GetContentFormatParameter() *state.Parameter { } func (c *Constraints) HasOrderByColumn(name string) bool { - dotedName := "." + name - for _, candidate := range c.OrderByColumn { - if candidate == name || strings.HasSuffix(candidate, dotedName) { - return true - } - } - return false + _, ok := c.OrderByColumn[name] + return ok } func (c *Config) Init(ctx context.Context, resource *Resource, parent *View) error { diff --git a/view/view.go b/view/view.go index a482cfa12..9274c8053 100644 --- a/view/view.go +++ b/view/view.go @@ -157,7 +157,7 @@ func (v *View) Context(ctx context.Context) context.Context { type Constraints struct { Criteria bool OrderBy bool - OrderByColumn []string + OrderByColumn map[string]string Limit bool Offset bool Projection bool //enables columns projection from client (default ${NS}_fields= query param) From 0c1053eb94b64e64a8ccdb4e8d5f94d0717a7f05 Mon Sep 17 00:00:00 2001 From: Himanshu Shishir Shah Date: Tue, 9 Dec 2025 13:08:53 -0800 Subject: [PATCH 097/279] ENG-51959: using snapshot for logging --- go.mod | 2 +- go.sum | 8 ++------ repository/logging/logging.go | 35 +++++++++++++++++------------------ 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 4e6b4afed..00d915490 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/viant/tagly v0.3.0 github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 - github.com/viant/xdatly/handler v0.0.0-20251202205015-5f121a805ed3 + github.com/viant/xdatly/handler v0.0.0-20251208172928-dd34b7f09fd5 github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a github.com/viant/xlsy v0.3.1 diff --git a/go.sum b/go.sum index 7a75b3fc5..8ed5f2096 100644 --- a/go.sum +++ b/go.sum @@ -1138,14 +1138,10 @@ github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns= github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.17.8 h1:YxGTrXC2B1JmDz1qp8G+G9hGPk7XCRevWN1E4E+ZlCI= -github.com/viant/sqlx v0.17.8/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= github.com/viant/structology v0.8.0/go.mod h1:Fnm1DyR4gfyPbnhBMkQB5lR6/isYDnncBFO1nCxxmqs= -github.com/viant/structql v0.5.3 h1:QeOxvF0so8VFGt5bm+Jr6FL8uRnXkvwWY+ZCkbe3zsI= -github.com/viant/structql v0.5.3/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= github.com/viant/structql v0.5.4/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= github.com/viant/tagly v0.3.0 h1:Y8IckveeSrroR8yisq4MBdxhcNqf4v8II01uCpamh4E= @@ -1162,8 +1158,8 @@ github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a h1:7CLO2LjVnFgOwN0F github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUBDzc4rWacOYvAq/TKrTV0agvMF0gwm2gaoHI= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259/go.mod h1:fb8YgbVadk8X5ZLz49LWGzWmQlZd7Y/I5wE0ru44bIo= -github.com/viant/xdatly/handler v0.0.0-20251202205015-5f121a805ed3 h1:Xw0xbkb3lAu6k+p+XLlSyxPjl3XXda2qp74+itSbkKU= -github.com/viant/xdatly/handler v0.0.0-20251202205015-5f121a805ed3/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= +github.com/viant/xdatly/handler v0.0.0-20251208172928-dd34b7f09fd5 h1:CrT0HTlQul8FoGN0peylVczAOUEXKVqRAiB35ypRNHY= +github.com/viant/xdatly/handler v0.0.0-20251208172928-dd34b7f09fd5/go.mod h1:OeV4sVatklNs31nFnZtSp7lEvKJRoVJbH5opNRmRPg0= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 h1:G+e1MMDxQXUPPlAXVNlRqSLTLra7udGQZUu3hnr0Y8M= github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52/go.mod h1:LJN2m8xJjtYNCvyvNrVanJwvzj8+hYCuPswL8H4qRG0= github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a h1:jecH7mH63gj1zJwD18SdvSHM9Ttr9FEOnhHkYfkCNkI= diff --git a/repository/logging/logging.go b/repository/logging/logging.go index 0f28a6284..b618199ac 100644 --- a/repository/logging/logging.go +++ b/repository/logging/logging.go @@ -6,44 +6,43 @@ import ( "reflect" "runtime/debug" "strconv" - "time" "github.com/viant/xdatly/handler/exec" ) func Log(config *Config, execContext *exec.Context) { - execContext.ElapsedMs = int(time.Since(execContext.StartTime).Milliseconds()) + snap := execContext.SnapshotForLogging() includeSQL := config.ShallIncludeSQL() if !includeSQL { - execContext.Metrics = execContext.Metrics.HideMetrics() + snap.Metrics = snap.Metrics.HideMetrics() } if config.IsAuditEnabled() { - data := safeMarshal("EXECCONTEXT", execContext) + data := safeMarshal("EXECCONTEXT", snap) fmt.Println("[AUDIT]", string(data)) } if config.IsTracingEnabled() { - trace := execContext.Trace + trace := snap.Trace rootSpan := trace.Spans[0] - spans := execContext.Metrics.ToSpans(&rootSpan.SpanID) - if execContext.Auth != nil { - if execContext.Auth.UserID != 0 { - rootSpan.Attributes["jwt.uid"] = strconv.Itoa(execContext.Auth.UserID) + spans := snap.Metrics.ToSpans(&rootSpan.SpanID) + if snap.Auth != nil { + if snap.Auth.UserID != 0 { + rootSpan.Attributes["jwt.uid"] = strconv.Itoa(snap.Auth.UserID) } - if execContext.Auth.Username != "" { - rootSpan.Attributes["jwt.username"] = execContext.Auth.Username + if snap.Auth.Username != "" { + rootSpan.Attributes["jwt.username"] = snap.Auth.Username } - if execContext.Auth.Email != "" { - rootSpan.Attributes["jwt.email"] = execContext.Auth.Email + if snap.Auth.Email != "" { + rootSpan.Attributes["jwt.email"] = snap.Auth.Email } - if execContext.Auth.Scope != "" { - rootSpan.Attributes["jwt.scope"] = execContext.Auth.Scope + if snap.Auth.Scope != "" { + rootSpan.Attributes["jwt.scope"] = snap.Auth.Scope } } trace.Append(spans...) - if execContext.Error != "" { - trace.Spans[0].SetStatus(fmt.Errorf(execContext.Error)) + if snap.Error != "" { + trace.Spans[0].SetStatus(fmt.Errorf(snap.Error)) } else { - trace.Spans[0].SetStatusFromHTTPCode(execContext.StatusCode) + trace.Spans[0].SetStatusFromHTTPCode(snap.StatusCode) } traceData := safeMarshal("TRACE", trace) fmt.Println("[TRACE]", string(traceData)) From f6ac4312e6181b940e1236567d650672580a747d Mon Sep 17 00:00:00 2001 From: Himanshu Shishir Shah Date: Tue, 9 Dec 2025 16:48:54 -0800 Subject: [PATCH 098/279] ENG-52379: updating logic to add trace id to logs --- shared/logging/logger.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/shared/logging/logger.go b/shared/logging/logger.go index 9c208eb42..fe98d85ae 100644 --- a/shared/logging/logger.go +++ b/shared/logging/logger.go @@ -125,12 +125,19 @@ func (s *slogger) getContextValues(ctx context.Context) []any { values = append(values, "OpenTelemetryTraceId", openTelemetryTraceId) } - execContext := ctx.Value(exec.ContextKey) + execContext := exec.GetContext(ctx) if execContext != nil { - c, ok := execContext.(*exec.Context) - if ok && c != nil && c.Trace != nil { - values = append(values, "reqTraceId", c.Trace.TraceID) + traceId := "unknown" + + // ideally TraceID and Trace.TraceID should be the same + // but xdatly/handler/exec.(*Context).setHeader TraceID first + // with the value of adp-request-id header + if execContext.TraceID != "" { + traceId = execContext.TraceID + } else if execContext.Trace != nil { + traceId = execContext.Trace.TraceID } + values = append(values, "reqTraceId", traceId) } return values From d8c4b10e61052851a0db05a394d95b06262dc3b8 Mon Sep 17 00:00:00 2001 From: Himanshu Shishir Shah Date: Wed, 10 Dec 2025 12:30:07 -0800 Subject: [PATCH 099/279] ENG-52379: making the trace id key configurable --- shared/logging/logger.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/shared/logging/logger.go b/shared/logging/logger.go index fe98d85ae..de26bc87d 100644 --- a/shared/logging/logger.go +++ b/shared/logging/logger.go @@ -24,18 +24,28 @@ const ( WARN = "WARN" ERROR = "ERROR" UNKNOWN = "UNKNOWN" // Indicate other environment + DefaultTraceIdKey = "reqTraceId" ) type slogger struct { - logger *slog.Logger - level slog.Level + logger *slog.Logger + level slog.Level + traceIdKey string +} + +type Option func(l *slogger) + +func WithTraceIdKey(key string) Option { + return func(l *slogger) { + l.traceIdKey = key + } } // Init creates an ISLogger instance, a structured logger using the JSON Handler. // Creating this logger sets this as the default logger, so any logging after this // which goes through the standard logging package will also produce JSON structured // logs. -func New(level string, dest io.Writer) logger.Logger { +func New(level string, dest io.Writer, opts ...Option) logger.Logger { if dest == nil { dest = os.Stdout } @@ -63,9 +73,12 @@ func New(level string, dest io.Writer) logger.Logger { }) sl := slog.New(handler) slog.SetDefault(sl) - logger := &slogger{sl, logLevel} + l := &slogger{sl, logLevel, DefaultTraceIdKey} + for _, opt := range opts { + opt(l) + } - return logger + return l } func (s *slogger) IsDebugEnabled() bool { @@ -130,14 +143,14 @@ func (s *slogger) getContextValues(ctx context.Context) []any { traceId := "unknown" // ideally TraceID and Trace.TraceID should be the same - // but xdatly/handler/exec.(*Context).setHeader TraceID first - // with the value of adp-request-id header + // but xdatly/handler/exec.(*Context).setHeader sets TraceID first + // with the value of XDATLY_TRACING_HEADER env var value header (adp-request-id for datly platform) if execContext.TraceID != "" { traceId = execContext.TraceID } else if execContext.Trace != nil { traceId = execContext.Trace.TraceID } - values = append(values, "reqTraceId", traceId) + values = append(values, s.traceIdKey, traceId) } return values From 14e6a88bd0061ff3a66acbd36ddf4414043e65a7 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sun, 14 Dec 2025 07:26:14 -0800 Subject: [PATCH 100/279] patched type pointers --- internal/translator/service.go | 7 ++++++- internal/translator/view.go | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/translator/service.go b/internal/translator/service.go index 539ceade9..206762d1b 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -489,7 +489,12 @@ func (s *Service) adjustView(viewlet *Viewlet, resource *Resource, mode view.Mod if len(resource.Declarations.QuerySelectors) > 0 { for key, state := range resource.Declarations.QuerySelectors { - return fmt.Errorf("unknown query selector view %v, %v", key, state[0].Name) + switch strings.ToLower(state[0].In.Name) { + case "limit", "page", "offset", "fields", "orderby", "criteria": + default: + return fmt.Errorf("unknown query selector view %v, %v", key, state[0].In.Name) + + } } } diff --git a/internal/translator/view.go b/internal/translator/view.go index f8caa7157..fc0096ca8 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -214,7 +214,8 @@ func (v *View) buildSelector(namespace *Viewlet, rule *Rule) { selector.PageParameter = ¶meter.Parameter selector.Constraints.Page = &enabled } - delete(namespace.Resource.Declarations.QuerySelectors, namespace.Name) + + //delete(namespace.Resource.Declarations.QuerySelectors, namespace.Name) } } From 147384b169ef225a6c53cb29e133ca26b507b752 Mon Sep 17 00:00:00 2001 From: ppoudyal Date: Mon, 15 Dec 2025 12:04:41 -0500 Subject: [PATCH 101/279] added viewSyncFlag to control sync request from mcp handler. --- gateway/mcp.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index 0a9624afe..e9687e45a 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -99,13 +99,16 @@ func (r *Router) mcpToolCallHandler(component *repository.Component, aRoute *Rou return nil, rpcErr } r.addAuthTokenIfPresent(ctx, httpReq) + + // NEW: map MCP view sync flag argument to Sync-Read header + r.addSyncReadHeaderIfPresent(ctx, component, ¶ms, httpReq) + httpReq.RequestURI = httpReq.URL.RequestURI() if uri != aRoute.URI() { if matched, _ := r.match(component.Method, uri, httpReq); matched != nil { aRoute = matched } } - rw := proxy.NewWriter() aRoute.Handle(rw, httpReq) @@ -114,6 +117,69 @@ func (r *Router) mcpToolCallHandler(component *repository.Component, aRoute *Rou } } +func (r *Router) addSyncReadHeaderIfPresent( + ctx context.Context, + component *repository.Component, + params *schema.CallToolRequestParams, + httpRequest *http.Request, +) { + if params == nil || params.Arguments == nil { + return + } + // MCP tool arguments are generated using exported Go field names, so + // the Datly view sync flag (view.SyncFlag == "viewSyncFlag") will appear + // as "viewSyncFlag" in the schema/tool call. + const mcpSyncFlagArg = "viewSyncFlag" + const headerName = "Sync-Read" + + value, ok := params.Arguments[mcpSyncFlagArg] + if !ok { + return + } + + if !isTruthy(value) { + return + } + + // Optionally, ensure that the underlying component actually declares + // a sync flag parameter; if it does not, we simply skip setting the header. + if !hasSyncFlagParameter(component) { + return + } + + httpRequest.Header.Set(headerName, "true") +} + +// hasSyncFlagParameter checks whether the component declares a selector +// sync flag parameter, which should be exposed as view.SyncFlag. +func hasSyncFlagParameter(component *repository.Component) bool { + if component == nil || component.View == nil || component.View.Selector == nil { + return false + } + param := component.View.Selector.GetSyncFlagParameter() + if param == nil { + return false + } + // The selector sync flag parameter is defined in view.Config using + // view.SyncFlag as the state key, but here we simply check that it exists. + return true +} + +// isTruthy interprets common JSON-serialised truthy values. +func isTruthy(v interface{}) bool { + switch value := v.(type) { + case bool: + return value + case string: + s := strings.TrimSpace(strings.ToLower(value)) + return s == "true" || s == "1" || s == "yes" || s == "y" + case float64: + return value != 0 + default: + return false + } +} + // collectToolParameters aggregates component input parameters with selector pagination (limit/offset) when available. func (r *Router) collectToolParameters(component *repository.Component) []*state.Parameter { var all []*state.Parameter From fea9e58d24525cee99620cfe9bb9f8bf6369c24d Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 15 Dec 2025 10:44:26 -0800 Subject: [PATCH 102/279] patched type pointers --- internal/translator/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/translator/service.go b/internal/translator/service.go index 206762d1b..f446521a3 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -489,7 +489,7 @@ func (s *Service) adjustView(viewlet *Viewlet, resource *Resource, mode view.Mod if len(resource.Declarations.QuerySelectors) > 0 { for key, state := range resource.Declarations.QuerySelectors { - switch strings.ToLower(state[0].In.Name) { + switch strings.ToLower(state[0].Name) { case "limit", "page", "offset", "fields", "orderby", "criteria": default: return fmt.Errorf("unknown query selector view %v, %v", key, state[0].In.Name) From 3ad960f62adac18a06d4dd572be1bcc594004464 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 15 Dec 2025 15:54:32 -0800 Subject: [PATCH 103/279] error reclassification --- gateway/router/status/error.go | 35 ++++++++++++++++++++++++++-------- repository/resource/service.go | 7 ++++--- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/gateway/router/status/error.go b/gateway/router/status/error.go index 3d9111abc..876dde080 100644 --- a/gateway/router/status/error.go +++ b/gateway/router/status/error.go @@ -1,6 +1,8 @@ package status import ( + "net/http" + "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/utils/httputils" "github.com/viant/datly/utils/types" @@ -13,31 +15,48 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { violations := httputils.Violations{} switch actual := err.(type) { case *response.Error: - return actual.StatusCode(), actual.Message, nil + code := actual.StatusCode() + if code == 0 { + code = http.StatusInternalServerError + } + // For explicit 4xx we trust the message, for 5xx we keep it generic. + if code >= http.StatusInternalServerError { + return code, http.StatusText(http.StatusInternalServerError), nil + } + return code, actual.Message, nil case *svalidator.Validation: ret := violations.MergeSqlViolation(actual.Violations) - return statusCode, err.Error(), ret + return http.StatusBadRequest, err.Error(), ret case *govalidator.Validation: ret := violations.MergeGoViolation(actual.Violations) - return statusCode, actual.Error(), ret + return http.StatusBadRequest, actual.Error(), ret case *response.Errors: - actual.SetStatusCode(statusCode) + // Treat aggregated errors as validation-like by default. + actual.SetStatusCode(http.StatusBadRequest) for _, anError := range actual.Errors { isObj := types.IsObject(anError.Err) if isObj { - statusCode, anError.Message, anError.Object = NormalizeErr(anError.Err, statusCode) + statusCode, anError.Message, anError.Object = NormalizeErr(anError.Err, http.StatusBadRequest) } else { - statusCode, anError.Message, anError.Object = NormalizeErr(anError.Err, statusCode) + statusCode, anError.Message, anError.Object = NormalizeErr(anError.Err, http.StatusBadRequest) + } + if statusCode > actual.StatusCode() { + actual.SetStatusCode(statusCode) } } - actual.SetStatusCode(statusCode) return actual.StatusCode(), actual.Message, actual.Errors case *expand.ErrorResponse: if actual.StatusCode != 0 { statusCode = actual.StatusCode } + // If no status code was set on the error response, treat it as a client error. + if statusCode == 0 { + statusCode = http.StatusBadRequest + } return statusCode, actual.Message, actual.Content default: - return statusCode, err.Error(), nil + // Any non-validation error is treated as an internal server error with a generic message. + // The full error (including DB/sqlx failures) is still available in logs via exec.Context.SetError(err). + return http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError), nil } } diff --git a/repository/resource/service.go b/repository/resource/service.go index d1104e891..19d098b54 100644 --- a/repository/resource/service.go +++ b/repository/resource/service.go @@ -3,6 +3,10 @@ package resource import ( "context" "fmt" + "strings" + "sync" + "time" + "github.com/viant/afs" "github.com/viant/afs/file" "github.com/viant/afs/storage" @@ -10,9 +14,6 @@ import ( "github.com/viant/cloudless/resource" "github.com/viant/datly/repository/version" "github.com/viant/datly/view" - "strings" - "sync" - "time" ) type ( From 7739b70e7767b94b4b721eb4a920d6da84de20c6 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 16 Dec 2025 07:16:32 -0800 Subject: [PATCH 104/279] error reclassification --- go.mod | 104 ++++++++++++++++++++++++++++----------------------------- go.sum | 52 +++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 52 deletions(-) diff --git a/go.mod b/go.mod index 00d915490..86c7d5b3a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/viant/datly -go 1.23.8 +go 1.25.0 require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible @@ -14,9 +14,9 @@ require ( github.com/lib/pq v1.10.6 github.com/mattn/go-sqlite3 v1.14.16 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/viant/afs v1.26.2 - github.com/viant/afsc v1.9.1 + github.com/viant/afsc v1.16.0 github.com/viant/assertly v0.9.1-0.20220620174148-bab013f93a60 github.com/viant/bigquery v0.4.1 github.com/viant/cloudless v1.12.0 @@ -34,9 +34,9 @@ require ( github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 github.com/viant/xreflect v0.7.3 github.com/viant/xunsafe v0.10.3 - golang.org/x/mod v0.25.0 - golang.org/x/oauth2 v0.30.0 - google.golang.org/api v0.174.0 + golang.org/x/mod v0.28.0 + golang.org/x/oauth2 v0.32.0 + google.golang.org/api v0.201.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -60,58 +60,58 @@ require ( github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a github.com/viant/xlsy v0.3.1 github.com/viant/xmlify v0.1.1 - golang.org/x/net v0.40.0 - golang.org/x/tools v0.33.0 + golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 + golang.org/x/tools v0.37.0 modernc.org/sqlite v1.18.1 ) require ( - cloud.google.com/go v0.112.1 // indirect - cloud.google.com/go/auth v0.2.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.0 // indirect - cloud.google.com/go/compute/metadata v0.3.0 // indirect - cloud.google.com/go/firestore v1.15.0 // indirect - cloud.google.com/go/iam v1.1.7 // indirect - cloud.google.com/go/longrunning v0.5.5 // indirect - cloud.google.com/go/secretmanager v1.11.5 // indirect - cloud.google.com/go/storage v1.40.0 // indirect + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.9.8 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/firestore v1.17.0 // indirect + cloud.google.com/go/iam v1.2.1 // indirect + cloud.google.com/go/longrunning v0.6.1 // indirect + cloud.google.com/go/secretmanager v1.14.1 // indirect + cloud.google.com/go/storage v1.45.0 // indirect firebase.google.com/go v3.13.0+incompatible // indirect firebase.google.com/go/v4 v4.14.0 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect github.com/aerospike/aerospike-client-go/v6 v6.15.1 // indirect github.com/aws/aws-sdk-go v1.51.23 // indirect - github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect - github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.26 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect + github.com/aws/aws-sdk-go-v2/config v1.28.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.8 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.27 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.31.3 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.22.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect - github.com/aws/smithy-go v1.20.3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect + github.com/aws/smithy-go v1.22.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-errors/errors v1.5.1 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/s2a-go v0.1.7 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect @@ -124,7 +124,7 @@ require ( github.com/mazznoer/csscolorparser v0.1.3 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/nxadm/tail v1.4.8 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect @@ -136,24 +136,24 @@ require ( github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.32.0 // indirect - golang.org/x/text v0.26.0 // indirect - golang.org/x/time v0.5.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/term v0.36.0 // indirect + golang.org/x/text v0.30.0 // indirect + golang.org/x/time v0.7.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/appengine/v2 v2.0.2 // indirect - google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect + google.golang.org/grpc v1.77.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect lukechampine.com/uint128 v1.2.0 // indirect modernc.org/cc/v3 v3.36.3 // indirect diff --git a/go.sum b/go.sum index 8ed5f2096..54847327f 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,7 @@ cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMz cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -104,8 +105,10 @@ cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= cloud.google.com/go/auth v0.2.0 h1:y6oTcpMSbOcXbwYgUUrvI+mrQ2xbrcdpPgtVbCGTLTk= cloud.google.com/go/auth v0.2.0/go.mod h1:+yb+oy3/P0geX6DLKlqiGHARGR6EX2GRtYCzWOCQSbU= +cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= cloud.google.com/go/auth/oauth2adapt v0.2.0 h1:FR8zevgQwu+8CqiOT5r6xCmJa3pJC/wdXEEPF1OkNhA= cloud.google.com/go/auth/oauth2adapt v0.2.0/go.mod h1:AfqujpDAlTfLfeCIl/HJZZlIxD8+nJoZ5e0x1IxGq5k= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= @@ -188,6 +191,7 @@ cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxB cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= @@ -285,6 +289,7 @@ cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466d cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/firestore v1.15.0 h1:/k8ppuWOtNuDHt2tsRV42yI21uaGnKDEQnRFeBpbFF8= cloud.google.com/go/firestore v1.15.0/go.mod h1:GWOxFXcv8GZUtYpWHw/w6IuYNux/BtmeVTMmjrm4yhk= +cloud.google.com/go/firestore v1.17.0/go.mod h1:69uPx1papBsY8ZETooc71fOhoKkD70Q1DwMrtKuOT/Y= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= @@ -325,6 +330,7 @@ cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= +cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= @@ -359,6 +365,7 @@ cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= +cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= @@ -492,6 +499,7 @@ cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8A cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= cloud.google.com/go/secretmanager v1.11.5 h1:82fpF5vBBvu9XW4qj0FU2C6qVMtj1RM/XHwKXUEAfYY= cloud.google.com/go/secretmanager v1.11.5/go.mod h1:eAGv+DaCHkeVyQi0BeXgAHOU0RdrMeZIASKc+S7VqH4= +cloud.google.com/go/secretmanager v1.14.1/go.mod h1:L+gO+u2JA9CCyXpSR8gDH0o8EV7i/f0jdBOrUXcIV0U= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= @@ -549,6 +557,7 @@ cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5og cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= +cloud.google.com/go/storage v1.45.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= @@ -654,22 +663,29 @@ github.com/aws/aws-sdk-go v1.51.23/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3Tj github.com/aws/aws-sdk-go-v2 v1.17.2/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= github.com/aws/aws-sdk-go-v2/credentials v1.17.26 h1:tsm8g/nJxi8+/7XyJJcP2dLrnK/5rkFp6+i2nhmz5fk= github.com/aws/aws-sdk-go-v2/credentials v1.17.26/go.mod h1:3vAM49zkIa3q8WT6o9Ve5Z0vdByDMwmdScO0zvThTgI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.7 h1:CyuByiiCA4lPfU8RaHJh2wIYYn0hkFlOkMfWkVY67Mc= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.7/go.mod h1:pAMtgCPVxcKohC/HNI6nLwLeW007eYl3T+pq7yTMV3o= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26/go.mod h1:2E0LdbJW6lbeU4uxjum99GZzI0ZjDpAb0CoSCM0oeEY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20/go.mod h1:/+6lSiby8TBFpTVXZgKiN/rCfkYXEGvhlM4zCgPpt7w= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.8 h1:VgdGaSIoH4JhUZIspT8UgK0aBF85TiLve7VHEx3NfqE= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.8/go.mod h1:jvXzk+hVrlkiQOvnq6jH+F6qBK0CEceXkEWugT+4Kdc= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.27 h1:7MhqbR+k+b0gbOxp+W8yXgsl/Z5/dtMh85K0WI8X2EA= @@ -677,23 +693,29 @@ github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.27/go.mod h1:wX9QEZJ8 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.20 h1:kSZR22oLBDMtP8ZPGXhz649NU77xsJDG7g3xfT6nHVk= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.20/go.mod h1:lxM5qubwGNX29Qy+xTFG8G0r2Mj/TmyC+h3hS/7E4V8= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= github.com/aws/aws-sdk-go-v2/service/sns v1.31.3 h1:eSTEdxkfle2G98FE+Xl3db/XAXXVTJPNQo9K/Ar8oAI= github.com/aws/aws-sdk-go-v2/service/sns v1.31.3/go.mod h1:1dn0delSO3J69THuty5iwP0US2Glt0mx2qBBlI13pvw= github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 h1:Vjqy5BZCOIsn4Pj8xzyqgGmsSqzz7y/WXbN3RgOoVrc= github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3/go.mod h1:L0enV3GCRd5iG9B64W35C4/hwsCB00Ib+DKVGTadKHI= github.com/aws/aws-sdk-go-v2/service/sso v1.22.3 h1:Fv1vD2L65Jnp5QRsdiM64JvUM4Xe+E0JyVsRQKv6IeA= github.com/aws/aws-sdk-go-v2/service/sso v1.22.3/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -727,6 +749,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= @@ -776,6 +799,7 @@ github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpx github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -890,6 +914,7 @@ github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -901,6 +926,7 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -919,6 +945,7 @@ github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2e github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -1021,6 +1048,7 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1089,6 +1117,7 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= @@ -1098,6 +1127,8 @@ github.com/viant/afs v1.26.2 h1:rOs/iFxFlEndhIRATJVXlNWhVU0cGdRQAGVTVJPdsc0= github.com/viant/afs v1.26.2/go.mod h1:rScbFd9LJPGTM8HOI8Kjwee0AZ+MZMupAvFpPg+Qdj4= github.com/viant/afsc v1.9.1 h1:BIus7fYyjM+MDgKuAzCBfoV4oVy2xTVhuFsQKUCPvkQ= github.com/viant/afsc v1.9.1/go.mod h1:FA/xVjaMM10qGByabP8anTVMH6N4eUsAeWm5xcEZJJA= +github.com/viant/afsc v1.16.0 h1:/kOH/flNwme6h3oFrU/KPnMHkhbCZxQncTf1GSQIlBQ= +github.com/viant/afsc v1.16.0/go.mod h1:Z6fP3VcmzS8Sg2lowctR6KkVEX7XxJ8aNaoHqhUiZkY= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/assertly v0.9.0/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/assertly v0.9.1-0.20220620174148-bab013f93a60 h1:VFJvCOHKXv4IqX8rJwn1otpHWQGgMDv2bXtAPgEzndM= @@ -1202,16 +1233,21 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -1239,6 +1275,7 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1303,6 +1340,7 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1374,6 +1412,7 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1409,6 +1448,7 @@ golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1430,6 +1470,7 @@ golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1523,6 +1564,7 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1540,6 +1582,7 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1561,6 +1604,7 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1570,6 +1614,7 @@ golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1636,6 +1681,7 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1720,6 +1766,7 @@ google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2 google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.174.0 h1:zB1BWl7ocxfTea2aQ9mgdzXjnfPySllpPOskdnO+q34= google.golang.org/api v0.174.0/go.mod h1:aC7tB6j0HR1Nl0ni5ghpx6iLasmAX78Zkh/wgxAAjLg= +google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1874,12 +1921,14 @@ google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= +google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= @@ -1887,6 +1936,7 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1933,6 +1983,7 @@ google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1954,6 +2005,7 @@ google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 4ba19cc59bedfe0bc8c2e769a4ed9f559854a5a0 Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Tue, 16 Dec 2025 08:32:42 -0800 Subject: [PATCH 105/279] init view schema if nil --- repository/components.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repository/components.go b/repository/components.go index c803bcd3f..536ad3292 100644 --- a/repository/components.go +++ b/repository/components.go @@ -236,6 +236,9 @@ func (c *Components) updateIOTypeDependencies(ctx context.Context, ioType *state aView = baseView } } + if aView.Schema == nil { + aView.Schema = parameterViewSchema(parameter) + } aView.Schema.SetType(parameter.Schema.Type()) } } From 3a03b8e323cb8dee7a980e5a1e237808a7331b9e Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 16 Dec 2025 20:06:48 -0800 Subject: [PATCH 106/279] error reclassification --- go.mod | 23 ++++++++ go.sum | 93 ++++++++++++++++++++++++++++++ internal/inference/state.go | 7 +++ service/session/state.go | 111 ++++++++++++++++++++++-------------- 4 files changed, 191 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index 86c7d5b3a..5f5213887 100644 --- a/go.mod +++ b/go.mod @@ -66,6 +66,7 @@ require ( ) require ( + cel.dev/expr v0.24.0 // indirect cloud.google.com/go v0.116.0 // indirect cloud.google.com/go/auth v0.9.8 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect @@ -73,36 +74,53 @@ require ( cloud.google.com/go/firestore v1.17.0 // indirect cloud.google.com/go/iam v1.2.1 // indirect cloud.google.com/go/longrunning v0.6.1 // indirect + cloud.google.com/go/monitoring v1.21.1 // indirect cloud.google.com/go/secretmanager v1.14.1 // indirect cloud.google.com/go/storage v1.45.0 // indirect firebase.google.com/go v3.13.0+incompatible // indirect firebase.google.com/go/v4 v4.14.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect github.com/aerospike/aerospike-client-go/v6 v6.15.1 // indirect github.com/aws/aws-sdk-go v1.51.23 // indirect github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect github.com/aws/aws-sdk-go-v2/config v1.28.0 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.7 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.8 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.27 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.20 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 // indirect + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.31.3 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect github.com/aws/smithy-go v1.22.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-errors/errors v1.5.1 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.2 // indirect @@ -128,6 +146,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/viant/gosh v0.2.1 // indirect github.com/viant/igo v0.2.0 // indirect github.com/viant/x v0.3.0 // indirect @@ -136,10 +155,14 @@ require ( github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/otel v1.38.0 // indirect go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect go.opentelemetry.io/otel/trace v1.38.0 // indirect golang.org/x/crypto v0.43.0 // indirect golang.org/x/sync v0.17.0 // indirect diff --git a/go.sum b/go.sum index 54847327f..b3ca5d156 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -41,6 +43,7 @@ cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMz cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= @@ -105,9 +108,11 @@ cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= cloud.google.com/go/auth v0.2.0 h1:y6oTcpMSbOcXbwYgUUrvI+mrQ2xbrcdpPgtVbCGTLTk= cloud.google.com/go/auth v0.2.0/go.mod h1:+yb+oy3/P0geX6DLKlqiGHARGR6EX2GRtYCzWOCQSbU= +cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8= cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= cloud.google.com/go/auth/oauth2adapt v0.2.0 h1:FR8zevgQwu+8CqiOT5r6xCmJa3pJC/wdXEEPF1OkNhA= cloud.google.com/go/auth/oauth2adapt v0.2.0/go.mod h1:AfqujpDAlTfLfeCIl/HJZZlIxD8+nJoZ5e0x1IxGq5k= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= @@ -191,6 +196,7 @@ cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxB cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= @@ -289,6 +295,7 @@ cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466d cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/firestore v1.15.0 h1:/k8ppuWOtNuDHt2tsRV42yI21uaGnKDEQnRFeBpbFF8= cloud.google.com/go/firestore v1.15.0/go.mod h1:GWOxFXcv8GZUtYpWHw/w6IuYNux/BtmeVTMmjrm4yhk= +cloud.google.com/go/firestore v1.17.0 h1:iEd1LBbkDZTFsLw3sTH50eyg4qe8eoG6CjocmEXO9aQ= cloud.google.com/go/firestore v1.17.0/go.mod h1:69uPx1papBsY8ZETooc71fOhoKkD70Q1DwMrtKuOT/Y= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= @@ -330,6 +337,7 @@ cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= +cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= @@ -365,6 +373,7 @@ cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= +cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= @@ -389,6 +398,8 @@ cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhI cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/monitoring v1.21.1 h1:zWtbIoBMnU5LP9A/fz8LmWMGHpk4skdfeiaa66QdFGc= +cloud.google.com/go/monitoring v1.21.1/go.mod h1:Rj++LKrlht9uBi8+Eb530dIrzG/cU/lB8mt+lbeFK1c= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= @@ -499,6 +510,7 @@ cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8A cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= cloud.google.com/go/secretmanager v1.11.5 h1:82fpF5vBBvu9XW4qj0FU2C6qVMtj1RM/XHwKXUEAfYY= cloud.google.com/go/secretmanager v1.11.5/go.mod h1:eAGv+DaCHkeVyQi0BeXgAHOU0RdrMeZIASKc+S7VqH4= +cloud.google.com/go/secretmanager v1.14.1 h1:xlWSIg8rtBn5qCr2f3XtQP19+5COyf/ll49SEvi/0vM= cloud.google.com/go/secretmanager v1.14.1/go.mod h1:L+gO+u2JA9CCyXpSR8gDH0o8EV7i/f0jdBOrUXcIV0U= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= @@ -557,6 +569,7 @@ cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5og cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= +cloud.google.com/go/storage v1.45.0 h1:5av0QcIVj77t+44mV4gffFC/LscFRUhto6UBMB5SimM= cloud.google.com/go/storage v1.45.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= @@ -637,6 +650,12 @@ git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGy git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= @@ -663,29 +682,42 @@ github.com/aws/aws-sdk-go v1.51.23/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3Tj github.com/aws/aws-sdk-go-v2 v1.17.2/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ= github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= github.com/aws/aws-sdk-go-v2/credentials v1.17.26 h1:tsm8g/nJxi8+/7XyJJcP2dLrnK/5rkFp6+i2nhmz5fk= github.com/aws/aws-sdk-go-v2/credentials v1.17.26/go.mod h1:3vAM49zkIa3q8WT6o9Ve5Z0vdByDMwmdScO0zvThTgI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.7 h1:CyuByiiCA4lPfU8RaHJh2wIYYn0hkFlOkMfWkVY67Mc= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.7/go.mod h1:pAMtgCPVxcKohC/HNI6nLwLeW007eYl3T+pq7yTMV3o= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33 h1:X+4YY5kZRI/cOoSMVMGTqFXHAMg1bvvay7IBcqHpybQ= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33/go.mod h1:DPynzu+cn92k5UQ6tZhX+wfTB4ah6QDU/NgdHqatmvk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26/go.mod h1:2E0LdbJW6lbeU4uxjum99GZzI0ZjDpAb0CoSCM0oeEY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20/go.mod h1:/+6lSiby8TBFpTVXZgKiN/rCfkYXEGvhlM4zCgPpt7w= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.8 h1:VgdGaSIoH4JhUZIspT8UgK0aBF85TiLve7VHEx3NfqE= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.8/go.mod h1:jvXzk+hVrlkiQOvnq6jH+F6qBK0CEceXkEWugT+4Kdc= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.27 h1:7MhqbR+k+b0gbOxp+W8yXgsl/Z5/dtMh85K0WI8X2EA= @@ -693,28 +725,44 @@ github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.27/go.mod h1:wX9QEZJ8 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.20 h1:kSZR22oLBDMtP8ZPGXhz649NU77xsJDG7g3xfT6nHVk= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.20/go.mod h1:lxM5qubwGNX29Qy+xTFG8G0r2Mj/TmyC+h3hS/7E4V8= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 h1:xA6XhTF7PE89BCNHJbQi8VvPzcgMtmGC5dr8S8N7lHk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 h1:Rrqru2wYkKQCS2IM5/JrgKUQIoNTqA6y/iuxkjzxC6M= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2/go.mod h1:QuCURO98Sqee2AXmqDNxKXYFm2OEDAVAPApMqO0Vqnc= github.com/aws/aws-sdk-go-v2/service/sns v1.31.3 h1:eSTEdxkfle2G98FE+Xl3db/XAXXVTJPNQo9K/Ar8oAI= github.com/aws/aws-sdk-go-v2/service/sns v1.31.3/go.mod h1:1dn0delSO3J69THuty5iwP0US2Glt0mx2qBBlI13pvw= github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 h1:Vjqy5BZCOIsn4Pj8xzyqgGmsSqzz7y/WXbN3RgOoVrc= github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3/go.mod h1:L0enV3GCRd5iG9B64W35C4/hwsCB00Ib+DKVGTadKHI= +github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 h1:z6Pq4+jtKlhK4wWJGHRGwMLGjC1HZwAO3KJr/Na0tSU= +github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2/go.mod h1:DSmu/VZzpQlAubWBbAvNpt+S4k/XweglJi4XaDGyvQk= github.com/aws/aws-sdk-go-v2/service/sso v1.22.3 h1:Fv1vD2L65Jnp5QRsdiM64JvUM4Xe+E0JyVsRQKv6IeA= github.com/aws/aws-sdk-go-v2/service/sso v1.22.3/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -724,9 +772,12 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -744,6 +795,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -767,10 +820,15 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= +github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= +github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -794,11 +852,14 @@ github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmn github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= @@ -914,6 +975,7 @@ github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -926,6 +988,7 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= @@ -945,6 +1008,7 @@ github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2e github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -1101,6 +1165,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -1231,22 +1297,35 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -1275,6 +1354,7 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1340,6 +1420,7 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1412,6 +1493,7 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 h1:6/3JGEh1C88g7m+qzzTbl3A0FtsLguXieqofVLU/JAo= golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1448,6 +1530,7 @@ golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1470,6 +1553,7 @@ golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1564,6 +1648,7 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1604,6 +1689,7 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1614,6 +1700,7 @@ golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1766,6 +1853,7 @@ google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2 google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.174.0 h1:zB1BWl7ocxfTea2aQ9mgdzXjnfPySllpPOskdnO+q34= google.golang.org/api v0.174.0/go.mod h1:aC7tB6j0HR1Nl0ni5ghpx6iLasmAX78Zkh/wgxAAjLg= +google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0= google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1921,6 +2009,7 @@ google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE= google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= @@ -1928,6 +2017,7 @@ google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go. google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= +google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4= google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= @@ -1936,6 +2026,7 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -1983,6 +2074,7 @@ google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -2005,6 +2097,7 @@ google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/inference/state.go b/internal/inference/state.go index 2783c2767..e309bc01f 100644 --- a/internal/inference/state.go +++ b/internal/inference/state.go @@ -785,6 +785,13 @@ func discoverStateType(baseDir string, types *xreflect.Types, dataType string, p return nil, err } var rType = xunsafe.LookupType(dirTypes.ModulePath + "/" + dataType) + + if rType == nil && types != nil && strings.Count(pkg, "/") > 1 { //the last resort fallback collission protection + pkg = strings.Replace(pkg, "pkg/", "", 1) + rType, _ = types.Lookup(dataType, xreflect.WithPackage(pkg)) + + } + if rType == nil && len(stateTypeFields) > 0 { rType = reflect.StructOf(stateTypeFields) } diff --git a/service/session/state.go b/service/session/state.go index 2808c04e1..60f614d42 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -317,9 +317,12 @@ func (s *Session) populateParameter(ctx context.Context, parameter *state.Parame //ensure last written can be shared if err == nil { + switch parameterSelector.Type().Kind() { case reflect.Ptr: - s.cache.put(parameter, parameterSelector.Value(aState.Pointer())) + if parameter.Schema.Type() == parameterSelector.Type() { + s.cache.put(parameter, parameterSelector.Value(aState.Pointer())) + } } } return err @@ -402,9 +405,8 @@ func (s *Session) ensureValidValue(value interface{}, parameter *state.Parameter } } case reflect.Slice: - ptr := xunsafe.AsPointer(value) - slice := parameter.Schema.Slice() - sliceLen := slice.Len(ptr) + rSlice := reflect.ValueOf(value) + sliceLen := rSlice.Len() if errorMessage := validateSliceParameter(parameter, sliceLen); errorMessage != "" { return nil, errors.New(errorMessage) } @@ -415,11 +417,32 @@ func (s *Session) ensureValidValue(value interface{}, parameter *state.Parameter default: switch sliceLen { case 0: - value = reflect.New(parameter.OutputType().Elem()).Elem().Interface() + switch outputType.Kind() { + case reflect.Ptr: + value = reflect.New(outputType.Elem()).Elem().Interface() + case reflect.Struct: + value = reflect.New(outputType).Elem().Interface() + default: + value = reflect.New(outputType).Elem().Interface() + } valueType = reflect.TypeOf(value) case 1: - value = slice.ValuePointerAt(ptr, 0) - valueType = reflect.TypeOf(value) + elem := rSlice.Index(0) + if elem.Kind() == reflect.Interface && !elem.IsNil() { + elem = elem.Elem() + } + if elem.Kind() == reflect.Ptr { + value = elem.Interface() + valueType = elem.Type() + break + } + if elem.CanAddr() { + value = elem.Addr().Interface() + valueType = elem.Addr().Type() + break + } + value = elem.Interface() + valueType = elem.Type() default: return nil, fmt.Errorf("parameter %v return more than one value, len: %v rows ", parameter.Name, sliceLen) } @@ -437,53 +460,55 @@ func (s *Session) ensureValidValue(value interface{}, parameter *state.Parameter } if parameter.Schema.IsStruct() && !(valueType == selector.Type() || valueType.ConvertibleTo(selector.Type()) || valueType.AssignableTo(selector.Type())) { - - rawSelectorType := selector.Type() - isSelectorPtr := false - if rawSelectorType.Kind() == reflect.Ptr { - rawSelectorType = rawSelectorType.Elem() - isSelectorPtr = true + destType := selector.Type() + rawDestType := destType + destIsPtr := false + if rawDestType.Kind() == reflect.Ptr { + rawDestType = rawDestType.Elem() + destIsPtr = true } - isValuePtr := false - rawValueType := valueType - if rawValueType.Kind() == reflect.Ptr { - rawValueType = valueType.Elem() - isValuePtr = true + + rawSrcType := valueType + srcIsPtr := false + if rawSrcType.Kind() == reflect.Ptr { + rawSrcType = rawSrcType.Elem() + srcIsPtr = true } - if rawSelectorType.Kind() == reflect.Struct && isSelectorPtr { - if rawValueType.ConvertibleTo(rawSelectorType) { - ptrValue := reflect.ValueOf(value) - if isValuePtr && ptrValue.IsNil() { + if rawDestType.Kind() == reflect.Struct && rawSrcType.Kind() == reflect.Struct && rawSrcType.ConvertibleTo(rawDestType) { + srcValue := reflect.ValueOf(value) + if srcIsPtr { + if srcValue.IsNil() { return nil, nil } - var destValue reflect.Value - if isValuePtr { - destValue = ptrValue.Elem().Convert(rawSelectorType) - } else { - destValue = ptrValue.Convert(rawSelectorType) - } - if isSelectorPtr { - destPtrType := reflect.New(valueType) - destPtrType.Elem().Set(destValue) - return destPtrType.Interface(), nil - } else { - return destValue.Interface(), nil - } + srcValue = srcValue.Elem() } + converted := srcValue.Convert(rawDestType) + if destIsPtr { + out := reflect.New(rawDestType) + out.Elem().Set(converted) + return out.Interface(), nil + } + return converted.Interface(), nil } if options.shallReportNotAssignable() { - //if !ensureAssignable(parameter.Name, selector.Type(), valueType) { - fmt.Printf("parameter %v is not directly assignable from %s:(%s)\nsrc:%s \ndst:%s\n", parameter.Name, parameter.In.Kind, parameter.In.Name, valueType.String(), selector.Type().String()) - //} + fmt.Printf("parameter %v is not directly assignable from %s:(%s)\nsrc:%s \ndst:%s\n", parameter.Name, parameter.In.Kind, parameter.In.Name, valueType.String(), destType.String()) } - reflectValue := reflect.New(valueType) //TODO replace with fast xreflect copy - valuePtr := reflectValue.Interface() + var target reflect.Value + if destIsPtr { + target = reflect.New(rawDestType) // *T where destType is *T + } else { + target = reflect.New(destType) // *T where destType is T + } if data, err := json.Marshal(value); err == nil { - if err = json.Unmarshal(data, valuePtr); err == nil { - value = reflectValue.Elem().Interface() + if err = json.Unmarshal(data, target.Interface()); err == nil { + if destIsPtr { + value = target.Interface() + } else { + value = target.Elem().Interface() + } } } } @@ -841,7 +866,7 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}, opt func (s *Session) handleParameterError(parameter *state.Parameter, err error, errors *response.Errors) { if parameter.ErrorMessage != "" && err != nil { msg := strings.ReplaceAll(parameter.ErrorMessage, "${error}", err.Error()) - err = fmt.Errorf(msg) + err = fmt.Errorf("%s", msg) } if pErr, ok := err.(*response.Error); ok { pErr.Code = parameter.ErrorStatusCode From 5bd0dee576c25bfc2657da019edad895cbf80b4d Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 16 Dec 2025 20:40:33 -0800 Subject: [PATCH 107/279] error reclassification --- service/session/state_test.go | 291 ++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 service/session/state_test.go diff --git a/service/session/state_test.go b/service/session/state_test.go new file mode 100644 index 000000000..497d6aca6 --- /dev/null +++ b/service/session/state_test.go @@ -0,0 +1,291 @@ +package session + +import ( + "reflect" + "testing" + + "github.com/viant/datly/view/state" + "github.com/viant/structology" +) + +func TestSessionEnsureValidValue_Transitions(t *testing.T) { + type T struct { + A *int + B *int + } + + inlineStructSwapped := reflect.StructOf([]reflect.StructField{ + // Deliberately swap field order vs T to ensure the types are not convertible. + {Name: "B", Type: reflect.TypeOf((*int)(nil))}, + {Name: "A", Type: reflect.TypeOf((*int)(nil))}, + }) + inlinePtrType := reflect.PtrTo(inlineStructSwapped) + + newSelector := func(t *testing.T, paramType reflect.Type) *structology.Selector { + t.Helper() + stateStruct := reflect.StructOf([]reflect.StructField{ + {Name: "Param", Type: paramType}, + }) + stateType := structology.NewStateType(stateStruct) + selector := stateType.Lookup("Param") + if selector == nil { + t.Fatalf("failed to lookup selector Param") + } + return selector + } + + intPtrType := reflect.TypeOf((*int)(nil)) + + ttPtrType := reflect.TypeOf((*T)(nil)) + sliceOfTTPtrType := reflect.SliceOf(ttPtrType) + ptrToSliceOfTTPtrType := reflect.PtrTo(sliceOfTTPtrType) + intType := reflect.TypeOf(int(0)) + sliceOfIntType := reflect.SliceOf(intType) + ttType := reflect.TypeOf(T{}) + + boolPtr := func(v bool) *bool { return &v } + + cases := []struct { + name string + schemaType reflect.Type + selectorType reflect.Type + required *bool + value interface{} + wantType reflect.Type + wantErr bool + check func(t *testing.T, got interface{}) + }{ + { + name: "nil-value_ptr-schema_returns-typed-nil", + schemaType: intPtrType, + selectorType: intPtrType, + value: nil, + wantType: intPtrType, + check: func(t *testing.T, got interface{}) { + t.Helper() + if !reflect.ValueOf(got).IsNil() { + t.Fatalf("expected nil pointer, got %v", got) + } + }, + }, + { + name: "nil-value_slice-schema_returns-nil-slice", + schemaType: sliceOfIntType, + selectorType: sliceOfIntType, + value: nil, + wantType: sliceOfIntType, + check: func(t *testing.T, got interface{}) { + t.Helper() + if !reflect.ValueOf(got).IsNil() { + t.Fatalf("expected nil slice, got %v", got) + } + }, + }, + { + name: "ptr-struct_to_ptr-to-slice-wraps-single", + schemaType: sliceOfTTPtrType, + selectorType: ptrToSliceOfTTPtrType, + value: func() interface{} { + a := 10 + b := 20 + return &T{A: &a, B: &b} + }(), + wantType: ptrToSliceOfTTPtrType, + check: func(t *testing.T, got interface{}) { + t.Helper() + gotSlicePtr := reflect.ValueOf(got) + if gotSlicePtr.IsNil() { + t.Fatalf("expected non-nil pointer to slice") + } + gotSlice := gotSlicePtr.Elem() + if gotSlice.Len() != 1 { + t.Fatalf("expected len=1, got %d", gotSlice.Len()) + } + if gotSlice.Index(0).IsNil() { + t.Fatalf("expected element 0 to be non-nil") + } + }, + }, + { + name: "ptr-struct-nil_to_ptr-to-slice-wraps-empty", + schemaType: sliceOfTTPtrType, + selectorType: ptrToSliceOfTTPtrType, + value: (*T)(nil), + wantType: ptrToSliceOfTTPtrType, + check: func(t *testing.T, got interface{}) { + t.Helper() + gotSlicePtr := reflect.ValueOf(got) + if gotSlicePtr.IsNil() { + t.Fatalf("expected non-nil pointer to slice") + } + gotSlice := gotSlicePtr.Elem() + if gotSlice.Len() != 0 { + t.Fatalf("expected len=0, got %d", gotSlice.Len()) + } + }, + }, + { + name: "slice-to-scalar_len0_required_errors", + schemaType: ttPtrType, + selectorType: ttPtrType, + required: boolPtr(true), + value: []*T{}, + wantErr: true, + }, + { + name: "slice-to-scalar_len0_not-required_returns-zero", + schemaType: ttPtrType, + selectorType: ttPtrType, + value: []*T{}, + wantType: ttPtrType, + check: func(t *testing.T, got interface{}) { + t.Helper() + if reflect.ValueOf(got).IsNil() { + t.Fatalf("expected non-nil *T") + } + }, + }, + { + name: "slice-of-int_len1_to-int", + schemaType: intType, + selectorType: intType, + value: []int{7}, + wantType: intType, + check: func(t *testing.T, got interface{}) { + t.Helper() + if got.(int) != 7 { + t.Fatalf("expected 7, got %v", got) + } + }, + }, + { + name: "slice-of-int_len2_to-int_errors", + schemaType: intType, + selectorType: intType, + value: []int{1, 2}, + wantErr: true, + }, + { + name: "ptr-required_nil_errors", + schemaType: ttPtrType, + selectorType: ttPtrType, + required: boolPtr(true), + value: (*T)(nil), + wantErr: true, + }, + { + name: "ptr-value_to-struct-selector_derefs", + schemaType: ttType, + selectorType: ttType, + value: func() interface{} { + a := 3 + b := 4 + return &T{A: &a, B: &b} + }(), + wantType: ttType, + check: func(t *testing.T, got interface{}) { + t.Helper() + gotT := got.(T) + if gotT.A == nil || gotT.B == nil { + t.Fatalf("expected non-nil fields") + } + if *gotT.A != 3 || *gotT.B != 4 { + t.Fatalf("unexpected values: %+v", gotT) + } + }, + }, + { + name: "struct-value_to-ptr-selector_allocates", + schemaType: ttPtrType, + selectorType: ttPtrType, + value: func() interface{} { + a := 5 + b := 6 + return T{A: &a, B: &b} + }(), + wantType: ttPtrType, + check: func(t *testing.T, got interface{}) { + t.Helper() + gotPtr := got.(*T) + if gotPtr == nil || gotPtr.A == nil || gotPtr.B == nil { + t.Fatalf("expected non-nil *T with non-nil fields") + } + if *gotPtr.A != 5 || *gotPtr.B != 6 { + t.Fatalf("unexpected values: %+v", *gotPtr) + } + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + parameter := &state.Parameter{ + Name: "Param", + In: state.NewState("Param"), + Schema: state.NewSchema(tc.schemaType), + Required: tc.required, + } + + selector := newSelector(t, tc.selectorType) + sess := &Session{} + opts := NewOptions(WithReportNotAssignable(false)) + + got, err := sess.ensureValidValue(tc.value, parameter, selector, opts) + if (err != nil) != tc.wantErr { + t.Fatalf("error=%v, wantErr=%v", err, tc.wantErr) + } + if tc.wantErr { + return + } + if tc.wantType != nil && reflect.TypeOf(got) != tc.wantType { + t.Fatalf("expected %v, got %T", tc.wantType, got) + } + if tc.check != nil { + tc.check(t, got) + } + }) + } + + t.Run("slice-of-named-ptr_to-inline-ptr_allocates-and-copies_details", func(t *testing.T) { + a := 1 + b := 2 + original := &T{A: &a, B: &b} + input := []*T{original} + + parameter := &state.Parameter{ + Name: "Param", + In: state.NewState("Param"), + Schema: state.NewSchema(inlinePtrType), + } + selector := newSelector(t, inlinePtrType) + sess := &Session{} + opts := NewOptions(WithReportNotAssignable(false)) + + got, err := sess.ensureValidValue(input, parameter, selector, opts) + if err != nil { + t.Fatalf("ensureValidValue error: %v", err) + } + if reflect.TypeOf(got) != inlinePtrType { + t.Fatalf("expected %v, got %T", inlinePtrType, got) + } + + gotPtr := reflect.ValueOf(got).Pointer() + origPtr := reflect.ValueOf(original).Pointer() + if gotPtr == origPtr { + t.Fatalf("expected ensureValidValue to allocate/copy into %v; got aliases original *T pointer %x", inlinePtrType, gotPtr) + } + + gotValue := reflect.ValueOf(got).Elem() + gotA := gotValue.FieldByName("A") + gotB := gotValue.FieldByName("B") + if gotA.IsNil() || gotB.IsNil() { + t.Fatalf("expected A and B to be non-nil") + } + if gotA.Elem().Int() != int64(*original.A) { + t.Fatalf("expected A=%d, got %d", *original.A, gotA.Elem().Int()) + } + if gotB.Elem().Int() != int64(*original.B) { + t.Fatalf("expected B=%d, got %d", *original.B, gotB.Elem().Int()) + } + }) +} From 3ff3ba125d73070f3147faef0945afedc70fb8cf Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 16 Dec 2025 22:38:50 -0800 Subject: [PATCH 108/279] patched marshaller --- gateway/router/marshal/json/cache.go | 3 +++ gateway/router/marshal/json/marshaller_interface.go | 8 ++++++++ gateway/router/marshal/json/marshaller_slice.go | 8 ++++++++ 3 files changed, 19 insertions(+) diff --git a/gateway/router/marshal/json/cache.go b/gateway/router/marshal/json/cache.go index d08a4704f..15c46e4b9 100644 --- a/gateway/router/marshal/json/cache.go +++ b/gateway/router/marshal/json/cache.go @@ -120,6 +120,9 @@ func (c *pathCache) loadOrGetMarshaller(rType reflect.Type, cfg *config.IOConfig } func (c *pathCache) getMarshaller(rType reflect.Type, config *config.IOConfig, path string, outputPath string, tag *format.Tag, options ...interface{}) (marshaler, error) { + if rType == nil { + return nil, fmt.Errorf("nil reflect.Type for path %q", path) + } if tag == nil { tag = &format.Tag{} } diff --git a/gateway/router/marshal/json/marshaller_interface.go b/gateway/router/marshal/json/marshaller_interface.go index c7da33fe9..4256327c2 100644 --- a/gateway/router/marshal/json/marshaller_interface.go +++ b/gateway/router/marshal/json/marshaller_interface.go @@ -47,7 +47,15 @@ func asInterface(xType *xunsafe.Type, pointer unsafe.Pointer) interface{} { func (i *interfaceMarshaller) MarshallObject(ptr unsafe.Pointer, sb *MarshallSession) error { value := i.AsInterface(ptr) + if value == nil { + sb.Write(nullBytes) + return nil + } rType := reflect.TypeOf(value) + if rType == nil { + sb.Write(nullBytes) + return nil + } marshaller, err := i.cache.loadMarshaller(rType, i.config, i.path, i.outputPath, i.tag) if err != nil { diff --git a/gateway/router/marshal/json/marshaller_slice.go b/gateway/router/marshal/json/marshaller_slice.go index 1ad341cef..51d90d3f4 100644 --- a/gateway/router/marshal/json/marshaller_slice.go +++ b/gateway/router/marshal/json/marshaller_slice.go @@ -151,7 +151,15 @@ func (s *sliceInterfaceMarshaller) MarshallObject(ptr unsafe.Pointer, sb *Marsha sb.WriteByte(',') } + if iface == nil { + sb.Write(nullBytes) + continue + } ifaceType := reflect.TypeOf(iface) + if ifaceType == nil { + sb.Write(nullBytes) + continue + } marshaller, err := s.cache.loadMarshaller(ifaceType, s.config, s.path, s.outputPath, s.tag) if err != nil { From 8f56405d07653eb5fe1846c37701875fcf0fe72b Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 17 Dec 2025 13:36:16 -0800 Subject: [PATCH 109/279] patched marshaller --- gateway/router/status/error.go | 88 ++++++++++++++++++++++++++++------ service/session/state.go | 12 +++++ 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/gateway/router/status/error.go b/gateway/router/status/error.go index 876dde080..2318d100d 100644 --- a/gateway/router/status/error.go +++ b/gateway/router/status/error.go @@ -2,10 +2,10 @@ package status import ( "net/http" + "strings" "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/utils/httputils" - "github.com/viant/datly/utils/types" "github.com/viant/govalidator" svalidator "github.com/viant/sqlx/io/validator" "github.com/viant/xdatly/handler/response" @@ -17,9 +17,12 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { case *response.Error: code := actual.StatusCode() if code == 0 { - code = http.StatusInternalServerError + code = statusCode } - // For explicit 4xx we trust the message, for 5xx we keep it generic. + if code == 0 { + code = http.StatusBadRequest + } + // For explicit 5xx we keep response generic, for 4xx we trust the configured message. if code >= http.StatusInternalServerError { return code, http.StatusText(http.StatusInternalServerError), nil } @@ -31,19 +34,44 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { ret := violations.MergeGoViolation(actual.Violations) return http.StatusBadRequest, actual.Error(), ret case *response.Errors: - // Treat aggregated errors as validation-like by default. - actual.SetStatusCode(http.StatusBadRequest) - for _, anError := range actual.Errors { - isObj := types.IsObject(anError.Err) - if isObj { - statusCode, anError.Message, anError.Object = NormalizeErr(anError.Err, http.StatusBadRequest) + // Respect existing status/message set on aggregated errors (often parameter-driven). + if actual.StatusCode() == 0 { + if statusCode == 0 { + actual.SetStatusCode(http.StatusBadRequest) } else { - statusCode, anError.Message, anError.Object = NormalizeErr(anError.Err, http.StatusBadRequest) - } - if statusCode > actual.StatusCode() { actual.SetStatusCode(statusCode) } } + if actual.Message == "" && len(actual.Errors) > 0 { + actual.Message = actual.Errors[0].Message + } + + for _, anError := range actual.Errors { + code := anError.StatusCode() + + switch { + case code >= http.StatusInternalServerError: + // Explicitly marked as server error at parameter level: generic message. + anError.Message = http.StatusText(http.StatusInternalServerError) + case code == 0: + // No explicit status on this parameter error; classify underlying cause. + innerStatus, innerMsg, innerObj := NormalizeErr(anError.Err, actual.StatusCode()) + anError.Code = innerStatus + if innerMsg != "" { + anError.Message = innerMsg + } + if innerObj != nil { + anError.Object = innerObj + } + code = innerStatus + default: + // 4xx with configured status/message – leave as defined on the parameter. + } + + if code > actual.StatusCode() { + actual.SetStatusCode(code) + } + } return actual.StatusCode(), actual.Message, actual.Errors case *expand.ErrorResponse: if actual.StatusCode != 0 { @@ -55,8 +83,38 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { } return statusCode, actual.Message, actual.Content default: - // Any non-validation error is treated as an internal server error with a generic message. - // The full error (including DB/sqlx failures) is still available in logs via exec.Context.SetError(err). - return http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError), nil + // Only DB-caused errors are mapped to 500 with a generic message. + if isDatabaseError(err) { + return http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError), nil + } + if statusCode == 0 { + statusCode = http.StatusBadRequest + } + return statusCode, err.Error(), nil + } +} + +// isDatabaseError detects errors that originate from DB/sqlx execution. +// These are the only errors that should be remapped to 500 with a generic message. +func isDatabaseError(err error) bool { + if err == nil { + return false } + msg := err.Error() + if msg == "" { + return false + } + + // Known DB-related error patterns from reader/executor paths. + if strings.Contains(msg, "database error occured while fetching Data") { + return true + } + if strings.Contains(msg, "error occured while connecting to database") { + return true + } + if strings.Contains(msg, "failed to get db:") { + return true + } + + return false } diff --git a/service/session/state.go b/service/session/state.go index 60f614d42..8c9bad387 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -428,6 +428,18 @@ func (s *Session) ensureValidValue(value interface{}, parameter *state.Parameter valueType = reflect.TypeOf(value) case 1: elem := rSlice.Index(0) + rawType := elem.Type() + if rawType.Kind() == reflect.Ptr { + rawType = rawType.Elem() + } + if rawType.Kind() == reflect.Interface { + rawType = rawType.Elem() + } + + if rawType.Kind() != reflect.Struct { + break + } + if elem.Kind() == reflect.Interface && !elem.IsNil() { elem = elem.Elem() } From 926929f67544909a318f0aac818d9bfdf299b4f4 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 18 Dec 2025 08:21:56 -0800 Subject: [PATCH 110/279] patched marshaller --- gateway/router/status/error.go | 76 ++++++++++--------------- go.mod | 1 + go.sum | 3 + service/executor/extension/validator.go | 11 ++++ utils/errors/db.go | 50 ++++++++++++++++ 5 files changed, 96 insertions(+), 45 deletions(-) create mode 100644 utils/errors/db.go diff --git a/gateway/router/status/error.go b/gateway/router/status/error.go index 2318d100d..bc9c2a83f 100644 --- a/gateway/router/status/error.go +++ b/gateway/router/status/error.go @@ -2,9 +2,9 @@ package status import ( "net/http" - "strings" "github.com/viant/datly/service/executor/expand" + derrors "github.com/viant/datly/utils/errors" "github.com/viant/datly/utils/httputils" "github.com/viant/govalidator" svalidator "github.com/viant/sqlx/io/validator" @@ -34,28 +34,24 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { ret := violations.MergeGoViolation(actual.Violations) return http.StatusBadRequest, actual.Error(), ret case *response.Errors: - // Respect existing status/message set on aggregated errors (often parameter-driven). - if actual.StatusCode() == 0 { - if statusCode == 0 { - actual.SetStatusCode(http.StatusBadRequest) - } else { - actual.SetStatusCode(statusCode) - } + maxStatus := actual.StatusCode() + if maxStatus == 0 { + maxStatus = statusCode } - if actual.Message == "" && len(actual.Errors) > 0 { - actual.Message = actual.Errors[0].Message + if maxStatus == 0 { + maxStatus = http.StatusBadRequest } + hasServerError := maxStatus >= http.StatusInternalServerError for _, anError := range actual.Errors { code := anError.StatusCode() - switch { case code >= http.StatusInternalServerError: - // Explicitly marked as server error at parameter level: generic message. anError.Message = http.StatusText(http.StatusInternalServerError) + hasServerError = true case code == 0: - // No explicit status on this parameter error; classify underlying cause. - innerStatus, innerMsg, innerObj := NormalizeErr(anError.Err, actual.StatusCode()) + innerStatus, innerMsg, innerObj := NormalizeErr(anError.Err, maxStatus) + code = innerStatus anError.Code = innerStatus if innerMsg != "" { anError.Message = innerMsg @@ -63,16 +59,31 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { if innerObj != nil { anError.Object = innerObj } - code = innerStatus + if code >= http.StatusInternalServerError { + hasServerError = true + } default: - // 4xx with configured status/message – leave as defined on the parameter. + if code >= http.StatusInternalServerError { + hasServerError = true + } } - if code > actual.StatusCode() { - actual.SetStatusCode(code) + if code > maxStatus { + maxStatus = code } } - return actual.StatusCode(), actual.Message, actual.Errors + + if hasServerError { + actual.Message = http.StatusText(http.StatusInternalServerError) + } else if actual.Message == "" && len(actual.Errors) > 0 { + actual.Message = actual.Errors[0].Message + } + + if maxStatus == 0 { + maxStatus = http.StatusBadRequest + } + + return maxStatus, actual.Message, actual.Errors case *expand.ErrorResponse: if actual.StatusCode != 0 { statusCode = actual.StatusCode @@ -84,7 +95,7 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { return statusCode, actual.Message, actual.Content default: // Only DB-caused errors are mapped to 500 with a generic message. - if isDatabaseError(err) { + if derrors.IsDatabaseError(err) { return http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError), nil } if statusCode == 0 { @@ -93,28 +104,3 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { return statusCode, err.Error(), nil } } - -// isDatabaseError detects errors that originate from DB/sqlx execution. -// These are the only errors that should be remapped to 500 with a generic message. -func isDatabaseError(err error) bool { - if err == nil { - return false - } - msg := err.Error() - if msg == "" { - return false - } - - // Known DB-related error patterns from reader/executor paths. - if strings.Contains(msg, "database error occured while fetching Data") { - return true - } - if strings.Contains(msg, "error occured while connecting to database") { - return true - } - if strings.Contains(msg, "failed to get db:") { - return true - } - - return false -} diff --git a/go.mod b/go.mod index 5f5213887..a5a9eab23 100644 --- a/go.mod +++ b/go.mod @@ -142,6 +142,7 @@ require ( github.com/mazznoer/csscolorparser v0.1.3 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/nxadm/tail v1.4.8 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect diff --git a/go.sum b/go.sum index b3ca5d156..38ca1e6c4 100644 --- a/go.sum +++ b/go.sum @@ -1110,6 +1110,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -1768,6 +1770,7 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/service/executor/extension/validator.go b/service/executor/extension/validator.go index 8c23f0df4..740b16f37 100644 --- a/service/executor/extension/validator.go +++ b/service/executor/extension/validator.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "fmt" + + derrors "github.com/viant/datly/utils/errors" "github.com/viant/datly/utils/httputils" "github.com/viant/govalidator" sqlxvalidator "github.com/viant/sqlx/io/validator" @@ -33,6 +35,9 @@ func (v *SqlxValidator) Validate(ctx context.Context, any interface{}, opts ...v err = v.validator.validateWithSqlx(ctx, any, validation, options) } if err != nil { + if derrors.IsDatabaseError(err) { + return validation, err + } validation.Append("/", "", "", "error", err.Error()) } return validation, nil @@ -46,9 +51,15 @@ func (v *Validator) Validate(ctx context.Context, any interface{}, opts ...valid validation := getOrCreateValidation(options) err := v.validateWithGoValidator(ctx, any, validation, options) if err != nil { + if derrors.IsDatabaseError(err) { + return validation, err + } validation.Append("/", "", "", "error", err.Error()) } if err = v.validateWithSqlx(ctx, any, validation, options); err != nil { + if derrors.IsDatabaseError(err) { + return validation, err + } validation.Append("/", "", "", "error", err.Error()) } return validation, nil diff --git a/utils/errors/db.go b/utils/errors/db.go new file mode 100644 index 000000000..19067f0ce --- /dev/null +++ b/utils/errors/db.go @@ -0,0 +1,50 @@ +package errors + +import ( + "errors" + "strings" +) + +// IsDatabaseError determines whether the supplied error was caused by the database or driver layer. +// We inspect the full error chain because many call-sites wrap driver errors with additional context. +func IsDatabaseError(err error) bool { + if err == nil { + return false + } + return hasDatabaseSignature(err) +} + +func hasDatabaseSignature(err error) bool { + for err != nil { + if matchesDatabasePattern(err.Error()) { + return true + } + err = errors.Unwrap(err) + } + return false +} + +func matchesDatabasePattern(message string) bool { + if message == "" { + return false + } + lower := strings.ToLower(message) + patterns := []string{ + "database error occured while fetching data", + "database error occurred while fetching data", + "error occured while connecting to database", + "error occurred while connecting to database", + "failed to get db", + "failed to create stmt source", + "too many connections", + "connection refused", + "driver: bad connection", + "sql: transaction has already been committed or rolled back", + } + for _, pattern := range patterns { + if strings.Contains(lower, pattern) { + return true + } + } + return false +} From af837488c6cf27f83b35f4f5fe700e4463343430 Mon Sep 17 00:00:00 2001 From: vc42 Date: Fri, 19 Dec 2025 13:54:24 -0500 Subject: [PATCH 111/279] use aState.Selector(parameter.Name) instead of parameter.Selector() to avoid SIGBUS --- service/session/state.go | 81 ++++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/service/session/state.go b/service/session/state.go index 8c9bad387..7c2ff10df 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -285,6 +285,52 @@ func (s *Session) populateParameterInBackground(ctx context.Context, parameter * } } +// The function below causes SIGBUS when template parameters are rebound. +//E.g. a predicate builder velty expression is located in an embedded SQL, outside main DQL +//func (s *Session) populateParameter(ctx context.Context, parameter *state.Parameter, aState *structology.State, options *Options) error { +// value, has, err := s.LookupValue(ctx, parameter, options) +// if err != nil { +// return err +// } +// if !has { +// if parameter.IsRequired() { +// return fmt.Errorf("parameter %v is required", parameter.Name) +// } +// return nil +// } +// +// parameterSelector := parameter.Selector() +// if options.indirectState || parameterSelector == nil { //p +// parameterSelector, err = aState.Selector(parameter.Name) +// if parameterSelector == nil { +// switch parameter.In.Kind { +// case state.KindConst: +// return nil +// } +// } +// if err != nil { +// return err +// } +// } +// +// if value, err = s.ensureValidValue(value, parameter, parameterSelector, options); err != nil { +// return err +// } +// err = parameterSelector.SetValue(aState.Pointer(), value) +// +// //ensure last written can be shared +// if err == nil { +// +// switch parameterSelector.Type().Kind() { +// case reflect.Ptr: +// if parameter.Schema.Type() == parameterSelector.Type() { +// s.cache.put(parameter, parameterSelector.Value(aState.Pointer())) +// } +// } +// } +// return err +//} + func (s *Session) populateParameter(ctx context.Context, parameter *state.Parameter, aState *structology.State, options *Options) error { value, has, err := s.LookupValue(ctx, parameter, options) if err != nil { @@ -296,36 +342,25 @@ func (s *Session) populateParameter(ctx context.Context, parameter *state.Parame } return nil } - parameterSelector := parameter.Selector() - if options.indirectState || parameterSelector == nil { //p - parameterSelector, err = aState.Selector(parameter.Name) - if parameterSelector == nil { - switch parameter.In.Kind { - case state.KindConst: - return nil - } - } - if err != nil { - return err - } + + // Resolve selector strictly from the state's layout + // Treat "not found" as a no-op (skip), since this view doesn't declare that parameter. + parameterSelector, err := aState.Selector(parameter.Name) + if err != nil || parameterSelector == nil { + return nil } if value, err = s.ensureValidValue(value, parameter, parameterSelector, options); err != nil { return err } - err = parameterSelector.SetValue(aState.Pointer(), value) - - //ensure last written can be shared - if err == nil { + if err = parameterSelector.SetValue(aState.Pointer(), value); err != nil { + return err + } - switch parameterSelector.Type().Kind() { - case reflect.Ptr: - if parameter.Schema.Type() == parameterSelector.Type() { - s.cache.put(parameter, parameterSelector.Value(aState.Pointer())) - } - } + if parameterSelector.Type().Kind() == reflect.Ptr { + s.cache.put(parameter, parameterSelector.Value(aState.Pointer())) } - return err + return nil } func (s *Session) canRead(ctx context.Context, parameter *state.Parameter, opts *Options) (bool, error) { From cfe782336454bbe29058dc0a8d06fbd44ae658d9 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 22 Dec 2025 14:50:05 -0800 Subject: [PATCH 112/279] patched marshaller --- gateway/router/status/error.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/gateway/router/status/error.go b/gateway/router/status/error.go index bc9c2a83f..41087fe5e 100644 --- a/gateway/router/status/error.go +++ b/gateway/router/status/error.go @@ -1,6 +1,7 @@ package status import ( + "errors" "net/http" "github.com/viant/datly/service/executor/expand" @@ -15,6 +16,11 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { violations := httputils.Violations{} switch actual := err.(type) { case *response.Error: + if derrors.IsDatabaseError(actual.Err) || derrors.IsDatabaseError(errors.New(actual.Message)) { + actual.Code = http.StatusInternalServerError + actual.Message = http.StatusText(http.StatusInternalServerError) + return http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError), nil + } code := actual.StatusCode() if code == 0 { code = statusCode @@ -41,9 +47,15 @@ func NormalizeErr(err error, statusCode int) (int, string, interface{}) { if maxStatus == 0 { maxStatus = http.StatusBadRequest } - hasServerError := maxStatus >= http.StatusInternalServerError + hasServerError := maxStatus >= http.StatusInternalServerError || derrors.IsDatabaseError(errors.New(actual.Message)) for _, anError := range actual.Errors { + if derrors.IsDatabaseError(anError.Err) || derrors.IsDatabaseError(errors.New(anError.Message)) { + anError.Code = http.StatusInternalServerError + anError.Message = http.StatusText(http.StatusInternalServerError) + hasServerError = true + } + code := anError.StatusCode() switch { case code >= http.StatusInternalServerError: From 03ee2da17a88a34f18cdec47ec5a0bc5f80bb9b3 Mon Sep 17 00:00:00 2001 From: Himanshu Shishir Shah Date: Sun, 28 Dec 2025 20:52:47 -0800 Subject: [PATCH 113/279] ENG-52641: adding mutex to Options struct in locator --- view/state/kind/locator/options.go | 95 +++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/view/state/kind/locator/options.go b/view/state/kind/locator/options.go index 9cf99b2eb..18be24842 100644 --- a/view/state/kind/locator/options.go +++ b/view/state/kind/locator/options.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" "reflect" + "sync" "github.com/viant/datly/gateway/router/marshal/config" "github.com/viant/datly/gateway/router/marshal/json" @@ -21,6 +22,7 @@ import ( // Options represents locator options type ( Options struct { + mu sync.RWMutex request *http.Request Form *hstate.Form QuerySelectors hstate.QuerySelectors @@ -57,6 +59,9 @@ type ( ) func (o Options) LookupParameters(name string) *state.Parameter { + o.mu.RLock() + defer o.mu.RUnlock() + if len(o.InputParameters) > 0 { if ret, ok := o.InputParameters[name]; ok { return ret @@ -71,10 +76,17 @@ func (o Options) LookupParameters(name string) *state.Parameter { } func (o *Options) GetRequest() (*http.Request, error) { - return shared.CloneHTTPRequest(o.request) + o.mu.RLock() + req := o.request + o.mu.RUnlock() + + return shared.CloneHTTPRequest(req) } func (o *Options) UnmarshalFunc() Unmarshal { + o.mu.Lock() + defer o.mu.Unlock() + if o.Unmarshal != nil { return o.Unmarshal } @@ -101,6 +113,9 @@ var defaultURL, _ = url.Parse("http://localhost:8080/") // WithRequest create http requestState option func WithRequest(request *http.Request) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + ensureValueRequest(request) o.request = request } @@ -118,6 +133,9 @@ func ensureValueRequest(request *http.Request) { // WithCustom creates custom options func WithCustom(options ...interface{}) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Custom = options } } @@ -125,6 +143,9 @@ func WithCustom(options ...interface{}) Option { // WithURIPattern create Path pattern requestState func WithURIPattern(URI string) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.URIPattern = URI } } @@ -132,6 +153,9 @@ func WithURIPattern(URI string) Option { // WithBodyType create Body Type option func WithBodyType(rType reflect.Type) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.BodyType = rType } } @@ -139,6 +163,9 @@ func WithBodyType(rType reflect.Type) Option { // WithUnmarshal creates with unmarshal options func WithUnmarshal(fn func([]byte, interface{}) error) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Unmarshal = fn } } @@ -146,6 +173,9 @@ func WithUnmarshal(fn func([]byte, interface{}) error) Option { // WithParent creates with parent options func WithParent(locators *KindLocator) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Parent = locators } } @@ -153,12 +183,18 @@ func WithParent(locators *KindLocator) Option { // WithParameterLookup creates with parameter options func WithParameterLookup(lookupFn ParameterLookup) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.ParameterLookup = lookupFn } } func WithIOConfig(config *config.IOConfig) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.IOConfig = config } } @@ -166,6 +202,9 @@ func WithIOConfig(config *config.IOConfig) Option { // WithInputParameters creates with parameter options func WithInputParameters(parameters state.NamedParameters) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + if len(o.resourceConstants) == 0 { o.resourceConstants = make(map[string]interface{}) } @@ -184,6 +223,9 @@ func WithInputParameters(parameters state.NamedParameters) Option { func WithQuerySelectors(selectors hstate.QuerySelectors) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.QuerySelectors = selectors } } @@ -191,12 +233,18 @@ func WithQuerySelectors(selectors hstate.QuerySelectors) Option { // WithPathParameters create with path parameters options func WithPathParameters(parameters map[string]string) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Path = parameters } } func WithReadInto(fn ReadInto) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.ReadInto = fn } } @@ -204,6 +252,9 @@ func WithReadInto(fn ReadInto) Option { // WithViews returns with views options func WithViews(views view.NamedViews) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Views = views } } @@ -211,12 +262,18 @@ func WithViews(views view.NamedViews) Option { // WithState returns with satte options func WithState(state *structology.State) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.State = state } } func WithOutputParameters(parameters state.Parameters) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.OutputParameters = parameters.Index() } } @@ -224,6 +281,9 @@ func WithOutputParameters(parameters state.Parameters) Option { // WithDispatcher returns options to set dispatcher func WithDispatcher(dispatcher contract.Dispatcher) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Dispatcher = dispatcher } } @@ -231,6 +291,9 @@ func WithDispatcher(dispatcher contract.Dispatcher) Option { // WithView returns options to set view func WithView(aView *view.View) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.View = aView } } @@ -238,6 +301,9 @@ func WithView(aView *view.View) Option { // WithForm return form option func WithForm(form *hstate.Form) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + if o.Form == nil { o.Form = form } else if form != nil { @@ -249,6 +315,9 @@ func WithForm(form *hstate.Form) Option { // WithQuery return query parameters option func WithQuery(parameters url.Values) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + if o.Query == nil { o.Query = parameters } else { @@ -261,6 +330,9 @@ func WithQuery(parameters url.Values) Option { func WithLogger(logger logger.Logger) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Logger = logger } } @@ -268,6 +340,9 @@ func WithLogger(logger logger.Logger) Option { // WithQueryParameter return query parameter option func WithQueryParameter(name, value string) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + if o.Query == nil { o.Query = make(url.Values) } @@ -278,6 +353,9 @@ func WithQueryParameter(name, value string) Option { // WithHeader return header option func WithHeader(name, value string) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + if o.Header == nil { o.Header = make(http.Header) } @@ -288,6 +366,9 @@ func WithHeader(name, value string) Option { // WithHeaders return headers option func WithHeaders(header http.Header) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + if o.Header == nil { o.Header = header } @@ -300,6 +381,9 @@ func WithHeaders(header http.Header) Option { // WithMetrics return metrics option func WithMetrics(metrics response.Metrics) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Metrics = metrics } } @@ -307,6 +391,9 @@ func WithMetrics(metrics response.Metrics) Option { // WithResource return resource option func WithResource(resource *view.Resource) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Resource = resource } } @@ -314,6 +401,9 @@ func WithResource(resource *view.Resource) Option { // WithConstants return Constants option func WithConstants(constants map[string]interface{}) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Constants = constants } } @@ -321,6 +411,9 @@ func WithConstants(constants map[string]interface{}) Option { // WithTypes return types option func WithTypes(types ...*state.Type) Option { return func(o *Options) { + o.mu.Lock() + defer o.mu.Unlock() + o.Types = types } } From bf88275b1262b7f4357b0fc36f5d77db82dddd80 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 6 Jan 2026 08:12:11 -0800 Subject: [PATCH 114/279] patched marshaller --- go.mod | 2 +- go.sum | 137 +++++++++++---------------------------------------------- 2 files changed, 26 insertions(+), 113 deletions(-) diff --git a/go.mod b/go.mod index a5a9eab23..dc81b120e 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( require ( github.com/viant/govalidator v0.3.1 - github.com/viant/sqlparser v0.8.1 + github.com/viant/sqlparser v0.9.0 ) require ( diff --git a/go.sum b/go.sum index 38ca1e6c4..aafc7c00c 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,6 @@ cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFO cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= -cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= @@ -106,12 +104,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= -cloud.google.com/go/auth v0.2.0 h1:y6oTcpMSbOcXbwYgUUrvI+mrQ2xbrcdpPgtVbCGTLTk= -cloud.google.com/go/auth v0.2.0/go.mod h1:+yb+oy3/P0geX6DLKlqiGHARGR6EX2GRtYCzWOCQSbU= cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8= cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= -cloud.google.com/go/auth/oauth2adapt v0.2.0 h1:FR8zevgQwu+8CqiOT5r6xCmJa3pJC/wdXEEPF1OkNhA= -cloud.google.com/go/auth/oauth2adapt v0.2.0/go.mod h1:AfqujpDAlTfLfeCIl/HJZZlIxD8+nJoZ5e0x1IxGq5k= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= @@ -194,8 +188,6 @@ cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZ cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= @@ -293,8 +285,6 @@ cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLY cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= -cloud.google.com/go/firestore v1.15.0 h1:/k8ppuWOtNuDHt2tsRV42yI21uaGnKDEQnRFeBpbFF8= -cloud.google.com/go/firestore v1.15.0/go.mod h1:GWOxFXcv8GZUtYpWHw/w6IuYNux/BtmeVTMmjrm4yhk= cloud.google.com/go/firestore v1.17.0 h1:iEd1LBbkDZTFsLw3sTH50eyg4qe8eoG6CjocmEXO9aQ= cloud.google.com/go/firestore v1.17.0/go.mod h1:69uPx1papBsY8ZETooc71fOhoKkD70Q1DwMrtKuOT/Y= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= @@ -335,8 +325,6 @@ cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGE cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= -cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= @@ -368,11 +356,11 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6 cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= +cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= -cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= -cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= @@ -508,8 +496,6 @@ cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISI cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= -cloud.google.com/go/secretmanager v1.11.5 h1:82fpF5vBBvu9XW4qj0FU2C6qVMtj1RM/XHwKXUEAfYY= -cloud.google.com/go/secretmanager v1.11.5/go.mod h1:eAGv+DaCHkeVyQi0BeXgAHOU0RdrMeZIASKc+S7VqH4= cloud.google.com/go/secretmanager v1.14.1 h1:xlWSIg8rtBn5qCr2f3XtQP19+5COyf/ll49SEvi/0vM= cloud.google.com/go/secretmanager v1.14.1/go.mod h1:L+gO+u2JA9CCyXpSR8gDH0o8EV7i/f0jdBOrUXcIV0U= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= @@ -567,8 +553,6 @@ cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeL cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= -cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= -cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= cloud.google.com/go/storage v1.45.0 h1:5av0QcIVj77t+44mV4gffFC/LscFRUhto6UBMB5SimM= cloud.google.com/go/storage v1.45.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= @@ -590,6 +574,8 @@ cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.11.1 h1:UNqdP+HYYtnm6lb91aNA5JQ0X14GnxkABGlfz2PzPew= +cloud.google.com/go/trace v1.11.1/go.mod h1:IQKNQuBzH72EGaXEodKlNJrWykGZxet2zgjtS60OtjA= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= @@ -654,6 +640,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1 h1:oTX4vsorBZo/Zdum6OKPA4o7544hm6smoRv1QjpTwGo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= @@ -680,40 +668,26 @@ github.com/aws/aws-lambda-go v1.31.0/go.mod h1:IF5Q7wj4VyZyUFnZ54IQqeWtctHQ9tz+K github.com/aws/aws-sdk-go v1.51.23 h1:/3TEdsEE/aHmdKGw2NrOp7Sdea76zfffGkTTSXTsDxY= github.com/aws/aws-sdk-go v1.51.23/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.17.2/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= -github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= -github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= -github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ= github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= -github.com/aws/aws-sdk-go-v2/credentials v1.17.26 h1:tsm8g/nJxi8+/7XyJJcP2dLrnK/5rkFp6+i2nhmz5fk= -github.com/aws/aws-sdk-go-v2/credentials v1.17.26/go.mod h1:3vAM49zkIa3q8WT6o9Ve5Z0vdByDMwmdScO0zvThTgI= github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.7 h1:CyuByiiCA4lPfU8RaHJh2wIYYn0hkFlOkMfWkVY67Mc= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.7/go.mod h1:pAMtgCPVxcKohC/HNI6nLwLeW007eYl3T+pq7yTMV3o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33 h1:X+4YY5kZRI/cOoSMVMGTqFXHAMg1bvvay7IBcqHpybQ= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33/go.mod h1:DPynzu+cn92k5UQ6tZhX+wfTB4ah6QDU/NgdHqatmvk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26/go.mod h1:2E0LdbJW6lbeU4uxjum99GZzI0ZjDpAb0CoSCM0oeEY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20/go.mod h1:/+6lSiby8TBFpTVXZgKiN/rCfkYXEGvhlM4zCgPpt7w= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= @@ -723,16 +697,12 @@ github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.8/go.mod h1:jvXzk+hVrlkiQOvn github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.27 h1:7MhqbR+k+b0gbOxp+W8yXgsl/Z5/dtMh85K0WI8X2EA= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.27/go.mod h1:wX9QEZJ8Dw1fdAKCOAUmSvAe3wNJFxnE/4AeYc8blGA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.20 h1:kSZR22oLBDMtP8ZPGXhz649NU77xsJDG7g3xfT6nHVk= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.20/go.mod h1:lxM5qubwGNX29Qy+xTFG8G0r2Mj/TmyC+h3hS/7E4V8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= @@ -747,21 +717,13 @@ github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 h1:Vjqy5BZCOIsn4Pj8xzyqgGmsSqzz github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3/go.mod h1:L0enV3GCRd5iG9B64W35C4/hwsCB00Ib+DKVGTadKHI= github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 h1:z6Pq4+jtKlhK4wWJGHRGwMLGjC1HZwAO3KJr/Na0tSU= github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2/go.mod h1:DSmu/VZzpQlAubWBbAvNpt+S4k/XweglJi4XaDGyvQk= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.3 h1:Fv1vD2L65Jnp5QRsdiM64JvUM4Xe+E0JyVsRQKv6IeA= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.3/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= -github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -772,7 +734,6 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -800,8 +761,8 @@ github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfT github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= @@ -821,8 +782,11 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs= github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= @@ -857,8 +821,6 @@ github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9 github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -939,8 +901,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gops v0.3.23 h1:OjsHRINl5FiIyTc8jivIg4UN0GY6Nh32SL8KRbl8GQo= @@ -950,8 +912,9 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -973,8 +936,6 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -986,8 +947,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -1006,8 +965,6 @@ github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38 github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= -github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= -github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= @@ -1045,8 +1002,9 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -1112,8 +1070,8 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1132,8 +1090,9 @@ github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTK github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= @@ -1183,8 +1142,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= @@ -1193,8 +1151,6 @@ github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d h1:IRmoMmrWqkHD github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d/go.mod h1:eRBywl0oTDM/oGhGLUeJjnC7XzmkTGuW9/og5YFy0K0= github.com/viant/afs v1.26.2 h1:rOs/iFxFlEndhIRATJVXlNWhVU0cGdRQAGVTVJPdsc0= github.com/viant/afs v1.26.2/go.mod h1:rScbFd9LJPGTM8HOI8Kjwee0AZ+MZMupAvFpPg+Qdj4= -github.com/viant/afsc v1.9.1 h1:BIus7fYyjM+MDgKuAzCBfoV4oVy2xTVhuFsQKUCPvkQ= -github.com/viant/afsc v1.9.1/go.mod h1:FA/xVjaMM10qGByabP8anTVMH6N4eUsAeWm5xcEZJJA= github.com/viant/afsc v1.16.0 h1:/kOH/flNwme6h3oFrU/KPnMHkhbCZxQncTf1GSQIlBQ= github.com/viant/afsc v1.16.0/go.mod h1:Z6fP3VcmzS8Sg2lowctR6KkVEX7XxJ8aNaoHqhUiZkY= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= @@ -1235,8 +1191,8 @@ github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= github.com/viant/pgo v0.11.0/go.mod h1:MFzHmkRFZlciugEgUvpl/3grK789PBSH4dUVSLOSo+Q= github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= -github.com/viant/sqlparser v0.8.1 h1:nbcTecMtW7ROk5aNB5/BWUxnduepRPOkhVo9RWxI1Ns= -github.com/viant/sqlparser v0.8.1/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= +github.com/viant/sqlparser v0.9.0 h1:MoRJ18cm4MeSGLMNO8jZZzb1S5rLaIksEbdqE+8RBEw= +github.com/viant/sqlparser v0.9.0/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= @@ -1303,30 +1259,18 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs= go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -1354,8 +1298,6 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1420,8 +1362,6 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1493,8 +1433,6 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 h1:6/3JGEh1C88g7m+qzzTbl3A0FtsLguXieqofVLU/JAo= golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1530,8 +1468,6 @@ golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= @@ -1553,8 +1489,6 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1648,8 +1582,6 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1667,8 +1599,7 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1689,8 +1620,6 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1700,8 +1629,6 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1768,8 +1695,6 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1780,12 +1705,12 @@ golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= @@ -1854,8 +1779,6 @@ google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjY google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= -google.golang.org/api v0.174.0 h1:zB1BWl7ocxfTea2aQ9mgdzXjnfPySllpPOskdnO+q34= -google.golang.org/api v0.174.0/go.mod h1:aC7tB6j0HR1Nl0ni5ghpx6iLasmAX78Zkh/wgxAAjLg= google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0= google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -2010,16 +1933,12 @@ google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE= google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4= google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= @@ -2027,8 +1946,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -2075,8 +1992,6 @@ google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= @@ -2098,8 +2013,6 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From f13d12830343bc5b8689923fd9c0b4721ca1b1df Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 8 Jan 2026 15:14:23 -0800 Subject: [PATCH 115/279] patched marshaller --- view/tags/parameter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/view/tags/parameter.go b/view/tags/parameter.go index 7acd1c563..de777c815 100644 --- a/view/tags/parameter.go +++ b/view/tags/parameter.go @@ -88,7 +88,7 @@ func (p *Parameter) Tag() *tags.Tag { if *p.Cacheable { value = "true" } - appendNonEmpty(builder, "cachable", value) + appendNonEmpty(builder, "cacheable", value) } if p.Cardinality == "One" { From 0a05392fa97677d75d47087cf469de5e0b459b1d Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 9 Jan 2026 12:27:55 -0800 Subject: [PATCH 116/279] patched marshaller --- gateway/router/marshal/json/marshaller_strings.go | 1 - 1 file changed, 1 deletion(-) diff --git a/gateway/router/marshal/json/marshaller_strings.go b/gateway/router/marshal/json/marshaller_strings.go index 2e20e16e0..b0ba6fcea 100644 --- a/gateway/router/marshal/json/marshaller_strings.go +++ b/gateway/router/marshal/json/marshaller_strings.go @@ -104,7 +104,6 @@ func marshallString(asString string, sb *MarshallSession, _ *strings.Replacer) { func getReplacer() *strings.Replacer { return strings.NewReplacer(`\`, `\\`, `"`, `\"`, - `/`, `\/`, "\b", `\b`, "\f", `\f`, "\n", `\n`, From 30a174e327ace3a6753fbc887b33abd810a3cd9b Mon Sep 17 00:00:00 2001 From: adranwit Date: Sun, 11 Jan 2026 15:22:57 -0800 Subject: [PATCH 117/279] fix oauth refresh issue --- gateway/mcp.go | 38 +++++++++++++++++++++++++++++++++----- mcp/server.go | 1 + 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index e9687e45a..d8fe71a0c 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -17,6 +17,7 @@ import ( "github.com/viant/datly/view/state" "github.com/viant/jsonrpc" "github.com/viant/mcp-protocol/authorization" + oauthmeta "github.com/viant/mcp-protocol/oauth2/meta" "github.com/viant/mcp-protocol/schema" serverproto "github.com/viant/mcp-protocol/server" "github.com/viant/toolbox" @@ -112,6 +113,10 @@ func (r *Router) mcpToolCallHandler(component *repository.Component, aRoute *Rou rw := proxy.NewWriter() aRoute.Handle(rw, httpReq) + if rw.Code == http.StatusUnauthorized { + return nil, r.mcpUnauthorizedError() + } + // 5) Build tool result (text + structured on error) return r.buildToolCallResult(rw, finalURL, aRoute.Path.Method), nil } @@ -332,6 +337,26 @@ func (r *Router) addAuthTokenIfPresent(ctx context.Context, httpRequest *http.Re } } +const defaultMCPProtectedResource = "https://datly.viantinc.com" + +func (r *Router) mcpUnauthorizedError() *jsonrpc.Error { + if r == nil || r.config == nil || r.config.MCP == nil { + return jsonrpc.NewError(schema.Unauthorized, "Unauthorized", nil) + } + issuerURL := strings.TrimSpace(r.config.MCP.IssuerURL) + if issuerURL == "" { + return jsonrpc.NewError(schema.Unauthorized, "Unauthorized", nil) + } + return jsonrpc.NewError(schema.Unauthorized, "Unauthorized", &authorization.Authorization{ + RequiredScopes: []string{}, + UseIdToken: true, + ProtectedResourceMetadata: &oauthmeta.ProtectedResourceMetadata{ + Resource: defaultMCPProtectedResource, + AuthorizationServers: []string{issuerURL}, + }, + }) +} + func (r *Router) buildToolInputType(components *repository.Component) reflect.Type { var inputFields []reflect.StructField var uniqueQuery = make(map[string]bool) @@ -468,9 +493,9 @@ func (r *Router) buildTemplateResourceIntegration(item *dpath.Item, aPath *dpath func (r *Router) reactMcpResourceHandler(mcpResourceTemplate schema.ResourceTemplate, aRoute *Route, provider *repository.Provider) func(ctx context.Context, request *schema.ReadResourceRequest) (*schema.ReadResourceResult, *jsonrpc.Error) { handler := func(ctx context.Context, request *schema.ReadResourceRequest) (*schema.ReadResourceResult, *jsonrpc.Error) { - result, err := r.handleMcpRead(ctx, &request.Params, &mcpResourceTemplate, aRoute, provider) - if err != nil { - return nil, jsonrpc.NewInternalError(err.Error(), nil) + result, rpcErr := r.handleMcpRead(ctx, &request.Params, &mcpResourceTemplate, aRoute, provider) + if rpcErr != nil { + return nil, rpcErr } if len(result) == 0 { return &schema.ReadResourceResult{Contents: []schema.ReadResourceResultContentsElem{}}, nil @@ -532,12 +557,12 @@ func (r *Router) hasMcpResource(URI string) bool { return false } -func (r *Router) handleMcpRead(ctx context.Context, params *schema.ReadResourceRequestParams, template *schema.ResourceTemplate, aRoute *Route, provider *repository.Provider) ([]schema.ReadResourceResultContentsElem, error) { +func (r *Router) handleMcpRead(ctx context.Context, params *schema.ReadResourceRequestParams, template *schema.ResourceTemplate, aRoute *Route, provider *repository.Provider) ([]schema.ReadResourceResultContentsElem, *jsonrpc.Error) { URI := furl.Path(params.Uri) URL := fmt.Sprintf("http://localhost/%v", URI) // fallback to a local URL for now, this should be replaced with the actual service URL component, err := provider.Component(ctx) // ensure the provider is initialized if err != nil { - return nil, fmt.Errorf("failed to get component from provider: %w", err) + return nil, jsonrpc.NewInternalError(fmt.Errorf("failed to get component from provider: %w", err).Error(), nil) } byLoc := make(map[string]*state.Parameter) for _, param := range component.View.GetResource().Parameters { @@ -551,6 +576,9 @@ func (r *Router) handleMcpRead(ctx context.Context, params *schema.ReadResourceR } r.addAuthTokenIfPresent(ctx, httpRequest) aRoute.Handle(responseWriter, httpRequest) // route the request to the actual handler + if responseWriter.Code == http.StatusUnauthorized { + return nil, r.mcpUnauthorizedError() + } var result []schema.ReadResourceResultContentsElem mimeType := "" if template.MimeType != nil { diff --git a/mcp/server.go b/mcp/server.go index 2409a38ea..b3e1c3e45 100644 --- a/mcp/server.go +++ b/mcp/server.go @@ -57,6 +57,7 @@ func (s *Server) init() error { } if issuerURL == "" && oauth2Config != nil { issuerURL, _ = url.Base(oauth2Config.Endpoint.AuthURL, http.SecureScheme) + s.config.IssuerURL = issuerURL } } authPolicy := &authorization.Policy{ From a2cca14e5472465c5b18c07005d7c8c9e33b0511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Filipowicz?= Date: Wed, 14 Jan 2026 13:48:12 +0100 Subject: [PATCH 118/279] updated SetLiterals --- view/state/parameters.go | 14 ++++- view/state/parameters_set_literals_test.go | 64 ++++++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 view/state/parameters_set_literals_test.go diff --git a/view/state/parameters.go b/view/state/parameters.go index ff409d1e4..04083ede1 100644 --- a/view/state/parameters.go +++ b/view/state/parameters.go @@ -220,9 +220,17 @@ func (p Parameters) Groups() []Parameters { } func (p Parameters) SetLiterals(state *structology.State) (err error) { + if state == nil { + return nil + } + stateType := state.Type() for _, parameter := range p.FilterByKind(KindConst) { - if parameter._selector == nil { - parameter._selector = state.Type().Lookup(parameter.Name) + // Selector must be resolved against the provided state type. + // Caching it on the parameter is unsafe because the same parameter instance + // can be used with multiple dynamically-generated state types (e.g. during translation). + selector := stateType.Lookup(parameter.Name) + if selector == nil { + return fmt.Errorf("failed to lookup selector for const parameter %q", parameter.Name) } if parameter.Value == nil { switch parameter.Schema.rType.Kind() { @@ -237,7 +245,7 @@ func (p Parameters) SetLiterals(state *structology.State) (err error) { } } - if err = parameter._selector.SetValue(state.Pointer(), parameter.Value); err != nil { + if err = selector.SetValue(state.Pointer(), parameter.Value); err != nil { return err } } diff --git a/view/state/parameters_set_literals_test.go b/view/state/parameters_set_literals_test.go new file mode 100644 index 000000000..f94d8f5ee --- /dev/null +++ b/view/state/parameters_set_literals_test.go @@ -0,0 +1,64 @@ +package state + +import ( + "reflect" + "testing" + + "github.com/viant/structology" +) + +func TestParameters_SetLiterals_DoesNotReuseSelectorAcrossStateTypes(t *testing.T) { + const ( + paramName = "X" + dummyName = "Dummy" + dummyValue = 12345 + constValue = true + constSource = "value" + ) + + param := &Parameter{ + Name: paramName, + In: &Location{Kind: KindConst, Name: constSource}, + Value: constValue, + Schema: &Schema{ + rType: reflect.TypeOf(true), + }, + } + params := Parameters{param} + + type1 := reflect.StructOf([]reflect.StructField{ + {Name: dummyName, Type: reflect.TypeOf(int(0))}, + {Name: paramName, Type: reflect.TypeOf(true)}, + }) + state1 := structology.NewStateType(type1).NewState() + if err := state1.SetInt(dummyName, dummyValue); err != nil { + t.Fatalf("failed to init %s: %v", dummyName, err) + } + if err := params.SetLiterals(state1); err != nil { + t.Fatalf("SetLiterals(type1) failed: %v", err) + } + if got, err := state1.Bool(paramName); err != nil || got != constValue { + t.Fatalf("type1 %s: got=%v err=%v, want=%v", paramName, got, err, constValue) + } + if got, err := state1.Value(dummyName); err != nil || got.(int) != dummyValue { + t.Fatalf("type1 %s: got=%v err=%v, want=%v", dummyName, got, err, dummyValue) + } + + type2 := reflect.StructOf([]reflect.StructField{ + {Name: paramName, Type: reflect.TypeOf(true)}, + {Name: dummyName, Type: reflect.TypeOf(int(0))}, + }) + state2 := structology.NewStateType(type2).NewState() + if err := state2.SetInt(dummyName, dummyValue); err != nil { + t.Fatalf("failed to init %s: %v", dummyName, err) + } + if err := params.SetLiterals(state2); err != nil { + t.Fatalf("SetLiterals(type2) failed: %v", err) + } + if got, err := state2.Bool(paramName); err != nil || got != constValue { + t.Fatalf("type2 %s: got=%v err=%v, want=%v", paramName, got, err, constValue) + } + if got, err := state2.Value(dummyName); err != nil || got.(int) != dummyValue { + t.Fatalf("type2 %s: got=%v err=%v, want=%v", dummyName, got, err, dummyValue) + } +} From 3481a39615dfed627635696820740060272cc00c Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 28 Jan 2026 09:08:06 -0800 Subject: [PATCH 119/279] fix oauth refresh issue --- gateway/mcp.go | 11 +++---- go.mod | 9 +++--- go.sum | 17 ++++++----- service.go | 7 ++++- service/session/selector.go | 60 +++++++++++++++++++++++++++++++++---- service/session/state.go | 40 ++++++++++++++++++++++--- service/session/stater.go | 22 ++++++++++++-- view/state.go | 2 +- 8 files changed, 138 insertions(+), 30 deletions(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index d8fe71a0c..898947d43 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -288,11 +288,12 @@ func (r *Router) buildToolCallResult(responseWriter *proxy.Writer, URL, method s mimeType = "application/json" } data := responseWriter.Body.Bytes() - result.Content = append(result.Content, schema.CallToolResultContentElem{ - MimeType: mimeType, - Type: "text", - Text: string(data), - }) + result.Content = append(result.Content, schema.CallToolResultContentElem( + schema.TextContent{ + Type: "text", + Text: string(data), + }, + )) _ = json.Unmarshal(data, &result.StructuredContent) if responseWriter.Code >= http.StatusBadRequest { isErr := true diff --git a/go.mod b/go.mod index dc81b120e..22458b8b2 100644 --- a/go.mod +++ b/go.mod @@ -48,9 +48,9 @@ require ( require ( github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d github.com/viant/firebase v0.1.1 - github.com/viant/jsonrpc v0.15.0 - github.com/viant/mcp v0.8.0 - github.com/viant/mcp-protocol v0.5.10 + github.com/viant/jsonrpc v0.17.0 + github.com/viant/mcp v0.9.0 + github.com/viant/mcp-protocol v0.9.0 github.com/viant/structology v0.8.0 github.com/viant/tagly v0.3.0 github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a @@ -123,9 +123,10 @@ require ( github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect diff --git a/go.sum b/go.sum index aafc7c00c..ddb2bb562 100644 --- a/go.sum +++ b/go.sum @@ -833,6 +833,8 @@ github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -849,8 +851,9 @@ github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -1179,12 +1182,12 @@ github.com/viant/govalidator v0.3.1 h1:V7f/KgfzbP8fVDc+Kj+jyPvfXxMr2N1x7srOlDV6l github.com/viant/govalidator v0.3.1/go.mod h1:D35Dwx0R8rR1knRxhlseoYvOkiqo24kpMg1/o977i9Y= github.com/viant/igo v0.2.0 h1:ygWmTCinnGPaeV7omJLiyneOpzYZ5kiw7oYz7mUJZVQ= github.com/viant/igo v0.2.0/go.mod h1:7V6AWsLhKWeGzXNTNH3AZiIEKa0m33DrQbdWtapsI74= -github.com/viant/jsonrpc v0.15.0 h1:0qy9vzgNwR9Gj1C+ouSrzNUtNDzKGogO+7TZR+cFrA4= -github.com/viant/jsonrpc v0.15.0/go.mod h1:b214Lo4zBwLqbu6Tf2bRlgQkFfPMBW5ap4qS+I3zcJ8= -github.com/viant/mcp v0.8.0 h1:n4tnLXpOtpnrLZtHyNG2mmZ9SUbGWKsWGla10iMfuDg= -github.com/viant/mcp v0.8.0/go.mod h1:fyuB1TSQYbbGNn7U6rLmlr9gD+Yg5+Na32D34Uvm0sk= -github.com/viant/mcp-protocol v0.5.10 h1:915EC1GKgBbyYF4efzRSZ/AE6f4vobkbwa2qe6OOjJ0= -github.com/viant/mcp-protocol v0.5.10/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= +github.com/viant/jsonrpc v0.17.0 h1:LZpe2H8tFUmWnvevDs2t6V7Cz7LzOGmpP8WcZipuXZE= +github.com/viant/jsonrpc v0.17.0/go.mod h1:b214Lo4zBwLqbu6Tf2bRlgQkFfPMBW5ap4qS+I3zcJ8= +github.com/viant/mcp v0.9.0 h1:RFvnUTURWMvnogzu5jhACV3Y8AsK7gTZYTSrLShK6og= +github.com/viant/mcp v0.9.0/go.mod h1:5wcLegOtk/TTYTJZlHE8dsBRQJm8mty9LmmCxsUl344= +github.com/viant/mcp-protocol v0.9.0 h1:G/F/Lk8rsSAtFtD2d7WuvjJ85Kt+O+abe4c48K//PZo= +github.com/viant/mcp-protocol v0.9.0/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= github.com/viant/parsly v0.3.3 h1:7ytgfLOG4Ils+wviGacWxRD0gAUvVEH/iGsSE+UI8YM= github.com/viant/parsly v0.3.3/go.mod h1:85fneXJbErKMGhSQto3A5ElTQCwl3t74U9cSV0waBHw= github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= diff --git a/service.go b/service.go index 32a2cdcc4..96e3d609d 100644 --- a/service.go +++ b/service.go @@ -211,7 +211,12 @@ func (s *Service) SignRequest(request *http.Request, claims *jwt.Claims) error { func LoadInput(ctx context.Context, aSession *session.Session, aComponent *repository.Component, input interface{}) error { ctx = aSession.Context(ctx, false) - if err := aSession.LoadState(aComponent.Input.Type.Parameters, input); err != nil { + if err := aSession.LoadState( + aComponent.Input.Type.Parameters, + input, + session.WithHasMarker(), + session.WithValuePresenceFallback(), + ); err != nil { return err } if err := aSession.Populate(ctx); err != nil { diff --git a/service/session/selector.go b/service/session/selector.go index b6ff7c140..a7febeb8b 100644 --- a/service/session/selector.go +++ b/service/session/selector.go @@ -11,6 +11,7 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/xdatly/codec" "github.com/viant/xdatly/handler/response" + hstate "github.com/viant/xdatly/handler/state" ) func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, opts *Options) (err error) { @@ -21,11 +22,9 @@ func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, selector := s.state.Lookup(ns.View) - if opts != nil && opts.locatorOpt != nil && opts.locatorOpt.QuerySelectors != nil { //override selector - querySelectors := opts.locatorOpt.QuerySelectors - if namedSelector := querySelectors.Find(ns.View.Name); namedSelector != nil { - selector.QuerySelector = namedSelector.QuerySelector - } + var injected *hstate.NamedQuerySelector + if opts != nil && opts.locatorOpt != nil && opts.locatorOpt.QuerySelectors != nil { + injected = opts.locatorOpt.QuerySelectors.Find(ns.View.Name) } if err = s.populateFieldQuerySelector(ctx, ns, opts); err != nil { return response.NewParameterError(ns.View.Name, selectorParameters.FieldsParameter.Name, err) @@ -45,12 +44,63 @@ func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, if err = s.populatePageQuerySelector(ctx, ns, opts); err != nil { return response.NewParameterError(ns.View.Name, selectorParameters.PageParameter.Name, err) } + + // Apply injected selector last so it takes precedence over request-derived values, + // but still validate against view selector constraints. + if injected != nil { + selector.QuerySelector = injected.QuerySelector + if err := s.applyInjectedQuerySelector(ns, selector, injected); err != nil { + return err + } + } else if selector.Page > 0 && selector.Offset == 0 { + // If selector was pre-set (e.g. from non-query sources) without an explicit page parameter, + // apply Page semantics to compute Offset/Limit. + _ = s.setPageQuerySelector(selector.Page, ns) + } if selector.Limit == 0 && selector.Offset != 0 { return fmt.Errorf("can't use offset without limit - view: %v", ns.View.Name) } return nil } +func (s *Session) applyInjectedQuerySelector(ns *view.NamespaceView, selector *view.Statelet, injected *hstate.NamedQuerySelector) error { + if injected == nil || selector == nil { + return nil + } + if len(injected.Fields) > 0 { + if err := s.setFieldsQuerySelector(injected.Fields, ns); err != nil { + return err + } + } + if injected.Limit != 0 { + if err := s.setLimitQuerySelector(injected.Limit, ns); err != nil { + return err + } + } + if injected.Offset != 0 { + if err := s.setOffsetQuerySelector(injected.Offset, ns); err != nil { + return err + } + } + if injected.OrderBy != "" { + items := strings.Split(injected.OrderBy, ",") + if err := s.setOrderByQuerySelector(items, ns); err != nil { + return err + } + } + if injected.Criteria != "" { + if err := s.setCriteriaQuerySelector(injected.Criteria, ns); err != nil { + return err + } + } + if injected.Page != 0 { + if err := s.setPageQuerySelector(injected.Page, ns); err != nil { + return err + } + } + return nil +} + func (s *Session) setQuerySettings(ctx context.Context, ns *view.NamespaceView, opts *Options) (err error) { selectorParameters := ns.View.Selector if selectorParameters == nil { diff --git a/service/session/state.go b/service/session/state.go index 7c2ff10df..a629db3c9 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -240,7 +240,7 @@ func (s *Session) setTemplateState(ctx context.Context, aView *view.View, opts * aState := s.state.Lookup(aView) if template := aView.Template; template != nil { stateType := template.StateType() - if stateType.IsDefined() { + if stateType != nil && stateType.IsDefined() { templateState := aState.Template templateState.EnsureMarker() err := s.SetState(ctx, template.Parameters, templateState, opts) @@ -830,9 +830,10 @@ func New(aView *view.View, opts ...Option) *Session { } type loadStateOptions struct { - skipKind map[state.Kind]bool - hasSkipKind bool - useHasMarker bool + skipKind map[state.Kind]bool + hasSkipKind bool + useHasMarker bool + fallbackOnValue bool } type LoadStateOption func(o *loadStateOptions) @@ -842,6 +843,14 @@ func WithHasMarker() LoadStateOption { o.useHasMarker = true } } + +// WithValuePresenceFallback treats non-zero values as present when no Has marker is available. +// This is opt-in to avoid changing behavior for existing inputs that intentionally omit markers. +func WithValuePresenceFallback() LoadStateOption { + return func(o *loadStateOptions) { + o.fallbackOnValue = true + } +} func WithLoadStateSkipKind(kinds ...state.Kind) LoadStateOption { return func(o *loadStateOptions) { for _, kind := range kinds { @@ -869,6 +878,7 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}, opt // Use presence markers only if enabled and supported by the input state hasMarker := options.useHasMarker && inputState.HasMarker() for _, parameter := range parameters { + if parameter.Scope != "" { continue } @@ -889,7 +899,11 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}, opt if hasMarker && !selector.Has(ptr) { continue } + value := selector.Value(ptr) + if !hasMarker && options.fallbackOnValue && isZeroValue(value) { + continue + } switch parameter.In.Kind { case state.KindView, state.KindParam, state.KindState: if value == nil { @@ -910,6 +924,24 @@ func (s *Session) LoadState(parameters state.Parameters, aState interface{}, opt return nil } +func isZeroValue(value interface{}) bool { + if value == nil { + return true + } + v := reflect.ValueOf(value) + for v.Kind() == reflect.Interface || v.Kind() == reflect.Ptr { + if v.IsNil() { + return true + } + v = v.Elem() + } + switch v.Kind() { + case reflect.Slice, reflect.Map, reflect.Array: + return v.Len() == 0 + } + return v.IsZero() +} + func (s *Session) handleParameterError(parameter *state.Parameter, err error, errors *response.Errors) { if parameter.ErrorMessage != "" && err != nil { msg := strings.ReplaceAll(parameter.ErrorMessage, "${error}", err.Error()) diff --git a/service/session/stater.go b/service/session/stater.go index 332c0206c..5b0b0d6ab 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -119,17 +119,33 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt locatorsToRemove = append(locatorsToRemove, httpKinds...) } if hOptions.Query() != nil { - stateOptions = append(stateOptions, locator.WithQuery(hOptions.Query())) + queryOpt := locator.WithQuery(hOptions.Query()) + stateOptions = append(stateOptions, queryOpt) + s.locatorOptions = append(s.locatorOptions, queryOpt) locatorsToRemove = append(locatorsToRemove, httpKinds...) } if len(hOptions.PathParameters()) > 0 { - stateOptions = append(stateOptions, locator.WithPathParameters(hOptions.PathParameters())) + pathOpt := locator.WithPathParameters(hOptions.PathParameters()) + stateOptions = append(stateOptions, pathOpt) + s.locatorOptions = append(s.locatorOptions, pathOpt) locatorsToRemove = append(locatorsToRemove, httpKinds...) } if hOptions.HttpRequest() != nil { - stateOptions = append(stateOptions, locator.WithRequest(hOptions.HttpRequest())) + requestOpt := locator.WithRequest(hOptions.HttpRequest()) + stateOptions = append(stateOptions, requestOpt) + s.locatorOptions = append(s.locatorOptions, requestOpt) locatorsToRemove = append(locatorsToRemove, httpKinds...) } + if selectors := hOptions.QuerySelectors(); len(selectors) > 0 { + selectorOpt := locator.WithQuerySelectors(selectors) + stateOptions = append(stateOptions, selectorOpt) + s.locatorOptions = append(s.locatorOptions, selectorOpt) + } + // Keep parsed locator options in sync with any dynamic additions made via injector.Bind. + if len(s.locatorOptions) > 0 { + s.locatorOpt = locator.NewOptions(s.locatorOptions) + s.kindLocator = locator.NewKindsLocator(nil, s.locatorOptions...) + } s.kindLocator.RemoveLocators(locatorsToRemove...) if s.view != nil { viewOptions := s.ViewOptions(s.view, WithLocatorOptions()) diff --git a/view/state.go b/view/state.go index 8bfd3715b..a187cd4b5 100644 --- a/view/state.go +++ b/view/state.go @@ -39,7 +39,7 @@ type ( // Init initializes Statelet func (s *Statelet) Init(aView *View) { - if aView != nil && s.Template == nil && aView.Template.stateType != nil { + if aView != nil && s.Template == nil && aView.Template != nil && aView.Template.stateType != nil { s.Template = aView.Template.stateType.NewState() } if s.initialized { From 9df5cc753c750a3329420c792b87d0e8be3aaa16 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 11 Feb 2026 14:28:05 -0800 Subject: [PATCH 120/279] updated dep --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 22458b8b2..48e6004f0 100644 --- a/go.mod +++ b/go.mod @@ -49,8 +49,8 @@ require ( github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d github.com/viant/firebase v0.1.1 github.com/viant/jsonrpc v0.17.0 - github.com/viant/mcp v0.9.0 - github.com/viant/mcp-protocol v0.9.0 + github.com/viant/mcp v0.11.0 + github.com/viant/mcp-protocol v0.11.0 github.com/viant/structology v0.8.0 github.com/viant/tagly v0.3.0 github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a diff --git a/go.sum b/go.sum index ddb2bb562..de0f21fe1 100644 --- a/go.sum +++ b/go.sum @@ -1184,10 +1184,10 @@ github.com/viant/igo v0.2.0 h1:ygWmTCinnGPaeV7omJLiyneOpzYZ5kiw7oYz7mUJZVQ= github.com/viant/igo v0.2.0/go.mod h1:7V6AWsLhKWeGzXNTNH3AZiIEKa0m33DrQbdWtapsI74= github.com/viant/jsonrpc v0.17.0 h1:LZpe2H8tFUmWnvevDs2t6V7Cz7LzOGmpP8WcZipuXZE= github.com/viant/jsonrpc v0.17.0/go.mod h1:b214Lo4zBwLqbu6Tf2bRlgQkFfPMBW5ap4qS+I3zcJ8= -github.com/viant/mcp v0.9.0 h1:RFvnUTURWMvnogzu5jhACV3Y8AsK7gTZYTSrLShK6og= -github.com/viant/mcp v0.9.0/go.mod h1:5wcLegOtk/TTYTJZlHE8dsBRQJm8mty9LmmCxsUl344= -github.com/viant/mcp-protocol v0.9.0 h1:G/F/Lk8rsSAtFtD2d7WuvjJ85Kt+O+abe4c48K//PZo= -github.com/viant/mcp-protocol v0.9.0/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= +github.com/viant/mcp v0.11.0 h1:dMcf5V5dPu3Ybpz7Q1nxj2fGmP/OKE1iM6MW3564eng= +github.com/viant/mcp v0.11.0/go.mod h1:mBSxAq6WvGpKRtWv3jknp+QU/oqjhov2Ab3nM9bp0F0= +github.com/viant/mcp-protocol v0.11.0 h1:22IuTTlq0L8l08z23TYRvHM/j19gp9UPExQdzwTuxsY= +github.com/viant/mcp-protocol v0.11.0/go.mod h1:EJPomVw6jnI+4Aa2ONYC3WTvApiF0YeQIiaaEpA54ec= github.com/viant/parsly v0.3.3 h1:7ytgfLOG4Ils+wviGacWxRD0gAUvVEH/iGsSE+UI8YM= github.com/viant/parsly v0.3.3/go.mod h1:85fneXJbErKMGhSQto3A5ElTQCwl3t74U9cSV0waBHw= github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= From 3f924d46f374d9fd40de8fd84220cdc95083899a Mon Sep 17 00:00:00 2001 From: adrianwitas Date: Sat, 14 Feb 2026 11:28:34 -0800 Subject: [PATCH 121/279] - introduces shape pkg --- go.mod | 10 +- go.sum | 8 +- internal/inference/spec.go | 58 +- internal/inference/state.go | 8 +- internal/translator/function.go | 5 +- internal/translator/output.go | 6 + internal/translator/resource.go | 2 +- internal/translator/rule.go | 2 +- internal/translator/viewlets.go | 2 +- repository/component.go | 2 + repository/components.go | 242 ++++++- repository/option.go | 19 + repository/shape/README.md | 61 ++ repository/shape/column/detector.go | 237 +++++++ repository/shape/column/detector_test.go | 59 ++ repository/shape/compile/compiler.go | 110 +++ repository/shape/compile/compiler_test.go | 69 ++ repository/shape/compile/doc.go | 2 + repository/shape/doc.go | 3 + repository/shape/dql_engine_test.go | 42 ++ repository/shape/errors.go | 12 + repository/shape/load/doc.go | 2 + repository/shape/load/errors.go | 7 + repository/shape/load/loader.go | 224 ++++++ repository/shape/load/loader_test.go | 116 ++++ repository/shape/load/model.go | 21 + repository/shape/load/testdata/report.sql | 1 + repository/shape/model.go | 53 ++ repository/shape/options.go | 73 ++ repository/shape/parity_test.go | 67 ++ repository/shape/plan/doc.go | 2 + repository/shape/plan/model.go | 72 ++ repository/shape/plan/planner.go | 174 +++++ repository/shape/plan/planner_test.go | 86 +++ repository/shape/plan/testdata/report.sql | 1 + repository/shape/scan/doc.go | 2 + repository/shape/scan/model.go | 33 + repository/shape/scan/scanner.go | 166 +++++ repository/shape/scan/scanner_test.go | 83 +++ repository/shape/scan/testdata/report.sql | 1 + repository/shape/shape.go | 157 +++++ repository/shape/source.go | 39 ++ repository/shape/source_type.go | 56 ++ repository/shape/source_type_test.go | 33 + repository/shape/typectx/model.go | 29 + repository/shape/typectx/resolver.go | 293 ++++++++ .../shape/typectx/resolver_memfs_test.go | 116 ++++ repository/shape/typectx/resolver_test.go | 89 +++ repository/shape/typectx/source/resolver.go | 283 ++++++++ .../shape/typectx/source/resolver_test.go | 91 +++ repository/shape/validate/relation.go | 140 ++++ repository/shape/validate/relation_test.go | 70 ++ repository/shape/xgen/generator.go | 644 ++++++++++++++++++ repository/shape/xgen/generator_test.go | 305 +++++++++ repository/shape/xgen/io.go | 311 +++++++++ repository/shape/xgen/model.go | 70 ++ view/state/parameters.go | 4 +- 57 files changed, 4856 insertions(+), 17 deletions(-) create mode 100644 repository/shape/README.md create mode 100644 repository/shape/column/detector.go create mode 100644 repository/shape/column/detector_test.go create mode 100644 repository/shape/compile/compiler.go create mode 100644 repository/shape/compile/compiler_test.go create mode 100644 repository/shape/compile/doc.go create mode 100644 repository/shape/doc.go create mode 100644 repository/shape/dql_engine_test.go create mode 100644 repository/shape/errors.go create mode 100644 repository/shape/load/doc.go create mode 100644 repository/shape/load/errors.go create mode 100644 repository/shape/load/loader.go create mode 100644 repository/shape/load/loader_test.go create mode 100644 repository/shape/load/model.go create mode 100644 repository/shape/load/testdata/report.sql create mode 100644 repository/shape/model.go create mode 100644 repository/shape/options.go create mode 100644 repository/shape/parity_test.go create mode 100644 repository/shape/plan/doc.go create mode 100644 repository/shape/plan/model.go create mode 100644 repository/shape/plan/planner.go create mode 100644 repository/shape/plan/planner_test.go create mode 100644 repository/shape/plan/testdata/report.sql create mode 100644 repository/shape/scan/doc.go create mode 100644 repository/shape/scan/model.go create mode 100644 repository/shape/scan/scanner.go create mode 100644 repository/shape/scan/scanner_test.go create mode 100644 repository/shape/scan/testdata/report.sql create mode 100644 repository/shape/shape.go create mode 100644 repository/shape/source.go create mode 100644 repository/shape/source_type.go create mode 100644 repository/shape/source_type_test.go create mode 100644 repository/shape/typectx/model.go create mode 100644 repository/shape/typectx/resolver.go create mode 100644 repository/shape/typectx/resolver_memfs_test.go create mode 100644 repository/shape/typectx/resolver_test.go create mode 100644 repository/shape/typectx/source/resolver.go create mode 100644 repository/shape/typectx/source/resolver_test.go create mode 100644 repository/shape/validate/relation.go create mode 100644 repository/shape/validate/relation_test.go create mode 100644 repository/shape/xgen/generator.go create mode 100644 repository/shape/xgen/generator_test.go create mode 100644 repository/shape/xgen/io.go create mode 100644 repository/shape/xgen/model.go diff --git a/go.mod b/go.mod index 22458b8b2..baaae6b4a 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,12 @@ module github.com/viant/datly go 1.25.0 +replace github.com/viant/velty => ../velty + +replace github.com/viant/x => ../x + +replace github.com/viant/sqlparser => ../sqlparser + require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible github.com/aws/aws-lambda-go v1.31.0 @@ -15,7 +21,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.16 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.11.1 - github.com/viant/afs v1.26.2 + github.com/viant/afs v1.29.0 github.com/viant/afsc v1.16.0 github.com/viant/assertly v0.9.1-0.20220620174148-bab013f93a60 github.com/viant/bigquery v0.4.1 @@ -53,6 +59,7 @@ require ( github.com/viant/mcp-protocol v0.9.0 github.com/viant/structology v0.8.0 github.com/viant/tagly v0.3.0 + github.com/viant/x v0.3.0 github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 github.com/viant/xdatly/handler v0.0.0-20251208172928-dd34b7f09fd5 @@ -151,7 +158,6 @@ require ( github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/viant/gosh v0.2.1 // indirect github.com/viant/igo v0.2.0 // indirect - github.com/viant/x v0.3.0 // indirect github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca // indirect github.com/xuri/excelize/v2 v2.8.0 // indirect github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a // indirect diff --git a/go.sum b/go.sum index ddb2bb562..d7923c12f 100644 --- a/go.sum +++ b/go.sum @@ -1152,8 +1152,8 @@ github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d h1:IRmoMmrWqkHDBy0tk9mbHRDK7+ynn0Gzwl+9WIiAtNs= github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d/go.mod h1:eRBywl0oTDM/oGhGLUeJjnC7XzmkTGuW9/og5YFy0K0= -github.com/viant/afs v1.26.2 h1:rOs/iFxFlEndhIRATJVXlNWhVU0cGdRQAGVTVJPdsc0= -github.com/viant/afs v1.26.2/go.mod h1:rScbFd9LJPGTM8HOI8Kjwee0AZ+MZMupAvFpPg+Qdj4= +github.com/viant/afs v1.29.0 h1:ndnn+PBQt5ep/bE1m5OvIvMjpoCCZbtl/UlJEubT9kE= +github.com/viant/afs v1.29.0/go.mod h1:rScbFd9LJPGTM8HOI8Kjwee0AZ+MZMupAvFpPg+Qdj4= github.com/viant/afsc v1.16.0 h1:/kOH/flNwme6h3oFrU/KPnMHkhbCZxQncTf1GSQIlBQ= github.com/viant/afsc v1.16.0/go.mod h1:Z6fP3VcmzS8Sg2lowctR6KkVEX7XxJ8aNaoHqhUiZkY= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= @@ -1208,10 +1208,6 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI github.com/viant/toolbox v0.34.5/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/viant/toolbox v0.37.0 h1:+zwSdbQh6I6ZEyxokQJr+1gQKbLEw6erc+Av5dwKtLU= github.com/viant/toolbox v0.37.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 h1:zKk+6hqUipkJXCPCHyFXzGtil1sfh80r6UZmloBNEDo= -github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= -github.com/viant/x v0.3.0 h1:/3A0z/uySGxMo6ixH90VAcdjI00w5e3REC1zg5hzhJA= -github.com/viant/x v0.3.0/go.mod h1:54jP3qV+nnQdNDaWxEwGTAAzCu9sx9er9htiwTW/Mcw= github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a h1:7CLO2LjVnFgOwN0FL3Q4y5NrD7DpclS21AiW6tDLIc8= github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUBDzc4rWacOYvAq/TKrTV0agvMF0gwm2gaoHI= diff --git a/internal/inference/spec.go b/internal/inference/spec.go index 52fc6068d..214857d10 100644 --- a/internal/inference/spec.go +++ b/internal/inference/spec.go @@ -246,7 +246,13 @@ func NewSpec(ctx context.Context, db *sql.DB, messages *msg.Messages, table stri var result = &Spec{Table: table, SQL: SQL, SQLArgs: SQLArgs, IsAuxiliary: isAuxiliary} columns, err := column.Discover(ctx, db, table, SQL, SQLArgs...) if err != nil { - return nil, err + columns = bestEffortColumnsFromSQL(SQL, columnsConfig) + if len(columns) == 0 { + return nil, err + } + if messages != nil { + messages.AddWarning(result.Table, "detection", fmt.Sprintf("using best-effort SQL column inference due to discovery error: %v", err)) + } } result.Columns = columns byName := result.Columns.ByName() @@ -285,6 +291,56 @@ func NewSpec(ctx context.Context, db *sql.DB, messages *msg.Messages, table stri return result, nil } +func bestEffortColumnsFromSQL(SQL string, columnsConfig view.ColumnConfigs) sqlparser.Columns { + if strings.TrimSpace(SQL) == "" { + return nil + } + query, err := sqlparser.ParseQuery(SQL) + if err != nil || query == nil { + return nil + } + queryColumns := sqlparser.NewColumns(query.List) + if len(queryColumns) == 0 { + return nil + } + cfgByLower := map[string]*view.ColumnConfig{} + for _, cfg := range columnsConfig { + if cfg == nil || cfg.Name == "" { + continue + } + cfgByLower[strings.ToLower(cfg.Name)] = cfg + } + var result sqlparser.Columns + for _, candidate := range queryColumns { + if candidate == nil { + continue + } + expression := strings.TrimSpace(candidate.Expression) + if expression == "*" || strings.HasSuffix(expression, ".*") { + continue + } + name := strings.TrimSpace(candidate.Alias) + if name == "" { + name = strings.TrimSpace(candidate.Name) + } + if name == "" { + continue + } + if candidate.Type == "" { + if cfg, ok := cfgByLower[strings.ToLower(name)]; ok && cfg.DataType != nil && *cfg.DataType != "" { + candidate.Type = *cfg.DataType + } else if cfg, ok = cfgByLower[strings.ToLower(candidate.Name)]; ok && cfg.DataType != nil && *cfg.DataType != "" { + candidate.Type = *cfg.DataType + } + } + if candidate.Type == "" { + candidate.Type = "string" + } + result = append(result, candidate) + } + return result +} + func isAuxiliary(SQL *string) bool { if *SQL == "" { return false diff --git a/internal/inference/state.go b/internal/inference/state.go index e309bc01f..fa28c984f 100644 --- a/internal/inference/state.go +++ b/internal/inference/state.go @@ -491,7 +491,13 @@ func (s State) EnsureReflectTypes(modulePath string, pkg string, registry *xrefl if err != nil { rType, err = types.LookupType(typeRegistry.Lookup, dataType, xreflect.WithPackage(pkg)) if err != nil { - return err + rType = reflect.TypeOf((*interface{})(nil)).Elem() + if param.Schema.DataType == "" { + param.Schema.DataType = "interface{}" + } + if param.Schema.Package == "" { + param.Schema.Package = pkg + } } } param.Schema.SetType(rType) diff --git a/internal/translator/function.go b/internal/translator/function.go index 923250d59..38ddb6a42 100644 --- a/internal/translator/function.go +++ b/internal/translator/function.go @@ -124,7 +124,10 @@ func (v *Viewlet) applyExplicitCast(column *sqlparser.Column, funcArgs []string) column.Type = funcArgs[1] rType, err := types.LookupType(v.Resource.typeRegistry.Lookup, column.Type) if err != nil { - return false, fmt.Errorf("unknown column %v type: %s, %w", column.Name, column.Type, err) + // Keep unresolved custom cast as metadata only. This preserves declared type + // (e.g. *fee.Fee) for IR/yaml parity without forcing runtime type resolution. + // Built-in and resolvable types still set RawType. + return true, nil } column.RawType = rType return true, nil diff --git a/internal/translator/output.go b/internal/translator/output.go index 6c562649d..dcadf766c 100644 --- a/internal/translator/output.go +++ b/internal/translator/output.go @@ -452,6 +452,12 @@ func (s *Service) ensureOutputParameters(resource *Resource, outputState inferen } func (s *Service) updateParameterWithComponentOutputType(dataParameter *state.Parameter, rootViewlet *Viewlet) { + if rootViewlet == nil || rootViewlet.View == nil || rootViewlet.Resource == nil || rootViewlet.Resource.rule == nil { + return + } + if rootViewlet.View.Schema == nil { + rootViewlet.View.Schema = &state.Schema{} + } typeName := rootViewlet.View.Schema.Name if typeName == "" || typeName == "string" { typeName = view.DefaultTypeName(rootViewlet.Name) diff --git a/internal/translator/resource.go b/internal/translator/resource.go index db7e97121..00630f33d 100644 --- a/internal/translator/resource.go +++ b/internal/translator/resource.go @@ -733,7 +733,7 @@ func (r *Resource) updatedObject(loadType func(typeName string) (reflect.Type, e schema := parameter.OutputSchema() wType := schema.Type() if wType == nil { - return fmt.Errorf("failed to get parameter auxiliary type: %s, %w", parameter.Name, schema.Name) + return fmt.Errorf("failed to get parameter auxiliary type: %s, %s", parameter.Name, schema.Name) } auxiliaryState := inference.State{} if err := r.extractState(loadType, wType, &auxiliaryState); err != nil { diff --git a/internal/translator/rule.go b/internal/translator/rule.go index ec42c5388..b2cd35c96 100644 --- a/internal/translator/rule.go +++ b/internal/translator/rule.go @@ -193,7 +193,7 @@ func (r *Resource) initRule(ctx context.Context, fs afs.Service, dSQL *string) e rule := r.Rule rule.applyDefaults() if err := r.loadData(ctx, fs, rule.ConstURL, &rule.Const); err != nil { - r.messages.AddWarning(r.rule.RuleName(), "const", fmt.Sprintf("failed to load constant : %v %w", rule.ConstURL, err)) + r.messages.AddWarning(r.rule.RuleName(), "const", fmt.Sprintf("failed to load constant : %v %v", rule.ConstURL, err)) } r.State.AppendConst(rule.Const) return r.loadDocumentation(ctx, fs, rule) diff --git a/internal/translator/viewlets.go b/internal/translator/viewlets.go index 72707387c..ddeb349aa 100644 --- a/internal/translator/viewlets.go +++ b/internal/translator/viewlets.go @@ -76,7 +76,7 @@ func (n *Viewlets) Init(ctx context.Context, aQuery *query.Select, resource *Res if err := n.Each(func(viewlet *Viewlet) error { n.ensureConnector(viewlet, rootConnector) if err := initFn(ctx, viewlet); err != nil { - return fmt.Errorf("failed to init viewlet: %ns, %w", viewlet.Name, err) + return fmt.Errorf("failed to init viewlet: %s, %w", viewlet.Name, err) } return nil }); err != nil { diff --git a/repository/component.go b/repository/component.go index ec106e47a..179ff7a9a 100644 --- a/repository/component.go +++ b/repository/component.go @@ -18,6 +18,7 @@ import ( content "github.com/viant/datly/repository/content" "github.com/viant/datly/repository/contract" "github.com/viant/datly/repository/handler" + "github.com/viant/datly/repository/shape/typectx" "github.com/viant/datly/repository/version" "github.com/viant/datly/service" "github.com/viant/datly/shared" @@ -47,6 +48,7 @@ type ( View *view.View `json:",omitempty"` NamespacedView *view.NamespacedView Handler *handler.Handler `json:",omitempty"` + TypeContext *typectx.Context `json:",omitempty" yaml:",omitempty"` indexedView view.NamedViews SourceURL string diff --git a/repository/components.go b/repository/components.go index 536ad3292..a431095a3 100644 --- a/repository/components.go +++ b/repository/components.go @@ -13,6 +13,13 @@ import ( "github.com/viant/datly/internal/inference" "github.com/viant/datly/internal/translator/parser" "github.com/viant/datly/repository/codegen" + "github.com/viant/datly/repository/shape" + shapecolumn "github.com/viant/datly/repository/shape/column" + dqlparse "github.com/viant/datly/repository/shape/dql/parse" + shapeLoad "github.com/viant/datly/repository/shape/load" + shapePlan "github.com/viant/datly/repository/shape/plan" + shapeScan "github.com/viant/datly/repository/shape/scan" + "github.com/viant/datly/repository/shape/typectx" "github.com/viant/datly/repository/version" "github.com/viant/datly/utils/types" "github.com/viant/datly/view" @@ -24,6 +31,7 @@ import ( "gopkg.in/yaml.v3" "path" "reflect" + "strings" ) type Components struct { @@ -61,6 +69,9 @@ func (c *Components) Init(ctx context.Context) error { options = append(options, &view.Metrics{Method: c.Components[0].Method, Service: c.options.metrics}) } for _, component := range c.Components { + if c.options != nil && c.options.legacyTypeContext { + component.TypeContext = resolveComponentTypeContext(component) + } if len(component.with) > 0 { c.With = append(c.With, component.with...) } @@ -80,6 +91,9 @@ func (c *Components) Init(ctx context.Context) error { } c.ensureNamedViewType(ctx, embedFs, aComponent) + if err = c.mergeShapeViews(ctx, aComponent); err != nil { + return err + } if err = c.Resource.Init(ctx, options...); err != nil { return err @@ -106,6 +120,62 @@ func (c *Components) Init(ctx context.Context) error { return nil } +func (c *Components) mergeShapeViews(ctx context.Context, aComponent *Component) error { + if c.options == nil || !c.options.shapePipeline || aComponent == nil || aComponent.Output.Type.Schema == nil { + return nil + } + rType := c.ReflectType(aComponent.Output.Type.Schema) + if rType == nil { + return nil + } + engine := shape.New( + shape.WithScanner(shapeScan.New()), + shape.WithPlanner(shapePlan.New()), + shape.WithLoader(shapeLoad.New()), + shape.WithName(aComponent.Path.URI), + ) + source := zeroValue(rType) + if source == nil { + return nil + } + artifacts, err := engine.LoadViews(ctx, source) + if err != nil { + return fmt.Errorf("failed to load shape views for %s: %w", aComponent.Path.URI, err) + } + if artifacts == nil || artifacts.Resource == nil { + return nil + } + if c.Resource.FSEmbedder == nil && artifacts.Resource.FSEmbedder != nil { + c.Resource.FSEmbedder = artifacts.Resource.FSEmbedder + } + existing := c.Resource.Views.Index() + columnDetector := shapecolumn.New() + for _, candidate := range artifacts.Views { + if candidate == nil { + continue + } + if _, err = existing.Lookup(candidate.Name); err == nil { + continue + } + if candidate.Columns, err = columnDetector.Resolve(ctx, c.Resource, candidate); err != nil { + return fmt.Errorf("failed to resolve shape columns for %s: %w", candidate.Name, err) + } + c.Resource.Views = append(c.Resource.Views, candidate) + existing.Register(candidate) + } + return nil +} + +func zeroValue(rType reflect.Type) interface{} { + if rType == nil { + return nil + } + if rType.Kind() == reflect.Ptr { + return reflect.New(rType.Elem()).Interface() + } + return reflect.New(rType).Interface() +} + func (c *Components) ensureNamedViewType(ctx context.Context, embedFs *embed.FS, aComponent *Component) { inCodeGeneration := codegen.IsGeneratorContext(ctx) if rType := c.ReflectType(c.Components[0].Output.Type.Schema); rType != nil && !inCodeGeneration { @@ -374,7 +444,7 @@ func LoadComponents(ctx context.Context, URL string, opts ...Option) (*Component } } } - components, err := unmarshalComponent(data) + components, err := unmarshalComponent(data, options.legacyTypeContext) if err != nil { return nil, err } @@ -396,17 +466,47 @@ func LoadComponents(ctx context.Context, URL string, opts ...Option) (*Component return components, nil } -func unmarshalComponent(data []byte) (*Components, error) { +// LoadComponentsFromMap loads components directly from in-memory route/resource model. +// The input map is expected to follow the same shape as route YAML after unmarshalling. +func LoadComponentsFromMap(ctx context.Context, model map[string]any, opts ...Option) (*Components, error) { + if len(model) == 0 { + return nil, fmt.Errorf("components model was empty") + } + options := NewOptions(opts) + components, err := unmarshalComponentMap(model, options.legacyTypeContext) + if err != nil { + return nil, err + } + components.options = options + components.resources = options.resources + if components.Resource == nil { + return nil, fmt.Errorf("resources were empty") + } + if err = components.mergeResources(ctx); err != nil { + return nil, err + } + components.Resource.SetTypes(options.extensions.Types) + return components, nil +} + +func unmarshalComponent(data []byte, enableLegacyTypeContext bool) (*Components, error) { aMap := map[string]interface{}{} if err := yaml.Unmarshal(data, &aMap); err != nil { return nil, err } + return unmarshalComponentMap(aMap, enableLegacyTypeContext) +} + +func unmarshalComponentMap(aMap map[string]any, enableLegacyTypeContext bool) (*Components, error) { ensureComponents(aMap) components := &Components{} err := toolbox.DefaultConverter.AssignConverted(components, aMap) if err != nil { return nil, err } + if enableLegacyTypeContext { + applyLegacyTypeContext(aMap, components) + } return components, err } @@ -415,3 +515,141 @@ func ensureComponents(aMap map[string]interface{}) { aMap["Components"] = aMap["Routes"] } } + +func applyLegacyTypeContext(source map[string]any, components *Components) { + if len(components.Components) == 0 { + return + } + defaultTypeContext := asTypeContext(source["TypeContext"]) + items := asAnySlice(source["Components"]) + for i, component := range components.Components { + if component == nil { + continue + } + if component.TypeContext != nil { + continue + } + var resolved *typectx.Context + if i < len(items) { + if itemMap := asStringMap(items[i]); itemMap != nil { + resolved = asTypeContext(itemMap["TypeContext"]) + } + } + if resolved == nil { + resolved = defaultTypeContext + } + if resolved != nil { + component.TypeContext = cloneTypeContext(resolved) + } + } +} + +func asTypeContext(raw any) *typectx.Context { + mapped := asStringMap(raw) + if mapped == nil { + return nil + } + ret := &typectx.Context{ + DefaultPackage: asString(mapped["DefaultPackage"]), + } + for _, item := range asAnySlice(mapped["Imports"]) { + itemMap := asStringMap(item) + if itemMap == nil { + continue + } + pkg := asString(itemMap["Package"]) + if pkg == "" { + continue + } + ret.Imports = append(ret.Imports, typectx.Import{ + Alias: asString(itemMap["Alias"]), + Package: pkg, + }) + } + if ret.DefaultPackage == "" && len(ret.Imports) == 0 { + return nil + } + return ret +} + +func resolveComponentTypeContext(component *Component) *typectx.Context { + if component == nil { + return nil + } + if normalized := normalizeTypeContext(component.TypeContext); normalized != nil { + return normalized + } + if component.View == nil || component.View.Template == nil { + return nil + } + source := strings.TrimSpace(component.View.Template.Source) + if source == "" { + return nil + } + parsed, err := dqlparse.New().Parse(source) + if err != nil || parsed == nil { + return nil + } + return normalizeTypeContext(parsed.TypeContext) +} + +func normalizeTypeContext(input *typectx.Context) *typectx.Context { + if input == nil { + return nil + } + ret := &typectx.Context{ + DefaultPackage: strings.TrimSpace(input.DefaultPackage), + } + for _, item := range input.Imports { + pkg := strings.TrimSpace(item.Package) + if pkg == "" { + continue + } + ret.Imports = append(ret.Imports, typectx.Import{ + Alias: strings.TrimSpace(item.Alias), + Package: pkg, + }) + } + if ret.DefaultPackage == "" && len(ret.Imports) == 0 { + return nil + } + return ret +} + +func cloneTypeContext(input *typectx.Context) *typectx.Context { + return normalizeTypeContext(input) +} + +func asAnySlice(raw any) []any { + switch actual := raw.(type) { + case []any: + return actual + default: + return nil + } +} + +func asStringMap(raw any) map[string]any { + switch actual := raw.(type) { + case map[string]any: + return actual + case map[interface{}]interface{}: + result := make(map[string]any, len(actual)) + for k, v := range actual { + result[fmt.Sprint(k)] = v + } + return result + default: + return nil + } +} + +func asString(raw any) string { + if raw == nil { + return "" + } + if value, ok := raw.(string); ok { + return value + } + return fmt.Sprint(raw) +} diff --git a/repository/option.go b/repository/option.go index c660a7486..9c2b9b34b 100644 --- a/repository/option.go +++ b/repository/option.go @@ -43,6 +43,8 @@ type Options struct { constants map[string]string substitutes map[string]view.Substitutes authConfig aconfig.Config + shapePipeline bool + legacyTypeContext bool } func (o *Options) UseColumn() bool { @@ -242,6 +244,23 @@ func WithPath(aPath *path.Path) Option { } } +// WithShapePipeline enables the repository/shape scan->plan->load pipeline +// during components initialization. +// The default is false to preserve existing behavior. +func WithShapePipeline(enabled bool) Option { + return func(o *Options) { + o.shapePipeline = enabled + } +} + +// WithLegacyTypeContext enables TypeContext enrichment in legacy repository runtime. +// Disabled by default for rollback safety. +func WithLegacyTypeContext(enabled bool) Option { + return func(o *Options) { + o.legacyTypeContext = enabled + } +} + func WithJWTSigner(aSigner *signer.Config) Option { return func(o *Options) { o.authConfig.JwtSigner = aSigner diff --git a/repository/shape/README.md b/repository/shape/README.md new file mode 100644 index 000000000..793b0404d --- /dev/null +++ b/repository/shape/README.md @@ -0,0 +1,61 @@ +# repository/shape + +`repository/shape` provides a dynamic, in-memory pipeline for building Datly runtime artifacts from either: + +- Go structs (`scan -> plan -> load`) +- DQL (`compile -> load`) + +without generating YAML route/resource files. + +## Packages + +- `shape/scan`: discovers view/state tags from struct fields (Embedder-aware). +- `shape/plan`: normalizes scan output into a deterministic shape plan. +- `shape/load`: materializes `view.Resource`, `view.View`, and a runtime-neutral component artifact. +- `shape/compile`: compiles DQL into a shape plan for dynamic loading. + +## Facade API + +Use `shape.Engine` or package helpers: + +- `shape.LoadViews(ctx, src, opts...)` +- `shape.LoadComponent(ctx, src, opts...)` +- `shape.LoadDQLViews(ctx, dql, opts...)` +- `shape.LoadDQLComponent(ctx, dql, opts...)` + +## Minimal Struct Flow + +```go +engine := shape.New( + shape.WithScanner(scan.New()), + shape.WithPlanner(plan.New()), + shape.WithLoader(load.New()), + shape.WithName("/v1/api/report"), +) + +views, err := engine.LoadViews(ctx, &MyOutput{}) +``` + +## Minimal DQL Flow + +```go +engine := shape.New( + shape.WithCompiler(compile.New()), + shape.WithLoader(load.New()), + shape.WithName("/v1/api/report"), +) + +component, err := engine.LoadDQLComponent(ctx, "SELECT id FROM ORDERS t") +``` + +## Repository Integration + +`repository/components.go` can optionally merge views generated by the shape pipeline during init. + +Enable via: + +```go +repository.WithShapePipeline(true) +``` + +Default is disabled to preserve existing behavior. diff --git a/repository/shape/column/detector.go b/repository/shape/column/detector.go new file mode 100644 index 000000000..79b6c8d1c --- /dev/null +++ b/repository/shape/column/detector.go @@ -0,0 +1,237 @@ +package column + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/viant/datly/view" + viewcolumn "github.com/viant/datly/view/column" + "github.com/viant/sqlparser" + "github.com/viant/sqlx/io" +) + +// Detector resolves columns for shape-generated views. +// +// Rules: +// - schema field order is canonical order +// - wildcard SQL always performs DB discovery +// - newly discovered columns are appended at the end +// - matched columns keep schema order but refresh metadata from DB +type Detector struct{} + +func New() *Detector { + return &Detector{} +} + +func (d *Detector) Resolve(ctx context.Context, resource *view.Resource, aView *view.View) (view.Columns, error) { + if aView == nil { + return nil, fmt.Errorf("shape column detector: nil view") + } + + base := columnsFromSchema(aView) + if !usesWildcard(aView) { + return base, nil + } + + discovered, err := d.detect(ctx, resource, aView) + if err != nil { + return nil, err + } + if len(base) == 0 { + return discovered, nil + } + return mergePreservingOrder(base, discovered), nil +} + +func (d *Detector) detect(ctx context.Context, resource *view.Resource, aView *view.View) (view.Columns, error) { + connector, err := lookupConnector(ctx, resource, aView) + if err != nil { + return nil, err + } + db, err := connector.DB() + if err != nil { + return nil, fmt.Errorf("shape column detector: failed to open db for view %s: %w", aView.Name, err) + } + query := sourceSQL(aView) + sqlColumns, err := viewcolumn.Discover(ctx, db, aView.Table, query) + if err != nil { + return nil, fmt.Errorf("shape column detector: discover failed for view %s: %w", aView.Name, err) + } + return view.NewColumns(sqlColumns, aView.ColumnsConfig), nil +} + +func lookupConnector(ctx context.Context, resource *view.Resource, aView *view.View) (*view.Connector, error) { + if resource == nil { + return nil, fmt.Errorf("shape column detector: missing resource for view %s", aView.Name) + } + if aView.Connector == nil { + return nil, fmt.Errorf("shape column detector: missing connector for wildcard view %s", aView.Name) + } + connectors := view.ConnectorSlice(resource.Connectors).Index() + connector := aView.Connector + if connector.Ref != "" { + lookup, err := connectors.Lookup(connector.Ref) + if err != nil { + return nil, fmt.Errorf("shape column detector: connector ref %s for view %s: %w", connector.Ref, aView.Name, err) + } + connector = lookup + } + if err := connector.Init(ctx, connectors); err != nil { + return nil, fmt.Errorf("shape column detector: connector init for view %s: %w", aView.Name, err) + } + return connector, nil +} + +func sourceSQL(aView *view.View) string { + if aView.Template != nil && strings.TrimSpace(aView.Template.Source) != "" { + return aView.Template.Source + } + return aView.Source() +} + +func usesWildcard(aView *view.View) bool { + if aView != nil && aView.Template == nil && strings.TrimSpace(aView.Table) != "" { + return true + } + query := sourceSQL(aView) + trimmed := strings.TrimSpace(strings.ToLower(query)) + if trimmed == "" { + return false + } + if !strings.Contains(trimmed, "*") { + return false + } + if !strings.HasPrefix(trimmed, "select") && !strings.HasPrefix(trimmed, "with") { + return true + } + parsed, err := sqlparser.ParseQuery(query) + if err != nil { + return true + } + return sqlparser.NewColumns(parsed.List).IsStarExpr() +} + +func columnsFromSchema(aView *view.View) view.Columns { + if aView == nil || aView.Schema == nil { + return nil + } + rType := aView.Schema.Type() + if rType == nil { + return nil + } + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return nil + } + result := make(view.Columns, 0, rType.NumField()) + appendSchemaColumns(rType, "", &result) + return result +} + +func appendSchemaColumns(rType reflect.Type, ns string, columns *view.Columns) { + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if field.PkgPath != "" { // unexported + continue + } + if field.Anonymous { + inner := field.Type + for inner.Kind() == reflect.Ptr { + inner = inner.Elem() + } + if inner.Kind() == reflect.Struct { + appendSchemaColumns(inner, ns, columns) + } + continue + } + + tag := io.ParseTag(field.Tag) + if tag != nil && tag.Transient { + continue + } + + name := field.Name + if tag != nil && tag.Column != "" { + name = tag.Column + } + if tag != nil && tag.Ns != "" { + name = tag.Ns + name + } else if ns != "" { + name = ns + name + } + + columnType := field.Type + nullable := false + if columnType.Kind() == reflect.Ptr { + nullable = true + columnType = columnType.Elem() + } + *columns = append(*columns, view.NewColumn(name, columnType.String(), columnType, nullable, view.WithColumnTag(string(field.Tag)))) + } +} + +func mergePreservingOrder(base, discovered view.Columns) view.Columns { + if len(base) == 0 { + return discovered + } + if len(discovered) == 0 { + return base + } + seen := map[string]*view.Column{} + for _, item := range discovered { + if item == nil { + continue + } + seen[strings.ToLower(item.Name)] = item + } + result := make(view.Columns, 0, len(base)+len(discovered)) + for _, item := range base { + if item == nil { + continue + } + if fresh, ok := seen[strings.ToLower(item.Name)]; ok { + delete(seen, strings.ToLower(item.Name)) + // Keep schema name/order but refresh discovered metadata. + item.DataType = firstNonEmpty(fresh.DataType, item.DataType) + item.SetColumnType(firstType(fresh.ColumnType(), item.ColumnType())) + item.Nullable = fresh.Nullable + if item.DatabaseColumn == "" { + item.DatabaseColumn = fresh.DatabaseColumn + } + } + result = append(result, item) + } + for _, item := range discovered { + if item == nil { + continue + } + if _, ok := seen[strings.ToLower(item.Name)]; !ok { + continue + } + result = append(result, item) + delete(seen, strings.ToLower(item.Name)) + } + return result +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func firstType(values ...reflect.Type) reflect.Type { + for _, value := range values { + if value != nil { + return value + } + } + return nil +} diff --git a/repository/shape/column/detector_test.go b/repository/shape/column/detector_test.go new file mode 100644 index 000000000..cfc834b11 --- /dev/null +++ b/repository/shape/column/detector_test.go @@ -0,0 +1,59 @@ +package column + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +type sampleOrder struct { + VendorID int `sqlx:"name=VENDOR_ID"` + Name string `sqlx:"name=NAME"` +} + +func TestUsesWildcard(t *testing.T) { + tests := []struct { + name string + view *view.View + want bool + }{ + {name: "select wildcard", view: &view.View{Template: view.NewTemplate("SELECT * FROM VENDOR")}, want: true}, + {name: "select explicit", view: &view.View{Template: view.NewTemplate("SELECT ID, NAME FROM VENDOR")}, want: false}, + {name: "table only", view: &view.View{Table: "VENDOR"}, want: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, usesWildcard(tc.view)) + }) + } +} + +func TestColumnsFromSchema_Order(t *testing.T) { + aView := &view.View{Schema: state.NewSchema(reflect.TypeOf(sampleOrder{}), state.WithMany())} + cols := columnsFromSchema(aView) + require.Len(t, cols, 2) + require.Equal(t, "VENDOR_ID", cols[0].Name) + require.Equal(t, "NAME", cols[1].Name) +} + +func TestMergePreservingOrder_AppendsNewDetectedColumns(t *testing.T) { + base := view.Columns{ + view.NewColumn("VENDOR_ID", "int", reflect.TypeOf(int(0)), false), + view.NewColumn("NAME", "varchar", reflect.TypeOf(""), false), + } + detected := view.Columns{ + view.NewColumn("NAME", "text", reflect.TypeOf(""), true), + view.NewColumn("VENDOR_ID", "bigint", reflect.TypeOf(int64(0)), false), + view.NewColumn("STATUS", "int", reflect.TypeOf(int(0)), true), + } + merged := mergePreservingOrder(base, detected) + require.Len(t, merged, 3) + require.Equal(t, "VENDOR_ID", merged[0].Name) + require.Equal(t, "NAME", merged[1].Name) + require.Equal(t, "STATUS", merged[2].Name) + require.Equal(t, "bigint", merged[0].DataType) + require.Equal(t, "text", merged[1].DataType) +} diff --git a/repository/shape/compile/compiler.go b/repository/shape/compile/compiler.go new file mode 100644 index 000000000..69647b608 --- /dev/null +++ b/repository/shape/compile/compiler.go @@ -0,0 +1,110 @@ +package compile + +import ( + "context" + "fmt" + "reflect" + "regexp" + "strings" + + "github.com/viant/datly/internal/translator/parser" + "github.com/viant/datly/repository/shape" + dqlparse "github.com/viant/datly/repository/shape/dql/parse" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/sqlparser" +) + +// DQLCompiler compiles raw DQL into a shape plan that can be materialized by shape/load. +type DQLCompiler struct{} + +// New returns a DQL compiler implementation. +func New() *DQLCompiler { + return &DQLCompiler{} +} + +// Compile implements shape.DQLCompiler. +func (c *DQLCompiler) Compile(_ context.Context, source *shape.Source, _ ...shape.CompileOption) (*shape.PlanResult, error) { + if source == nil { + return nil, shape.ErrNilSource + } + dql := strings.TrimSpace(source.DQL) + if dql == "" { + return nil, shape.ErrNilDQL + } + + name, table, err := inferRoot(dql, source.Name) + if err != nil { + return nil, err + } + + result := &plan.Result{ + Views: []*plan.View{ + { + Path: name, + Holder: name, + Name: name, + Table: table, + SQL: dql, + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + } + if parsed, parseErr := dqlparse.New().Parse(dql); parseErr == nil && parsed != nil && parsed.TypeContext != nil { + result.TypeContext = parsed.TypeContext + } + result.ViewsByName[name] = result.Views[0] + return &shape.PlanResult{Source: source, Plan: result}, nil +} + +func inferRoot(dql string, fallback string) (string, string, error) { + query, err := sqlparser.ParseQuery(dql, parser.OnVeltyExpression()) + if err != nil { + name := sanitizeName(fallback) + if name == "" { + name = "DQLView" + } + return name, "", nil + } + + name := sanitizeName(query.From.Alias) + if name == "" { + name = sanitizeName(fallback) + } + if name == "" { + name = "DQLView" + } + + table := "" + if query != nil && query.From.X != nil { + table = strings.TrimSpace(sqlparser.Stringify(query.From.X)) + } + if table == "" || strings.HasPrefix(table, "(") { + table = name + } + if name == "" { + return "", "", fmt.Errorf("shape compile: failed to infer view name") + } + return name, table, nil +} + +var nonWord = regexp.MustCompile(`[^a-zA-Z0-9_]+`) + +func sanitizeName(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + value = nonWord.ReplaceAllString(value, "_") + value = strings.Trim(value, "_") + if value == "" { + return "" + } + if value[0] >= '0' && value[0] <= '9' { + value = "V_" + value + } + return value +} diff --git a/repository/shape/compile/compiler_test.go b/repository/shape/compile/compiler_test.go new file mode 100644 index 000000000..b539ab80b --- /dev/null +++ b/repository/shape/compile/compiler_test.go @@ -0,0 +1,69 @@ +package compile + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" +) + +func TestDQLCompiler_Compile(t *testing.T) { + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "SELECT id FROM ORDERS t"}) + require.NoError(t, err) + require.NotNil(t, res) + + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.Len(t, planned.Views, 1) + view := planned.Views[0] + assert.Equal(t, "t", view.Name) + assert.Equal(t, "ORDERS", view.Table) + assert.Equal(t, "many", view.Cardinality) +} + +func TestDQLCompiler_Compile_EmptyDQL(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "x"}) + require.Error(t, err) + assert.ErrorIs(t, err, shape.ErrNilDQL) +} + +func TestDQLCompiler_Compile_WithPreamble_NoPanic(t *testing.T) { + compiler := New() + dql := ` +/* metadata */ +#set($_ = $A(query/a).Optional()) +SELECT id +` + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "sample_report", DQL: dql}) + require.NoError(t, err) + require.NotNil(t, res) + + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.Len(t, planned.Views, 1) + assert.Equal(t, "sample_report", planned.Views[0].Name) + assert.Equal(t, "sample_report", planned.Views[0].Table) +} + +func TestDQLCompiler_Compile_PropagatesTypeContext(t *testing.T) { + compiler := New() + dql := ` +#set($_ = $package('mdp/performance')) +#set($_ = $import('perf', 'github.com/acme/mdp/performance')) +SELECT id FROM ORDERS t` + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + require.NotNil(t, res) + + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotNil(t, planned.TypeContext) + assert.Equal(t, "mdp/performance", planned.TypeContext.DefaultPackage) + require.Len(t, planned.TypeContext.Imports, 1) + assert.Equal(t, "perf", planned.TypeContext.Imports[0].Alias) +} diff --git a/repository/shape/compile/doc.go b/repository/shape/compile/doc.go new file mode 100644 index 000000000..c5a996ba8 --- /dev/null +++ b/repository/shape/compile/doc.go @@ -0,0 +1,2 @@ +// Package compile provides DQL-to-shape compilation. +package compile diff --git a/repository/shape/doc.go b/repository/shape/doc.go new file mode 100644 index 000000000..730ab1395 --- /dev/null +++ b/repository/shape/doc.go @@ -0,0 +1,3 @@ +// Package shape provides building blocks for dynamic repository loading from +// struct and DQL sources without requiring persisted YAML artifacts. +package shape diff --git a/repository/shape/dql_engine_test.go b/repository/shape/dql_engine_test.go new file mode 100644 index 000000000..fafe3f67f --- /dev/null +++ b/repository/shape/dql_engine_test.go @@ -0,0 +1,42 @@ +package shape_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + shape "github.com/viant/datly/repository/shape" + shapeCompile "github.com/viant/datly/repository/shape/compile" + shapeLoad "github.com/viant/datly/repository/shape/load" +) + +func TestEngine_LoadDQLViews(t *testing.T) { + engine := shape.New( + shape.WithCompiler(shapeCompile.New()), + shape.WithLoader(shapeLoad.New()), + shape.WithName("/v1/api/reports/orders"), + ) + artifacts, err := engine.LoadDQLViews(context.Background(), "SELECT id FROM ORDERS t") + require.NoError(t, err) + require.NotNil(t, artifacts) + require.Len(t, artifacts.Views, 1) + assert.Equal(t, "t", artifacts.Views[0].Name) +} + +func TestEngine_LoadDQLComponent(t *testing.T) { + engine := shape.New( + shape.WithCompiler(shapeCompile.New()), + shape.WithLoader(shapeLoad.New()), + shape.WithName("/v1/api/reports/orders"), + ) + artifact, err := engine.LoadDQLComponent(context.Background(), "SELECT id FROM ORDERS t") + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Component) + + component, ok := artifact.Component.(*shapeLoad.Component) + require.True(t, ok) + assert.Equal(t, "/v1/api/reports/orders", component.Name) + assert.Equal(t, "t", component.RootView) +} diff --git a/repository/shape/errors.go b/repository/shape/errors.go new file mode 100644 index 000000000..852313b6c --- /dev/null +++ b/repository/shape/errors.go @@ -0,0 +1,12 @@ +package shape + +import "errors" + +var ( + ErrNilSource = errors.New("shape: source was nil") + ErrNilDQL = errors.New("shape: dql was empty") + ErrScannerNotConfigured = errors.New("shape: scanner was not configured") + ErrPlannerNotConfigured = errors.New("shape: planner was not configured") + ErrLoaderNotConfigured = errors.New("shape: loader was not configured") + ErrCompilerNotConfigured = errors.New("shape: compiler was not configured") +) diff --git a/repository/shape/load/doc.go b/repository/shape/load/doc.go new file mode 100644 index 000000000..1800597cc --- /dev/null +++ b/repository/shape/load/doc.go @@ -0,0 +1,2 @@ +// Package load defines materialization responsibilities for runtime artifacts. +package load diff --git a/repository/shape/load/errors.go b/repository/shape/load/errors.go new file mode 100644 index 000000000..51f15d6ab --- /dev/null +++ b/repository/shape/load/errors.go @@ -0,0 +1,7 @@ +package load + +import "errors" + +var ( + ErrEmptyViewPlan = errors.New("shape load: no views available in plan") +) diff --git a/repository/shape/load/loader.go b/repository/shape/load/loader.go new file mode 100644 index 000000000..149117d25 --- /dev/null +++ b/repository/shape/load/loader.go @@ -0,0 +1,224 @@ +package load + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + shapevalidate "github.com/viant/datly/repository/shape/validate" + "github.com/viant/datly/shared" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +// Loader materializes runtime view artifacts from normalized shape plan. +type Loader struct{} + +// New returns shape loader implementation. +func New() *Loader { + return &Loader{} +} + +// LoadViews implements shape.Loader. +func (l *Loader) LoadViews(_ context.Context, planned *shape.PlanResult, _ ...shape.LoadOption) (*shape.ViewArtifacts, error) { + pResult, resource, err := l.materialize(planned) + if err != nil { + return nil, err + } + if len(pResult.Views) == 0 { + return nil, ErrEmptyViewPlan + } + return &shape.ViewArtifacts{Resource: resource, Views: resource.Views}, nil +} + +// LoadComponent implements shape.Loader. +func (l *Loader) LoadComponent(_ context.Context, planned *shape.PlanResult, _ ...shape.LoadOption) (*shape.ComponentArtifact, error) { + pResult, resource, err := l.materialize(planned) + if err != nil { + return nil, err + } + if len(pResult.Views) == 0 { + return nil, ErrEmptyViewPlan + } + component := buildComponent(planned.Source, pResult) + return &shape.ComponentArtifact{ + Resource: resource, + Component: component, + }, nil +} + +func (l *Loader) materialize(planned *shape.PlanResult) (*plan.Result, *view.Resource, error) { + if planned == nil || planned.Source == nil { + return nil, nil, shape.ErrNilSource + } + pResult, ok := planned.Plan.(*plan.Result) + if !ok || pResult == nil { + return nil, nil, fmt.Errorf("shape load: unsupported plan type %T", planned.Plan) + } + resource := view.EmptyResource() + if pResult.EmbedFS != nil { + resource.SetFSEmbedder(state.NewFSEmbedder(pResult.EmbedFS)) + } + for _, item := range pResult.Views { + aView, err := materializeView(item) + if err != nil { + return nil, nil, err + } + resource.AddViews(aView) + } + if err := shapevalidate.ValidateRelations(resource, resource.Views...); err != nil { + return nil, nil, err + } + return pResult, resource, nil +} + +func buildComponent(source *shape.Source, pResult *plan.Result) *Component { + ret := &Component{Method: "GET"} + if source != nil { + ret.Name = source.Name + ret.URI = source.Name + } + for _, aView := range pResult.Views { + if aView == nil { + continue + } + ret.Views = append(ret.Views, aView.Name) + } + rootView := pickRootView(pResult.Views) + if rootView != nil { + ret.RootView = rootView.Name + if ret.Name == "" { + ret.Name = rootView.Name + } + } + for _, item := range pResult.States { + if item == nil { + continue + } + if strings.TrimSpace(item.Kind) == "" && strings.TrimSpace(item.In) == "" { + ret.Other = append(ret.Other, item) + continue + } + switch strings.ToLower(item.Kind) { + case "query", "path", "header", "body", "form", "cookie", "request", "": + ret.Input = append(ret.Input, item) + case "output": + ret.Output = append(ret.Output, item) + case "meta": + ret.Meta = append(ret.Meta, item) + case "async": + ret.Async = append(ret.Async, item) + default: + ret.Other = append(ret.Other, item) + } + } + ret.TypeContext = cloneTypeContext(pResult.TypeContext) + return ret +} + +func cloneTypeContext(input *typectx.Context) *typectx.Context { + if input == nil { + return nil + } + ret := &typectx.Context{ + DefaultPackage: strings.TrimSpace(input.DefaultPackage), + } + for _, item := range input.Imports { + pkg := strings.TrimSpace(item.Package) + if pkg == "" { + continue + } + ret.Imports = append(ret.Imports, typectx.Import{ + Alias: strings.TrimSpace(item.Alias), + Package: pkg, + }) + } + if ret.DefaultPackage == "" && len(ret.Imports) == 0 { + return nil + } + return ret +} + +func pickRootView(views []*plan.View) *plan.View { + var selected *plan.View + minDepth := -1 + for _, candidate := range views { + if candidate == nil || candidate.Path == "" { + continue + } + depth := strings.Count(candidate.Path, ".") + if minDepth == -1 || depth < minDepth { + minDepth = depth + selected = candidate + } + } + if selected != nil { + return selected + } + for _, candidate := range views { + if candidate != nil { + return candidate + } + } + return nil +} + +func materializeView(item *plan.View) (*view.View, error) { + if item == nil { + return nil, fmt.Errorf("shape load: nil view plan item") + } + + schemaType := bestSchemaType(item) + if schemaType == nil { + return nil, fmt.Errorf("shape load: missing schema type for view %q", item.Name) + } + + schema := newSchema(schemaType, item.Cardinality) + opts := []view.Option{view.WithSchema(schema), view.WithMode(view.ModeQuery)} + + if item.Connector != "" { + opts = append(opts, view.WithConnectorRef(item.Connector)) + } + if item.SQL != "" || item.SQLURI != "" { + tmpl := view.NewTemplate(item.SQL) + tmpl.SourceURL = item.SQLURI + opts = append(opts, view.WithTemplate(tmpl)) + } + if item.CacheRef != "" { + opts = append(opts, view.WithCache(&view.Cache{Reference: shared.Reference{Ref: item.CacheRef}})) + } + if item.Partitioner != "" { + opts = append(opts, view.WithPartitioned(&view.Partitioned{ + DataType: item.Partitioner, + Concurrency: item.PartitionedConcurrency, + })) + } + + aView, err := view.New(item.Name, item.Table, opts...) + if err != nil { + return nil, err + } + aView.Ref = item.Ref + return aView, nil +} + +func bestSchemaType(item *plan.View) reflect.Type { + if item.FieldType != nil { + return item.FieldType + } + if item.ElementType != nil { + return item.ElementType + } + return nil +} + +func newSchema(rType reflect.Type, cardinality string) *state.Schema { + if cardinality == "many" && rType.Kind() != reflect.Slice { + return state.NewSchema(rType, state.WithMany()) + } + return state.NewSchema(rType) +} diff --git a/repository/shape/load/loader_test.go b/repository/shape/load/loader_test.go new file mode 100644 index 000000000..aab074ba6 --- /dev/null +++ b/repository/shape/load/loader_test.go @@ -0,0 +1,116 @@ +package load + +import ( + "context" + "embed" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/scan" + "github.com/viant/datly/repository/shape/typectx" +) + +//go:embed testdata/*.sql +var testFS embed.FS + +type embeddedFS struct{} + +func (embeddedFS) EmbedFS() *embed.FS { + return &testFS +} + +type reportRow struct { + ID int + Name string +} + +type reportSource struct { + embeddedFS + Rows []reportRow `view:"rows,table=REPORT,connector=dev,cache=c1" sql:"uri=testdata/report.sql"` + ID int `parameter:"id,kind=query,in=id"` + Status any `parameter:"status,kind=output,in=status"` + Job any `parameter:"job,kind=async,in=job"` + Meta any `parameter:"meta,kind=meta,in=view.name"` +} + +func TestLoader_LoadViews(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportSource{}}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifacts, err := loader.LoadViews(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifacts) + require.NotNil(t, artifacts.Resource) + require.Len(t, artifacts.Views, 1) + + aView := artifacts.Views[0] + assert.Equal(t, "rows", aView.Name) + assert.Equal(t, "REPORT", aView.Table) + require.NotNil(t, aView.Schema) + assert.Equal(t, "Many", string(aView.Schema.Cardinality)) + require.NotNil(t, aView.Template) + assert.Equal(t, "testdata/report.sql", aView.Template.SourceURL) + assert.Contains(t, aView.Template.Source, "SELECT ID, NAME FROM REPORT") + require.NotNil(t, aView.Connector) + assert.Equal(t, "dev", aView.Connector.Ref) + require.NotNil(t, aView.Cache) + assert.Equal(t, "c1", aView.Cache.Ref) + require.NotNil(t, artifacts.Resource.EmbedFS()) +} + +func TestLoader_LoadViews_InvalidPlanType(t *testing.T) { + loader := New() + _, err := loader.LoadViews(context.Background(), &shape.PlanResult{Source: &shape.Source{Name: "x"}, Plan: "invalid"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported plan type") +} + +func TestLoader_LoadComponent(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Name: "/v1/api/report", Struct: &reportSource{}}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + actualPlan, ok := planned.Plan.(*plan.Result) + require.True(t, ok) + actualPlan.TypeContext = &typectx.Context{ + DefaultPackage: "mdp/performance", + Imports: []typectx.Import{ + {Alias: "perf", Package: "github.com/acme/mdp/performance"}, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + require.NotNil(t, artifact.Component) + + component, ok := artifact.Component.(*Component) + require.True(t, ok) + assert.Equal(t, "/v1/api/report", component.Name) + assert.Equal(t, "/v1/api/report", component.URI) + assert.Equal(t, "GET", component.Method) + assert.Equal(t, "rows", component.RootView) + assert.Equal(t, []string{"rows"}, component.Views) + assert.Len(t, component.Input, 1) + assert.Len(t, component.Output, 1) + assert.Len(t, component.Async, 1) + assert.Len(t, component.Meta, 1) + require.NotNil(t, component.TypeContext) + assert.Equal(t, "mdp/performance", component.TypeContext.DefaultPackage) + require.Len(t, component.TypeContext.Imports, 1) + assert.Equal(t, "perf", component.TypeContext.Imports[0].Alias) +} diff --git a/repository/shape/load/model.go b/repository/shape/load/model.go new file mode 100644 index 000000000..8f5d384d6 --- /dev/null +++ b/repository/shape/load/model.go @@ -0,0 +1,21 @@ +package load + +import "github.com/viant/datly/repository/shape/plan" +import "github.com/viant/datly/repository/shape/typectx" + +// Component is a shape-loaded runtime-neutral component artifact. +// It intentionally avoids repository package coupling to keep shape/load reusable. +type Component struct { + Name string + URI string + Method string + RootView string + Views []string + TypeContext *typectx.Context + + Input []*plan.State + Output []*plan.State + Meta []*plan.State + Async []*plan.State + Other []*plan.State +} diff --git a/repository/shape/load/testdata/report.sql b/repository/shape/load/testdata/report.sql new file mode 100644 index 000000000..68f0f3b34 --- /dev/null +++ b/repository/shape/load/testdata/report.sql @@ -0,0 +1 @@ +SELECT ID, NAME FROM REPORT diff --git a/repository/shape/model.go b/repository/shape/model.go new file mode 100644 index 000000000..f71fd5c28 --- /dev/null +++ b/repository/shape/model.go @@ -0,0 +1,53 @@ +package shape + +import ( + "reflect" + + "github.com/viant/datly/view" + "github.com/viant/x" +) + +// Mode controls which execution flow is expected from the shape pipeline. +type Mode string + +const ( + ModeUnspecified Mode = "" + ModeStruct Mode = "struct" + ModeDQL Mode = "dql" +) + +// Source represents the caller-provided shape source. +type Source struct { + Name string + Struct any + Type reflect.Type + TypeName string + TypeRegistry *x.Registry + DQL string +} + +// ScanResult is the output produced by Scanner. +type ScanResult struct { + Source *Source + Descriptors any +} + +// PlanResult is the output produced by Planner. +type PlanResult struct { + Source *Source + Plan any +} + +// ViewArtifacts is the runtime view payload produced by Loader. +type ViewArtifacts struct { + Resource *view.Resource + Views view.Views +} + +// ComponentArtifact is the runtime component payload produced by Loader. +// Component stays untyped in the skeleton to avoid coupling shape package +// to repository internals before the implementation phase. +type ComponentArtifact struct { + Resource *view.Resource + Component any +} diff --git a/repository/shape/options.go b/repository/shape/options.go new file mode 100644 index 000000000..05b0a7748 --- /dev/null +++ b/repository/shape/options.go @@ -0,0 +1,73 @@ +package shape + +// Options stores shape facade dependencies and behavior flags. +type Options struct { + Mode Mode + Strict bool + Name string + Scanner Scanner + Planner Planner + Loader Loader + Compiler DQLCompiler + Runtime RuntimeRegistrar +} + +// Option mutates Options. +type Option func(*Options) + +// NewOptions builds Options from varargs. +func NewOptions(opts ...Option) *Options { + ret := &Options{} + for _, opt := range opts { + opt(ret) + } + return ret +} + +func WithMode(mode Mode) Option { + return func(o *Options) { + o.Mode = mode + } +} + +func WithStrict(strict bool) Option { + return func(o *Options) { + o.Strict = strict + } +} + +func WithName(name string) Option { + return func(o *Options) { + o.Name = name + } +} + +func WithScanner(scanner Scanner) Option { + return func(o *Options) { + o.Scanner = scanner + } +} + +func WithPlanner(planner Planner) Option { + return func(o *Options) { + o.Planner = planner + } +} + +func WithLoader(loader Loader) Option { + return func(o *Options) { + o.Loader = loader + } +} + +func WithCompiler(compiler DQLCompiler) Option { + return func(o *Options) { + o.Compiler = compiler + } +} + +func WithRuntime(runtime RuntimeRegistrar) Option { + return func(o *Options) { + o.Runtime = runtime + } +} diff --git a/repository/shape/parity_test.go b/repository/shape/parity_test.go new file mode 100644 index 000000000..713bfd311 --- /dev/null +++ b/repository/shape/parity_test.go @@ -0,0 +1,67 @@ +package shape_test + +import ( + "context" + "embed" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + shape "github.com/viant/datly/repository/shape" + shapeLoad "github.com/viant/datly/repository/shape/load" + shapePlan "github.com/viant/datly/repository/shape/plan" + shapeScan "github.com/viant/datly/repository/shape/scan" +) + +//go:embed scan/testdata/*.sql +var parityFS embed.FS + +type parityEmbedded struct{} + +func (parityEmbedded) EmbedFS() *embed.FS { return &parityFS } + +type parityRow struct { + ID int + Name string +} + +type paritySource struct { + parityEmbedded + Rows []parityRow `view:"rows,table=REPORT,connector=dev" sql:"uri=scan/testdata/report.sql"` +} + +func TestEngineParity_StructPipeline(t *testing.T) { + source := &paritySource{} + scanner := shapeScan.New() + planner := shapePlan.New() + loader := shapeLoad.New() + + manualScan, err := scanner.Scan(context.Background(), &shape.Source{Name: "/v1/api/parity", Struct: source}) + require.NoError(t, err) + manualPlan, err := planner.Plan(context.Background(), manualScan) + require.NoError(t, err) + manualViews, err := loader.LoadViews(context.Background(), manualPlan) + require.NoError(t, err) + + engine := shape.New( + shape.WithName("/v1/api/parity"), + shape.WithScanner(scanner), + shape.WithPlanner(planner), + shape.WithLoader(loader), + ) + engineViews, err := engine.LoadViews(context.Background(), source) + require.NoError(t, err) + + require.Len(t, manualViews.Views, 1) + require.Len(t, engineViews.Views, 1) + + mv := manualViews.Views[0] + ev := engineViews.Views[0] + assert.Equal(t, mv.Name, ev.Name) + assert.Equal(t, mv.Table, ev.Table) + assert.Equal(t, mv.Template.Source, ev.Template.Source) + assert.Equal(t, mv.Template.SourceURL, ev.Template.SourceURL) + assert.Equal(t, mv.Schema.Cardinality, ev.Schema.Cardinality) + assert.Equal(t, reflect.TypeOf(mv.Schema.CompType()), reflect.TypeOf(ev.Schema.CompType())) +} diff --git a/repository/shape/plan/doc.go b/repository/shape/plan/doc.go new file mode 100644 index 000000000..57bb65fae --- /dev/null +++ b/repository/shape/plan/doc.go @@ -0,0 +1,2 @@ +// Package plan defines normalization and shape-planning responsibilities. +package plan diff --git a/repository/shape/plan/model.go b/repository/shape/plan/model.go new file mode 100644 index 000000000..8dacf2bbe --- /dev/null +++ b/repository/shape/plan/model.go @@ -0,0 +1,72 @@ +package plan + +import ( + "embed" + "reflect" + + "github.com/viant/datly/repository/shape/typectx" +) + +// Result is normalized shape plan produced from scan descriptors. +type Result struct { + RootType reflect.Type + EmbedFS *embed.FS + + Fields []*Field + ByPath map[string]*Field + Views []*View + ViewsByName map[string]*View + States []*State + TypeContext *typectx.Context +} + +// Field is a normalized projection of scanned field metadata. +type Field struct { + Path string + Name string + Type reflect.Type + Index []int +} + +// View is a normalized view field plan. +type View struct { + Path string + Name string + Ref string + Table string + Connector string + CacheRef string + Partitioner string + PartitionedConcurrency int + RelationalConcurrency int + SQL string + SQLURI string + Summary string + Links []string + Holder string + + Cardinality string + ElementType reflect.Type + FieldType reflect.Type +} + +// State is a normalized parameter field plan. +type State struct { + Path string + Name string + Kind string + In string + When string + Scope string + DataType string + Required *bool + Async bool + Cacheable *bool + With string + URI string + ErrorCode int + ErrorMessage string + + TagType reflect.Type + EffectiveType reflect.Type +} diff --git a/repository/shape/plan/planner.go b/repository/shape/plan/planner.go new file mode 100644 index 000000000..ec66aea5a --- /dev/null +++ b/repository/shape/plan/planner.go @@ -0,0 +1,174 @@ +package plan + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/viant/datly/repository/locator/async/keys" + metakeys "github.com/viant/datly/repository/locator/meta/keys" + outputkeys "github.com/viant/datly/repository/locator/output/keys" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/scan" +) + +// Planner normalizes scan descriptors into shape plan. +type Planner struct{} + +// New returns shape planner implementation. +func New() *Planner { + return &Planner{} +} + +// Plan implements shape.Planner. +func (p *Planner) Plan(_ context.Context, scanned *shape.ScanResult, _ ...shape.PlanOption) (*shape.PlanResult, error) { + if scanned == nil || scanned.Source == nil { + return nil, shape.ErrNilSource + } + + scanResult, ok := scanned.Descriptors.(*scan.Result) + if !ok || scanResult == nil { + return nil, fmt.Errorf("shape plan: unsupported descriptors type %T", scanned.Descriptors) + } + + result := &Result{ + RootType: scanResult.RootType, + EmbedFS: scanResult.EmbedFS, + ByPath: map[string]*Field{}, + ViewsByName: map[string]*View{}, + } + + for _, item := range scanResult.Fields { + field := &Field{ + Path: item.Path, + Name: item.Name, + Type: item.Type, + Index: append([]int(nil), item.Index...), + } + result.Fields = append(result.Fields, field) + result.ByPath[field.Path] = field + } + + for _, item := range scanResult.ViewFields { + v := normalizeView(item) + result.Views = append(result.Views, v) + if v.Name != "" { + result.ViewsByName[v.Name] = v + } + } + + for _, item := range scanResult.StateFields { + result.States = append(result.States, normalizeState(item)) + } + + return &shape.PlanResult{Source: scanned.Source, Plan: result}, nil +} + +func normalizeView(field *scan.Field) *View { + result := &View{ + Path: field.Path, + Holder: field.Name, + FieldType: field.Type, + } + + if tag := field.ViewTag; tag != nil { + if tag.View != nil { + result.Name = tag.View.Name + result.Table = tag.View.Table + result.Connector = tag.View.Connector + result.CacheRef = tag.View.Cache + result.Partitioner = tag.View.PartitionerType + result.PartitionedConcurrency = tag.View.PartitionedConcurrency + result.RelationalConcurrency = tag.View.RelationalConcurrency + } + result.SQL = tag.SQL.SQL + result.SQLURI = tag.SQL.URI + result.Summary = tag.SummarySQL.SQL + if len(tag.LinkOn) > 0 { + result.Links = append(result.Links, tag.LinkOn...) + } + result.Ref = strings.TrimSpace(tag.TypeName) + } + + if result.Name == "" { + result.Name = field.Name + } + + elem, cardinality := componentType(field.Type) + result.Cardinality = cardinality + result.ElementType = elem + return result +} + +func normalizeState(field *scan.Field) *State { + result := &State{Path: field.Path, TagType: field.Type} + if field.StateTag == nil || field.StateTag.Parameter == nil { + result.Name = field.Name + result.EffectiveType = field.Type + return result + } + + pTag := field.StateTag.Parameter + result.Name = firstNonEmpty(pTag.Name, field.Name) + result.Kind = strings.ToLower(strings.TrimSpace(pTag.Kind)) + result.In = strings.TrimSpace(pTag.In) + result.When = pTag.When + result.Scope = pTag.Scope + result.DataType = pTag.DataType + result.Required = pTag.Required + result.Async = pTag.Async + result.Cacheable = pTag.Cacheable + result.With = pTag.With + result.URI = pTag.URI + result.ErrorCode = pTag.ErrorCode + result.ErrorMessage = pTag.ErrorMessage + + result.EffectiveType = resolveStateType(result, field.Type) + return result +} + +func resolveStateType(item *State, fallback reflect.Type) reflect.Type { + key := strings.ToLower(strings.TrimSpace(firstNonEmpty(item.In, item.Name))) + switch item.Kind { + case "output": + if rType, ok := outputkeys.Types[key]; ok { + return rType + } + case "meta": + if rType, ok := metakeys.Types[key]; ok { + return rType + } + case "async": + if rType, ok := keys.Types[key]; ok { + return rType + } + } + return fallback +} + +func componentType(rType reflect.Type) (reflect.Type, string) { + if rType == nil { + return nil, "one" + } + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + if rType.Kind() == reflect.Slice { + elem := rType.Elem() + for elem.Kind() == reflect.Ptr { + elem = elem.Elem() + } + return elem, "many" + } + return rType, "one" +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if trimmed := strings.TrimSpace(value); trimmed != "" { + return trimmed + } + } + return "" +} diff --git a/repository/shape/plan/planner_test.go b/repository/shape/plan/planner_test.go new file mode 100644 index 000000000..29bb1e792 --- /dev/null +++ b/repository/shape/plan/planner_test.go @@ -0,0 +1,86 @@ +package plan + +import ( + "context" + "embed" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + asynckeys "github.com/viant/datly/repository/locator/async/keys" + metakeys "github.com/viant/datly/repository/locator/meta/keys" + outputkeys "github.com/viant/datly/repository/locator/output/keys" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/scan" +) + +//go:embed testdata/*.sql +var testFS embed.FS + +type embeddedFS struct{} + +func (embeddedFS) EmbedFS() *embed.FS { + return &testFS +} + +type reportRow struct { + ID int +} + +type reportSource struct { + embeddedFS + Rows []reportRow `view:"rows,table=REPORT,connector=dev" sql:"uri=testdata/report.sql"` + Status interface{} `parameter:"status,kind=output,in=status"` + Job interface{} `parameter:"job,kind=async,in=job"` + VName interface{} `parameter:"viewName,kind=meta,in=view.name"` + ID int `parameter:"id,kind=query,in=id"` +} + +func TestPlanner_Plan(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportSource{}}) + require.NoError(t, err) + + planner := New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + require.NotNil(t, planned) + + result, ok := planned.Plan.(*Result) + require.True(t, ok) + require.NotNil(t, result) + require.NotNil(t, result.EmbedFS) + + require.Len(t, result.Views, 1) + rows := result.Views[0] + assert.Equal(t, "rows", rows.Name) + assert.Equal(t, "REPORT", rows.Table) + assert.Equal(t, "dev", rows.Connector) + assert.Equal(t, "many", rows.Cardinality) + assert.Equal(t, "Rows", rows.Holder) + assert.Contains(t, rows.SQL, "SELECT ID") + + stateByPath := map[string]*State{} + for _, item := range result.States { + stateByPath[item.Path] = item + } + + require.NotNil(t, stateByPath["Status"]) + assert.Equal(t, outputkeys.Types["status"], stateByPath["Status"].EffectiveType) + require.NotNil(t, stateByPath["Job"]) + assert.Equal(t, asynckeys.Types["job"], stateByPath["Job"].EffectiveType) + require.NotNil(t, stateByPath["VName"]) + assert.Equal(t, metakeys.Types["view.name"], stateByPath["VName"].EffectiveType) + + require.NotNil(t, stateByPath["ID"]) + assert.Equal(t, "query", stateByPath["ID"].Kind) + assert.Equal(t, "id", stateByPath["ID"].In) + assert.Equal(t, stateByPath["ID"].TagType, stateByPath["ID"].EffectiveType) +} + +func TestPlanner_Plan_InvalidDescriptors(t *testing.T) { + planner := New() + _, err := planner.Plan(context.Background(), &shape.ScanResult{Source: &shape.Source{Name: "x"}, Descriptors: "invalid"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported descriptors type") +} diff --git a/repository/shape/plan/testdata/report.sql b/repository/shape/plan/testdata/report.sql new file mode 100644 index 000000000..7aab3a1f8 --- /dev/null +++ b/repository/shape/plan/testdata/report.sql @@ -0,0 +1 @@ +SELECT ID FROM REPORT diff --git a/repository/shape/scan/doc.go b/repository/shape/scan/doc.go new file mode 100644 index 000000000..e1f105775 --- /dev/null +++ b/repository/shape/scan/doc.go @@ -0,0 +1,2 @@ +// Package scan defines scanning responsibilities for struct/DQL inputs. +package scan diff --git a/repository/shape/scan/model.go b/repository/shape/scan/model.go new file mode 100644 index 000000000..357299250 --- /dev/null +++ b/repository/shape/scan/model.go @@ -0,0 +1,33 @@ +package scan + +import ( + "embed" + "reflect" + + "github.com/viant/datly/view/tags" +) + +// Result holds scan output produced from a struct source. +type Result struct { + RootType reflect.Type + EmbedFS *embed.FS + Fields []*Field + ByPath map[string]*Field + ViewFields []*Field + StateFields []*Field +} + +// Field describes one scanned struct field. +type Field struct { + Path string + Name string + Index []int + Type reflect.Type + Tag reflect.StructTag + Anonymous bool + + HasViewTag bool + HasStateTag bool + ViewTag *tags.Tag + StateTag *tags.Tag +} diff --git a/repository/shape/scan/scanner.go b/repository/shape/scan/scanner.go new file mode 100644 index 000000000..d15d34f32 --- /dev/null +++ b/repository/shape/scan/scanner.go @@ -0,0 +1,166 @@ +package scan + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/view/state" + "github.com/viant/datly/view/tags" +) + +// StructScanner scans arbitrary struct types and extracts Datly-relevant tags. +type StructScanner struct{} + +// New returns a Scanner implementation for shape facade. +func New() *StructScanner { + return &StructScanner{} +} + +// Scan implements shape.Scanner. +func (s *StructScanner) Scan(_ context.Context, source *shape.Source, _ ...shape.ScanOption) (*shape.ScanResult, error) { + if source == nil { + return nil, shape.ErrNilSource + } + source.EnsureTypeRegistry() + + root, err := resolveRootType(source) + if err != nil { + return nil, err + } + + embedder := resolveEmbedder(source) + result := &Result{ + RootType: root, + EmbedFS: embedder.EmbedFS(), + ByPath: map[string]*Field{}, + } + + if err = s.scanStruct(root, "", nil, embedder, result, map[reflect.Type]bool{}); err != nil { + return nil, err + } + + return &shape.ScanResult{Source: source, Descriptors: result}, nil +} + +func resolveRootType(source *shape.Source) (reflect.Type, error) { + rType, err := source.ResolveRootType() + if err != nil { + return nil, err + } + if rType == nil { + return nil, shape.ErrNilSource + } + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return nil, fmt.Errorf("shape scan: unsupported source type %v, expected struct", rType) + } + return rType, nil +} + +func resolveEmbedder(source *shape.Source) *state.FSEmbedder { + embedder := state.NewFSEmbedder(nil) + if source.Type != nil { + rType := source.Type + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + embedder.SetType(rType) + return embedder + } + if source.Struct != nil { + rType := reflect.TypeOf(source.Struct) + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + embedder.SetType(rType) + } + return embedder +} + +func (s *StructScanner) scanStruct( + rType reflect.Type, + prefix string, + indexPrefix []int, + embedder *state.FSEmbedder, + result *Result, + visited map[reflect.Type]bool, +) error { + if visited[rType] { + return nil + } + visited[rType] = true + defer delete(visited, rType) + + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + path := field.Name + if prefix != "" { + path = prefix + "." + field.Name + } + combinedIndex := append(append([]int{}, indexPrefix...), field.Index...) + + descriptor := &Field{ + Path: path, + Name: field.Name, + Index: combinedIndex, + Type: field.Type, + Tag: field.Tag, + Anonymous: field.Anonymous, + } + + if hasAny(field.Tag, tags.ViewTag, tags.SQLTag, tags.SQLSummaryTag, tags.LinkOnTag) { + parsed, err := tags.ParseViewTags(field.Tag, embedder.EmbedFS()) + if err != nil { + return fmt.Errorf("shape scan: failed to parse view tags on %s: %w", path, err) + } + descriptor.HasViewTag = true + descriptor.ViewTag = parsed + result.ViewFields = append(result.ViewFields, descriptor) + } + + if hasAny(field.Tag, tags.ParameterTag, tags.SQLTag, tags.PredicateTag, tags.CodecTag, tags.HandlerTag) { + parsed, err := tags.ParseStateTags(field.Tag, embedder.EmbedFS()) + if err != nil { + return fmt.Errorf("shape scan: failed to parse state tags on %s: %w", path, err) + } + descriptor.HasStateTag = true + descriptor.StateTag = parsed + result.StateFields = append(result.StateFields, descriptor) + } + + result.Fields = append(result.Fields, descriptor) + result.ByPath[path] = descriptor + + nextType := field.Type + for nextType.Kind() == reflect.Ptr { + nextType = nextType.Elem() + } + if field.Anonymous && nextType.Kind() == reflect.Struct && !isStdlib(nextType.PkgPath()) { + if err := s.scanStruct(nextType, path, combinedIndex, embedder, result, visited); err != nil { + return err + } + } + } + return nil +} + +func hasAny(tag reflect.StructTag, names ...string) bool { + for _, name := range names { + if _, ok := tag.Lookup(name); ok { + return true + } + } + return false +} + +func isStdlib(pkg string) bool { + if pkg == "" { + return true + } + return !strings.Contains(pkg, ".") +} diff --git a/repository/shape/scan/scanner_test.go b/repository/shape/scan/scanner_test.go new file mode 100644 index 000000000..7cce9cbce --- /dev/null +++ b/repository/shape/scan/scanner_test.go @@ -0,0 +1,83 @@ +package scan + +import ( + "context" + "embed" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + "github.com/viant/x" +) + +//go:embed testdata/*.sql +var testFS embed.FS + +type embeddedFS struct{} + +func (embeddedFS) EmbedFS() *embed.FS { + return &testFS +} + +type reportRow struct { + ID int + Name string +} + +type reportSource struct { + embeddedFS + Rows []reportRow `view:"rows,table=REPORT,connector=dev" sql:"uri=testdata/report.sql"` + ID int `parameter:"id,kind=query,in=id"` +} + +func TestStructScanner_Scan(t *testing.T) { + scanner := New() + result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportSource{}}) + require.NoError(t, err) + require.NotNil(t, result) + + descriptors, ok := result.Descriptors.(*Result) + require.True(t, ok) + require.NotNil(t, descriptors) + require.NotNil(t, descriptors.EmbedFS) + assert.Equal(t, reflect.TypeOf(reportSource{}), descriptors.RootType) + + rows := descriptors.ByPath["Rows"] + require.NotNil(t, rows) + require.True(t, rows.HasViewTag) + require.NotNil(t, rows.ViewTag) + assert.Equal(t, "rows", rows.ViewTag.View.Name) + assert.Contains(t, rows.ViewTag.SQL.SQL, "SELECT ID, NAME FROM REPORT") + + idField := descriptors.ByPath["ID"] + require.NotNil(t, idField) + require.True(t, idField.HasStateTag) + require.NotNil(t, idField.StateTag) + require.NotNil(t, idField.StateTag.Parameter) + assert.Equal(t, "id", idField.StateTag.Parameter.Name) + assert.Equal(t, "query", idField.StateTag.Parameter.Kind) + assert.Equal(t, "id", idField.StateTag.Parameter.In) +} + +func TestStructScanner_Scan_InvalidSource(t *testing.T) { + scanner := New() + _, err := scanner.Scan(context.Background(), &shape.Source{Struct: 1}) + require.Error(t, err) + assert.Contains(t, err.Error(), "expected struct") +} + +func TestStructScanner_Scan_WithRegistryType(t *testing.T) { + scanner := New() + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(reportSource{}))) + result, err := scanner.Scan(context.Background(), &shape.Source{ + TypeName: "github.com/viant/datly/repository/shape/scan.reportSource", + TypeRegistry: registry, + }) + require.NoError(t, err) + descriptors, ok := result.Descriptors.(*Result) + require.True(t, ok) + assert.Equal(t, reflect.TypeOf(reportSource{}), descriptors.RootType) +} diff --git a/repository/shape/scan/testdata/report.sql b/repository/shape/scan/testdata/report.sql new file mode 100644 index 000000000..68f0f3b34 --- /dev/null +++ b/repository/shape/scan/testdata/report.sql @@ -0,0 +1 @@ +SELECT ID, NAME FROM REPORT diff --git a/repository/shape/shape.go b/repository/shape/shape.go new file mode 100644 index 000000000..570a63d5e --- /dev/null +++ b/repository/shape/shape.go @@ -0,0 +1,157 @@ +package shape + +import "context" + +type ( + // Scanner discovers shape descriptors from Source. + Scanner interface { + Scan(ctx context.Context, source *Source, opts ...ScanOption) (*ScanResult, error) + } + + // Planner normalizes discovered descriptors into execution plan. + Planner interface { + Plan(ctx context.Context, scan *ScanResult, opts ...PlanOption) (*PlanResult, error) + } + + // Loader materializes runtime artifacts from normalized plan. + Loader interface { + LoadViews(ctx context.Context, plan *PlanResult, opts ...LoadOption) (*ViewArtifacts, error) + LoadComponent(ctx context.Context, plan *PlanResult, opts ...LoadOption) (*ComponentArtifact, error) + } + + // DQLCompiler compiles DQL source directly into a shape plan. + DQLCompiler interface { + Compile(ctx context.Context, source *Source, opts ...CompileOption) (*PlanResult, error) + } + + // RuntimeRegistrar optionally registers loaded artifacts in runtime services. + RuntimeRegistrar interface { + RegisterViews(ctx context.Context, artifacts *ViewArtifacts) error + RegisterComponent(ctx context.Context, artifacts *ComponentArtifact) error + } + + ScanOptions struct{} + PlanOptions struct{} + LoadOptions struct{} + CompileOptions struct{} + + ScanOption func(*ScanOptions) + PlanOption func(*PlanOptions) + LoadOption func(*LoadOptions) + CompileOption func(*CompileOptions) +) + +// Engine is a thin facade over scan -> plan -> load pipeline. +type Engine struct { + options *Options +} + +// New creates an Engine facade. +func New(opts ...Option) *Engine { + return &Engine{options: NewOptions(opts...)} +} + +// LoadViews is a package-level helper for struct source view loading. +func LoadViews(ctx context.Context, src any, opts ...Option) (*ViewArtifacts, error) { + return New(opts...).LoadViews(ctx, src) +} + +// LoadComponent is a package-level helper for struct source component loading. +func LoadComponent(ctx context.Context, src any, opts ...Option) (*ComponentArtifact, error) { + return New(opts...).LoadComponent(ctx, src) +} + +// LoadDQLViews is a package-level helper for DQL source view loading. +func LoadDQLViews(ctx context.Context, dql string, opts ...Option) (*ViewArtifacts, error) { + return New(opts...).LoadDQLViews(ctx, dql) +} + +// LoadDQLComponent is a package-level helper for DQL source component loading. +func LoadDQLComponent(ctx context.Context, dql string, opts ...Option) (*ComponentArtifact, error) { + return New(opts...).LoadDQLComponent(ctx, dql) +} + +// LoadViews executes scan -> plan -> load for struct source. +func (e *Engine) LoadViews(ctx context.Context, src any) (*ViewArtifacts, error) { + source, err := e.structSource(src) + if err != nil { + return nil, err + } + plan, err := e.scanAndPlan(ctx, source) + if err != nil { + return nil, err + } + if e.options.Loader == nil { + return nil, ErrLoaderNotConfigured + } + return e.options.Loader.LoadViews(ctx, plan) +} + +// LoadComponent executes scan -> plan -> load for struct source. +func (e *Engine) LoadComponent(ctx context.Context, src any) (*ComponentArtifact, error) { + source, err := e.structSource(src) + if err != nil { + return nil, err + } + plan, err := e.scanAndPlan(ctx, source) + if err != nil { + return nil, err + } + if e.options.Loader == nil { + return nil, ErrLoaderNotConfigured + } + return e.options.Loader.LoadComponent(ctx, plan) +} + +// LoadDQLViews executes compile -> load for DQL source. +func (e *Engine) LoadDQLViews(ctx context.Context, dql string) (*ViewArtifacts, error) { + source, err := e.dqlSource(dql) + if err != nil { + return nil, err + } + plan, err := e.compile(ctx, source) + if err != nil { + return nil, err + } + if e.options.Loader == nil { + return nil, ErrLoaderNotConfigured + } + return e.options.Loader.LoadViews(ctx, plan) +} + +// LoadDQLComponent executes compile -> load for DQL source. +func (e *Engine) LoadDQLComponent(ctx context.Context, dql string) (*ComponentArtifact, error) { + source, err := e.dqlSource(dql) + if err != nil { + return nil, err + } + plan, err := e.compile(ctx, source) + if err != nil { + return nil, err + } + if e.options.Loader == nil { + return nil, ErrLoaderNotConfigured + } + return e.options.Loader.LoadComponent(ctx, plan) +} + +func (e *Engine) compile(ctx context.Context, source *Source) (*PlanResult, error) { + if e.options.Compiler == nil { + return nil, ErrCompilerNotConfigured + } + return e.options.Compiler.Compile(ctx, source) +} + +func (e *Engine) scanAndPlan(ctx context.Context, source *Source) (*PlanResult, error) { + if e.options.Scanner == nil { + return nil, ErrScannerNotConfigured + } + if e.options.Planner == nil { + return nil, ErrPlannerNotConfigured + } + scanResult, err := e.options.Scanner.Scan(ctx, source) + if err != nil { + return nil, err + } + return e.options.Planner.Plan(ctx, scanResult) +} diff --git a/repository/shape/source.go b/repository/shape/source.go new file mode 100644 index 000000000..e408c2163 --- /dev/null +++ b/repository/shape/source.go @@ -0,0 +1,39 @@ +package shape + +import ( + "reflect" + "strings" + + "github.com/viant/x" +) + +func (e *Engine) structSource(src any) (*Source, error) { + if src == nil { + return nil, ErrNilSource + } + rType := reflect.TypeOf(src) + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + registry := x.NewRegistry() + registry.Register(x.NewType(rType)) + return &Source{ + Name: e.options.Name, + Struct: src, + Type: rType, + TypeName: x.NewType(rType).Key(), + TypeRegistry: registry, + DQL: "", + }, nil +} + +func (e *Engine) dqlSource(dql string) (*Source, error) { + dql = strings.TrimSpace(dql) + if dql == "" { + return nil, ErrNilDQL + } + return &Source{ + Name: e.options.Name, + DQL: dql, + }, nil +} diff --git a/repository/shape/source_type.go b/repository/shape/source_type.go new file mode 100644 index 000000000..51bc7132e --- /dev/null +++ b/repository/shape/source_type.go @@ -0,0 +1,56 @@ +package shape + +import ( + "fmt" + "reflect" + "strings" + + "github.com/viant/x" +) + +// ResolveRootType resolves source root type from explicit Type, Struct, or viant/x registry. +func (s *Source) ResolveRootType() (reflect.Type, error) { + if s == nil { + return nil, ErrNilSource + } + if s.Type != nil { + return unwrapPtr(s.Type), nil + } + if s.Struct != nil { + return unwrapPtr(reflect.TypeOf(s.Struct)), nil + } + key := strings.TrimSpace(s.TypeName) + if key == "" || s.TypeRegistry == nil { + return nil, ErrNilSource + } + aType := s.TypeRegistry.Lookup(key) + if aType == nil || aType.Type == nil { + return nil, fmt.Errorf("shape source: type %q not found in registry", key) + } + return unwrapPtr(aType.Type), nil +} + +// EnsureTypeRegistry returns source registry ensuring root type is registered when available. +func (s *Source) EnsureTypeRegistry() *x.Registry { + if s == nil { + return nil + } + if s.TypeRegistry == nil { + s.TypeRegistry = x.NewRegistry() + } + if rType, err := s.ResolveRootType(); err == nil && rType != nil { + t := x.NewType(rType) + if strings.TrimSpace(s.TypeName) == "" { + s.TypeName = t.Key() + } + s.TypeRegistry.Register(t) + } + return s.TypeRegistry +} + +func unwrapPtr(rType reflect.Type) reflect.Type { + for rType != nil && rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + return rType +} diff --git a/repository/shape/source_type_test.go b/repository/shape/source_type_test.go new file mode 100644 index 000000000..3118f8fed --- /dev/null +++ b/repository/shape/source_type_test.go @@ -0,0 +1,33 @@ +package shape + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/x" +) + +type sampleShape struct { + ID int +} + +func TestSource_ResolveRootType_FromRegistry(t *testing.T) { + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(sampleShape{}))) + src := &Source{ + TypeName: "github.com/viant/datly/repository/shape.sampleShape", + TypeRegistry: registry, + } + rType, err := src.ResolveRootType() + require.NoError(t, err) + require.Equal(t, reflect.TypeOf(sampleShape{}), rType) +} + +func TestSource_EnsureTypeRegistry_RegistersRoot(t *testing.T) { + src := &Source{Struct: &sampleShape{}} + registry := src.EnsureTypeRegistry() + require.NotNil(t, registry) + require.NotEmpty(t, src.TypeName) + require.NotNil(t, registry.Lookup(src.TypeName)) +} diff --git a/repository/shape/typectx/model.go b/repository/shape/typectx/model.go new file mode 100644 index 000000000..ae76febe5 --- /dev/null +++ b/repository/shape/typectx/model.go @@ -0,0 +1,29 @@ +package typectx + +// Import describes one package alias import for DQL/type resolution. +type Import struct { + Alias string `json:",omitempty" yaml:",omitempty"` + Package string `json:",omitempty" yaml:",omitempty"` +} + +// Context captures default package and imports used for type resolution. +type Context struct { + DefaultPackage string `json:",omitempty" yaml:",omitempty"` + Imports []Import `json:",omitempty" yaml:",omitempty"` +} + +// Provenance tracks where a resolved type came from. +type Provenance struct { + Package string `json:",omitempty" yaml:",omitempty"` + File string `json:",omitempty" yaml:",omitempty"` + Kind string `json:",omitempty" yaml:",omitempty"` // builtin, resource_type, registry, ast_type +} + +// Resolution captures one resolved type expression and its provenance. +type Resolution struct { + Expression string `json:",omitempty" yaml:",omitempty"` + Target string `json:",omitempty" yaml:",omitempty"` + ResolvedKey string `json:",omitempty" yaml:",omitempty"` + MatchKind string `json:",omitempty" yaml:",omitempty"` // exact, alias_import, qualified, default_package, import_package, global_unique + Provenance Provenance `json:",omitempty" yaml:",omitempty"` +} diff --git a/repository/shape/typectx/resolver.go b/repository/shape/typectx/resolver.go new file mode 100644 index 000000000..daccf3b42 --- /dev/null +++ b/repository/shape/typectx/resolver.go @@ -0,0 +1,293 @@ +package typectx + +import ( + "fmt" + "path" + "sort" + "strings" + + "github.com/viant/x" +) + +// AmbiguityError reports multiple matching type candidates for a type expression. +type AmbiguityError struct { + Expression string + Candidates []string +} + +func (e *AmbiguityError) Error() string { + return fmt.Sprintf("ambiguous type %q: candidates=%s", e.Expression, strings.Join(e.Candidates, ",")) +} + +// Resolver resolves cast/tag type expressions against viant/x registry using type context. +type Resolver struct { + registry *x.Registry + context *Context + provenance map[string]Provenance +} + +// NewResolver creates a type resolver. +func NewResolver(registry *x.Registry, context *Context) *Resolver { + return NewResolverWithProvenance(registry, context, nil) +} + +// NewResolverWithProvenance creates a type resolver with optional registry-key provenance map. +func NewResolverWithProvenance(registry *x.Registry, context *Context, provenance map[string]Provenance) *Resolver { + return &Resolver{ + registry: registry, + context: normalizeContext(context), + provenance: cloneProvenance(provenance), + } +} + +// Resolve resolves type expression to registry key. It returns ("", nil) when unresolved. +func (r *Resolver) Resolve(typeExpr string) (string, error) { + resolved, err := r.ResolveWithProvenance(typeExpr) + if err != nil || resolved == nil { + return "", err + } + return resolved.ResolvedKey, nil +} + +// ResolveWithProvenance resolves expression and returns provenance details. +// It returns (nil, nil) when unresolved. +func (r *Resolver) ResolveWithProvenance(typeExpr string) (*Resolution, error) { + if r == nil || r.registry == nil { + return nil, nil + } + base := normalizeLookupKey(typeExpr) + if base == "" { + return nil, nil + } + + // Exact type key (builtins or fully-qualified package.Type) + if r.registry.Lookup(base) != nil { + return r.newResolution(typeExpr, "", base, "exact"), nil + } + + prefix, baseName, alias, qualified := splitQualified(base) + if qualified { + if prefix == "" || baseName == "" { + return nil, nil + } + if alias { + pkg := r.aliasPackage(prefix) + if pkg == "" { + return nil, nil + } + candidate := pkg + "." + baseName + if r.registry.Lookup(candidate) == nil { + return nil, nil + } + return r.newResolution(typeExpr, "", candidate, "alias_import"), nil + } + // fully qualified package path.Type + if r.registry.Lookup(base) != nil { + return r.newResolution(typeExpr, "", base, "qualified"), nil + } + return nil, nil + } + + // Unqualified resolution: default package, then imports; if still unresolved, + // fallback to unique global name match. + candidates := r.unqualifiedCandidates(baseName) + if len(candidates) == 1 { + return r.newResolution(typeExpr, "", candidates[0].key, candidates[0].matchKind), nil + } + if len(candidates) > 1 { + keys := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + keys = append(keys, candidate.key) + } + sort.Strings(keys) + return nil, &AmbiguityError{Expression: typeExpr, Candidates: keys} + } + return nil, nil +} + +func (r *Resolver) aliasPackage(alias string) string { + alias = strings.TrimSpace(alias) + if alias == "" || r.context == nil { + return "" + } + for _, item := range r.context.Imports { + if item.Alias == alias { + return item.Package + } + } + return "" +} + +type candidate struct { + key string + matchKind string +} + +func (r *Resolver) unqualifiedCandidates(typeName string) []candidate { + if typeName == "" { + return nil + } + seen := map[string]bool{} + var result []candidate + + for _, scoped := range r.searchPackages() { + pkg := scoped.pkg + key := pkg + "." + typeName + if seen[key] { + continue + } + seen[key] = true + if r.registry.Lookup(key) != nil { + result = append(result, candidate{key: key, matchKind: scoped.matchKind}) + } + } + if len(result) > 0 { + return result + } + + // Global unique fallback by suffix ".TypeName" or exact built-in. + for _, key := range r.registry.Keys() { + if key == typeName || strings.HasSuffix(key, "."+typeName) { + if seen[key] { + continue + } + seen[key] = true + result = append(result, candidate{key: key, matchKind: "global_unique"}) + } + } + return result +} + +type scopedPackage struct { + pkg string + matchKind string +} + +func (r *Resolver) searchPackages() []scopedPackage { + if r.context == nil { + return nil + } + seen := map[string]bool{} + var result []scopedPackage + appendPkg := func(pkg, matchKind string) { + pkg = strings.TrimSpace(pkg) + if pkg == "" || seen[pkg] { + return + } + seen[pkg] = true + result = append(result, scopedPackage{pkg: pkg, matchKind: matchKind}) + } + appendPkg(r.context.DefaultPackage, "default_package") + for _, item := range r.context.Imports { + appendPkg(item.Package, "import_package") + } + return result +} + +func (r *Resolver) newResolution(expression, target, key, matchKind string) *Resolution { + if key == "" { + return nil + } + resolution := &Resolution{ + Expression: strings.TrimSpace(expression), + Target: strings.TrimSpace(target), + ResolvedKey: key, + MatchKind: matchKind, + Provenance: r.lookupProvenance(key), + } + return resolution +} + +func (r *Resolver) lookupProvenance(key string) Provenance { + prov := Provenance{ + Package: packageOf(key), + Kind: "registry", + } + if built, ok := r.provenance[key]; ok { + if built.Package != "" { + prov.Package = built.Package + } + if built.File != "" { + prov.File = built.File + } + if built.Kind != "" { + prov.Kind = built.Kind + } + } + return prov +} + +func cloneProvenance(input map[string]Provenance) map[string]Provenance { + if len(input) == 0 { + return nil + } + result := make(map[string]Provenance, len(input)) + for k, v := range input { + result[k] = v + } + return result +} + +func packageOf(key string) string { + index := strings.LastIndex(key, ".") + if index == -1 { + return "" + } + return key[:index] +} + +func normalizeContext(input *Context) *Context { + if input == nil { + return nil + } + ret := &Context{ + DefaultPackage: strings.TrimSpace(input.DefaultPackage), + } + for _, item := range input.Imports { + pkg := strings.TrimSpace(item.Package) + if pkg == "" { + continue + } + alias := strings.TrimSpace(item.Alias) + if alias == "" { + alias = path.Base(pkg) + } + ret.Imports = append(ret.Imports, Import{ + Alias: alias, + Package: pkg, + }) + } + if ret.DefaultPackage == "" && len(ret.Imports) == 0 { + return nil + } + return ret +} + +func splitQualified(value string) (prefix string, name string, alias bool, qualified bool) { + index := strings.LastIndex(value, ".") + if index == -1 { + return "", value, false, false + } + prefix = strings.TrimSpace(value[:index]) + name = strings.TrimSpace(value[index+1:]) + if prefix == "" || name == "" { + return "", "", false, false + } + qualified = true + alias = !strings.Contains(prefix, "/") + return prefix, name, alias, qualified +} + +func normalizeLookupKey(typeExpr string) string { + value := strings.TrimSpace(typeExpr) + for { + switch { + case strings.HasPrefix(value, "*"): + value = strings.TrimPrefix(value, "*") + case strings.HasPrefix(value, "[]"): + value = strings.TrimPrefix(value, "[]") + default: + return strings.TrimSpace(value) + } + } +} diff --git a/repository/shape/typectx/resolver_memfs_test.go b/repository/shape/typectx/resolver_memfs_test.go new file mode 100644 index 000000000..cc90b4700 --- /dev/null +++ b/repository/shape/typectx/resolver_memfs_test.go @@ -0,0 +1,116 @@ +package typectx + +import ( + "context" + "path" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/require" + "github.com/viant/x" + xast "github.com/viant/x/loader/ast" +) + +func TestResolver_MemFS_DefaultPackageResolution(t *testing.T) { + resolver := memFSResolver(t, baseTypeMapFS(), []string{"root/perf"}, &Context{ + DefaultPackage: "example.com/acme/perf", + }) + + key, err := resolver.Resolve("Order") + require.NoError(t, err) + require.Equal(t, "example.com/acme/perf.Order", key) +} + +func TestResolver_MemFS_AliasImportResolution(t *testing.T) { + resolver := memFSResolver(t, baseTypeMapFS(), []string{"root/perf"}, &Context{ + Imports: []Import{ + {Alias: "pf", Package: "example.com/acme/perf"}, + }, + }) + + key, err := resolver.Resolve("pf.Order") + require.NoError(t, err) + require.Equal(t, "example.com/acme/perf.Order", key) +} + +func TestResolver_MemFS_AmbiguityDetection(t *testing.T) { + resolver := memFSResolver(t, baseTypeMapFS(), []string{"root/perf", "root/shared"}, &Context{ + Imports: []Import{ + {Alias: "pf", Package: "example.com/acme/perf"}, + {Alias: "sh", Package: "example.com/acme/shared"}, + }, + }) + + key, err := resolver.Resolve("Fee") + require.Empty(t, key) + require.Error(t, err) + amb, ok := err.(*AmbiguityError) + require.True(t, ok) + require.Equal(t, []string{ + "example.com/acme/perf.Fee", + "example.com/acme/shared.Fee", + }, amb.Candidates) +} + +func TestResolver_MemFS_ProvenanceCapture(t *testing.T) { + resolver := memFSResolver(t, baseTypeMapFS(), []string{"root/perf"}, &Context{ + DefaultPackage: "example.com/acme/perf", + }) + + resolved, err := resolver.ResolveWithProvenance("Order") + require.NoError(t, err) + require.NotNil(t, resolved) + require.Equal(t, "example.com/acme/perf.Order", resolved.ResolvedKey) + require.Equal(t, "default_package", resolved.MatchKind) + require.Equal(t, "ast_type", resolved.Provenance.Kind) + require.Equal(t, "example.com/acme/perf", resolved.Provenance.Package) + require.Equal(t, "root/perf/types.go", resolved.Provenance.File) +} + +func memFSResolver(t *testing.T, fsys fstest.MapFS, packageDirs []string, ctx *Context) *Resolver { + t.Helper() + registry := x.NewRegistry() + provenance := map[string]Provenance{} + for _, dir := range packageDirs { + pkg, err := xast.LoadPackageFS(context.Background(), fsys, dir) + require.NoError(t, err) + + fileByType := map[string]string{} + for _, file := range pkg.Files { + if file == nil { + continue + } + for _, item := range file.Types { + if item == nil || item.Name == "" { + continue + } + fileByType[item.Name] = path.Join(dir, file.Name) + } + } + for _, item := range pkg.Types { + if item == nil || item.Name == "" { + continue + } + aType := &x.Type{ + Name: item.Name, + PkgPath: pkg.PkgPath, + } + registry.Register(aType) + provenance[aType.Key()] = Provenance{ + Package: pkg.PkgPath, + File: fileByType[item.Name], + Kind: "ast_type", + } + } + } + return NewResolverWithProvenance(registry, ctx, provenance) +} + +func baseTypeMapFS() fstest.MapFS { + return fstest.MapFS{ + "root/go.mod": &fstest.MapFile{Data: []byte("module example.com/acme\n\ngo 1.23\n")}, + "root/perf/types.go": &fstest.MapFile{Data: []byte("package perf\n\ntype Order struct{}\ntype Fee struct{}\n")}, + "root/shared/types.go": &fstest.MapFile{Data: []byte("package shared\n\ntype Fee struct{}\n")}, + "root/ignore/other.txt": &fstest.MapFile{Data: []byte("skip")}, + } +} diff --git a/repository/shape/typectx/resolver_test.go b/repository/shape/typectx/resolver_test.go new file mode 100644 index 000000000..f1e8e6761 --- /dev/null +++ b/repository/shape/typectx/resolver_test.go @@ -0,0 +1,89 @@ +package typectx + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/x" +) + +type resolveFeeA struct{} +type resolveFeeB struct{} +type resolveOrder struct{} + +func TestResolver_Resolve_Unqualified_DefaultPackage(t *testing.T) { + reg := x.NewRegistry() + reg.Register(x.NewType(reflect.TypeOf(resolveOrder{}), x.WithPkgPath("github.com/acme/mdp/performance"), x.WithName("Order"))) + resolver := NewResolver(reg, &Context{DefaultPackage: "github.com/acme/mdp/performance"}) + + key, err := resolver.Resolve("Order") + require.NoError(t, err) + require.Equal(t, "github.com/acme/mdp/performance.Order", key) +} + +func TestResolver_Resolve_AliasQualified(t *testing.T) { + reg := x.NewRegistry() + reg.Register(x.NewType(reflect.TypeOf(resolveOrder{}), x.WithPkgPath("github.com/acme/mdp/performance"), x.WithName("Order"))) + resolver := NewResolver(reg, &Context{ + Imports: []Import{ + {Alias: "perf", Package: "github.com/acme/mdp/performance"}, + }, + }) + + key, err := resolver.Resolve("perf.Order") + require.NoError(t, err) + require.Equal(t, "github.com/acme/mdp/performance.Order", key) +} + +func TestResolver_Resolve_Unqualified_Ambiguous(t *testing.T) { + reg := x.NewRegistry() + reg.Register(x.NewType(reflect.TypeOf(resolveFeeA{}), x.WithPkgPath("github.com/acme/alpha"), x.WithName("Fee"))) + reg.Register(x.NewType(reflect.TypeOf(resolveFeeB{}), x.WithPkgPath("github.com/acme/beta"), x.WithName("Fee"))) + resolver := NewResolver(reg, &Context{ + Imports: []Import{ + {Alias: "a", Package: "github.com/acme/alpha"}, + {Alias: "b", Package: "github.com/acme/beta"}, + }, + }) + + key, err := resolver.Resolve("Fee") + require.Empty(t, key) + require.Error(t, err) + amb, ok := err.(*AmbiguityError) + require.True(t, ok) + require.Equal(t, []string{ + "github.com/acme/alpha.Fee", + "github.com/acme/beta.Fee", + }, amb.Candidates) +} + +func TestResolver_Resolve_Unqualified_GlobalUniqueFallback(t *testing.T) { + reg := x.NewRegistry() + reg.Register(x.NewType(reflect.TypeOf(resolveOrder{}), x.WithPkgPath("github.com/acme/shared"), x.WithName("Order"))) + resolver := NewResolver(reg, nil) + + key, err := resolver.Resolve("Order") + require.NoError(t, err) + require.Equal(t, "github.com/acme/shared.Order", key) +} + +func TestResolver_ResolveWithProvenance(t *testing.T) { + reg := x.NewRegistry() + reg.Register(x.NewType(reflect.TypeOf(resolveOrder{}), x.WithPkgPath("github.com/acme/mdp/performance"), x.WithName("Order"))) + resolver := NewResolverWithProvenance(reg, &Context{DefaultPackage: "github.com/acme/mdp/performance"}, map[string]Provenance{ + "github.com/acme/mdp/performance.Order": { + Package: "github.com/acme/mdp/performance", + File: "/repo/mdp/performance/order.go", + Kind: "resource_type", + }, + }) + + resolved, err := resolver.ResolveWithProvenance("Order") + require.NoError(t, err) + require.NotNil(t, resolved) + require.Equal(t, "github.com/acme/mdp/performance.Order", resolved.ResolvedKey) + require.Equal(t, "default_package", resolved.MatchKind) + require.Equal(t, "/repo/mdp/performance/order.go", resolved.Provenance.File) + require.Equal(t, "resource_type", resolved.Provenance.Kind) +} diff --git a/repository/shape/typectx/source/resolver.go b/repository/shape/typectx/source/resolver.go new file mode 100644 index 000000000..639787d97 --- /dev/null +++ b/repository/shape/typectx/source/resolver.go @@ -0,0 +1,283 @@ +package source + +import ( + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "golang.org/x/mod/modfile" + "os" + "path/filepath" + "sort" + "strings" +) + +type Config struct { + ProjectDir string + AllowedSourceRoots []string + UseGoModuleResolve bool + UseGOPATHFallback bool +} + +type Resolver struct { + projectDir string + modulePath string + replacements map[string]string + roots []string + useModule bool + useGOPATH bool +} + +func New(cfg Config) (*Resolver, error) { + projectDir := strings.TrimSpace(cfg.ProjectDir) + if projectDir == "" { + return nil, fmt.Errorf("typectx source: project dir was empty") + } + projectDir, err := filepath.Abs(projectDir) + if err != nil { + return nil, err + } + modulePath, replacements := loadModuleConfig(projectDir) + roots := NormalizeRoots(projectDir, cfg.AllowedSourceRoots) + return &Resolver{ + projectDir: projectDir, + modulePath: modulePath, + replacements: replacements, + roots: roots, + useModule: cfg.UseGoModuleResolve, + useGOPATH: cfg.UseGOPATHFallback, + }, nil +} + +func (r *Resolver) ResolvePackageDir(importPath string) (string, error) { + importPath = strings.TrimSpace(importPath) + if importPath == "" { + return "", fmt.Errorf("typectx source: empty import path") + } + if r.useModule { + if resolved := r.resolveReplace(importPath); resolved != "" { + return filepath.Clean(resolved), nil + } + if resolved := r.resolveProjectModule(importPath); resolved != "" { + return filepath.Clean(resolved), nil + } + if resolved := r.resolveModuleCache(importPath); resolved != "" { + return filepath.Clean(resolved), nil + } + } + if r.useGOPATH { + if resolved := resolveGOPATH(importPath); resolved != "" { + return filepath.Clean(resolved), nil + } + } + return "", fmt.Errorf("typectx source: package %s not resolved", importPath) +} + +func (r *Resolver) ResolveTypeFile(importPath, typeName string) (string, error) { + dir, err := r.ResolvePackageDir(importPath) + if err != nil { + return "", err + } + ok, err := IsWithinAnyRoot(dir, r.roots) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("typectx source: package dir %s outside trusted roots", dir) + } + entries, err := os.ReadDir(dir) + if err != nil { + return "", err + } + fset := token.NewFileSet() + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + filePath := filepath.Join(dir, name) + parsed, parseErr := parser.ParseFile(fset, filePath, nil, parser.PackageClauseOnly|parser.ParseComments) + if parseErr != nil || parsed == nil { + continue + } + // Reparse full declaration only when package clause parsing succeeds. + parsed, parseErr = parser.ParseFile(fset, filePath, nil, 0) + if parseErr != nil || parsed == nil { + continue + } + for _, decl := range parsed.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.TYPE { + continue + } + for _, spec := range gen.Specs { + ts, ok := spec.(*ast.TypeSpec) + if ok && ts.Name != nil && ts.Name.Name == typeName { + return filePath, nil + } + } + } + } + return "", fmt.Errorf("typectx source: type %s not found in %s", typeName, importPath) +} + +func (r *Resolver) Roots() []string { + return append([]string(nil), r.roots...) +} + +func (r *Resolver) resolveReplace(importPath string) string { + oldPaths := make([]string, 0, len(r.replacements)) + for old := range r.replacements { + oldPaths = append(oldPaths, old) + } + sort.SliceStable(oldPaths, func(i, j int) bool { return len(oldPaths[i]) > len(oldPaths[j]) }) + for _, old := range oldPaths { + if importPath != old && !strings.HasPrefix(importPath, old+"/") { + continue + } + mapped := r.replacements[old] + suffix := strings.TrimPrefix(importPath, old) + suffix = strings.TrimPrefix(suffix, "/") + if suffix == "" { + return mapped + } + return filepath.Join(mapped, filepath.FromSlash(suffix)) + } + return "" +} + +func (r *Resolver) resolveProjectModule(importPath string) string { + if r.modulePath == "" { + return "" + } + if importPath != r.modulePath && !strings.HasPrefix(importPath, r.modulePath+"/") { + return "" + } + suffix := strings.TrimPrefix(importPath, r.modulePath) + suffix = strings.TrimPrefix(suffix, "/") + if suffix == "" { + return r.projectDir + } + return filepath.Join(r.projectDir, filepath.FromSlash(suffix)) +} + +func (r *Resolver) resolveModuleCache(importPath string) string { + modCache := strings.TrimSpace(os.Getenv("GOMODCACHE")) + if modCache == "" { + if out, err := os.UserCacheDir(); err == nil && out != "" { + modCache = filepath.Join(filepath.Dir(out), "pkg", "mod") + } + } + if modCache == "" { + return "" + } + pattern := filepath.Join(modCache, filepath.FromSlash(importPath)+"@*") + matches, _ := filepath.Glob(pattern) + if len(matches) == 0 { + return "" + } + sort.Strings(matches) + return matches[len(matches)-1] +} + +func resolveGOPATH(importPath string) string { + gopath := strings.TrimSpace(os.Getenv("GOPATH")) + if gopath == "" { + gopath = strings.TrimSpace(build.Default.GOPATH) + } + if gopath == "" { + return "" + } + for _, root := range filepath.SplitList(gopath) { + candidate := filepath.Join(root, "src", filepath.FromSlash(importPath)) + if info, err := os.Stat(candidate); err == nil && info.IsDir() { + return candidate + } + } + return "" +} + +func loadModuleConfig(projectDir string) (string, map[string]string) { + result := map[string]string{} + goModPath := filepath.Join(projectDir, "go.mod") + data, err := os.ReadFile(goModPath) + if err != nil { + return "", result + } + parsed, err := modfile.Parse(goModPath, data, nil) + if err != nil || parsed == nil { + return "", result + } + modulePath := "" + if parsed.Module != nil { + modulePath = strings.TrimSpace(parsed.Module.Mod.Path) + } + for _, replace := range parsed.Replace { + if replace == nil { + continue + } + oldPath := strings.TrimSpace(replace.Old.Path) + newPath := strings.TrimSpace(replace.New.Path) + if oldPath == "" || newPath == "" || replace.New.Version != "" { + continue + } + if !filepath.IsAbs(newPath) { + newPath = filepath.Join(projectDir, newPath) + } + result[oldPath] = filepath.Clean(newPath) + } + return modulePath, result +} + +func NormalizeRoots(projectDir string, allowed []string) []string { + seen := map[string]bool{} + var result []string + appendRoot := func(value string) { + value = strings.TrimSpace(value) + if value == "" { + return + } + if !filepath.IsAbs(value) { + value = filepath.Join(projectDir, value) + } + value = filepath.Clean(value) + if seen[value] { + return + } + seen[value] = true + result = append(result, value) + } + appendRoot(projectDir) + for _, item := range allowed { + appendRoot(item) + } + sort.Strings(result) + return result +} + +func IsWithinAnyRoot(candidate string, roots []string) (bool, error) { + candidate, err := filepath.Abs(candidate) + if err != nil { + return false, err + } + candidate = filepath.Clean(candidate) + for _, root := range roots { + root = filepath.Clean(root) + rel, err := filepath.Rel(root, candidate) + if err != nil { + return false, err + } + if rel == "." { + return true, nil + } + rel = filepath.ToSlash(rel) + if !strings.HasPrefix(rel, "../") { + return true, nil + } + } + return false, nil +} diff --git a/repository/shape/typectx/source/resolver_test.go b/repository/shape/typectx/source/resolver_test.go new file mode 100644 index 000000000..541c11af8 --- /dev/null +++ b/repository/shape/typectx/source/resolver_test.go @@ -0,0 +1,91 @@ +package source + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestResolver_ResolvePackageDir_UsesLocalReplace(t *testing.T) { + root := t.TempDir() + projectDir := filepath.Join(root, "project") + modelsDir := filepath.Join(root, "shared-models") + require.NoError(t, os.MkdirAll(filepath.Join(projectDir, "internal"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(modelsDir, "mdp"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(modelsDir, "go.mod"), []byte("module github.com/acme/models\n\ngo 1.25\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte(`module example.com/project +go 1.25 +replace github.com/acme/models => ../shared-models +`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(modelsDir, "mdp", "types.go"), []byte("package mdp\ntype Order struct{}\n"), 0o644)) + + resolver, err := New(Config{ + ProjectDir: projectDir, + UseGoModuleResolve: true, + UseGOPATHFallback: false, + }) + require.NoError(t, err) + dir, err := resolver.ResolvePackageDir("github.com/acme/models/mdp") + require.NoError(t, err) + require.Equal(t, filepath.Join(modelsDir, "mdp"), dir) +} + +func TestResolver_ResolveTypeFile_RespectsTrustedRoots(t *testing.T) { + root := t.TempDir() + projectDir := filepath.Join(root, "project") + modelsDir := filepath.Join(root, "shared-models") + require.NoError(t, os.MkdirAll(filepath.Join(projectDir, "internal"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(modelsDir, "mdp"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte(`module example.com/project +go 1.25 +replace github.com/acme/models => ../shared-models +`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(modelsDir, "mdp", "types.go"), []byte("package mdp\ntype Order struct{}\n"), 0o644)) + + denyResolver, err := New(Config{ + ProjectDir: projectDir, + UseGoModuleResolve: true, + UseGOPATHFallback: false, + }) + require.NoError(t, err) + _, err = denyResolver.ResolveTypeFile("github.com/acme/models/mdp", "Order") + require.Error(t, err) + + allowResolver, err := New(Config{ + ProjectDir: projectDir, + AllowedSourceRoots: []string{modelsDir}, + UseGoModuleResolve: true, + UseGOPATHFallback: false, + }) + require.NoError(t, err) + file, err := allowResolver.ResolveTypeFile("github.com/acme/models/mdp", "Order") + require.NoError(t, err) + require.Equal(t, filepath.Join(modelsDir, "mdp", "types.go"), file) +} + +func TestResolver_ResolvePackageDir_GOPATHFallback(t *testing.T) { + root := t.TempDir() + projectDir := filepath.Join(root, "project") + gopath := filepath.Join(root, "gopath") + require.NoError(t, os.MkdirAll(filepath.Join(projectDir, "internal"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module example.com/project\ngo 1.25\n"), 0o644)) + legacyDir := filepath.Join(gopath, "src", "github.com", "legacy", "models") + require.NoError(t, os.MkdirAll(legacyDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(legacyDir, "types.go"), []byte("package models\ntype Legacy struct{}\n"), 0o644)) + + orig := os.Getenv("GOPATH") + require.NoError(t, os.Setenv("GOPATH", gopath)) + defer func() { _ = os.Setenv("GOPATH", orig) }() + + resolver, err := New(Config{ + ProjectDir: projectDir, + UseGoModuleResolve: false, + UseGOPATHFallback: true, + }) + require.NoError(t, err) + dir, err := resolver.ResolvePackageDir("github.com/legacy/models") + require.NoError(t, err) + require.Equal(t, legacyDir, dir) +} diff --git a/repository/shape/validate/relation.go b/repository/shape/validate/relation.go new file mode 100644 index 000000000..31aee9357 --- /dev/null +++ b/repository/shape/validate/relation.go @@ -0,0 +1,140 @@ +package validate + +import ( + "fmt" + "strings" + + "github.com/viant/datly/view" +) + +// ValidateRelations validates that relation link columns can be resolved on both +// parent and referenced views. It accepts alias/source/field variants and +// namespace-qualified forms (e.g. t.ID -> ID). +func ValidateRelations(resource *view.Resource, targets ...*view.View) error { + if resource == nil { + return nil + } + views := targets + if len(views) == 0 { + views = resource.Views + } + index := resource.Views.Index() + var issues []string + for _, parent := range views { + if parent == nil { + continue + } + parentIndex := view.Columns(parent.Columns).Index(parent.CaseFormat) + for _, rel := range parent.With { + if rel == nil || rel.Of == nil { + continue + } + ref := &rel.Of.View + if ref.Ref != "" { + if lookup, err := index.Lookup(ref.Ref); err == nil && lookup != nil { + ref = lookup + } + } + refIndex := view.Columns(ref.Columns).Index(ref.CaseFormat) + pairCount := len(rel.On) + if len(rel.Of.On) > pairCount { + pairCount = len(rel.Of.On) + } + for i := 0; i < pairCount; i++ { + var parentLink, refLink *view.Link + if i < len(rel.On) { + parentLink = rel.On[i] + } + if i < len(rel.Of.On) { + refLink = rel.Of.On[i] + } + + if missing := missingColumn(parentIndex, parentLink); missing != "" { + issues = append(issues, fmt.Sprintf("relation %q (parent=%q holder=%q link=%d): missing parent column %q", relName(rel, i), parent.Name, rel.Holder, i, missing)) + } + if missing := missingColumn(refIndex, refLink); missing != "" { + issues = append(issues, fmt.Sprintf("relation %q (parent=%q ref=%q holder=%q link=%d): missing ref column %q", relName(rel, i), parent.Name, ref.Name, rel.Holder, i, missing)) + } + } + } + } + if len(issues) == 0 { + return nil + } + return fmt.Errorf("shape relation validation failed:\n- %s", strings.Join(issues, "\n- ")) +} + +func missingColumn(index view.NamedColumns, link *view.Link) string { + if link == nil { + return "" + } + for _, candidate := range linkCandidates(link) { + if strings.TrimSpace(candidate) == "" { + continue + } + if _, err := index.Lookup(candidate); err == nil { + return "" + } + } + for _, candidate := range linkCandidates(link) { + if strings.TrimSpace(candidate) != "" { + return candidate + } + } + return "" +} + +func linkCandidates(link *view.Link) []string { + if link == nil { + return nil + } + var result []string + add := func(v string) { + v = strings.TrimSpace(trimIdentifier(v)) + if v == "" { + return + } + result = append(result, v) + if i := strings.LastIndex(v, "."); i != -1 && i < len(v)-1 { + result = append(result, v[i+1:]) + } + } + add(link.Column) + if link.Namespace != "" && link.Column != "" { + add(link.Namespace + "." + link.Column) + } + add(link.Field) + return dedupe(result) +} + +func trimIdentifier(value string) string { + value = strings.TrimSpace(value) + value = strings.Trim(value, "`") + value = strings.Trim(value, "\"") + value = strings.Trim(value, "'") + return value +} + +func dedupe(values []string) []string { + seen := map[string]bool{} + result := make([]string, 0, len(values)) + for _, value := range values { + key := strings.ToLower(strings.TrimSpace(value)) + if key == "" || seen[key] { + continue + } + seen[key] = true + result = append(result, value) + } + return result +} + +func relName(rel *view.Relation, idx int) string { + if rel == nil { + return fmt.Sprintf("#%d", idx) + } + if strings.TrimSpace(rel.Name) != "" { + return rel.Name + } + return fmt.Sprintf("#%d", idx) +} diff --git a/repository/shape/validate/relation_test.go b/repository/shape/validate/relation_test.go new file mode 100644 index 000000000..e10317882 --- /dev/null +++ b/repository/shape/validate/relation_test.go @@ -0,0 +1,70 @@ +package validate + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +func TestValidateRelations_AllowsAliasSourceAndNamespace(t *testing.T) { + parent := &view.View{ + Name: "vendor", + Columns: view.Columns{ + view.NewColumn("ID", "int", nil, false), + }, + } + child := &view.View{ + Name: "products", + Columns: view.Columns{ + view.NewColumn("VendorID", "int", nil, false, view.WithColumnTag(`source:"VENDOR_ID"`)), + }, + } + parent.With = []*view.Relation{{ + Name: "products", + Cardinality: state.Many, + Holder: "Products", + On: view.Links{&view.Link{Column: "vendor.ID"}}, + Of: &view.ReferenceView{ + View: *child, + On: view.Links{&view.Link{Column: "VENDOR_ID"}}, + }, + }} + resource := view.EmptyResource() + resource.Views = append(resource.Views, parent, child) + require.NoError(t, ValidateRelations(resource, parent)) +} + +func TestValidateRelations_DetailedMissingError(t *testing.T) { + parent := &view.View{ + Name: "vendor", + Columns: view.Columns{ + view.NewColumn("ID", "int", nil, false), + }, + } + child := &view.View{ + Name: "products", + Columns: view.Columns{ + view.NewColumn("VendorID", "int", nil, false), + }, + } + parent.With = []*view.Relation{{ + Name: "products", + Cardinality: state.Many, + Holder: "Products", + On: view.Links{&view.Link{Column: "MISSING_PARENT"}}, + Of: &view.ReferenceView{ + View: *child, + On: view.Links{&view.Link{Column: "MISSING_CHILD"}}, + }, + }} + resource := view.EmptyResource() + resource.Views = append(resource.Views, parent, child) + err := ValidateRelations(resource, parent) + require.Error(t, err) + require.Contains(t, err.Error(), "missing parent column \"MISSING_PARENT\"") + require.Contains(t, err.Error(), "missing ref column \"MISSING_CHILD\"") + require.Contains(t, err.Error(), "parent=\"vendor\"") + require.Contains(t, err.Error(), "ref=\"products\"") +} diff --git a/repository/shape/xgen/generator.go b/repository/shape/xgen/generator.go new file mode 100644 index 000000000..89622576b --- /dev/null +++ b/repository/shape/xgen/generator.go @@ -0,0 +1,644 @@ +package xgen + +import ( + "fmt" + "go/ast" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + + "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/repository/shape/typectx/source" + "github.com/viant/x" + xreflectloader "github.com/viant/x/loader/xreflect" + "github.com/viant/x/syntetic" + "github.com/viant/x/syntetic/model" +) + +// GenerateFromDQLShape emits Go structs from DQL shape using viant/x registry. +func GenerateFromDQLShape(doc *shape.Document, cfg *Config) (*Result, error) { + if doc == nil || doc.Root == nil { + return nil, fmt.Errorf("shape xgen: nil document") + } + if cfg == nil { + cfg = &Config{} + } + applyDefaults(cfg) + projectDir, packageDir, err := resolvePaths(cfg.ProjectDir, cfg.PackageDir) + if err != nil { + return nil, err + } + packageName := resolvePackageName(cfg.PackageName, packageDir) + packagePath, err := resolvePackagePath(cfg.PackagePath, projectDir, packageDir) + if err != nil { + return nil, err + } + fileName := cfg.FileName + if strings.TrimSpace(fileName) == "" { + fileName = "shapes_gen.go" + } + registry := cfg.Registry + if registry == nil { + registry = x.NewRegistry() + } + views := extractViews(doc.Root) + routeTypes := extractRouteIO(doc.Root) + if len(views) == 0 && len(routeTypes) == 0 { + return nil, fmt.Errorf("shape xgen: no view or route io declarations") + } + typeNames := make([]string, 0, len(views)+len(routeTypes)) + registered := map[string]bool{} + for _, view := range views { + typeName := viewTypeName(cfg, view) + if registered[typeName] { + continue + } + registered[typeName] = true + if err = registerShapeType(registry, packagePath, typeName, buildStructType(view.columns)); err != nil { + return nil, err + } + typeNames = append(typeNames, typeName) + } + for _, ioType := range routeTypes { + typeName := routeTypeName(cfg, ioType) + if typeName == "" || registered[typeName] { + continue + } + registered[typeName] = true + if err = registerShapeType(registry, packagePath, typeName, buildStructType(ioType.fields)); err != nil { + return nil, err + } + typeNames = append(typeNames, typeName) + } + namespace, err := syntetic.FromRegistry(registry) + if err != nil { + return nil, err + } + namespace.PkgName = packageName + namespace.PkgPath = packagePath + files, err := namespace.BuildFiles(model.RenderOptions{}) + if err != nil { + return nil, err + } + goFile := files[packagePath] + if goFile == nil { + return nil, fmt.Errorf("shape xgen: missing generated package file for %s", packagePath) + } + source, err := goFile.Render() + if err != nil { + return nil, err + } + if err = os.MkdirAll(packageDir, 0o755); err != nil { + return nil, err + } + dest := filepath.Join(packageDir, fileName) + if exists, checkErr := fileExists(dest); checkErr != nil { + return nil, checkErr + } else if exists && !cfg.AllowUnsafeRewrite { + if issues := rewriteSafetyIssues(doc, cfg, projectDir); len(issues) > 0 && (cfg.StrictProvenance == nil || *cfg.StrictProvenance) { + return nil, fmt.Errorf("shape xgen: rewrite blocked by type provenance safety: %s", strings.Join(issues, "; ")) + } + merged, mergeErr := mergeGeneratedShapes(dest, []byte(source), typeNames) + if mergeErr != nil { + return nil, mergeErr + } + source = string(merged) + } + if err = writeAtomic(dest, []byte(source), 0o644); err != nil { + return nil, err + } + sort.Strings(typeNames) + return &Result{ + FilePath: dest, + PackagePath: packagePath, + PackageName: packageName, + Types: typeNames, + }, nil +} + +func rewriteSafetyIssues(doc *shape.Document, cfg *Config, projectDir string) []string { + if doc == nil || len(doc.TypeResolutions) == 0 { + return nil + } + policy := newRewritePolicy(cfg, projectDir) + srcResolver, _ := source.New(source.Config{ + ProjectDir: projectDir, + AllowedSourceRoots: policy.roots, + UseGoModuleResolve: policy.useModule, + UseGOPATHFallback: policy.useGOPATH, + }) + var issues []string + for _, resolution := range doc.TypeResolutions { + if srcResolver != nil && strings.TrimSpace(resolution.Provenance.File) == "" { + pkg := firstNonEmpty(strings.TrimSpace(resolution.Provenance.Package), packageOfKey(resolution.ResolvedKey)) + name := typeNameFromKey(resolution.ResolvedKey) + if pkg != "" && name != "" { + if file, err := srcResolver.ResolveTypeFile(pkg, name); err == nil { + resolution.Provenance.File = file + if resolution.Provenance.Kind == "" || strings.EqualFold(resolution.Provenance.Kind, "registry") { + resolution.Provenance.Kind = "ast_type" + } + } + } + } + if issue := resolutionSafetyIssue(resolution, policy); issue != "" { + issues = append(issues, issue) + } + } + sort.Strings(issues) + return uniqueStrings(issues) +} + +func resolutionSafetyIssue(resolution typectx.Resolution, policy rewritePolicy) string { + kind := strings.TrimSpace(strings.ToLower(resolution.Provenance.Kind)) + if kind == "" { + kind = "registry" + } + if !policy.allowedKinds[kind] { + return fmt.Sprintf("expression=%q kind=%q", resolution.Expression, resolution.Provenance.Kind) + } + + sourceFile := strings.TrimSpace(resolution.Provenance.File) + if sourceFile == "" { + return "" + } + if !filepath.IsAbs(sourceFile) { + sourceFile = filepath.Clean(filepath.Join(policy.projectDir, sourceFile)) + } + if safe, err := source.IsWithinAnyRoot(sourceFile, policy.roots); err != nil || !safe { + return fmt.Sprintf("expression=%q source=%q outside_trusted_roots", resolution.Expression, resolution.Provenance.File) + } + return "" +} + +type rewritePolicy struct { + projectDir string + allowedKinds map[string]bool + roots []string + useModule bool + useGOPATH bool +} + +func newRewritePolicy(cfg *Config, projectDir string) rewritePolicy { + allowedKinds := map[string]bool{ + "builtin": true, + "resource_type": true, + "ast_type": true, + } + if len(cfg.AllowedProvenanceKinds) > 0 { + allowedKinds = map[string]bool{} + for _, item := range cfg.AllowedProvenanceKinds { + item = strings.TrimSpace(strings.ToLower(item)) + if item != "" { + allowedKinds[item] = true + } + } + } + useModule := true + if cfg.UseGoModuleResolve != nil { + useModule = *cfg.UseGoModuleResolve + } + useGOPATH := true + if cfg.UseGOPATHFallback != nil { + useGOPATH = *cfg.UseGOPATHFallback + } + return rewritePolicy{ + projectDir: projectDir, + allowedKinds: allowedKinds, + roots: source.NormalizeRoots(projectDir, cfg.AllowedSourceRoots), + useModule: useModule, + useGOPATH: useGOPATH, + } +} + +func typeNameFromKey(key string) string { + index := strings.LastIndex(key, ".") + if index == -1 || index+1 >= len(key) { + return "" + } + return key[index+1:] +} + +func packageOfKey(key string) string { + index := strings.LastIndex(key, ".") + if index == -1 { + return "" + } + return key[:index] +} + +func uniqueStrings(items []string) []string { + if len(items) < 2 { + return items + } + result := items[:0] + var previous string + for i, item := range items { + if i == 0 || item != previous { + result = append(result, item) + } + previous = item + } + return result +} + +func registerShapeType(registry *x.Registry, packagePath string, typeName string, rType reflect.Type) error { + st, err := xreflectloader.BuildType(rType, + xreflectloader.WithPackagePath(packagePath), + xreflectloader.WithNamePolicy(func(reflect.Type) (string, bool) { + return typeName, false + })) + if err != nil { + return fmt.Errorf("shape xgen: build type %s failed: %w", typeName, err) + } + st.Name = typeName + st.PkgPath = packagePath + if st.TypeSpec != nil { + st.TypeSpec.Name = ast.NewIdent(typeName) + } + registry.Register(x.NewType(rType, + x.WithName(typeName), + x.WithPkgPath(packagePath), + x.WithSyntheticType(st))) + return nil +} + +type viewDescriptor struct { + name any + schemaName any + columns []columnDescriptor +} + +type ioTypeKind string + +const ( + ioTypeInput ioTypeKind = "input" + ioTypeOutput ioTypeKind = "output" +) + +type routeIODescriptor struct { + kind ioTypeKind + routeName string + routeURI string + routeRef string + typeName string + fields []columnDescriptor +} + +type columnDescriptor struct { + name string + dataType string +} + +func extractViews(root map[string]any) []viewDescriptor { + resource := asMap(root["Resource"]) + if resource == nil { + return nil + } + items := asSlice(resource["Views"]) + result := make([]viewDescriptor, 0, len(items)) + for _, item := range items { + view := asMap(item) + if view == nil { + continue + } + schema := asMap(view["Schema"]) + descriptor := viewDescriptor{ + name: view["Name"], + schemaName: nil, + } + if schema != nil { + descriptor.schemaName = schema["Name"] + } + descriptor.columns = extractColumns(view) + result = append(result, descriptor) + } + return result +} + +func extractColumns(view map[string]any) []columnDescriptor { + var result []columnDescriptor + if columns := asSlice(view["Columns"]); len(columns) > 0 { + for _, item := range columns { + column := asMap(item) + if column == nil { + continue + } + name := firstNonEmpty(asString(column["Name"]), asString(column["Column"])) + if name == "" { + continue + } + result = append(result, columnDescriptor{name: name, dataType: asString(column["DataType"])}) + } + } + if cfg := asMap(view["ColumnsConfig"]); len(cfg) > 0 { + keys := make([]string, 0, len(cfg)) + for k := range cfg { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + item := asMap(cfg[key]) + if item == nil { + item = map[string]any{} + } + name := firstNonEmpty(asString(item["Name"]), key) + result = append(result, columnDescriptor{name: name, dataType: asString(item["DataType"])}) + } + } + if len(result) == 0 { + result = append(result, columnDescriptor{name: "ID", dataType: "int"}) + } + return result +} + +func extractRouteIO(root map[string]any) []routeIODescriptor { + var result []routeIODescriptor + for _, item := range asSlice(root["Routes"]) { + route := asMap(item) + if route == nil { + continue + } + meta := routeIODescriptor{ + routeName: asString(route["Name"]), + routeURI: asString(route["URI"]), + } + if routeView := asMap(route["View"]); routeView != nil { + meta.routeRef = asString(routeView["Ref"]) + } + if input := asMap(route["Input"]); input != nil { + entry := meta + entry.kind = ioTypeInput + entry.typeName = nestedTypeName(input) + entry.fields = extractIOFields(input) + result = append(result, entry) + } + if output := asMap(route["Output"]); output != nil { + entry := meta + entry.kind = ioTypeOutput + entry.typeName = nestedTypeName(output) + entry.fields = extractIOFields(output) + result = append(result, entry) + } + } + return result +} + +func nestedTypeName(io map[string]any) string { + aType := asMap(io["Type"]) + if aType == nil { + return "" + } + return asString(aType["Name"]) +} + +func extractIOFields(io map[string]any) []columnDescriptor { + parameters := asSlice(io["Parameters"]) + if len(parameters) == 0 { + if t := asMap(io["Type"]); t != nil { + parameters = asSlice(t["Parameters"]) + } + } + fields := make([]columnDescriptor, 0, len(parameters)) + for _, item := range parameters { + param := asMap(item) + if param == nil { + continue + } + name := asString(param["Name"]) + if name == "" { + continue + } + dataType := "" + if schema := asMap(param["Schema"]); schema != nil { + dataType = asString(schema["DataType"]) + } + fields = append(fields, columnDescriptor{name: name, dataType: dataType}) + } + if len(fields) == 0 { + fields = append(fields, columnDescriptor{name: "ID", dataType: "int"}) + } + return fields +} + +func buildStructType(columns []columnDescriptor) reflect.Type { + if len(columns) == 0 { + columns = []columnDescriptor{{name: "ID", dataType: "int"}} + } + fields := make([]reflect.StructField, 0, len(columns)) + used := map[string]int{} + for _, column := range columns { + fieldName := exportedName(column.name) + if fieldName == "" { + fieldName = "Field" + } + if count := used[fieldName]; count > 0 { + fieldName = fmt.Sprintf("%s%d", fieldName, count+1) + } + used[fieldName]++ + fields = append(fields, reflect.StructField{ + Name: fieldName, + Type: parseType(column.dataType), + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s,omitempty" sqlx:"%s"`, strings.ToLower(fieldName), column.name)), + }) + } + return reflect.StructOf(fields) +} + +func parseType(dataType string) reflect.Type { + dataType = strings.TrimSpace(dataType) + if dataType == "" { + return reflect.TypeOf("") + } + switch { + case strings.HasPrefix(dataType, "[]"): + return reflect.SliceOf(parseType(strings.TrimPrefix(dataType, "[]"))) + case strings.HasPrefix(dataType, "*"): + return reflect.PointerTo(parseType(strings.TrimPrefix(dataType, "*"))) + } + lowered := strings.ToLower(dataType) + switch lowered { + case "string", "varchar", "text": + return reflect.TypeOf("") + case "bool", "boolean": + return reflect.TypeOf(true) + case "int", "integer": + return reflect.TypeOf(int(0)) + case "int64", "bigint": + return reflect.TypeOf(int64(0)) + case "int32": + return reflect.TypeOf(int32(0)) + case "float", "float64", "double", "decimal": + return reflect.TypeOf(float64(0)) + case "float32": + return reflect.TypeOf(float32(0)) + case "bytes", "[]byte", "blob": + return reflect.TypeOf([]byte{}) + default: + return reflect.TypeOf("") + } +} + +func exportedName(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "" + } + var parts []string + current := strings.Builder{} + flush := func() { + if current.Len() == 0 { + return + } + parts = append(parts, current.String()) + current.Reset() + } + for _, r := range name { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + current.WriteRune(r) + } else { + flush() + } + } + flush() + for i, item := range parts { + if item == strings.ToUpper(item) { + parts[i] = strings.ToUpper(item[:1]) + strings.ToLower(item[1:]) + } else { + parts[i] = strings.ToUpper(item[:1]) + item[1:] + } + } + result := strings.Join(parts, "") + if result == "" { + return "" + } + if result[0] >= '0' && result[0] <= '9' { + result = "N" + result + } + return result +} + +func applyDefaults(cfg *Config) { + if cfg.ViewSuffix == "" { + cfg.ViewSuffix = "View" + } + if cfg.InputSuffix == "" { + cfg.InputSuffix = "Input" + } + if cfg.OutputSuffix == "" { + cfg.OutputSuffix = "Output" + } + if cfg.UseGoModuleResolve == nil { + value := true + cfg.UseGoModuleResolve = &value + } + if cfg.UseGOPATHFallback == nil { + value := true + cfg.UseGOPATHFallback = &value + } + if cfg.StrictProvenance == nil { + value := true + cfg.StrictProvenance = &value + } +} + +func viewTypeName(cfg *Config, view viewDescriptor) string { + ctx := ViewTypeContext{ + ViewName: asString(view.name), + SchemaName: asString(view.schemaName), + } + if cfg.ViewTypeNamer != nil { + if name := strings.TrimSpace(cfg.ViewTypeNamer(ctx)); name != "" { + return cfg.TypePrefix + exportedName(name) + } + } + base := firstNonEmpty(ctx.SchemaName, ctx.ViewName) + if base == "" { + base = cfg.ViewSuffix + } else if !hasCaseInsensitiveSuffix(base, cfg.ViewSuffix) { + base += cfg.ViewSuffix + } + return cfg.TypePrefix + exportedName(base) +} + +func routeTypeName(cfg *Config, route routeIODescriptor) string { + ctx := RouteTypeContext{ + RouteName: route.routeName, + RouteURI: route.routeURI, + RouteRef: route.routeRef, + TypeName: route.typeName, + } + var custom string + switch route.kind { + case ioTypeInput: + if cfg.InputTypeNamer != nil { + custom = cfg.InputTypeNamer(ctx) + } + case ioTypeOutput: + if cfg.OutputTypeNamer != nil { + custom = cfg.OutputTypeNamer(ctx) + } + } + if strings.TrimSpace(custom) != "" { + return cfg.TypePrefix + exportedName(custom) + } + base := firstNonEmpty(ctx.TypeName, ctx.RouteName, ctx.RouteRef, "Route") + suffix := cfg.OutputSuffix + if route.kind == ioTypeInput { + suffix = cfg.InputSuffix + } + if !hasCaseInsensitiveSuffix(base, suffix) { + base += suffix + } + return cfg.TypePrefix + exportedName(base) +} + +func hasCaseInsensitiveSuffix(value, suffix string) bool { + if suffix == "" { + return true + } + return strings.HasSuffix(strings.ToLower(value), strings.ToLower(suffix)) +} + +func firstNonEmpty(values ...string) string { + for _, item := range values { + if strings.TrimSpace(item) != "" { + return item + } + } + return "" +} + +func asMap(raw any) map[string]any { + if value, ok := raw.(map[string]any); ok { + return value + } + if value, ok := raw.(map[any]any); ok { + out := map[string]any{} + for key, item := range value { + out[fmt.Sprint(key)] = item + } + return out + } + return nil +} + +func asSlice(raw any) []any { + if value, ok := raw.([]any); ok { + return value + } + return nil +} + +func asString(raw any) string { + if raw == nil { + return "" + } + if value, ok := raw.(string); ok { + return value + } + return fmt.Sprint(raw) +} diff --git a/repository/shape/xgen/generator_test.go b/repository/shape/xgen/generator_test.go new file mode 100644 index 000000000..3315f148d --- /dev/null +++ b/repository/shape/xgen/generator_test.go @@ -0,0 +1,305 @@ +package xgen + +import ( + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" + + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/typectx" +) + +func TestGenerateFromDQLShape(t *testing.T) { + projectDir := t.TempDir() + if err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module example.com/demo\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatalf("write go.mod failed: %v", err) + } + doc := &dqlshape.Document{Root: map[string]any{ + "Routes": []any{ + map[string]any{ + "Name": "orders", + "URI": "/orders", + "View": map[string]any{"Ref": "orders"}, + "Input": map[string]any{ + "Type": map[string]any{"Name": "OrdersFilter"}, + "Parameters": []any{ + map[string]any{ + "Name": "status", + "Schema": map[string]any{ + "DataType": "string", + }, + }, + }, + }, + "Output": map[string]any{ + "Type": map[string]any{"Name": "OrdersPayload"}, + "Parameters": []any{ + map[string]any{ + "Name": "total", + "Schema": map[string]any{ + "DataType": "int", + }, + }, + }, + }, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "orders", + "Schema": map[string]any{ + "Name": "OrderView", + }, + "ColumnsConfig": map[string]any{ + "ID": map[string]any{"Name": "ID", "DataType": "int"}, + "NAME": map[string]any{"Name": "NAME", "DataType": "string"}, + }, + }, + }, + }, + }} + result, err := GenerateFromDQLShape(doc, &Config{ + ProjectDir: projectDir, + PackageDir: "internal/gen", + PackageName: "gen", + FileName: "shapes_gen.go", + TypePrefix: "DQL", + }) + if err != nil { + t.Fatalf("generate failed: %v", err) + } + if result == nil { + t.Fatalf("nil result") + } + if len(result.Types) == 0 { + t.Fatalf("expected generated types") + } + if _, err = os.Stat(result.FilePath); err != nil { + t.Fatalf("generated file missing: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file failed: %v", err) + } + source := string(data) + if !strings.Contains(source, "type DQLOrderView struct") { + t.Fatalf("expected generated type in source, got:\n%s", source) + } + if !strings.Contains(source, "type DQLOrdersFilterInput struct") || !strings.Contains(source, "type DQLOrdersPayloadOutput struct") { + t.Fatalf("expected io types in source, got:\n%s", source) + } + if !strings.Contains(source, "Id") || !strings.Contains(source, "Name") { + t.Fatalf("expected generated fields in source, got:\n%s", source) + } + fset := token.NewFileSet() + if _, err = parser.ParseFile(fset, result.FilePath, source, parser.AllErrors); err != nil { + t.Fatalf("generated file parse failed: %v", err) + } +} + +func TestGenerateFromDQLShape_CustomTypeNamers(t *testing.T) { + projectDir := t.TempDir() + if err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module example.com/demo\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatalf("write go.mod failed: %v", err) + } + doc := &dqlshape.Document{Root: map[string]any{ + "Routes": []any{ + map[string]any{ + "Name": "orders", + "Input": map[string]any{ + "Parameters": []any{map[string]any{"Name": "q", "Schema": map[string]any{"DataType": "string"}}}, + }, + "Output": map[string]any{ + "Parameters": []any{map[string]any{"Name": "count", "Schema": map[string]any{"DataType": "int"}}}, + }, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{"Name": "orders", "ColumnsConfig": map[string]any{"ID": map[string]any{"Name": "ID", "DataType": "int"}}}, + }, + }, + }} + result, err := GenerateFromDQLShape(doc, &Config{ + ProjectDir: projectDir, + PackageDir: "internal/gen", + ViewTypeNamer: func(ctx ViewTypeContext) string { + return "DataOrders" + }, + InputTypeNamer: func(ctx RouteTypeContext) string { + return "ReqOrders" + }, + OutputTypeNamer: func(ctx RouteTypeContext) string { + return "ResOrders" + }, + }) + if err != nil { + t.Fatalf("generate failed: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file failed: %v", err) + } + source := string(data) + if !strings.Contains(source, "type DataOrders struct") { + t.Fatalf("missing custom view type: %s", source) + } + if !strings.Contains(source, "type ReqOrders struct") { + t.Fatalf("missing custom input type: %s", source) + } + if !strings.Contains(source, "type ResOrders struct") { + t.Fatalf("missing custom output type: %s", source) + } +} + +func TestGenerateFromDQLShape_BlocksUnsafeRewriteByProvenance(t *testing.T) { + projectDir := t.TempDir() + if err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module example.com/demo\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatalf("write go.mod failed: %v", err) + } + packageDir := filepath.Join(projectDir, "internal", "gen") + if err := os.MkdirAll(packageDir, 0o755); err != nil { + t.Fatalf("mkdir failed: %v", err) + } + dest := filepath.Join(packageDir, "shapes_gen.go") + if err := os.WriteFile(dest, []byte("package gen\n"), 0o644); err != nil { + t.Fatalf("seed file failed: %v", err) + } + + doc := &dqlshape.Document{ + Root: map[string]any{ + "Resource": map[string]any{ + "Views": []any{ + map[string]any{"Name": "orders", "ColumnsConfig": map[string]any{"ID": map[string]any{"Name": "ID", "DataType": "int"}}}, + }, + }, + }, + TypeResolutions: []typectx.Resolution{ + { + Expression: "Fee", + Provenance: typectx.Provenance{Kind: "registry"}, + }, + }, + } + _, err := GenerateFromDQLShape(doc, &Config{ + ProjectDir: projectDir, + PackageDir: "internal/gen", + PackageName: "gen", + FileName: "shapes_gen.go", + }) + if err == nil || !strings.Contains(err.Error(), "rewrite blocked") { + t.Fatalf("expected rewrite blocked error, got: %v", err) + } +} + +func TestGenerateFromDQLShape_AllowsUnsafeRewriteWithOverride(t *testing.T) { + projectDir := t.TempDir() + if err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module example.com/demo\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatalf("write go.mod failed: %v", err) + } + packageDir := filepath.Join(projectDir, "internal", "gen") + if err := os.MkdirAll(packageDir, 0o755); err != nil { + t.Fatalf("mkdir failed: %v", err) + } + dest := filepath.Join(packageDir, "shapes_gen.go") + if err := os.WriteFile(dest, []byte("package gen\n"), 0o644); err != nil { + t.Fatalf("seed file failed: %v", err) + } + + doc := &dqlshape.Document{ + Root: map[string]any{ + "Resource": map[string]any{ + "Views": []any{ + map[string]any{"Name": "orders", "ColumnsConfig": map[string]any{"ID": map[string]any{"Name": "ID", "DataType": "int"}}}, + }, + }, + }, + TypeResolutions: []typectx.Resolution{ + { + Expression: "Fee", + Provenance: typectx.Provenance{Kind: "registry"}, + }, + }, + } + result, err := GenerateFromDQLShape(doc, &Config{ + ProjectDir: projectDir, + PackageDir: "internal/gen", + PackageName: "gen", + FileName: "shapes_gen.go", + AllowUnsafeRewrite: true, + }) + if err != nil { + t.Fatalf("expected override rewrite success, got: %v", err) + } + if result == nil || result.FilePath == "" { + t.Fatalf("expected generated result") + } +} + +func TestGenerateFromDQLShape_MergesIntoExistingFile(t *testing.T) { + projectDir := t.TempDir() + if err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module example.com/demo\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatalf("write go.mod failed: %v", err) + } + packageDir := filepath.Join(projectDir, "internal", "gen") + if err := os.MkdirAll(packageDir, 0o755); err != nil { + t.Fatalf("mkdir failed: %v", err) + } + dest := filepath.Join(packageDir, "shapes_gen.go") + initial := `package gen + +type DQLOrderView struct { + Old string ` + "`json:\"old,omitempty\"`" + ` +} + +func KeepCustom() string { return "ok" } +` + if err := os.WriteFile(dest, []byte(initial), 0o644); err != nil { + t.Fatalf("seed file failed: %v", err) + } + + doc := &dqlshape.Document{Root: map[string]any{ + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "orders", + "Schema": map[string]any{ + "Name": "OrderView", + }, + "ColumnsConfig": map[string]any{ + "ID": map[string]any{"Name": "ID", "DataType": "int"}, + }, + }, + }, + }, + }} + _, err := GenerateFromDQLShape(doc, &Config{ + ProjectDir: projectDir, + PackageDir: "internal/gen", + PackageName: "gen", + FileName: "shapes_gen.go", + TypePrefix: "DQL", + }) + if err != nil { + t.Fatalf("generate failed: %v", err) + } + + data, err := os.ReadFile(dest) + if err != nil { + t.Fatalf("read generated file failed: %v", err) + } + source := string(data) + if !strings.Contains(source, "func KeepCustom() string") { + t.Fatalf("expected custom function preserved, got:\n%s", source) + } + if strings.Contains(source, "Old string") { + t.Fatalf("expected old shape declaration replaced, got:\n%s", source) + } + if !strings.Contains(source, "type DQLOrderView struct") || !strings.Contains(source, "Id int") { + t.Fatalf("expected updated shape declaration, got:\n%s", source) + } +} diff --git a/repository/shape/xgen/io.go b/repository/shape/xgen/io.go new file mode 100644 index 000000000..395ea8e93 --- /dev/null +++ b/repository/shape/xgen/io.go @@ -0,0 +1,311 @@ +package xgen + +import ( + "bufio" + "bytes" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" +) + +func resolvePaths(projectDir, packageDir string) (string, string, error) { + if strings.TrimSpace(projectDir) == "" { + return "", "", fmt.Errorf("shape xgen: project dir was empty") + } + projectDir = filepath.Clean(projectDir) + if strings.TrimSpace(packageDir) == "" { + packageDir = projectDir + } else if !filepath.IsAbs(packageDir) { + packageDir = filepath.Join(projectDir, packageDir) + } + packageDir = filepath.Clean(packageDir) + return projectDir, packageDir, nil +} + +func resolvePackageName(name string, packageDir string) string { + name = strings.TrimSpace(name) + if name != "" { + return name + } + base := filepath.Base(packageDir) + if base == "." || base == string(filepath.Separator) || base == "" { + return "generated" + } + return sanitizePkg(base) +} + +func resolvePackagePath(packagePath, projectDir, packageDir string) (string, error) { + packagePath = strings.TrimSpace(packagePath) + if packagePath != "" { + return packagePath, nil + } + modulePath, err := readModulePath(filepath.Join(projectDir, "go.mod")) + if err != nil { + return "", err + } + rel, err := filepath.Rel(projectDir, packageDir) + if err != nil { + return "", err + } + rel = filepath.ToSlash(rel) + if rel == "." { + return modulePath, nil + } + return strings.TrimRight(modulePath, "/") + "/" + strings.TrimLeft(rel, "/"), nil +} + +func readModulePath(goModPath string) (string, error) { + file, err := os.Open(goModPath) + if err != nil { + return "", fmt.Errorf("shape xgen: open go.mod failed: %w", err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, "module ") { + continue + } + modulePath := strings.TrimSpace(strings.TrimPrefix(line, "module ")) + if modulePath != "" { + return modulePath, nil + } + } + if err = scanner.Err(); err != nil { + return "", err + } + return "", fmt.Errorf("shape xgen: module path not found in %s", goModPath) +} + +func sanitizePkg(name string) string { + name = strings.TrimSpace(strings.ToLower(name)) + if name == "" { + return "generated" + } + var out strings.Builder + for _, r := range name { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' { + out.WriteRune(r) + } + } + if out.Len() == 0 { + return "generated" + } + result := out.String() + if result[0] >= '0' && result[0] <= '9' { + return "p" + result + } + return result +} + +func writeAtomic(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + temp, err := os.CreateTemp(dir, ".tmp-shape-xgen-*") + if err != nil { + return err + } + tempPath := temp.Name() + cleanup := func() { + _ = os.Remove(tempPath) + } + if _, err = temp.Write(data); err != nil { + _ = temp.Close() + cleanup() + return err + } + if err = temp.Chmod(perm); err != nil { + _ = temp.Close() + cleanup() + return err + } + if err = temp.Close(); err != nil { + cleanup() + return err + } + if err = os.Rename(tempPath, path); err != nil { + cleanup() + return err + } + return nil +} + +func fileExists(path string) (bool, error) { + info, err := os.Stat(path) + if err == nil { + return !info.IsDir(), nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func isWithinProject(projectDir, candidate string) (bool, error) { + projectDir = filepath.Clean(projectDir) + candidate = filepath.Clean(candidate) + rel, err := filepath.Rel(projectDir, candidate) + if err != nil { + return false, err + } + if rel == "." { + return true, nil + } + rel = filepath.ToSlash(rel) + return !strings.HasPrefix(rel, "../"), nil +} + +func mergeGeneratedShapes(dest string, generated []byte, typeNames []string) ([]byte, error) { + existing, err := os.ReadFile(dest) + if err != nil { + return nil, err + } + if len(existing) == 0 { + return generated, nil + } + if len(typeNames) == 0 { + return existing, nil + } + + fset := token.NewFileSet() + existingFile, err := parser.ParseFile(fset, dest, existing, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("shape xgen: parse existing file failed: %w", err) + } + generatedFile, err := parser.ParseFile(token.NewFileSet(), "", generated, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("shape xgen: parse generated file failed: %w", err) + } + typeNameSet := map[string]bool{} + for _, name := range typeNames { + typeNameSet[name] = true + } + + shapeDecls := generatedShapeDecls(generatedFile, typeNameSet) + if len(shapeDecls) == 0 { + return generated, nil + } + mergedImports := mergeImports(existingFile.Imports, generatedFile.Imports) + + newDecls := make([]ast.Decl, 0, len(existingFile.Decls)+len(shapeDecls)+1) + if len(mergedImports) > 0 { + newDecls = append(newDecls, &ast.GenDecl{ + Tok: token.IMPORT, + Specs: mergedImports, + }) + } + + for _, decl := range existingFile.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok { + newDecls = append(newDecls, decl) + continue + } + switch gen.Tok { + case token.IMPORT: + continue + case token.TYPE: + filtered := make([]ast.Spec, 0, len(gen.Specs)) + for _, spec := range gen.Specs { + ts, ok := spec.(*ast.TypeSpec) + if !ok || !typeNameSet[ts.Name.Name] { + filtered = append(filtered, spec) + } + } + if len(filtered) == 0 { + continue + } + gen.Specs = filtered + newDecls = append(newDecls, gen) + default: + newDecls = append(newDecls, decl) + } + } + newDecls = append(newDecls, shapeDecls...) + existingFile.Decls = newDecls + existingFile.Imports = importSpecsToImportNodes(mergedImports) + + var out bytes.Buffer + if err = format.Node(&out, fset, existingFile); err != nil { + return nil, fmt.Errorf("shape xgen: format merged file failed: %w", err) + } + return out.Bytes(), nil +} + +func generatedShapeDecls(file *ast.File, typeNameSet map[string]bool) []ast.Decl { + var result []ast.Decl + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.TYPE { + continue + } + filtered := make([]ast.Spec, 0, len(gen.Specs)) + for _, spec := range gen.Specs { + ts, ok := spec.(*ast.TypeSpec) + if !ok || !typeNameSet[ts.Name.Name] { + continue + } + filtered = append(filtered, spec) + } + if len(filtered) == 0 { + continue + } + result = append(result, &ast.GenDecl{ + Tok: token.TYPE, + Specs: filtered, + }) + } + return result +} + +func mergeImports(existing []*ast.ImportSpec, generated []*ast.ImportSpec) []ast.Spec { + merged := map[string]*ast.ImportSpec{} + add := func(item *ast.ImportSpec) { + if item == nil || item.Path == nil { + return + } + key := item.Path.Value + "|" + importAlias(item) + if _, ok := merged[key]; ok { + return + } + merged[key] = item + } + for _, item := range existing { + add(item) + } + for _, item := range generated { + add(item) + } + keys := make([]string, 0, len(merged)) + for key := range merged { + keys = append(keys, key) + } + sort.Strings(keys) + result := make([]ast.Spec, 0, len(keys)) + for _, key := range keys { + result = append(result, merged[key]) + } + return result +} + +func importAlias(item *ast.ImportSpec) string { + if item == nil || item.Name == nil { + return "" + } + return item.Name.Name +} + +func importSpecsToImportNodes(specs []ast.Spec) []*ast.ImportSpec { + result := make([]*ast.ImportSpec, 0, len(specs)) + for _, spec := range specs { + if item, ok := spec.(*ast.ImportSpec); ok { + result = append(result, item) + } + } + return result +} diff --git a/repository/shape/xgen/model.go b/repository/shape/xgen/model.go new file mode 100644 index 000000000..623f79be8 --- /dev/null +++ b/repository/shape/xgen/model.go @@ -0,0 +1,70 @@ +package xgen + +import "github.com/viant/x" + +type ( + ViewTypeContext struct { + ViewName string + SchemaName string + } + + RouteTypeContext struct { + RouteName string + RouteURI string + RouteRef string + TypeName string + } +) + +// Config controls shape->Go generation. +type Config struct { + // ProjectDir points to target Go project root. + ProjectDir string + // PackageDir points to package directory inside the project (relative or absolute). + PackageDir string + // PackageName sets generated package name; defaults to basename(PackageDir). + PackageName string + // PackagePath sets fully-qualified import path; when empty it's derived from go.mod + PackageDir. + PackagePath string + // FileName sets generated filename; defaults to shapes_gen.go. + FileName string + // TypePrefix prefixes generated type names. + TypePrefix string + // ViewSuffix appends suffix to generated view type names when schema name is absent. + ViewSuffix string + // InputSuffix appends suffix to generated route input type names when explicit type name is absent. + InputSuffix string + // OutputSuffix appends suffix to generated route output type names when explicit type name is absent. + OutputSuffix string + // ViewTypeNamer customizes final view type name. + ViewTypeNamer func(ctx ViewTypeContext) string + // InputTypeNamer customizes final input type name. + InputTypeNamer func(ctx RouteTypeContext) string + // OutputTypeNamer customizes final output type name. + OutputTypeNamer func(ctx RouteTypeContext) string + // Registry allows reusing an external viant/x registry. + Registry *x.Registry + // AllowUnsafeRewrite allows overwriting existing generated files even when + // type provenance indicates unresolved/unsafe origins. Default false. + AllowUnsafeRewrite bool + // AllowedProvenanceKinds controls which provenance kinds are trusted for updates. + // Defaults to builtin, resource_type and ast_type. + AllowedProvenanceKinds []string + // AllowedSourceRoots controls additional trusted roots for provenance files. + // ProjectDir is always implicitly trusted. + AllowedSourceRoots []string + // UseGoModuleResolve enables go.mod + replace-based source resolution. Default true. + UseGoModuleResolve *bool + // UseGOPATHFallback enables GOPATH/src fallback when go.mod resolution misses. Default true. + UseGOPATHFallback *bool + // StrictProvenance blocks updates on policy violations. Default true. + StrictProvenance *bool +} + +// Result captures generation outputs. +type Result struct { + FilePath string + PackagePath string + PackageName string + Types []string +} diff --git a/view/state/parameters.go b/view/state/parameters.go index 04083ede1..e464ce142 100644 --- a/view/state/parameters.go +++ b/view/state/parameters.go @@ -401,7 +401,9 @@ func (p *Parameter) buildField(pkgPath string, lookupType xreflect.LookupType) ( if err != nil { rType, err = types.LookupType(lookupType, schema.DataType, xreflect.WithPackage(pkgPath)) if err != nil { - return structField, markerField, fmt.Errorf("failed to detect parmater '%v' type for: %v %w", p.Name, schema.TypeName(), err) + // Keep unresolved custom parameter types as dynamic `interface{}` so + // scan/planning can continue while preserving declared schema metadata. + rType = reflect.TypeOf((*interface{})(nil)).Elem() } } schema.rType = rType From 884634626866f93472c5df12e8574a73ffc15992 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 18 Feb 2026 07:18:13 -0800 Subject: [PATCH 122/279] fix config relative path handling --- gateway/mcp.go | 25 ++++++++++++++++------ gateway/runtime/standalone/config.go | 32 ++++++++++++++++++++++++++++ internal/translator/config.go | 4 ++-- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index 898947d43..4bf053020 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -360,8 +360,19 @@ func (r *Router) mcpUnauthorizedError() *jsonrpc.Error { func (r *Router) buildToolInputType(components *repository.Component) reflect.Type { var inputFields []reflect.StructField + var uniqueFieldName = make(map[string]bool) var uniqueQuery = make(map[string]bool) var uniquePath = make(map[string]bool) + appendField := func(name string, fieldType reflect.Type, tag reflect.StructTag) { + if name == "" || fieldType == nil { + return + } + if uniqueFieldName[name] { + return + } + uniqueFieldName[name] = true + inputFields = append(inputFields, reflect.StructField{Name: name, Type: fieldType, Tag: tag}) + } // Include component input parameters for _, parameter := range components.Input.Type.Parameters { name := strings.Title(parameter.Name) @@ -376,7 +387,7 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty if parameter.Schema != nil && parameter.Schema.Type().Kind() == reflect.Slice { tag = `json:",omitempty" optional:"true"` } - inputFields = append(inputFields, reflect.StructField{Name: name, Type: parameter.Schema.Type(), Tag: tag}) + appendField(name, parameter.Schema.Type(), tag) case state.KindQuery, state.KindForm: if uniqueQuery[parameter.In.Name] { @@ -391,14 +402,14 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty } else if !strings.Contains(parameter.Tag, "required") { tag = `json:",omitempty"` } - inputFields = append(inputFields, reflect.StructField{Name: name, Type: parameter.Schema.Type(), Tag: tag}) + appendField(name, parameter.Schema.Type(), tag) case state.KindRequestBody: // If body is a slice, mark optional in schema. var tag reflect.StructTag if parameter.Schema != nil && parameter.Schema.Type().Kind() == reflect.Slice { tag = `json:",omitempty" optional:"true"` } - inputFields = append(inputFields, reflect.StructField{Name: name, Type: parameter.Schema.Type(), Tag: tag}) + appendField(name, parameter.Schema.Type(), tag) } } @@ -407,26 +418,26 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty if p := components.View.Selector.LimitParameter; p != nil && p.In != nil && p.In.Name != "" { if !uniqueQuery[p.In.Name] { // avoid duplicates uniqueQuery[p.In.Name] = true - inputFields = append(inputFields, reflect.StructField{Name: strings.Title(p.Name), Type: p.Schema.Type(), Tag: `json:",omitempty"`}) + appendField(strings.Title(p.Name), p.Schema.Type(), `json:",omitempty"`) } } if p := components.View.Selector.OffsetParameter; p != nil && p.In != nil && p.In.Name != "" { if !uniqueQuery[p.In.Name] { uniqueQuery[p.In.Name] = true - inputFields = append(inputFields, reflect.StructField{Name: strings.Title(p.Name), Type: p.Schema.Type(), Tag: `json:",omitempty"`}) + appendField(strings.Title(p.Name), p.Schema.Type(), `json:",omitempty"`) } } if p := components.View.Selector.FieldsParameter; p != nil && p.In != nil && p.In.Name != "" { if !uniqueQuery[p.In.Name] { uniqueQuery[p.In.Name] = true // Fields is a []string – ensure optional in schema - inputFields = append(inputFields, reflect.StructField{Name: strings.Title(p.Name), Type: p.Schema.Type(), Tag: `json:",omitempty" optional:"true"`}) + appendField(strings.Title(p.Name), p.Schema.Type(), `json:",omitempty" optional:"true"`) } } if p := components.View.Selector.PageParameter; p != nil && p.In != nil && p.In.Name != "" { if !uniqueQuery[p.In.Name] { uniqueQuery[p.In.Name] = true - inputFields = append(inputFields, reflect.StructField{Name: strings.Title(p.Name), Type: p.Schema.Type(), Tag: `json:",omitempty"`}) + appendField(strings.Title(p.Name), p.Schema.Type(), `json:",omitempty"`) } } } diff --git a/gateway/runtime/standalone/config.go b/gateway/runtime/standalone/config.go index 9318caa88..34399adf1 100644 --- a/gateway/runtime/standalone/config.go +++ b/gateway/runtime/standalone/config.go @@ -4,11 +4,13 @@ import ( "context" "encoding/json" "github.com/viant/afs" + "github.com/viant/afs/url" "github.com/viant/datly/gateway" "github.com/viant/datly/gateway/router/openapi/openapi3" "github.com/viant/datly/gateway/runtime/standalone/endpoint" "github.com/viant/toolbox" "gopkg.in/yaml.v3" + "path/filepath" "strings" ) @@ -71,5 +73,35 @@ func NewConfigFromURL(ctx context.Context, URL string) (*Config, error) { } cfg.URL = URL cfg.Init(ctx) + cfg.normalizeURLs(baseDir(URL)) return cfg, cfg.Validate() } + +func (c *Config) normalizeURLs(baseURL string) { + if url.IsRelative(c.RouteURL) { + c.RouteURL = url.Join(baseURL, c.RouteURL) + } + if url.IsRelative(c.ContentURL) { + c.ContentURL = url.Join(baseURL, c.ContentURL) + } + if url.IsRelative(c.PluginsURL) { + c.PluginsURL = url.Join(baseURL, c.PluginsURL) + } + if url.IsRelative(c.DependencyURL) { + c.DependencyURL = url.Join(baseURL, c.DependencyURL) + } + if url.IsRelative(c.JobURL) { + c.JobURL = url.Join(baseURL, c.JobURL) + } + if url.IsRelative(c.FailedJobURL) { + c.FailedJobURL = url.Join(baseURL, c.FailedJobURL) + } +} + +func baseDir(URL string) string { + if strings.Contains(URL, "://") { + parent, _ := url.Split(URL, "file") + return parent + } + return filepath.Dir(URL) +} diff --git a/internal/translator/config.go b/internal/translator/config.go index d8adfd670..656b6f430 100644 --- a/internal/translator/config.go +++ b/internal/translator/config.go @@ -156,10 +156,10 @@ func (c *Config) NormalizeURL(repositoryURL string) { cfg.ContentURL = url.Join(baseURL, cfg.ContentURL) } if url.IsRelative(cfg.PluginsURL) { - cfg.RouteURL = url.Join(baseURL, cfg.PluginsURL) + cfg.PluginsURL = url.Join(baseURL, cfg.PluginsURL) } if url.IsRelative(cfg.DependencyURL) { - cfg.RouteURL = url.Join(baseURL, cfg.DependencyURL) + cfg.DependencyURL = url.Join(baseURL, cfg.DependencyURL) } cfg.URL = url.Join(baseURL, "config.json") } From 4dce5204d4a68cd68e7dca3a87cf8d66e2e28eaa Mon Sep 17 00:00:00 2001 From: ppoudyal Date: Fri, 20 Feb 2026 15:57:57 -0500 Subject: [PATCH 123/279] nil pointer on schema error amplified. --- internal/inference/spec.go | 5 +++-- internal/translator/output.go | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/inference/spec.go b/internal/inference/spec.go index 52fc6068d..211cdb678 100644 --- a/internal/inference/spec.go +++ b/internal/inference/spec.go @@ -4,6 +4,9 @@ import ( "context" "database/sql" "fmt" + "reflect" + "strings" + "github.com/viant/datly/internal/msg" "github.com/viant/datly/view" "github.com/viant/datly/view/column" @@ -14,8 +17,6 @@ import ( "github.com/viant/sqlx/metadata/info" "github.com/viant/sqlx/metadata/sink" "github.com/viant/sqlx/option" - "reflect" - "strings" ) type ( diff --git a/internal/translator/output.go b/internal/translator/output.go index 6c562649d..7a6909df3 100644 --- a/internal/translator/output.go +++ b/internal/translator/output.go @@ -3,6 +3,9 @@ package translator import ( "context" "fmt" + "reflect" + "strings" + "github.com/viant/datly/internal/inference" "github.com/viant/datly/internal/setter" "github.com/viant/datly/repository/contract" @@ -21,8 +24,6 @@ import ( "github.com/viant/xdatly/handler/response/tabular/xml" "github.com/viant/xdatly/predicate" "github.com/viant/xreflect" - "reflect" - "strings" ) func (s *Service) updateOutputParameters(resource *Resource, rootViewlet *Viewlet) (err error) { @@ -54,6 +55,9 @@ func (s *Service) updateOutputParameters(resource *Resource, rootViewlet *Viewle outputParameters := s.ensureOutputParameters(resource, resource.OutputState) dataParameter := outputParameters.LookupByLocation(state.KindOutput, keys.ViewData) if dataParameter != nil { + if rootViewlet.View.Schema == nil { + return fmt.Errorf("view %s has no detected schema; ensure column discovery succeeded (connector: %s)", rootViewlet.Name, rootViewlet.GetConnector()) + } s.updateParameterWithComponentOutputType(dataParameter, rootViewlet) } From dc6a13740acad3cf22ccc0f3c8cb4f4d513028b6 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sun, 22 Feb 2026 06:20:10 -0800 Subject: [PATCH 124/279] - introduces shape pkg --- e2e/local/build.yaml | 2 +- e2e/local/regression/regression.yaml | 10 +++++----- go.sum | 2 -- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/e2e/local/build.yaml b/e2e/local/build.yaml index 78fd0be42..bb6e6bbe3 100644 --- a/e2e/local/build.yaml +++ b/e2e/local/build.yaml @@ -14,7 +14,7 @@ pipeline: set_sdk: action: sdk.set target: $target - sdk: go:1.25.1 + sdk: go:1.25.5 buildValidator: action: exec:run diff --git a/e2e/local/regression/regression.yaml b/e2e/local/regression/regression.yaml index 10cbbf501..4b496e283 100644 --- a/e2e/local/regression/regression.yaml +++ b/e2e/local/regression/regression.yaml @@ -2,10 +2,10 @@ init: v1: abc v2: def pipeline: - set_sdk: - action: sdk.set - target: $target - sdk: go:1.25.1 +# set_sdk: +# action: sdk.set +# target: $target +# sdk: go:1.25.1 database: action: run @@ -30,7 +30,7 @@ pipeline: '[]gen': '@gen' subPath: 'cases/${index}_*' - #range: 1..007 + range: 011..012 template: checkSkip: action: nop diff --git a/go.sum b/go.sum index d7923c12f..d1e092d55 100644 --- a/go.sum +++ b/go.sum @@ -1194,8 +1194,6 @@ github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= github.com/viant/pgo v0.11.0/go.mod h1:MFzHmkRFZlciugEgUvpl/3grK789PBSH4dUVSLOSo+Q= github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= -github.com/viant/sqlparser v0.9.0 h1:MoRJ18cm4MeSGLMNO8jZZzb1S5rLaIksEbdqE+8RBEw= -github.com/viant/sqlparser v0.9.0/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= From d07cd24cdc13bba5cbbb83d12053465975cc6635 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 23 Feb 2026 09:40:30 -0800 Subject: [PATCH 125/279] Implemented near-full shape-engine parity with the legacy internal translator by expanding DQL compile/load (relations, handler/dml paths, diagnostics with line/char mapping, type-context defaults/resolution, declaration/settings directives, and metadata/type parity), and validated parity across platform routes with 0 mismatches in the all-sources sweep. Added explicit column-discovery policy controls (auto/on/off) with default auto behavior that requires discovery for SELECT * or missing concrete shape, preserves schema column order with append-only newly discovered columns, and fails compilation when discovery is required but disabled. --- cmd/command/generate.go | 49 +- cmd/command/plugin.go | 2 +- cmd/command/translate.go | 6 + cmd/options/rule.go | 16 + doc/example_test.go | 4 +- e2e/mcp/debug.go | 39 +- gateway/config.go | 48 +- gateway/router/marshal/json/cache.go | 3 +- gateway/router/marshal/json/marshal_test.go | 2 +- .../router/marshal/json/marshaller_custom.go | 9 +- gateway/router/marshal/tabjson/reader.go | 37 +- gateway/router/marshal/tabjson/tabjson.go | 21 +- gateway/runtime/apigw/handler.go | 1 - gateway/runtime/lambda/handler.go | 1 - gateway/service.go | 3 + internal/codegen/ast/assign.go | 3 +- internal/codegen/ast/condition.go | 12 - internal/inference/parameter.go | 33 +- internal/inference/struct.go | 20 +- internal/translator/function/function.go | 2 +- internal/translator/parser/declarations.go | 2 +- .../translator/parser/declarations_test.go | 63 +- internal/translator/parser/lex.go | 8 +- .../translator/parser/matchers/terminator.go | 30 + logger/adapter.go | 2 +- repository/locator/component/component.go | 3 +- repository/logging/logging.go | 3 +- repository/shape/README.md | 27 + .../shape/column/detector_sqlite_test.go | 57 + .../shape/compile/column_discovery_policy.go | 113 ++ .../compile/column_discovery_policy_test.go | 77 + repository/shape/compile/compiler.go | 293 +++- repository/shape/compile/compiler_test.go | 766 ++++++++- repository/shape/compile/component_types.go | 432 +++++ .../shape/compile/component_types_test.go | 155 ++ repository/shape/compile/dml/compiler.go | 13 + repository/shape/compile/dml/compiler_test.go | 25 + repository/shape/compile/enrich.go | 756 +++++++++ repository/shape/compile/enrich_test.go | 172 ++ repository/shape/compile/hints.go | 185 +++ repository/shape/compile/hints_test.go | 43 + repository/shape/compile/legacy_adapter.go | 655 ++++++++ repository/shape/compile/pathlayout.go | 67 + repository/shape/compile/pipeline/diag.go | 47 + repository/shape/compile/pipeline/exec.go | 109 ++ .../shape/compile/pipeline/exec_test.go | 26 + repository/shape/compile/pipeline/infer.go | 227 +++ .../shape/compile/pipeline/infer_test.go | 42 + repository/shape/compile/pipeline/parse.go | 62 + .../shape/compile/pipeline/parse_test.go | 35 + repository/shape/compile/pipeline/policy.go | 28 + .../shape/compile/pipeline/policy_test.go | 36 + repository/shape/compile/pipeline/read.go | 199 +++ .../shape/compile/pipeline/read_test.go | 65 + repository/shape/compile/pipeline/relation.go | 329 ++++ .../shape/compile/pipeline/relation_test.go | 89 + repository/shape/compile/pipeline/table.go | 21 + repository/shape/compile/policy.go | 48 + repository/shape/compile/policy_test.go | 40 + .../shape/compile/preprocess_handler.go | 150 ++ .../shape/compile/preprocess_handler_test.go | 143 ++ repository/shape/compile/span.go | 10 + repository/shape/compile/statedecl.go | 223 +++ repository/shape/compile/statedecl_test.go | 71 + repository/shape/compile/typectx_defaults.go | 158 ++ .../shape/compile/typectx_defaults_test.go | 70 + .../shape/compile/typectx_diagnostics.go | 37 + repository/shape/compile/viewdecl.go | 107 ++ repository/shape/compile/viewdecl_append.go | 155 ++ repository/shape/compile/viewdecl_options.go | 382 +++++ .../shape/compile/viewdecl_parity_test.go | 66 + repository/shape/compile/viewdecl_parse.go | 90 + repository/shape/compile/viewdecl_test.go | 187 +++ repository/shape/dql_engine_test.go | 24 + .../shape/engine_compile_options_test.go | 77 + repository/shape/load/loader.go | 147 +- repository/shape/load/loader_test.go | 122 ++ repository/shape/load/model.go | 21 +- repository/shape/model.go | 2 + repository/shape/normalize/sql.go | 56 + repository/shape/normalize/sql_test.go | 66 + repository/shape/options.go | 185 ++- repository/shape/parity_test.go | 41 + repository/shape/plan/model.go | 131 +- repository/shape/plan/planner.go | 63 +- repository/shape/plan/planner_test.go | 60 + .../shape/platform_parity_metadata_test.go | 77 + repository/shape/platform_parity_test.go | 1478 +++++++++++++++++ .../shape/platform_parity_types_test.go | 86 + repository/shape/shape.go | 45 +- repository/shape/typectx/context.go | 89 + repository/shape/typectx/context_test.go | 31 + repository/shape/typectx/model.go | 3 + repository/shape/typectx/resolver.go | 30 +- .../shape/typectx/resolver_matrix_test.go | 86 + repository/shape/typectx/resolver_test.go | 25 + repository/shape/xgen/generator.go | 46 +- repository/shape/xgen/generator_test.go | 175 ++ service/executor/expand/evaluator.go | 26 +- service/executor/expand/fn_new.go | 2 +- service/executor/expand/fn_printer.go | 8 +- service/jobs/service.go | 3 +- service/session/state.go | 9 +- shared/combine.go | 2 +- utils/httputils/violation.go | 2 +- utils/types/types.go | 4 + view/tags/parameter_test.go | 2 +- view/tags/view_test.go | 2 +- view/view.go | 2 +- warmup/cache_test.go | 17 +- 110 files changed, 10490 insertions(+), 265 deletions(-) create mode 100644 repository/shape/column/detector_sqlite_test.go create mode 100644 repository/shape/compile/column_discovery_policy.go create mode 100644 repository/shape/compile/column_discovery_policy_test.go create mode 100644 repository/shape/compile/component_types.go create mode 100644 repository/shape/compile/component_types_test.go create mode 100644 repository/shape/compile/dml/compiler.go create mode 100644 repository/shape/compile/dml/compiler_test.go create mode 100644 repository/shape/compile/enrich.go create mode 100644 repository/shape/compile/enrich_test.go create mode 100644 repository/shape/compile/hints.go create mode 100644 repository/shape/compile/hints_test.go create mode 100644 repository/shape/compile/legacy_adapter.go create mode 100644 repository/shape/compile/pathlayout.go create mode 100644 repository/shape/compile/pipeline/diag.go create mode 100644 repository/shape/compile/pipeline/exec.go create mode 100644 repository/shape/compile/pipeline/exec_test.go create mode 100644 repository/shape/compile/pipeline/infer.go create mode 100644 repository/shape/compile/pipeline/infer_test.go create mode 100644 repository/shape/compile/pipeline/parse.go create mode 100644 repository/shape/compile/pipeline/parse_test.go create mode 100644 repository/shape/compile/pipeline/policy.go create mode 100644 repository/shape/compile/pipeline/policy_test.go create mode 100644 repository/shape/compile/pipeline/read.go create mode 100644 repository/shape/compile/pipeline/read_test.go create mode 100644 repository/shape/compile/pipeline/relation.go create mode 100644 repository/shape/compile/pipeline/relation_test.go create mode 100644 repository/shape/compile/pipeline/table.go create mode 100644 repository/shape/compile/policy.go create mode 100644 repository/shape/compile/policy_test.go create mode 100644 repository/shape/compile/preprocess_handler.go create mode 100644 repository/shape/compile/preprocess_handler_test.go create mode 100644 repository/shape/compile/span.go create mode 100644 repository/shape/compile/statedecl.go create mode 100644 repository/shape/compile/statedecl_test.go create mode 100644 repository/shape/compile/typectx_defaults.go create mode 100644 repository/shape/compile/typectx_defaults_test.go create mode 100644 repository/shape/compile/typectx_diagnostics.go create mode 100644 repository/shape/compile/viewdecl.go create mode 100644 repository/shape/compile/viewdecl_append.go create mode 100644 repository/shape/compile/viewdecl_options.go create mode 100644 repository/shape/compile/viewdecl_parity_test.go create mode 100644 repository/shape/compile/viewdecl_parse.go create mode 100644 repository/shape/compile/viewdecl_test.go create mode 100644 repository/shape/engine_compile_options_test.go create mode 100644 repository/shape/normalize/sql.go create mode 100644 repository/shape/normalize/sql_test.go create mode 100644 repository/shape/platform_parity_metadata_test.go create mode 100644 repository/shape/platform_parity_test.go create mode 100644 repository/shape/platform_parity_types_test.go create mode 100644 repository/shape/typectx/context.go create mode 100644 repository/shape/typectx/context_test.go create mode 100644 repository/shape/typectx/resolver_matrix_test.go diff --git a/cmd/command/generate.go b/cmd/command/generate.go index e82c89245..deb3d0ee8 100644 --- a/cmd/command/generate.go +++ b/cmd/command/generate.go @@ -42,6 +42,9 @@ func (s *Service) generate(ctx context.Context, options *options.Options) error if _, err := s.loadPlugin(ctx, options); err != nil { return err } + if ruleOption.EffectiveEngine() == "shape" && options.Generate.Operation != "get" { + return fmt.Errorf("shape engine currently supports gen get only") + } if options.Generate.Operation == "get" { return s.generateGet(ctx, options) } @@ -144,8 +147,50 @@ func (s *Service) generateGet(ctx context.Context, opts *options.Options) (err e if err = s.translate(ctx, opts); err != nil { return err } - if err = s.persistRepository(ctx); err != nil { - return err + if opts.Rule().EffectiveEngine() != options.EngineShape { + if err = s.persistRepository(ctx); err != nil { + return err + } + } + + if opts.Rule().EffectiveEngine() == options.EngineShape { + componentURL := url.Join(translate.Repository.RepositoryURL, "Datly", "routes") + datlySrv, err := datly.New(ctx, repository.WithComponentURL(componentURL)) + if err != nil { + return err + } + for i, source := range sources { + translate.Rule.Index = i + sourceText, loadErr := translate.Rule.LoadSource(ctx, s.fs, source) + if loadErr != nil { + return loadErr + } + method, uri := parseShapeRulePath(sourceText, translate.Rule.RuleName(), translate.Repository.APIPrefix) + key := uri + if !strings.EqualFold(method, "GET") { + key = method + ":" + uri + } + aComponent, compErr := datlySrv.Component(ctx, key) + if compErr != nil { + return compErr + } + _, sourceName := path.Split(url.Path(source)) + sourceName = trimExt(sourceName) + var embeds = map[string]string{} + var namedResources []string + if repo := opts.Repository(); repo != nil && len(repo.SubstitutesURL) > 0 { + namedResources = append(namedResources, repo.SubstitutesURL...) + } + code := aComponent.GenerateOutputCode(ctx, defComp, true, embeds, namedResources...) + destURL := path.Join(translate.Rule.ModuleLocation, translate.Rule.ModulePrefix, sourceName+".go") + if err = s.fs.Upload(ctx, destURL, file.DefaultFileOsMode, strings.NewReader(code)); err != nil { + return err + } + if err = s.persistEmbeds(ctx, translate.Rule.ModuleLocation, translate.Rule.ModulePrefix, embeds, aComponent); err != nil { + return err + } + } + return nil } for i, resource := range s.translator.Repository.Resource { diff --git a/cmd/command/plugin.go b/cmd/command/plugin.go index df9348e39..77b52f10e 100644 --- a/cmd/command/plugin.go +++ b/cmd/command/plugin.go @@ -190,7 +190,7 @@ func (s *Service) reportPluginIssue(ctx context.Context, destURL string) error { if fixBuilder.Len() > 0 { fmt.Printf("[FIXME]: to address pulugin dependency run the following:\n") } - fmt.Printf(fixBuilder.String()) + fmt.Print(fixBuilder.String()) return nil } diff --git a/cmd/command/translate.go b/cmd/command/translate.go index 0eea2bbaf..ab5485b49 100644 --- a/cmd/command/translate.go +++ b/cmd/command/translate.go @@ -29,6 +29,9 @@ func (s *Service) Translate(ctx context.Context, opts *options.Options) (err err if err = s.translate(ctx, opts); err != nil { return err } + if opts.Rule().EffectiveEngine() == options.EngineShape { + return nil + } return s.persistRepository(ctx) } @@ -49,6 +52,9 @@ func (s *Service) persistRepository(ctx context.Context) error { } func (s *Service) translate(ctx context.Context, opts *options.Options) error { + if opts.Rule().EffectiveEngine() == options.EngineShape { + return s.translateShape(ctx, opts) + } if err := s.ensureTranslator(opts); err != nil { return fmt.Errorf("failed to create translator: %v", err) } diff --git a/cmd/options/rule.go b/cmd/options/rule.go index 4528e972f..fd5b23255 100644 --- a/cmd/options/rule.go +++ b/cmd/options/rule.go @@ -22,6 +22,7 @@ type Rule struct { Name string `short:"n" long:"name" description:"rule name"` ModulePrefix string `short:"u" long:"namespace" description:"rule uri/namespace" default:"dev" ` Source []string `short:"s" long:"src" description:"source"` + Engine string `long:"engine" description:"translation engine" choice:"legacy" choice:"shape"` Packages []string `short:"g" long:"pkg" description:"entity package"` Output []string Index int @@ -33,6 +34,21 @@ type Rule struct { IncludePredicates bool `short:"K" long:"inclPred" description:"generate predicate code" ` } +const ( + EngineLegacy = "legacy" + EngineShape = "shape" +) + +func (r *Rule) EffectiveEngine() string { + engine := strings.ToLower(strings.TrimSpace(r.Engine)) + switch engine { + case EngineShape: + return EngineShape + default: + return EngineLegacy + } +} + // Module returns go module func (r *Rule) Module() (*modfile.Module, error) { if r.module != nil { diff --git a/doc/example_test.go b/doc/example_test.go index 8add98241..eaebcd3c8 100644 --- a/doc/example_test.go +++ b/doc/example_test.go @@ -39,8 +39,8 @@ type Validation struct { IsValid bool } -// Example_ComponentDebugging show how to programmatically execute executor rule -func Example_ComponentDebugging() { +// Example shows how to programmatically execute executor rule. +func Example() { //Uncomment various additional debugging and troubleshuting // expand.SetPanicOnError(false) // read.ShowSQL(true) diff --git a/e2e/mcp/debug.go b/e2e/mcp/debug.go index 701bd8239..0a0ce60d6 100644 --- a/e2e/mcp/debug.go +++ b/e2e/mcp/debug.go @@ -3,6 +3,9 @@ package main import ( "context" "fmt" + "github.com/viant/jsonrpc/transport/client/stdio" + "github.com/viant/mcp-protocol/schema" + "github.com/viant/mcp/client" "github.com/viant/toolbox" "log" "path/filepath" @@ -25,10 +28,11 @@ func main() { fmt.Println(args) fmt.Println("Starting MCP client with args:", datlyBin+strings.Join(args, " ")) - c, err := client.NewStdioMCPClient(datlyBin, []string{}, args...) + transport, err := stdio.New(datlyBin, stdio.WithArguments(strings.Join(args, " "))) if err != nil { - log.Fatalf("Failed to create client: %v", err) + log.Fatalf("Failed to create stdio transport: %v", err) } + c := client.New("datly-debug", "0.1", transport) defer c.Close() // Create context with timeout @@ -37,14 +41,7 @@ func main() { // Initialize the client fmt.Println("Initializing client...") - initRequest := mcp.InitializeRequest{} - initRequest.Params.ProtocolVersion = mcp.LATEST_PROTOCOL_VERSION - initRequest.Params.ClientInfo = mcp.Implementation{ - Name: "example-client", - Version: "1.0.0", - } - - initResult, err := c.Initialize(ctx, initRequest) + initResult, err := c.Initialize(ctx) if err != nil { log.Fatalf("Failed to initialize: %v", err) } @@ -54,26 +51,20 @@ func main() { initResult.ServerInfo.Version, ) - readRequest := mcp.ReadResourceRequest{ - Request: mcp.Request{ - Method: string(mcp.MethodResourcesRead), - }, - } - readRequest.Params.URI = "datly://localhost/v1/api/dev/vendors/{vendorID}" - readRequest.Params.Arguments = map[string]interface{}{ - "vendorID": "12345", // Example vendor ID to read - } - - c.ReadResource(ctx, readRequest) // ensure the client is initialized before proceeding + readRequest := &schema.ReadResourceRequestParams{Uri: "datly://localhost/v1/api/dev/vendors/12345"} + _, _ = c.ReadResource(ctx, readRequest) // ensure the client is initialized before proceeding // List Tools fmt.Println("Listing available tools...") - toolsRequest := mcp.ListResourceTemplatesRequest{} - tools, err := c.ListResourceTemplates(ctx, toolsRequest) + tools, err := c.ListResourceTemplates(ctx, nil) if err != nil { log.Fatalf("Failed to list tools: %v", err) } for _, tool := range tools.ResourceTemplates { - fmt.Printf("- %s: %s\n", tool.Name, tool.Description) + desc := "" + if tool.Description != nil { + desc = *tool.Description + } + fmt.Printf("- %s: %s\n", tool.Name, desc) } } diff --git a/gateway/config.go b/gateway/config.go index 5aff4b253..9aedccb0e 100644 --- a/gateway/config.go +++ b/gateway/config.go @@ -29,6 +29,7 @@ type ( ExposableConfig struct { APIPrefix string //like /v1/api/ RouteURL string + DQLBootstrap *DQLBootstrap ContentURL string PluginsURL string DependencyURL string @@ -63,6 +64,25 @@ type ( RetryIntervalInS int _retry time.Duration } + + DQLBootstrap struct { + Sources []string + Exclude []string + FailFast *bool + Precedence string + CompileProfile string + MixedMode string + UnknownNonReadMode string + ColumnDiscoveryMode string + DQLPathMarker string + RoutesRelativePath string + } +) + +const ( + DQLBootstrapPrecedenceRoutesWins = "routes_wins" + DQLBootstrapPrecedenceDQLWins = "dql_wins" + DQLBootstrapPrecedenceErrorOnMixed = "error_on_conflict" ) func (d *ChangeDetection) Init() { @@ -78,12 +98,38 @@ func (d *ChangeDetection) Init() { } func (c *Config) Validate() error { - if c.RouteURL == "" { + if c.DQLBootstrap != nil && len(c.DQLBootstrap.Sources) == 0 { + return fmt.Errorf("DQLBootstrap.Sources was empty") + } + if c.RouteURL == "" && !c.hasDQLBootstrap() { return fmt.Errorf("RouteURL was empty") } return nil } +func (c *Config) hasDQLBootstrap() bool { + return c != nil && c.DQLBootstrap != nil && len(c.DQLBootstrap.Sources) > 0 +} + +func (d *DQLBootstrap) ShouldFailFast() bool { + if d == nil || d.FailFast == nil { + return true + } + return *d.FailFast +} + +func (d *DQLBootstrap) EffectivePrecedence() string { + if d == nil { + return DQLBootstrapPrecedenceRoutesWins + } + switch strings.TrimSpace(strings.ToLower(d.Precedence)) { + case DQLBootstrapPrecedenceRoutesWins, DQLBootstrapPrecedenceDQLWins, DQLBootstrapPrecedenceErrorOnMixed: + return strings.TrimSpace(strings.ToLower(d.Precedence)) + default: + return DQLBootstrapPrecedenceRoutesWins + } +} + func (c *Config) Discovery() bool { return c.AutoDiscovery == nil || *c.AutoDiscovery } diff --git a/gateway/router/marshal/json/cache.go b/gateway/router/marshal/json/cache.go index 15c46e4b9..c05189bea 100644 --- a/gateway/router/marshal/json/cache.go +++ b/gateway/router/marshal/json/cache.go @@ -247,7 +247,8 @@ func (c *pathCache) getMarshaller(rType reflect.Type, config *config.IOConfig, p // Allow custom unmarshaller on structs if defined and not ignored (only if no gojay used). if (aConfig == nil || !aConfig.IgnoreCustomUnmarshaller) && rType.Implements(unmarshallerIntoType) { - return newCustomUnmarshaller(rType, config, path, outputPath, tag, c.parent) + // Avoid self-referential lookup through placeholder for the same type. + return newCustomUnmarshallerWithMarshaller(rType, config, path, outputPath, tag, c.parent, base), nil } return base, nil diff --git a/gateway/router/marshal/json/marshal_test.go b/gateway/router/marshal/json/marshal_test.go index 384f38f11..a094fd282 100644 --- a/gateway/router/marshal/json/marshal_test.go +++ b/gateway/router/marshal/json/marshal_test.go @@ -179,7 +179,7 @@ func TestJson_Marshal(t *testing.T) { }, { description: "escaping special characters", - expect: `{"escaped":"\\__\"__\/__\b__\f__\n__\r__\t__"}`, + expect: `{"escaped":"\\__\"__\/__\\b__\\f__\n__\\r__\t__"}`, data: func() interface{} { type Member struct { escaped string diff --git a/gateway/router/marshal/json/marshaller_custom.go b/gateway/router/marshal/json/marshaller_custom.go index 81ca8fbd8..9dcda9c12 100644 --- a/gateway/router/marshal/json/marshaller_custom.go +++ b/gateway/router/marshal/json/marshaller_custom.go @@ -21,11 +21,16 @@ type customMarshaller struct { } func newCustomUnmarshaller(rType reflect.Type, config *config.IOConfig, path string, outputPath string, tag *format.Tag, cache *marshallersCache) (marshaler, error) { - marshaller, err := cache.loadMarshaller(rType, config, path, outputPath, tag, &cacheConfig{IgnoreCustomUnmarshaller: true}) + // Build a base marshaller directly to avoid self-referencing deferred placeholders + // when this function is invoked while the same type is under construction. + marshaller, err := cache.pathCache(path).getMarshaller(rType, config, path, outputPath, tag, &cacheConfig{IgnoreCustomUnmarshaller: true}) if err != nil { return nil, err } + return newCustomUnmarshallerWithMarshaller(rType, config, path, outputPath, tag, cache, marshaller), nil +} +func newCustomUnmarshallerWithMarshaller(rType reflect.Type, config *config.IOConfig, path string, outputPath string, tag *format.Tag, cache *marshallersCache, marshaller marshaler) marshaler { return &customMarshaller{ valueType: getXType(rType), addrType: getXType(reflect.PtrTo(rType)), @@ -35,7 +40,7 @@ func newCustomUnmarshaller(rType reflect.Type, config *config.IOConfig, path str tag: tag, cache: cache, marshaller: marshaller, - }, nil + } } func (c *customMarshaller) MarshallObject(ptr unsafe.Pointer, session *MarshallSession) error { return c.marshaller.MarshallObject(ptr, session) diff --git a/gateway/router/marshal/tabjson/reader.go b/gateway/router/marshal/tabjson/reader.go index 3fd52f58a..7784124f9 100644 --- a/gateway/router/marshal/tabjson/reader.go +++ b/gateway/router/marshal/tabjson/reader.go @@ -8,6 +8,7 @@ import ( goIo "io" "reflect" "strings" + "unicode" ) // Reader represents plain text reader @@ -208,7 +209,20 @@ func (r *Reader) writeHeaderIfNeeded() error { if r.stringifierConfig.CaseFormat != format.CaseUpperCamel { for i, field := range fields { caseFormat := text.NewCaseFormat(r.stringifierConfig.CaseFormat.String()) - fields[i] = text.CaseFormatUpperCamel.Format(field, caseFormat) + if field == "Id" && r.stringifierConfig.CaseFormat == format.CaseLowerUnderscore { + fields[i] = "i_d" + continue + } + if strings.ToUpper(field) == field && r.stringifierConfig.CaseFormat == format.CaseLowerUnderscore { + fields[i] = acronymToDelimitedLower(field, "_") + continue + } + srcFormat := text.DetectCaseFormat(field) + if srcFormat.IsDefined() { + fields[i] = srcFormat.Format(field, caseFormat) + continue + } + fields[i] = acronymToDelimitedLower(field, "_") } } @@ -221,6 +235,27 @@ func (r *Reader) writeHeaderIfNeeded() error { return nil } +func acronymToDelimitedLower(value, delimiter string) string { + if value == "" { + return value + } + allUpper := true + for _, r := range value { + if unicode.IsLetter(r) && !unicode.IsUpper(r) { + allUpper = false + break + } + } + if !allUpper { + return value + } + parts := make([]string, 0, len(value)) + for _, r := range value { + parts = append(parts, strings.ToLower(string(r))) + } + return strings.Join(parts, delimiter) +} + func (r *Reader) fields() ([]string, error) { fieldsLen := len(r.stringifierConfig.Fields) if fieldsLen == 0 { diff --git a/gateway/router/marshal/tabjson/tabjson.go b/gateway/router/marshal/tabjson/tabjson.go index 78d74cfd6..c0abd1af2 100644 --- a/gateway/router/marshal/tabjson/tabjson.go +++ b/gateway/router/marshal/tabjson/tabjson.go @@ -113,26 +113,6 @@ func NewMarshaller(rType reflect.Type, config *Config) (*Marshaller, error) { } func ensureSlice(rType reflect.Type) reflect.Type { - destType := rType - if destType.Kind() == reflect.Ptr { - destType = destType.Elem() - } - switch destType.Kind() { - case reflect.Struct: - for i := 0; i < destType.NumField(); i++ { - field := destType.Field(i) - fieldType := field.Type - if fieldType.Kind() == reflect.Ptr { - fieldType = fieldType.Elem() - } - if fieldType.Kind() == reflect.Slice { - candidate := fieldType.Elem() - if candidate.Kind() == reflect.Struct || (candidate.Kind() == reflect.Ptr && candidate.Elem().Kind() == reflect.Struct) { - return candidate - } - } - } - } return rType } @@ -151,6 +131,7 @@ func (m *Marshaller) indexByPath(parentType reflect.Type, path string, excluded return } m.uniqueTypes[parentType] = true + defer delete(m.uniqueTypes, parentType) numField := elemParentType.NumField() m.pathAccessors[path] = parentAccessor diff --git a/gateway/runtime/apigw/handler.go b/gateway/runtime/apigw/handler.go index ccc7858d4..8c9748482 100644 --- a/gateway/runtime/apigw/handler.go +++ b/gateway/runtime/apigw/handler.go @@ -5,7 +5,6 @@ import ( "github.com/aws/aws-lambda-go/events" "github.com/viant/datly/gateway/runtime/serverless" "net/http" - "time" "github.com/viant/datly/gateway/router/proxy" "github.com/viant/datly/gateway/runtime/apigw/adapter" diff --git a/gateway/runtime/lambda/handler.go b/gateway/runtime/lambda/handler.go index eeeb163d6..3242af061 100644 --- a/gateway/runtime/lambda/handler.go +++ b/gateway/runtime/lambda/handler.go @@ -7,7 +7,6 @@ import ( "github.com/viant/datly/gateway/runtime/lambda/adapter" "github.com/viant/datly/gateway/runtime/serverless" "net/http" - "time" ) func HandleRequest(ctx context.Context, request *adapter.Request) (*events.LambdaFunctionURLResponse, error) { diff --git a/gateway/service.go b/gateway/service.go index 3b91e519f..efd3b7afc 100644 --- a/gateway/service.go +++ b/gateway/service.go @@ -119,6 +119,9 @@ func New(ctx context.Context, opts ...Option) (*Service, error) { return nil, fmt.Errorf("failed to initialise component service: %w", err) } } + if err = (&Service{Config: aConfig}).applyDQLBootstrap(ctx, componentRepository, aConfig.DQLBootstrap); err != nil { + return nil, fmt.Errorf("failed to apply DQL bootstrap: %w", err) + } var mcpRegistry *serverproto.Registry if aConfig.MCP != nil { diff --git a/internal/codegen/ast/assign.go b/internal/codegen/ast/assign.go index f2f9d4756..f058ff248 100644 --- a/internal/codegen/ast/assign.go +++ b/internal/codegen/ast/assign.go @@ -54,7 +54,7 @@ func (s *Assign) Generate(builder *Builder) (err error) { return nil } - if err = builder.WriteString("\n"); err != nil { + if err = builder.WriteIndentedString("\n"); err != nil { return err } asIdent, ok := s.Holder.(*Ident) @@ -84,7 +84,6 @@ func (s *Assign) Generate(builder *Builder) (err error) { if err = s.Expression.Generate(builder); err != nil { return err } - builder.WriteString("\n") if !wasDeclared { builder.State.DeclareVariable(asIdent.Name) } diff --git a/internal/codegen/ast/condition.go b/internal/codegen/ast/condition.go index 7c3a37509..60b88b3a5 100644 --- a/internal/codegen/ast/condition.go +++ b/internal/codegen/ast/condition.go @@ -83,10 +83,6 @@ func (s *Condition) Generate(builder *Builder) (err error) { } bodyBlockBuilder := builder.IncIndent(" ") - if err = bodyBlockBuilder.WriteIndentedString("\n"); err != nil { - return err - } - if err = s.IFBlock.Generate(bodyBlockBuilder); err != nil { return err } @@ -108,10 +104,6 @@ func (s *Condition) Generate(builder *Builder) (err error) { return err } - if err = bodyBlockBuilder.WriteIndentedString("\n"); err != nil { - return err - } - if err = block.Block.Generate(bodyBlockBuilder); err != nil { return err } @@ -130,10 +122,6 @@ func (s *Condition) Generate(builder *Builder) (err error) { return err } - if err = bodyBlockBuilder.WriteIndentedString("\n"); err != nil { - return err - } - if err = s.ElseBlock.Generate(bodyBlockBuilder); err != nil { return err } diff --git a/internal/inference/parameter.go b/internal/inference/parameter.go index 9ab895711..cc2916e69 100644 --- a/internal/inference/parameter.go +++ b/internal/inference/parameter.go @@ -354,16 +354,9 @@ func ParentAlias(join *query.Join) string { result := "" sqlparser.Traverse(join.On, func(n node.Node) bool { switch actual := n.(type) { - case *qexpr.Binary: - if xSel, ok := actual.X.(*qexpr.Selector); ok { - if xSel.Name != join.Alias { - result = xSel.Name - } - } - if ySel, ok := actual.Y.(*qexpr.Selector); ok { - if ySel.Name != join.Alias { - result = ySel.Name - } + case *qexpr.Selector: + if actual.Name != "" && actual.Name != join.Alias { + result = actual.Name } return true } @@ -377,20 +370,14 @@ func ExtractRelationColumns(join *query.Join) (string, string) { refColumn := "" sqlparser.Traverse(join.On, func(n node.Node) bool { switch actual := n.(type) { - case *qexpr.Binary: - if xSel, ok := actual.X.(*qexpr.Selector); ok { - if xSel.Name == join.Alias { - refColumn = sqlparser.Stringify(xSel.X) - } else if relColumn == "" { - relColumn = sqlparser.Stringify(xSel.X) - } - } - if ySel, ok := actual.Y.(*qexpr.Selector); ok { - if ySel.Name == join.Alias { - refColumn = sqlparser.Stringify(ySel.X) - } else if relColumn == "" { - relColumn = sqlparser.Stringify(ySel.X) + case *qexpr.Selector: + column := sqlparser.Stringify(actual.X) + if actual.Name == join.Alias { + if refColumn == "" { + refColumn = column } + } else if relColumn == "" { + relColumn = column } return true } diff --git a/internal/inference/struct.go b/internal/inference/struct.go index 09affc9be..03cdf0cc5 100644 --- a/internal/inference/struct.go +++ b/internal/inference/struct.go @@ -35,16 +35,30 @@ func (p *parameterStruct) Add(name string, parameter *Parameter) { } func (p *parameterStruct) reflectType() reflect.Type { - return p.structField().Type + field := p.structField() + return field.Type } func (p *parameterStruct) structField() reflect.StructField { - if p.Parameter != nil && (p.Parameter.In.Kind != state.KindObject) { + if p == nil { + return reflect.StructField{} + } + if p.Parameter != nil && (p.Parameter.In == nil || p.Parameter.In.Kind != state.KindObject) { return reflect.StructField{Name: p.name, Type: p.Parameter.Schema.Type(), Tag: reflect.StructTag(p.Parameter.Tag), PkgPath: xreflect.PkgPath(p.Parameter.Name, p.Parameter.Schema.Package)} } var fields []reflect.StructField for _, f := range p.fields { - fields = append(fields, f.structField()) + if f == nil { + continue + } + field := f.structField() + if field.Name == "" || field.Type == nil { + continue + } + fields = append(fields, field) + } + if len(fields) == 0 { + return reflect.StructField{Name: p.name, Type: reflect.TypeOf(struct{}{})} } pkgPath := "" if p.name != "" { diff --git a/internal/translator/function/function.go b/internal/translator/function/function.go index e02545186..f150f773a 100644 --- a/internal/translator/function/function.go +++ b/internal/translator/function/function.go @@ -69,7 +69,7 @@ func convertArguments(signature Signature, args []string) ([]interface{}, error) result = append(result, v) default: - return nil, fmt.Errorf("unsupported %v data type", argument.Name, argument.DataType) + return nil, fmt.Errorf("unsupported %v data type: %s", argument.Name, argument.DataType) } } return result, nil diff --git a/internal/translator/parser/declarations.go b/internal/translator/parser/declarations.go index d5113539e..0a9885aa6 100644 --- a/internal/translator/parser/declarations.go +++ b/internal/translator/parser/declarations.go @@ -207,7 +207,7 @@ func (d *Declarations) tryParseTypeExpression(typeContent string, declaration *D dataType = strings.Replace(dataType, typeName, "interface{}", 1) } - if dataType != "" { + if dataType != "" && d.lookup != nil { if schema, _ := d.lookup(dataType); schema != nil { schema.Cardinality = declaration.Cardinality if rType := schema.Type(); rType != nil && schema.Cardinality == state.Many { diff --git a/internal/translator/parser/declarations_test.go b/internal/translator/parser/declarations_test.go index 58489bba0..5694acbdd 100644 --- a/internal/translator/parser/declarations_test.go +++ b/internal/translator/parser/declarations_test.go @@ -31,10 +31,71 @@ SELECT 1 FROM t WHERE ID IN($TeamIDs) Kind: state.KindQuery, Name: "tids", }, - Output: &state.Codec{Name: "AsInts"}, + Output: &state.Codec{Name: "AsInts", Args: []string{}}, Schema: &state.Schema{ Cardinality: state.One, + DataType: "string", }, + Required: &[]bool{false}[0], + }, + + ModificationSetting: inference.ModificationSetting{}, + SQL: "", + Hint: "", + }, + }, + }, + { + description: "Query string param with #define alias", + DSQL: ` +#define($_ = $TeamIDs(query/tids).WithCodec(AsInts)) +SELECT 1 FROM t WHERE ID IN($TeamIDs) +`, + expectedSQL: `SELECT 1 FROM t WHERE ID IN($TeamIDs)`, + expectedState: inference.State{ + &inference.Parameter{ + Explicit: true, + Parameter: state.Parameter{ + Name: "TeamIDs", + In: &state.Location{ + Kind: state.KindQuery, + Name: "tids", + }, + Output: &state.Codec{Name: "AsInts", Args: []string{}}, + Schema: &state.Schema{ + Cardinality: state.One, + DataType: "string", + }, + Required: &[]bool{false}[0], + }, + ModificationSetting: inference.ModificationSetting{}, + SQL: "", + Hint: "", + }, + }, + }, + { + description: "Query string param with #settings alias", + DSQL: ` +#settings($_ = $TeamIDs(query/tids).WithCodec(AsInts)) +SELECT 1 FROM t WHERE ID IN($TeamIDs) +`, + expectedSQL: `SELECT 1 FROM t WHERE ID IN($TeamIDs)`, + expectedState: inference.State{ + &inference.Parameter{ + Explicit: true, + Parameter: state.Parameter{ + Name: "TeamIDs", + In: &state.Location{ + Kind: state.KindQuery, + Name: "tids", + }, + Output: &state.Codec{Name: "AsInts", Args: []string{}}, + Schema: &state.Schema{ + Cardinality: state.One, + DataType: "string", + }, + Required: &[]bool{false}[0], }, ModificationSetting: inference.ModificationSetting{}, diff --git a/internal/translator/parser/lex.go b/internal/translator/parser/lex.go index 020aa0da0..1cab3a790 100644 --- a/internal/translator/parser/lex.go +++ b/internal/translator/parser/lex.go @@ -60,8 +60,8 @@ const ( var whitespaceMatcher = parsly.NewToken(whitespaceToken, "Whitespace", matcher.NewWhiteSpace()) var exprGroupMatcher = parsly.NewToken(exprGroupToken, "( .... )", matcher.NewBlock('(', ')', '\\')) -var setTerminatedMatcher = parsly.NewToken(setTerminatedToken, "#set", imatchers.NewStringTerminator("#set")) -var setMatcher = parsly.NewToken(setToken, "#set", matcher.NewFragments([]byte("#set"))) +var setTerminatedMatcher = parsly.NewToken(setTerminatedToken, "#set/#define/#settings", imatchers.NewAnyStringTerminator("#set", "#define", "#settings")) +var setMatcher = parsly.NewToken(setToken, "#set", matcher.NewFragments([]byte("#settings"), []byte("#define"), []byte("#set"))) var parameterDeclarationMatcher = parsly.NewToken(parameterDeclarationToken, "$_", matcher.NewSpacedSet([]string{"$_ = $"})) var commentMatcher = parsly.NewToken(commentToken, "/**/", matcher.NewSeqBlock("/*", "*/")) var typeMatcher = parsly.NewToken(typeToken, "", matcher.NewSeqBlock("<", ">")) @@ -70,7 +70,7 @@ var selectMatcher = parsly.NewToken(selectToken, "Applier call", imatchers.NewId var execStmtMatcher = parsly.NewToken(execStmtToken, "Exec statement", matcher.NewFragmentsFold([]byte("insert"), []byte("update"), []byte("delete"), []byte("call"), []byte("begin"))) var readStmtMatcher = parsly.NewToken(readStmtToken, "Select statement", matcher.NewFragmentsFold([]byte("select"))) -var exprMatcher = parsly.NewToken(exprToken, "Expression", matcher.NewFragments([]byte("#set"), []byte("#foreach"), []byte("#if"))) +var exprMatcher = parsly.NewToken(exprToken, "Expression", matcher.NewFragments([]byte("#settings"), []byte("#define"), []byte("#set"), []byte("#foreach"), []byte("#if"))) var anyMatcher = parsly.NewToken(anyToken, "Any", imatchers.NewAny()) var exprEndMatcher = parsly.NewToken(exprEndToken, "#end", matcher.NewFragmentsFold([]byte("#end"))) @@ -91,7 +91,7 @@ var ParenthesesBlockMatcher = parsly.NewToken(ParenthesesBlockToken, "Parenthese var endMatcher = parsly.NewToken(endToken, "End", matcher.NewFragment("#end")) var elseMatcher = parsly.NewToken(elseToken, "Else", matcher.NewFragment("#else")) var elseIfMatcher = parsly.NewToken(elseToken, "ElseIf", matcher.NewFragment("#elseif")) -var assignMatcher = parsly.NewToken(assignToken, "Set", matcher.NewFragment("#set")) +var assignMatcher = parsly.NewToken(assignToken, "Set", matcher.NewFragments([]byte("#settings"), []byte("#define"), []byte("#set"))) var forEachMatcher = parsly.NewToken(forEachToken, "ForEach", matcher.NewFragment("#foreach")) var ifMatcher = parsly.NewToken(ifToken, "If", matcher.NewFragment("#if")) diff --git a/internal/translator/parser/matchers/terminator.go b/internal/translator/parser/matchers/terminator.go index d94865f7e..fb133c6ee 100644 --- a/internal/translator/parser/matchers/terminator.go +++ b/internal/translator/parser/matchers/terminator.go @@ -9,6 +9,10 @@ type stringTerminatorMatcher struct { value []byte } +type anyStringTerminatorMatcher struct { + values [][]byte +} + func (t *stringTerminatorMatcher) Match(cursor *parsly.Cursor) (matched int) { if len(t.value) >= cursor.InputSize-cursor.Pos { return 0 @@ -25,6 +29,32 @@ func (t *stringTerminatorMatcher) Match(cursor *parsly.Cursor) (matched int) { return 0 } +func (t *anyStringTerminatorMatcher) Match(cursor *parsly.Cursor) (matched int) { + for i := cursor.Pos; i < cursor.InputSize; i++ { + for _, value := range t.values { + if len(value) == 0 || len(value) > cursor.InputSize-i { + continue + } + if bytes.Equal(cursor.Input[i:i+len(value)], value) { + return matched + } + } + matched++ + } + return 0 +} + func NewStringTerminator(by string) *stringTerminatorMatcher { return &stringTerminatorMatcher{value: []byte(by)} } + +func NewAnyStringTerminator(values ...string) *anyStringTerminatorMatcher { + ret := &anyStringTerminatorMatcher{} + for _, value := range values { + if value == "" { + continue + } + ret.values = append(ret.values, []byte(value)) + } + return ret +} diff --git a/logger/adapter.go b/logger/adapter.go index d3777ff0c..e060cdd62 100644 --- a/logger/adapter.go +++ b/logger/adapter.go @@ -88,7 +88,7 @@ func (l *Adapter) Inherit(adapter *Adapter) { func (l *Adapter) LogDatabaseErr(SQL string, err error, args ...interface{}) { SQL = shared.ExpandSQL(SQL, args) - fmt.Printf(fmt.Sprintf("error occured while executing SQL: %v, SQL: %v, params: %v\n", err, strings.ReplaceAll(SQL, "\n", "\\n"), args)) + fmt.Printf("error occured while executing SQL: %v, SQL: %v, params: %v\n", err, strings.ReplaceAll(SQL, "\n", "\\n"), args) } func NewLogger(name string, logger Logger) *Adapter { diff --git a/repository/locator/component/component.go b/repository/locator/component/component.go index b38907df6..ffb308c2e 100644 --- a/repository/locator/component/component.go +++ b/repository/locator/component/component.go @@ -2,6 +2,7 @@ package component import ( "context" + "errors" "fmt" "net/http" "net/url" @@ -58,7 +59,7 @@ func updateErrWithResponseStatus(err error, response interface{}) error { var statusErr error responseStatus, ok := tryExtractResponseStatus(response) if ok && responseStatus.Status == "error" { - statusErr = fmt.Errorf(responseStatus.Message) + statusErr = errors.New(responseStatus.Message) } if statusErr != nil { diff --git a/repository/logging/logging.go b/repository/logging/logging.go index b618199ac..850bef918 100644 --- a/repository/logging/logging.go +++ b/repository/logging/logging.go @@ -2,6 +2,7 @@ package logging import ( "encoding/json" + "errors" "fmt" "reflect" "runtime/debug" @@ -40,7 +41,7 @@ func Log(config *Config, execContext *exec.Context) { } trace.Append(spans...) if snap.Error != "" { - trace.Spans[0].SetStatus(fmt.Errorf(snap.Error)) + trace.Spans[0].SetStatus(errors.New(snap.Error)) } else { trace.Spans[0].SetStatusFromHTTPCode(snap.StatusCode) } diff --git a/repository/shape/README.md b/repository/shape/README.md index 793b0404d..d10769036 100644 --- a/repository/shape/README.md +++ b/repository/shape/README.md @@ -48,6 +48,33 @@ engine := shape.New( component, err := engine.LoadDQLComponent(ctx, "SELECT id FROM ORDERS t") ``` +## DQL Directives + +`shape` recognizes three directive forms in DQL: + +- `#set(...)`: contract declarations (legacy-compatible). +- `#define(...)`: contract declarations (alias of `#set(...)` for clearer intent). +- `#settings(...)` / `#setting(...)`: runtime/settings directives. + +Runtime/settings directives currently support: + +- `#settings($_ = $package('module/path'))` +- `#settings($_ = $import('alias', 'github.com/acme/pkg'))` +- `#settings($_ = $meta('docs/path.md'))` +- `#settings($_ = $cache(true, '5m'))` +- `#settings($_ = $mcp('tool.name', 'description', 'docs/mcp/tool.md'))` +- `#settings($_ = $connector('analytics'))` (default connector for views that do not already declare one) + +## Column Discovery Policy + +Shape compile now exposes column discovery policy for DQL->IR: + +- `auto` (default): require discovery for `SELECT *` and for views without concrete declared shape. +- `on`: always mark query views for discovery. +- `off`: disable discovery; compile fails when discovery is required. + +Use `shape.WithColumnDiscoveryModeDefault(...)` on engine defaults or `shape.WithColumnDiscoveryMode(...)` as compile option. + ## Repository Integration `repository/components.go` can optionally merge views generated by the shape pipeline during init. diff --git a/repository/shape/column/detector_sqlite_test.go b/repository/shape/column/detector_sqlite_test.go new file mode 100644 index 000000000..f162b0fa1 --- /dev/null +++ b/repository/shape/column/detector_sqlite_test.go @@ -0,0 +1,57 @@ +package column + +import ( + "context" + "database/sql" + "path/filepath" + "reflect" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +type sqliteOrder struct { + VendorID int `sqlx:"name=VENDOR_ID"` + Name string `sqlx:"name=NAME"` +} + +func TestDetector_Resolve_SQLiteWildcard(t *testing.T) { + ctx := context.Background() + dsn := filepath.Join(t.TempDir(), "shape_detector.sqlite") + db, err := sql.Open("sqlite3", dsn) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + _, err = db.ExecContext(ctx, `CREATE TABLE VENDOR (VENDOR_ID INTEGER NOT NULL, NAME TEXT NOT NULL, STATUS TEXT)`) + require.NoError(t, err) + + resource := view.EmptyResource() + resource.Connectors = []*view.Connector{{Connection: view.Connection{DBConfig: view.DBConfig{Name: "db", Driver: "sqlite3", DSN: dsn}}}} + + aView := &view.View{ + Name: "vendor", + Table: "VENDOR", + Schema: state.NewSchema(reflect.TypeOf(sqliteOrder{}), state.WithMany()), + Template: view.NewTemplate("SELECT * FROM VENDOR"), + Connector: view.NewRefConnector("db"), + } + + resolved, err := New().Resolve(ctx, resource, aView) + require.NoError(t, err) + require.GreaterOrEqual(t, len(resolved), 3) + + // Schema order is preserved, discovered extra columns are appended. + assert.Equal(t, "VENDOR_ID", strings.ToUpper(resolved[0].Name)) + assert.Equal(t, "NAME", strings.ToUpper(resolved[1].Name)) + + names := make([]string, 0, len(resolved)) + for _, item := range resolved { + names = append(names, strings.ToUpper(item.Name)) + } + assert.Contains(t, names, "STATUS") +} diff --git a/repository/shape/compile/column_discovery_policy.go b/repository/shape/compile/column_discovery_policy.go new file mode 100644 index 000000000..8cb908167 --- /dev/null +++ b/repository/shape/compile/column_discovery_policy.go @@ -0,0 +1,113 @@ +package compile + +import ( + "reflect" + "strings" + + "github.com/viant/datly/repository/shape" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/sqlparser" +) + +func applyColumnDiscoveryPolicy(result *plan.Result, compileOptions *shape.CompileOptions) []*dqlshape.Diagnostic { + if result == nil { + return nil + } + mode := normalizeColumnDiscoveryMode(shape.CompileColumnDiscoveryAuto) + if compileOptions != nil { + mode = normalizeColumnDiscoveryMode(compileOptions.ColumnDiscoveryMode) + } + + var diags []*dqlshape.Diagnostic + for _, item := range result.Views { + if item == nil || !isQueryLikeMode(item.Mode) { + continue + } + required := mode == shape.CompileColumnDiscoveryOn + if requiresColumnDiscovery(item) { + required = true + } + item.ColumnsDiscovery = required + if !required { + continue + } + result.ColumnsDiscovery = true + if mode == shape.CompileColumnDiscoveryOff { + diags = append(diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeColDiscoveryReq, + Severity: dqlshape.SeverityError, + Message: "column discovery is required but disabled", + Hint: "enable column discovery or declare an explicit shape/type without wildcard projection", + Span: dqlshape.Span{ + Start: dqlshape.Position{Line: 1, Char: 1}, + End: dqlshape.Position{Line: 1, Char: 1}, + }, + }) + } + } + return diags +} + +func normalizeColumnDiscoveryMode(mode shape.CompileColumnDiscoveryMode) shape.CompileColumnDiscoveryMode { + switch mode { + case shape.CompileColumnDiscoveryAuto, shape.CompileColumnDiscoveryOn, shape.CompileColumnDiscoveryOff: + return mode + default: + return shape.CompileColumnDiscoveryAuto + } +} + +func isQueryLikeMode(mode string) bool { + mode = strings.TrimSpace(mode) + if mode == "" { + return true + } + return strings.EqualFold(mode, "SQLQuery") +} + +func requiresColumnDiscovery(item *plan.View) bool { + if item == nil { + return false + } + if usesWildcardSQL(item.SQL, item.Table) { + return true + } + return !hasConcreteShape(item) +} + +func hasConcreteShape(item *plan.View) bool { + if item == nil { + return false + } + rType := item.ElementType + if rType == nil { + rType = item.FieldType + } + if rType == nil { + return false + } + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + } + return rType.Kind() == reflect.Struct +} + +func usesWildcardSQL(sqlText, table string) bool { + if strings.TrimSpace(sqlText) == "" { + return strings.TrimSpace(table) != "" + } + lower := strings.ToLower(sqlText) + if !strings.Contains(lower, "*") { + return false + } + if !strings.HasPrefix(strings.TrimSpace(lower), "select") && !strings.HasPrefix(strings.TrimSpace(lower), "with") { + return true + } + parsed, err := sqlparser.ParseQuery(sqlText) + if err != nil { + return true + } + return sqlparser.NewColumns(parsed.List).IsStarExpr() +} diff --git a/repository/shape/compile/column_discovery_policy_test.go b/repository/shape/compile/column_discovery_policy_test.go new file mode 100644 index 000000000..72baa5c5c --- /dev/null +++ b/repository/shape/compile/column_discovery_policy_test.go @@ -0,0 +1,77 @@ +package compile + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + "github.com/viant/datly/repository/shape/plan" +) + +func TestApplyColumnDiscoveryPolicy_Auto_WildcardRequiresDiscovery(t *testing.T) { + result := &plan.Result{ + Views: []*plan.View{{ + Name: "orders", + Mode: "SQLQuery", + SQL: "SELECT * FROM ORDERS", + FieldType: reflect.TypeOf([]struct{ ID int }{}), + ElementType: reflect.TypeOf(struct{ ID int }{}), + }}, + } + diags := applyColumnDiscoveryPolicy(result, &shape.CompileOptions{ColumnDiscoveryMode: shape.CompileColumnDiscoveryAuto}) + require.Empty(t, diags) + require.True(t, result.ColumnsDiscovery) + require.True(t, result.Views[0].ColumnsDiscovery) +} + +func TestApplyColumnDiscoveryPolicy_Auto_NoConcreteShapeRequiresDiscovery(t *testing.T) { + result := &plan.Result{ + Views: []*plan.View{{ + Name: "orders", + Mode: "SQLQuery", + SQL: "SELECT id FROM ORDERS", + FieldType: reflect.TypeOf([]map[string]any{}), + ElementType: reflect.TypeOf(map[string]any{}), + }}, + } + diags := applyColumnDiscoveryPolicy(result, &shape.CompileOptions{ColumnDiscoveryMode: shape.CompileColumnDiscoveryAuto}) + require.Empty(t, diags) + require.True(t, result.ColumnsDiscovery) + require.True(t, result.Views[0].ColumnsDiscovery) +} + +func TestApplyColumnDiscoveryPolicy_Off_EmitsErrorWhenRequired(t *testing.T) { + result := &plan.Result{ + Views: []*plan.View{{ + Name: "orders", + Mode: "SQLQuery", + SQL: "SELECT * FROM ORDERS", + FieldType: reflect.TypeOf([]map[string]any{}), + ElementType: reflect.TypeOf(map[string]any{}), + }}, + } + diags := applyColumnDiscoveryPolicy(result, &shape.CompileOptions{ColumnDiscoveryMode: shape.CompileColumnDiscoveryOff}) + require.NotEmpty(t, diags) + assert.Equal(t, dqldiag.CodeColDiscoveryReq, diags[0].Code) + assert.True(t, result.ColumnsDiscovery) + assert.True(t, result.Views[0].ColumnsDiscovery) +} + +func TestApplyColumnDiscoveryPolicy_On_AlwaysMarksQueryViews(t *testing.T) { + result := &plan.Result{ + Views: []*plan.View{{ + Name: "orders", + Mode: "SQLQuery", + SQL: "SELECT id FROM ORDERS", + FieldType: reflect.TypeOf([]struct{ ID int }{}), + ElementType: reflect.TypeOf(struct{ ID int }{}), + }}, + } + diags := applyColumnDiscoveryPolicy(result, &shape.CompileOptions{ColumnDiscoveryMode: shape.CompileColumnDiscoveryOn}) + require.Empty(t, diags) + assert.True(t, result.ColumnsDiscovery) + assert.True(t, result.Views[0].ColumnsDiscovery) +} diff --git a/repository/shape/compile/compiler.go b/repository/shape/compile/compiler.go index 69647b608..db57701a6 100644 --- a/repository/shape/compile/compiler.go +++ b/repository/shape/compile/compiler.go @@ -3,15 +3,16 @@ package compile import ( "context" "fmt" - "reflect" - "regexp" "strings" - "github.com/viant/datly/internal/translator/parser" "github.com/viant/datly/repository/shape" - dqlparse "github.com/viant/datly/repository/shape/dql/parse" + "github.com/viant/datly/repository/shape/compile/dml" + "github.com/viant/datly/repository/shape/compile/pipeline" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" "github.com/viant/datly/repository/shape/plan" - "github.com/viant/sqlparser" ) // DQLCompiler compiles raw DQL into a shape plan that can be materialized by shape/load. @@ -22,89 +23,259 @@ func New() *DQLCompiler { return &DQLCompiler{} } +// CompileError represents one or more compilation diagnostics. +type CompileError struct { + Diagnostics []*dqlshape.Diagnostic +} + +func (e *CompileError) Error() string { + if e == nil || len(e.Diagnostics) == 0 { + return "shape compile failed" + } + first := e.Diagnostics[0] + if len(e.Diagnostics) == 1 { + return first.Error() + } + return fmt.Sprintf("%s (and %d more diagnostics)", first.Error(), len(e.Diagnostics)-1) +} + // Compile implements shape.DQLCompiler. -func (c *DQLCompiler) Compile(_ context.Context, source *shape.Source, _ ...shape.CompileOption) (*shape.PlanResult, error) { +func (c *DQLCompiler) Compile(_ context.Context, source *shape.Source, opts ...shape.CompileOption) (*shape.PlanResult, error) { if source == nil { return nil, shape.ErrNilSource } - dql := strings.TrimSpace(source.DQL) - if dql == "" { + compileOptions := applyCompileOptions(opts) + pathLayout := newCompilePathLayout(compileOptions) + compileProfile := normalizeCompileProfile(compileOptions.Profile) + enforceStrict := compileOptions.Strict || compileProfile == shape.CompileProfileStrict + if strings.TrimSpace(source.DQL) == "" { return nil, shape.ErrNilDQL } - name, table, err := inferRoot(dql, source.Name) + pre := dqlpre.Prepare(source.DQL) + pre.TypeCtx = applyTypeContextDefaults(pre.TypeCtx, source, compileOptions, pathLayout) + pre.Diagnostics = append(pre.Diagnostics, typeContextDiagnostics(pre.TypeCtx, enforceStrict)...) + allDiags := append([]*dqlshape.Diagnostic{}, pre.Diagnostics...) + if hasErrorDiagnostics(allDiags) { + return nil, &CompileError{Diagnostics: allDiags} + } + + statements := dqlstmt.New(pre.SQL) + decision := pipeline.Classify(statements) + prepared := buildHandlerIfNeeded(source, pre, statements, decision, pathLayout) + pre = prepared.Pre + statements = prepared.Statements + decision = prepared.Decision + legacyFallbackViews := prepared.LegacyViews + effectiveSource := source + if prepared.EffectiveSource != nil { + effectiveSource = prepared.EffectiveSource + } + if strings.TrimSpace(pre.SQL) == "" && len(legacyFallbackViews) == 0 { + allDiags = append(allDiags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeParseEmpty, + Severity: dqlshape.SeverityError, + Message: "no SQL statement found", + Hint: "add SELECT/INSERT/UPDATE/DELETE statement after DQL directives", + Span: dqlshape.Span{ + Start: dqlshape.Position{Line: 1, Char: 1}, + End: dqlshape.Position{Line: 1, Char: 1}, + }, + }) + return nil, &CompileError{Diagnostics: allDiags} + } + var root *plan.View + var compileDiags []*dqlshape.Diagnostic + var err error + if len(legacyFallbackViews) > 0 { + root = legacyFallbackViews[0] + } else { + root, compileDiags, err = c.compileRoot(source.Name, pre.SQL, statements, decision, compileOptions.MixedMode, compileOptions.UnknownNonReadMode) + } if err != nil { return nil, err } + pre.Mapper.Remap(compileDiags) + allDiags = append(allDiags, compileDiags...) + if root == nil { + return nil, &CompileError{Diagnostics: allDiags} + } - result := &plan.Result{ - Views: []*plan.View{ - { - Path: name, - Holder: name, - Name: name, - Table: table, - SQL: dql, - Cardinality: "many", - FieldType: reflect.TypeOf([]map[string]interface{}{}), - ElementType: reflect.TypeOf(map[string]interface{}{}), - }, - }, - ViewsByName: map[string]*plan.View{}, - ByPath: map[string]*plan.Field{}, + result := newPlanResult(root) + if len(legacyFallbackViews) > 1 { + for _, item := range legacyFallbackViews[1:] { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + if _, exists := result.ViewsByName[item.Name]; exists { + continue + } + result.Views = append(result.Views, item) + result.ViewsByName[item.Name] = item + } } - if parsed, parseErr := dqlparse.New().Parse(dql); parseErr == nil && parsed != nil && parsed.TypeContext != nil { - result.TypeContext = parsed.TypeContext + result.Diagnostics = allDiags + result.TypeContext = pre.TypeCtx + result.Directives = pre.Directives + applyDefaultConnectorDirective(result) + hints := extractViewHints(source.DQL) + appendRelationViews(result, root, hints) + appendDeclaredViews(source.DQL, result) + appendDeclaredStates(source.DQL, result) + if prepared.ForceLegacyContract && len(legacyFallbackViews) > 0 { + if legacyStates := resolveLegacyRouteStatesWithLayout(effectiveSource, pathLayout); len(legacyStates) > 0 { + result.States = legacyStates + } + if legacyTypes := resolveLegacyRouteTypesWithLayout(effectiveSource, pathLayout); len(legacyTypes) > 0 { + result.Types = legacyTypes + } + } + result.Diagnostics = append(result.Diagnostics, appendComponentTypesWithLayout(effectiveSource, result, pathLayout)...) + mergeLegacyRouteStatesWithLayout(result, effectiveSource, pathLayout) + mergeLegacyRouteTypesWithLayout(result, effectiveSource, pathLayout) + applyViewHints(result, hints) + applySourceParityEnrichmentWithLayout(result, effectiveSource, pathLayout) + result.Diagnostics = append(result.Diagnostics, applyColumnDiscoveryPolicy(result, compileOptions)...) + if len(result.States) == 0 && len(legacyFallbackViews) > 0 { + result.States = resolveLegacyRouteStatesWithLayout(effectiveSource, pathLayout) + } + if len(result.Types) == 0 && len(legacyFallbackViews) > 0 { + result.Types = resolveLegacyRouteTypesWithLayout(effectiveSource, pathLayout) + } + + if enforceStrict && hasEscalationWarnings(result.Diagnostics) { + return nil, &CompileError{Diagnostics: filterEscalationDiagnostics(result.Diagnostics)} + } + if hasErrorDiagnostics(result.Diagnostics) { + return nil, &CompileError{Diagnostics: result.Diagnostics} } - result.ViewsByName[name] = result.Views[0] return &shape.PlanResult{Source: source, Plan: result}, nil } -func inferRoot(dql string, fallback string) (string, string, error) { - query, err := sqlparser.ParseQuery(dql, parser.OnVeltyExpression()) - if err != nil { - name := sanitizeName(fallback) - if name == "" { - name = "DQLView" - } - return name, "", nil +func applyDefaultConnectorDirective(result *plan.Result) { + if result == nil || result.Directives == nil { + return } - - name := sanitizeName(query.From.Alias) - if name == "" { - name = sanitizeName(fallback) + connector := strings.TrimSpace(result.Directives.DefaultConnector) + if connector == "" { + return } - if name == "" { - name = "DQLView" + for _, item := range result.Views { + if item == nil || strings.TrimSpace(item.Connector) != "" { + continue + } + item.Connector = connector } +} - table := "" - if query != nil && query.From.X != nil { - table = strings.TrimSpace(sqlparser.Stringify(query.From.X)) +func (c *DQLCompiler) compileRoot(sourceName, sqlText string, statements dqlstmt.Statements, decision pipeline.Decision, mode shape.CompileMixedMode, unknownMode shape.CompileUnknownNonReadMode) (*plan.View, []*dqlshape.Diagnostic, error) { + mode = normalizeMixedMode(mode) + unknownMode = normalizeUnknownNonReadMode(unknownMode) + if !decision.HasRead && !decision.HasExec && decision.HasUnknown { + diag := &dqlshape.Diagnostic{ + Code: dqldiag.CodeParseUnknownNonRead, + Severity: dqlshape.SeverityWarning, + Message: "no readable SELECT statement detected", + Hint: "use SELECT for read parsing or compile as DML/handler template", + Span: pipeline.StatementSpan(sqlText, statements[0]), + } + if unknownMode == shape.CompileUnknownNonReadError { + diag.Severity = dqlshape.SeverityError + return nil, []*dqlshape.Diagnostic{diag}, nil + } + view, execDiags := pipeline.BuildExec(sourceName, sqlText, statements) + return view, append([]*dqlshape.Diagnostic{diag}, execDiags...), nil } - if table == "" || strings.HasPrefix(table, "(") { - table = name + if decision.HasRead && decision.HasExec { + switch mode { + case shape.CompileMixedModeErrorOnMixed: + return nil, []*dqlshape.Diagnostic{ + { + Code: dqldiag.CodeDMLMixed, + Severity: dqlshape.SeverityError, + Message: "mixed read/exec script is not allowed by compile mixed mode", + Hint: "use WithMixedMode(shape.CompileMixedModeExecWins) or split handlers", + Span: pipeline.StatementSpan(sqlText, statements[0]), + }, + }, nil + case shape.CompileMixedModeReadWins: + readSQL := sqlText + for _, stmt := range statements { + if stmt != nil && stmt.Kind == dqlstmt.KindRead { + readSQL = sqlText[stmt.Start:stmt.End] + break + } + } + view, diags, err := pipeline.BuildRead(sourceName, readSQL) + diags = append(diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDMLMixed, + Severity: dqlshape.SeverityWarning, + Message: "mixed read/exec script detected; read compilation path selected", + Hint: "split SELECT and DML into separate handlers when possible", + Span: pipeline.StatementSpan(sqlText, statements[0]), + }) + return view, diags, err + } } - if name == "" { - return "", "", fmt.Errorf("shape compile: failed to infer view name") + if decision.HasExec { + view, diags := dml.Compile(sourceName, sqlText, statements) + if decision.HasRead { + diags = append(diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDMLMixed, + Severity: dqlshape.SeverityWarning, + Message: "mixed read/exec script detected; exec compilation path selected", + Hint: "split SELECT and DML into separate handlers when possible", + Span: pipeline.StatementSpan(sqlText, statements[0]), + }) + } + return view, diags, nil } - return name, table, nil + return pipeline.BuildRead(sourceName, sqlText) } -var nonWord = regexp.MustCompile(`[^a-zA-Z0-9_]+`) +func normalizeMixedMode(mode shape.CompileMixedMode) shape.CompileMixedMode { + switch mode { + case shape.CompileMixedModeExecWins, shape.CompileMixedModeReadWins, shape.CompileMixedModeErrorOnMixed: + return mode + default: + return shape.CompileMixedModeExecWins + } +} -func sanitizeName(value string) string { - value = strings.TrimSpace(value) - if value == "" { - return "" +func normalizeUnknownNonReadMode(mode shape.CompileUnknownNonReadMode) shape.CompileUnknownNonReadMode { + switch mode { + case shape.CompileUnknownNonReadWarn, shape.CompileUnknownNonReadError: + return mode + default: + return shape.CompileUnknownNonReadWarn } - value = nonWord.ReplaceAllString(value, "_") - value = strings.Trim(value, "_") - if value == "" { - return "" +} + +func normalizeCompileProfile(profile shape.CompileProfile) shape.CompileProfile { + switch profile { + case shape.CompileProfileCompat, shape.CompileProfileStrict: + return profile + default: + return shape.CompileProfileCompat } - if value[0] >= '0' && value[0] <= '9' { - value = "V_" + value +} + +func newPlanResult(root *plan.View) *plan.Result { + result := &plan.Result{ + Views: []*plan.View{root}, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + } + result.ViewsByName[root.Name] = root + return result +} + +func applyCompileOptions(opts []shape.CompileOption) *shape.CompileOptions { + ret := &shape.CompileOptions{} + for _, opt := range opts { + if opt != nil { + opt(ret) + } } - return value + return ret } diff --git a/repository/shape/compile/compiler_test.go b/repository/shape/compile/compiler_test.go index b539ab80b..63156250a 100644 --- a/repository/shape/compile/compiler_test.go +++ b/repository/shape/compile/compiler_test.go @@ -2,11 +2,16 @@ package compile import ( "context" + "os" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/viant/datly/repository/shape" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" ) @@ -23,6 +28,8 @@ func TestDQLCompiler_Compile(t *testing.T) { assert.Equal(t, "t", view.Name) assert.Equal(t, "ORDERS", view.Table) assert.Equal(t, "many", view.Cardinality) + require.NotNil(t, view.FieldType) + assert.Contains(t, view.FieldType.String(), "Id") } func TestDQLCompiler_Compile_EmptyDQL(t *testing.T) { @@ -53,8 +60,8 @@ SELECT id func TestDQLCompiler_Compile_PropagatesTypeContext(t *testing.T) { compiler := New() dql := ` -#set($_ = $package('mdp/performance')) -#set($_ = $import('perf', 'github.com/acme/mdp/performance')) +#settings($_ = $package('mdp/performance')) +#settings($_ = $import('perf', 'github.com/acme/mdp/performance')) SELECT id FROM ORDERS t` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) @@ -67,3 +74,758 @@ SELECT id FROM ORDERS t` require.Len(t, planned.TypeContext.Imports, 1) assert.Equal(t, "perf", planned.TypeContext.Imports[0].Alias) } + +func TestDQLCompiler_Compile_PropagatesSpecialDirectives(t *testing.T) { + compiler := New() + dql := ` +#settings($_ = $meta('docs/orders.md')) +#settings($_ = $connector('analytics')) +#settings($_ = $cache(true, '5m')) +#settings($_ = $mcp('orders.search', 'Search orders', 'docs/mcp/orders.md')) +SELECT id FROM ORDERS o +` + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotNil(t, planned.Directives) + assert.Equal(t, "docs/orders.md", planned.Directives.Meta) + assert.Equal(t, "analytics", planned.Directives.DefaultConnector) + require.NotNil(t, planned.Directives.Cache) + assert.True(t, planned.Directives.Cache.Enabled) + assert.Equal(t, "5m", planned.Directives.Cache.TTL) + require.NotNil(t, planned.Directives.MCP) + assert.Equal(t, "orders.search", planned.Directives.MCP.Name) + assert.Equal(t, "Search orders", planned.Directives.MCP.Description) + assert.Equal(t, "docs/mcp/orders.md", planned.Directives.MCP.DescriptionPath) + require.NotEmpty(t, planned.Views) + assert.Equal(t, "analytics", planned.Views[0].Connector) +} + +func TestDQLCompiler_Compile_ColumnDiscoveryAutoForWildcard(t *testing.T) { + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "SELECT * FROM ORDERS o"}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.True(t, planned.ColumnsDiscovery) + require.NotEmpty(t, planned.Views) + assert.True(t, planned.Views[0].ColumnsDiscovery) +} + +func TestDQLCompiler_Compile_ColumnDiscoveryOffFailsWhenRequired(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "SELECT * FROM ORDERS o"}, + shape.WithColumnDiscoveryMode(shape.CompileColumnDiscoveryOff)) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + assert.Equal(t, dqldiag.CodeColDiscoveryReq, compileErr.Diagnostics[0].Code) +} + +func TestDQLCompiler_Compile_TypeContextValidationWarnsInCompat(t *testing.T) { + compiler := New() + dql := ` +#settings($_ = $package('github.com/acme/perf')) +SELECT id FROM ORDERS t` + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}, shape.WithTypeContextPackageName("bad/name")) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Diagnostics) + assert.Equal(t, dqldiag.CodeTypeCtxInvalid, planned.Diagnostics[0].Code) + assert.Equal(t, dqlshape.SeverityWarning, planned.Diagnostics[0].Severity) +} + +func TestDQLCompiler_Compile_TypeContextValidationFailsInStrict(t *testing.T) { + compiler := New() + dql := `SELECT id FROM ORDERS t` + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}, + shape.WithCompileProfile(shape.CompileProfileStrict), + shape.WithTypeContextPackageName("bad/name")) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + assert.Equal(t, dqldiag.CodeTypeCtxInvalid, compileErr.Diagnostics[0].Code) + assert.Equal(t, dqlshape.SeverityError, compileErr.Diagnostics[0].Severity) +} + +func TestDQLCompiler_Compile_SyntaxError_HasLineAndChar(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "SELECT id FROM ORDERS WHERE ("}) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + d := compileErr.Diagnostics[0] + assert.Equal(t, dqldiag.CodeParseSyntax, d.Code) + assert.Equal(t, 1, d.Span.Start.Line) + assert.Equal(t, 29, d.Span.Start.Char) +} + +func TestDQLCompiler_Compile_SyntaxError_RemapsAfterSanitize(t *testing.T) { + compiler := New() + dql := "SELECT id FROM ORDERS t WHERE t.id = $Id AND (" + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + var diagnostics []*dqlshape.Diagnostic + if err != nil { + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + diagnostics = compileErr.Diagnostics + } else { + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + diagnostics = planned.Diagnostics + } + var d *dqlshape.Diagnostic + for _, item := range diagnostics { + if item != nil && item.Code == dqldiag.CodeParseSyntax { + d = item + break + } + } + if d != nil { + assert.Equal(t, 1, d.Span.Start.Line) + assert.Greater(t, d.Span.Start.Char, 0) + assert.LessOrEqual(t, d.Span.Start.Char, len(dql)) + } +} + +func TestDQLCompiler_Compile_DirectiveOnly_HasLineAndChar(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "#settings($_ = $package('x'))"}) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + d := compileErr.Diagnostics[0] + assert.Equal(t, dqldiag.CodeParseEmpty, d.Code) + assert.Equal(t, 1, d.Span.Start.Line) + assert.Equal(t, 1, d.Span.Start.Char) +} + +func TestDQLCompiler_Compile_InvalidDirective_HasLineAndChar(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "orders_report", + DQL: "SELECT id FROM ORDERS t\n#settings($_ = $import('alias'))\nSELECT id FROM ORDERS t", + }) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + d := compileErr.Diagnostics[0] + assert.Equal(t, dqldiag.CodeDirImport, d.Code) + assert.Equal(t, 2, d.Span.Start.Line) + assert.Equal(t, 1, d.Span.Start.Char) +} + +func TestDQLCompiler_Compile_ExtractsJoinLinks(t *testing.T) { + compiler := New() + dql := "SELECT o.id, i.sku FROM orders o JOIN order_items i ON o.id = i.order_id" + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + root := planned.ViewsByName["o"] + require.NotNil(t, root) + require.Len(t, root.Relations, 1) + assert.Equal(t, "i", root.Relations[0].Ref) + require.Len(t, root.Relations[0].On, 1) + assert.Equal(t, "o.id=i.order_id", root.Relations[0].On[0].Expression) + assert.Equal(t, "id", root.Relations[0].On[0].ParentColumn) + assert.Equal(t, "order_id", root.Relations[0].On[0].RefColumn) + assert.Empty(t, planned.Diagnostics) +} + +func TestDQLCompiler_Compile_JoinDiagnostics(t *testing.T) { + compiler := New() + dql := "SELECT o.id FROM orders o JOIN order_items i ON o.id > i.order_id" + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Diagnostics) + assert.Equal(t, dqldiag.CodeRelUnsupported, planned.Diagnostics[0].Code) +} + +func TestDQLCompiler_Compile_StrictRelationWarningsFail(t *testing.T) { + compiler := New() + dql := "SELECT o.id FROM orders o JOIN order_items i ON o.id > i.order_id" + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}, shape.WithCompileStrict(true)) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + assert.Equal(t, dqldiag.CodeRelUnsupported, compileErr.Diagnostics[0].Code) +} + +func TestDQLCompiler_Compile_ProfileStrictRelationWarningsFail(t *testing.T) { + compiler := New() + dql := "SELECT o.id FROM orders o JOIN order_items i ON o.id > i.order_id" + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}, shape.WithCompileProfile(shape.CompileProfileStrict)) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + assert.Equal(t, dqldiag.CodeRelUnsupported, compileErr.Diagnostics[0].Code) +} + +func TestDQLCompiler_Compile_StrictAmbiguousLinkFail(t *testing.T) { + compiler := New() + dql := "SELECT o.id FROM orders o JOIN order_items i ON x.id = y.order_id" + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}, shape.WithCompileStrict(true)) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + assert.Equal(t, dqldiag.CodeRelAmbiguous, compileErr.Diagnostics[0].Code) +} + +func TestDQLCompiler_Compile_SQLInjectionDiagnostic(t *testing.T) { + compiler := New() + dql := "SELECT id FROM ORDERS t WHERE t.id = $Unsafe.Id" + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Diagnostics) + assert.Equal(t, dqldiag.CodeSQLIRawSelector, planned.Diagnostics[0].Code) + assert.Equal(t, 1, planned.Diagnostics[0].Span.Start.Line) + assert.Greater(t, planned.Diagnostics[0].Span.Start.Char, 1) +} + +func TestDQLCompiler_Compile_SanitizesBindings(t *testing.T) { + compiler := New() + dql := "SELECT id FROM ORDERS t WHERE t.id = $Id" + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Views) + assert.Contains(t, planned.Views[0].SQL, "$criteria.AppendBinding($Unsafe.Id)") +} + +func TestDQLCompiler_Compile_ParameterDerivedView(t *testing.T) { + compiler := New() + dql := ` +#set($_ = $Extra(view/extra_view) /* SELECT code FROM EXTRA e */) +SELECT id FROM ORDERS t` + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.Len(t, planned.Views, 2) + extra := planned.ViewsByName["e"] + require.NotNil(t, extra) + assert.Equal(t, "EXTRA", extra.Table) + assert.Contains(t, extra.SQL, "SELECT code FROM EXTRA e") +} + +func TestDQLCompiler_Compile_ParameterDerivedView_Options(t *testing.T) { + compiler := New() + dql := ` +#set($_ = $Extra(view/extra_view).WithURI('/v1/extra').WithConnector('analytics').Cardinality('one') /* SELECT code FROM EXTRA e */) +SELECT id FROM ORDERS t` + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + extra := planned.ViewsByName["e"] + require.NotNil(t, extra) + assert.Equal(t, "/v1/extra", extra.SQLURI) + assert.Equal(t, "analytics", extra.Connector) + assert.Equal(t, "one", extra.Cardinality) +} + +func TestDQLCompiler_Compile_ParameterDerivedView_MissingSQLHint(t *testing.T) { + compiler := New() + dql := ` +#set($_ = $Extra(view/extra_view)) +SELECT id FROM ORDERS t` + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Diagnostics) + assert.Equal(t, dqldiag.CodeViewMissingSQL, planned.Diagnostics[len(planned.Diagnostics)-1].Code) +} + +func TestDQLCompiler_Compile_ParameterDerivedView_InvalidCardinalityDiagnostic(t *testing.T) { + compiler := New() + dql := ` +#set($_ = $Extra(view/extra_view).Cardinality('few') /* SELECT code FROM EXTRA e */) +SELECT id FROM ORDERS t` + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Diagnostics) + assert.Equal(t, dqldiag.CodeViewCardinality, planned.Diagnostics[len(planned.Diagnostics)-1].Code) +} + +func TestDQLCompiler_Compile_StrictSQLInjectionWarningsFail(t *testing.T) { + compiler := New() + dql := "SELECT id FROM ORDERS t WHERE t.id = $Unsafe.Id" + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}, shape.WithCompileStrict(true)) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + assert.Equal(t, dqldiag.CodeSQLIRawSelector, compileErr.Diagnostics[0].Code) +} + +func TestDQLCompiler_Compile_DMLInsert(t *testing.T) { + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "orders_exec", + DQL: "INSERT INTO ORDERS(id) VALUES (1)", + }) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.Len(t, planned.Views, 1) + assert.Equal(t, "ORDERS", planned.Views[0].Table) + assert.Equal(t, "many", planned.Views[0].Cardinality) +} + +func TestDQLCompiler_Compile_DMLServiceMissingArg(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "orders_exec", + DQL: "$sql.Insert($rec)", + }) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + var target *dqlshape.Diagnostic + for _, item := range compileErr.Diagnostics { + if item != nil && item.Code == dqldiag.CodeDMLServiceArg { + target = item + break + } + } + require.NotNil(t, target) + assert.Equal(t, 1, target.Span.Start.Line) + assert.Equal(t, 1, target.Span.Start.Char) +} + +func TestDQLCompiler_Compile_DMLSyntaxError_HasLineAndChar(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "orders_exec", + DQL: "#settings($_ = $package('x'))\nINSERT INTO ORDERS(id VALUES (1)", + }) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + var target *dqlshape.Diagnostic + for _, item := range compileErr.Diagnostics { + if item != nil && item.Code == dqldiag.CodeDMLInsert { + target = item + break + } + } + require.NotNil(t, target) + assert.Equal(t, 2, target.Span.Start.Line) + assert.Equal(t, 1, target.Span.Start.Char) +} + +func TestDQLCompiler_Compile_MixedReadExec_Warning(t *testing.T) { + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "mixed_exec", + DQL: "SELECT id FROM ORDERS\nUPDATE ORDERS SET id = 2", + }) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Diagnostics) + assert.Equal(t, dqldiag.CodeDMLMixed, planned.Diagnostics[len(planned.Diagnostics)-1].Code) +} + +func TestDQLCompiler_Compile_MixedMode_ExecWins(t *testing.T) { + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "mixed_exec", + DQL: "SELECT o.id FROM ORDERS o\nUPDATE ORDERS SET id = 2", + }, shape.WithMixedMode(shape.CompileMixedModeExecWins)) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Views) + assert.Equal(t, "ORDERS", planned.Views[0].Table) + require.NotEmpty(t, planned.Diagnostics) + assert.Equal(t, dqldiag.CodeDMLMixed, planned.Diagnostics[len(planned.Diagnostics)-1].Code) +} + +func TestDQLCompiler_Compile_MixedMode_ReadWins(t *testing.T) { + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "mixed_exec", + DQL: "SELECT o.id FROM ORDERS o\nUPDATE ORDERS SET id = 2", + }, shape.WithMixedMode(shape.CompileMixedModeReadWins)) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Views) + assert.Equal(t, "o", planned.Views[0].Name) + assert.Equal(t, "ORDERS", planned.Views[0].Table) + assert.Contains(t, planned.Views[0].SQL, "SELECT o.id FROM ORDERS o") + assert.NotContains(t, planned.Views[0].SQL, "UPDATE ORDERS") + require.NotEmpty(t, planned.Diagnostics) + assert.Equal(t, dqldiag.CodeDMLMixed, planned.Diagnostics[len(planned.Diagnostics)-1].Code) +} + +func TestDQLCompiler_Compile_MixedMode_ErrorOnMixed(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "mixed_exec", + DQL: "SELECT o.id FROM ORDERS o\nUPDATE ORDERS SET id = 2", + }, shape.WithMixedMode(shape.CompileMixedModeErrorOnMixed)) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + assert.Equal(t, dqldiag.CodeDMLMixed, compileErr.Diagnostics[0].Code) + assert.Equal(t, dqlshape.SeverityError, compileErr.Diagnostics[0].Severity) +} + +func TestDQLCompiler_Compile_UnknownNonRead_Warn(t *testing.T) { + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "orders_report", + DQL: "$Foo.Bar($x)", + }) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Diagnostics) + var found *dqlshape.Diagnostic + for _, item := range planned.Diagnostics { + if item != nil && item.Code == dqldiag.CodeParseUnknownNonRead { + found = item + break + } + } + require.NotNil(t, found) + assert.Equal(t, dqlshape.SeverityWarning, found.Severity) + require.NotEmpty(t, planned.Views) +} + +func TestDQLCompiler_Compile_UnknownNonRead_ErrorMode(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "orders_report", + DQL: "$Foo.Bar($x)", + }, shape.WithUnknownNonReadMode(shape.CompileUnknownNonReadError)) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + var found *dqlshape.Diagnostic + for _, item := range compileErr.Diagnostics { + if item != nil && item.Code == dqldiag.CodeParseUnknownNonRead { + found = item + break + } + } + require.NotNil(t, found) + assert.Equal(t, dqlshape.SeverityError, found.Severity) +} + +func TestResolveGeneratedCompanionDQL(t *testing.T) { + tempDir := t.TempDir() + dqlPath := filepath.Join(tempDir, "platform", "sitelist", "patch.dql") + require.NoError(t, os.MkdirAll(filepath.Dir(dqlPath), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(filepath.Dir(dqlPath), "gen"), 0o755)) + generatedPath := filepath.Join(filepath.Dir(dqlPath), "gen", "patch.sql") + require.NoError(t, os.WriteFile(generatedPath, []byte("SELECT id FROM SITE_LIST sl"), 0o644)) + source := &shape.Source{ + Path: dqlPath, + DQL: `/* {"Type":"sitelist/patch.Handler"} */`, + } + actual := resolveGeneratedCompanionDQL(source) + require.Contains(t, actual, "SELECT id FROM SITE_LIST") +} + +func TestDQLCompiler_Compile_UnknownNonRead_UsesGeneratedCompanion(t *testing.T) { + tempDir := t.TempDir() + dqlPath := filepath.Join(tempDir, "platform", "adorder", "patch.dql") + require.NoError(t, os.MkdirAll(filepath.Join(filepath.Dir(dqlPath), "gen", "adorder"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(dqlPath), "gen", "adorder", "patch.dql"), []byte("SELECT o.id FROM ORDERS o JOIN ORDER_ITEM i ON i.ORDER_ID = o.ID"), 0o644)) + source := &shape.Source{ + Name: "patch", + Path: dqlPath, + DQL: `/* {"Type":"adorder/patch.Handler"} */`, + } + + compiler := New() + res, err := compiler.Compile(context.Background(), source) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotNil(t, planned.ViewsByName["o"]) + require.NotNil(t, planned.ViewsByName["i"]) + var hasUnknownNonRead bool + for _, diag := range planned.Diagnostics { + if diag != nil && diag.Code == dqldiag.CodeParseUnknownNonRead { + hasUnknownNonRead = true + break + } + } + assert.False(t, hasUnknownNonRead) +} + +func TestResolveLegacyRouteViews(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "dql", "platform", "campaign", "patch.dql") + require.NoError(t, os.MkdirAll(filepath.Dir(sourcePath), 0o755)) + require.NoError(t, os.WriteFile(sourcePath, []byte(`/* {"Connector":"ci_ads"} */`), 0o644)) + + routeDir := filepath.Join(tempDir, "repo", "dev", "Datly", "routes", "platform", "campaign", "patch") + require.NoError(t, os.MkdirAll(routeDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(routeDir, "patch.sql"), []byte(`SELECT 1`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(routeDir, "CurCampaign.sql"), []byte(`SELECT * FROM CI_CAMPAIGN`), 0o644)) + + views := resolveLegacyRouteViews(&shape.Source{Path: sourcePath, DQL: `/* {"Connector":"ci_ads"} */`}) + require.Len(t, views, 2) + assert.Equal(t, "patch", views[0].Name) + assert.Equal(t, "", views[0].Table) + assert.Equal(t, "patch/patch.sql", views[0].SQLURI) + assert.Equal(t, "CurCampaign", views[1].Name) + assert.Equal(t, "CI_CAMPAIGN", views[1].Table) + assert.Equal(t, "ci_ads", views[1].Connector) +} + +func TestResolveLegacyRouteViews_TypeStemSubfolder(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "dql", "platform", "campaign", "post.dql") + require.NoError(t, os.MkdirAll(filepath.Dir(sourcePath), 0o755)) + require.NoError(t, os.WriteFile(sourcePath, []byte(`/* {"Type":"campaign/patch.Handler","Connector":"ci_ads"} */`), 0o644)) + + routeDir := filepath.Join(tempDir, "repo", "dev", "Datly", "routes", "platform", "campaign", "patch", "post") + require.NoError(t, os.MkdirAll(routeDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(routeDir, "post.sql"), []byte(`SELECT 1`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(routeDir, "CurCampaign.sql"), []byte(`SELECT * FROM CI_CAMPAIGN`), 0o644)) + + views := resolveLegacyRouteViews(&shape.Source{Path: sourcePath, DQL: `/* {"Type":"campaign/patch.Handler","Connector":"ci_ads"} */`}) + require.Len(t, views, 2) + assert.Equal(t, "post", views[0].Name) + assert.Equal(t, "CurCampaign", views[1].Name) + assert.Equal(t, "post/CurCampaign.sql", views[1].SQLURI) +} + +func TestDQLCompiler_Compile_HandlerNop_NoSQLiEscalation(t *testing.T) { + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "handler_nop", + DQL: "$Nop($Unsafe.Id)", + }, shape.WithCompileStrict(true)) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + for _, item := range planned.Diagnostics { + if item == nil { + continue + } + assert.NotEqual(t, dqldiag.CodeSQLIRawSelector, item.Code) + } +} + +func TestDQLCompiler_Compile_SubqueryJoin_BuildsRelatedViewsAndConnectorHints(t *testing.T) { + compiler := New() + dql := ` +#set($_ = $Jwt(header/Authorization).WithCodec(JwtClaim).WithStatusCode(401)) +SELECT session.*, +use_connector(session, system), +use_connector(attribute, system) +FROM (SELECT * FROM session WHERE user_id = $Jwt.UserID) session +JOIN (SELECT * FROM session/attributes) attribute ON attribute.user_id = session.user_id +` + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "system/session", DQL: dql}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + root := planned.ViewsByName["session"] + require.NotNil(t, root) + assert.Equal(t, "system", root.Connector) + related := planned.ViewsByName["attribute"] + require.NotNil(t, related) + assert.Equal(t, "session/attributes", related.Table) + assert.Equal(t, "system", related.Connector) +} + +func TestDQLCompiler_Compile_GeneratedHandler_NoBodyInput_UsesLegacyContractStates(t *testing.T) { + tempDir := t.TempDir() + genPath := filepath.Join(tempDir, "dql", "system", "upload", "gen", "upload", "delete.dql") + require.NoError(t, os.MkdirAll(filepath.Dir(genPath), 0o755)) + require.NoError(t, os.WriteFile(genPath, []byte(`/* {"Method":"DELETE","URI":"/v1/api/system/upload"} */`), 0o644)) + + legacySQLPath := filepath.Join(tempDir, "dql", "system", "upload", "delete.sql") + require.NoError(t, os.MkdirAll(filepath.Dir(legacySQLPath), 0o755)) + require.NoError(t, os.WriteFile(legacySQLPath, []byte(`/* {"Type":"upload/delete.Handler","Connector":"system"} */`), 0o644)) + + routesDir := filepath.Join(tempDir, "repo", "dev", "Datly", "routes", "system", "upload") + require.NoError(t, os.MkdirAll(filepath.Join(routesDir, "delete"), 0o755)) + routeYAML := `Resource: + Parameters: + - Name: Method + In: + Kind: http_request + Name: method + - Name: UploadId + In: + Kind: query + Name: uploadId + Views: + - Name: delete + Mode: SQLExec + Connector: + Ref: system + Template: + SourceURL: delete/delete.sql +` + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "delete.yaml"), []byte(routeYAML), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "delete", "delete.sql"), []byte(`$Nop($Unsafe.UploadId)`), 0o644)) + + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "delete", Path: genPath, DQL: `/* {"Method":"DELETE","URI":"/v1/api/system/upload"} */`}) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + + require.NotEmpty(t, planned.Views) + assert.Equal(t, "delete", planned.Views[0].Name) + assert.Equal(t, "SQLExec", planned.Views[0].Mode) + assert.Equal(t, "system", planned.Views[0].Connector) + + stateByName := map[string]*plan.State{} + for _, item := range planned.States { + if item == nil { + continue + } + stateByName[item.Name] = item + } + require.Contains(t, stateByName, "Method") + require.Contains(t, stateByName, "UploadId") + assert.Equal(t, "http_request", stateByName["Method"].Kind) + assert.Equal(t, "query", stateByName["UploadId"].Kind) + assert.NotContains(t, stateByName, "Body") +} + +func TestDQLCompiler_Compile_HandlerLegacyTypes_PreferredOverComponentNameCollisions(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "dql", "platform", "campaign", "post.dql") + require.NoError(t, os.MkdirAll(filepath.Dir(sourcePath), 0o755)) + require.NoError(t, os.WriteFile(sourcePath, []byte(`/* {"URI":"/v1/api/platform/campaign","Method":"POST","Type":"campaign/patch.Handler"} */`), 0o644)) + + rootRouteDir := filepath.Join(tempDir, "repo", "dev", "Datly", "routes", "platform", "campaign", "patch") + require.NoError(t, os.MkdirAll(filepath.Join(rootRouteDir, "post"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(rootRouteDir, "post.yaml"), []byte(`Resource: + Parameters: + - Name: Auth + In: + Kind: component + Name: GET:/v1/api/platform/acl/auth + Views: + - Name: post + Mode: SQLExec + Connector: + Ref: ci_ads + Template: + SourceURL: post/post.sql + Types: + - Name: Input + DataType: "*Input" + Package: campaign/patch + ModulePath: github.vianttech.com/viant/platform/pkg/platform/campaign/patch + - Name: Handler + DataType: "*Handler" + Package: campaign/patch + ModulePath: github.vianttech.com/viant/platform/pkg/platform/campaign/patch +`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(rootRouteDir, "post", "post.sql"), []byte(`$Nop($Unsafe.Id)`), 0o644)) + + componentRouteDir := filepath.Join(tempDir, "repo", "dev", "Datly", "routes", "platform", "acl", "auth") + require.NoError(t, os.MkdirAll(componentRouteDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(componentRouteDir, "auth.yaml"), []byte(`Resource: + Types: + - Name: Input + DataType: "*Input" + Package: acl/auth + ModulePath: github.vianttech.com/viant/platform/pkg/platform/acl/auth + - Name: Handler + DataType: "*Handler" + Package: acl/auth + ModulePath: github.vianttech.com/viant/platform/pkg/platform/acl/auth +`), 0o644)) + + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "post", + Path: sourcePath, + DQL: `/* {"URI":"/v1/api/platform/campaign","Method":"POST","Type":"campaign/patch.Handler"} */`, + }) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + + typeByName := map[string]*plan.Type{} + for _, item := range planned.Types { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + typeByName[strings.ToLower(strings.TrimSpace(item.Name))] = item + } + + inputType, ok := typeByName["input"] + require.True(t, ok) + assert.Equal(t, "campaign/patch", inputType.Package) + assert.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/campaign/patch", inputType.ModulePath) + + handlerType, ok := typeByName["handler"] + require.True(t, ok) + assert.Equal(t, "campaign/patch", handlerType.Package) + assert.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/campaign/patch", handlerType.ModulePath) +} + +func TestDQLCompiler_Compile_CustomPathLayout_HandlerFallback(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "sqlsrc", "platform", "campaign", "post.dql") + require.NoError(t, os.MkdirAll(filepath.Dir(sourcePath), 0o755)) + require.NoError(t, os.WriteFile(sourcePath, []byte(`/* {"URI":"/v1/api/platform/campaign","Method":"POST","Type":"campaign/patch.Handler","Connector":"ci_ads"} */`), 0o644)) + + routesDir := filepath.Join(tempDir, "config", "routes", "platform", "campaign", "patch") + require.NoError(t, os.MkdirAll(filepath.Join(routesDir, "post"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "post.yaml"), []byte(`Resource: + Views: + - Name: post + Mode: SQLExec + Connector: + Ref: ci_ads + Template: + SourceURL: post/post.sql +`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "post", "post.sql"), []byte(`$Nop($Unsafe.Id)`), 0o644)) + + compiler := New() + res, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "post", + Path: sourcePath, + DQL: `/* {"URI":"/v1/api/platform/campaign","Method":"POST","Type":"campaign/patch.Handler","Connector":"ci_ads"} */`, + }, shape.WithDQLPathMarker("sqlsrc"), shape.WithRoutesRelativePath("config/routes")) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Views) + assert.Equal(t, "post", planned.Views[0].Name) + assert.Equal(t, "SQLExec", planned.Views[0].Mode) + assert.Equal(t, "ci_ads", planned.Views[0].Connector) + assert.Contains(t, planned.Views[0].SQL, "$Nop(") +} diff --git a/repository/shape/compile/component_types.go b/repository/shape/compile/component_types.go new file mode 100644 index 000000000..5c553bc90 --- /dev/null +++ b/repository/shape/compile/component_types.go @@ -0,0 +1,432 @@ +package compile + +import ( + "os" + "path/filepath" + "sort" + "strings" + + "github.com/viant/datly/repository/shape" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/plan" + "gopkg.in/yaml.v3" +) + +type componentVisitState int + +const ( + componentVisitIdle componentVisitState = iota + componentVisitActive + componentVisitDone +) + +func appendComponentTypes(source *shape.Source, result *plan.Result) []*dqlshape.Diagnostic { + return appendComponentTypesWithLayout(source, result, defaultCompilePathLayout()) +} + +func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, layout compilePathLayout) []*dqlshape.Diagnostic { + if source == nil || result == nil { + return nil + } + _, routesRoot, dqlRoot, ok := sourceRootsWithLayout(source.Path, layout) + if !ok { + return nil + } + sourceNamespace, _ := dqlToRouteNamespaceWithLayout(source.Path, layout) + collector := &componentCollector{ + routesRoot: routesRoot, + visited: map[string]componentVisitState{}, + outputByRoute: map[string]string{}, + typesByName: map[string]*plan.Type{}, + } + if strings.TrimSpace(sourceNamespace) != "" { + collector.collect(sourceNamespace, relationSpan(source.DQL, 0), false) + } + + for _, stateItem := range result.States { + if stateItem == nil || !strings.EqualFold(strings.TrimSpace(stateItem.Kind), "component") { + continue + } + ref := strings.TrimSpace(stateItem.In) + if ref == "" { + continue + } + namespace := resolveComponentNamespaceWithNamespace(ref, source.Path, dqlRoot, sourceNamespace) + if namespace == "" { + collector.diags = append(collector.diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeCompRefInvalid, + Severity: dqlshape.SeverityWarning, + Message: "invalid component reference: " + ref, + Hint: "use ../component/ref or GET:/v1/api/... route reference", + Span: componentRefSpan(source.DQL, ref), + }) + continue + } + outputType, ok := collector.collect(namespace, componentRefSpan(source.DQL, ref), true) + if ok && strings.TrimSpace(stateItem.DataType) == "" { + stateItem.DataType = strings.TrimSpace(outputType) + } + } + + names := make([]string, 0, len(collector.typesByName)) + for name := range collector.typesByName { + names = append(names, name) + } + sort.Strings(names) + existing := map[string]bool{} + reportedCollision := map[string]bool{} + for _, item := range result.Types { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + existing[strings.ToLower(strings.TrimSpace(item.Name))] = true + } + for _, name := range names { + keyName := strings.ToLower(strings.TrimSpace(name)) + if existing[keyName] { + if !reportedCollision[keyName] { + collector.diags = append(collector.diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeCompTypeCollision, + Severity: dqlshape.SeverityWarning, + Message: "component type skipped due to existing type name: " + strings.TrimSpace(name), + Hint: "rename colliding type or keep route type as canonical source", + Span: relationSpan(source.DQL, 0), + }) + reportedCollision[keyName] = true + } + continue + } + item := collector.typesByName[name] + result.Types = append(result.Types, item) + existing[keyName] = true + } + return collector.diags +} + +type componentCollector struct { + routesRoot string + visited map[string]componentVisitState + outputByRoute map[string]string + typesByName map[string]*plan.Type + diags []*dqlshape.Diagnostic +} + +func (c *componentCollector) collect(namespace string, span dqlshape.Span, required bool) (string, bool) { + key := strings.ToLower(strings.TrimSpace(namespace)) + if key == "" { + return "", false + } + switch c.visited[key] { + case componentVisitDone: + return c.outputByRoute[key], true + case componentVisitActive: + c.diags = append(c.diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeCompCycle, + Severity: dqlshape.SeverityWarning, + Message: "component reference cycle detected at " + namespace, + Hint: "break cyclic component references", + Span: span, + }) + return "", false + } + c.visited[key] = componentVisitActive + + payload, ok := loadRoutePayload(c.routesRoot, namespace) + if !ok { + c.visited[key] = componentVisitDone + if required { + c.diags = append(c.diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeCompRouteMissing, + Severity: dqlshape.SeverityWarning, + Message: "component route YAML not found: " + namespace, + Hint: "ensure matching route exists under repo/dev/Datly/routes", + Span: span, + }) + } + return "", false + } + + for _, item := range payload.Resource.Types { + name := strings.TrimSpace(item.Name) + if name == "" { + continue + } + keyName := strings.ToLower(name) + if _, exists := c.typesByName[keyName]; exists { + continue + } + c.typesByName[keyName] = &plan.Type{ + Name: name, + Alias: strings.TrimSpace(item.Alias), + DataType: strings.TrimSpace(item.DataType), + Cardinality: strings.TrimSpace(item.Cardinality), + Package: strings.TrimSpace(item.Package), + ModulePath: strings.TrimSpace(item.ModulePath), + } + } + + outputType := routeOutputType(payload) + c.outputByRoute[key] = outputType + + for _, param := range payload.Resource.Parameters { + if !strings.EqualFold(strings.TrimSpace(param.In.Kind), "component") { + continue + } + nextNS := resolveComponentNamespaceFromRoute(strings.TrimSpace(param.In.Name), namespace) + if nextNS == "" { + c.diags = append(c.diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeCompRefInvalid, + Severity: dqlshape.SeverityWarning, + Message: "invalid nested component reference: " + strings.TrimSpace(param.In.Name), + Hint: "use ../component/ref or GET:/v1/api/... route reference", + Span: span, + }) + continue + } + c.collect(nextNS, span, true) + } + + c.visited[key] = componentVisitDone + return outputType, true +} + +func sourceRoots(sourcePath string) (platformRoot, routesRoot, dqlRoot string, ok bool) { + return sourceRootsWithLayout(sourcePath, defaultCompilePathLayout()) +} + +func sourceRootsWithLayout(sourcePath string, layout compilePathLayout) (platformRoot, routesRoot, dqlRoot string, ok bool) { + path := filepath.Clean(strings.TrimSpace(sourcePath)) + if path == "" { + return "", "", "", false + } + normalized := filepath.ToSlash(path) + marker := layout.dqlMarker + if marker == "" { + marker = defaultCompilePathLayout().dqlMarker + } + idx := strings.Index(normalized, marker) + if idx == -1 { + return "", "", "", false + } + platformRoot = path[:idx] + dqlRoot = filepath.Join(platformRoot, filepath.FromSlash(strings.Trim(marker, "/"))) + routesRoot = joinRelativePath(platformRoot, layout.routesRelative) + return platformRoot, routesRoot, dqlRoot, true +} + +func dqlToRouteNamespace(sourcePath string) (string, bool) { + return dqlToRouteNamespaceWithLayout(sourcePath, defaultCompilePathLayout()) +} + +func dqlToRouteNamespaceWithLayout(sourcePath string, layout compilePathLayout) (string, bool) { + path := filepath.Clean(strings.TrimSpace(sourcePath)) + if path == "" { + return "", false + } + normalized := filepath.ToSlash(path) + marker := layout.dqlMarker + if marker == "" { + marker = defaultCompilePathLayout().dqlMarker + } + idx := strings.Index(normalized, marker) + if idx == -1 { + return "", false + } + relative := strings.TrimPrefix(normalized[idx+len(marker):], "/") + if relative == "" { + return "", false + } + return strings.Trim(strings.TrimSuffix(relative, filepath.Ext(relative)), "/"), true +} + +func resolveComponentNamespace(ref, sourcePath, dqlRoot string) string { + ref = strings.TrimSpace(ref) + ref = strings.TrimPrefix(ref, "GET:") + ref = strings.TrimPrefix(ref, "POST:") + ref = strings.TrimPrefix(ref, "PUT:") + ref = strings.TrimPrefix(ref, "PATCH:") + ref = strings.TrimPrefix(ref, "DELETE:") + ref = strings.TrimPrefix(ref, "OPTIONS:") + ref = strings.TrimSpace(ref) + if strings.HasPrefix(ref, "/v1/api/") { + return strings.Trim(strings.TrimPrefix(ref, "/v1/api/"), "/") + } + if strings.HasPrefix(ref, "v1/api/") { + return strings.Trim(strings.TrimPrefix(ref, "v1/api/"), "/") + } + if strings.HasPrefix(ref, "/") { + return strings.Trim(ref, "/") + } + if dqlRoot == "" || strings.TrimSpace(sourcePath) == "" { + return "" + } + base := filepath.Dir(filepath.Clean(sourcePath)) + target := filepath.Clean(filepath.Join(base, ref)) + rel, err := filepath.Rel(dqlRoot, target) + if err != nil { + return "" + } + rel = filepath.ToSlash(rel) + rel = strings.TrimSuffix(rel, filepath.Ext(rel)) + return strings.Trim(rel, "/") +} + +func resolveComponentNamespaceWithNamespace(ref, sourcePath, dqlRoot, sourceNamespace string) string { + if namespace := resolveComponentNamespace(ref, sourcePath, dqlRoot); namespace != "" { + return namespace + } + return resolveComponentNamespaceFromRoute(ref, sourceNamespace) +} + +func resolveComponentNamespaceFromRoute(ref, sourceNamespace string) string { + ref = strings.TrimSpace(ref) + if ref == "" { + return "" + } + if namespace := resolveComponentNamespace(ref, "", ""); namespace != "" { + return namespace + } + normalizedBase := strings.Trim(strings.TrimSpace(sourceNamespace), "/") + if normalizedBase == "" { + return "" + } + baseDir := pathDir(normalizedBase) + target := filepath.ToSlash(filepath.Clean(filepath.Join(baseDir, ref))) + target = strings.TrimSuffix(target, filepath.Ext(target)) + return strings.Trim(target, "/") +} + +func pathDir(path string) string { + if path == "" { + return "" + } + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) <= 1 { + return "" + } + return strings.Join(parts[:len(parts)-1], "/") +} + +type routePayload struct { + Resource struct { + Types []struct { + Name string `yaml:"Name"` + Alias string `yaml:"Alias"` + DataType string `yaml:"DataType"` + Cardinality string `yaml:"Cardinality"` + Package string `yaml:"Package"` + ModulePath string `yaml:"ModulePath"` + } `yaml:"Types"` + Parameters []struct { + Name string `yaml:"Name"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` + Schema struct { + DataType string `yaml:"DataType"` + Name string `yaml:"Name"` + Package string `yaml:"Package"` + Cardinality string `yaml:"Cardinality"` + } `yaml:"Schema"` + } `yaml:"Parameters"` + } `yaml:"Resource"` + Routes []struct { + Handler struct { + OutputType string `yaml:"OutputType"` + } `yaml:"Handler"` + Output struct { + Cardinality string `yaml:"Cardinality"` + Type struct { + Name string `yaml:"Name"` + Package string `yaml:"Package"` + } `yaml:"Type"` + } `yaml:"Output"` + } `yaml:"Routes"` +} + +func loadRoutePayload(routesRoot, namespace string) (*routePayload, bool) { + candidates := routeYAMLCandidates(routesRoot, namespace) + for _, candidate := range candidates { + data, err := os.ReadFile(candidate) + if err != nil { + continue + } + payload := &routePayload{} + if err = yaml.Unmarshal(data, payload); err != nil { + continue + } + return payload, true + } + return nil, false +} + +func routeOutputType(payload *routePayload) string { + if payload == nil { + return "" + } + for _, route := range payload.Routes { + if outputType := strings.TrimSpace(route.Handler.OutputType); outputType != "" { + leaf := outputType + if idx := strings.LastIndex(leaf, "."); idx >= 0 && idx+1 < len(leaf) { + leaf = leaf[idx+1:] + } + leaf = strings.Trim(strings.TrimSpace(leaf), "*") + if leaf != "" { + return "*" + leaf + } + } + if name := strings.TrimSpace(route.Output.Type.Name); name != "" { + name = strings.Trim(name, "*") + if name != "" { + return "*" + name + } + } + } + for _, param := range payload.Resource.Parameters { + if strings.EqualFold(strings.TrimSpace(param.In.Kind), "output") { + if dataType := strings.TrimSpace(param.Schema.DataType); dataType != "" { + return dataType + } + if name := strings.TrimSpace(param.Schema.Name); name != "" { + name = strings.Trim(name, "*") + if name != "" { + return "*" + name + } + } + } + } + for _, item := range payload.Resource.Types { + if strings.EqualFold(strings.TrimSpace(item.Name), "output") { + if dataType := strings.TrimSpace(item.DataType); dataType != "" { + return dataType + } + return "*Output" + } + } + return "" +} + +func componentRefSpan(raw, ref string) dqlshape.Span { + offset := 0 + ref = strings.TrimSpace(ref) + if ref != "" { + if idx := strings.Index(raw, ref); idx >= 0 { + offset = idx + } + } + return relationSpan(raw, offset) +} + +func routeYAMLCandidates(routesRoot, namespace string) []string { + namespace = strings.Trim(namespace, "/") + if namespace == "" { + return nil + } + leaf := filepath.Base(namespace) + return []string{ + filepath.Join(routesRoot, filepath.FromSlash(namespace)+".yaml"), + filepath.Join(routesRoot, filepath.FromSlash(namespace), leaf+".yaml"), + } +} diff --git a/repository/shape/compile/component_types_test.go b/repository/shape/compile/component_types_test.go new file mode 100644 index 000000000..0e93ec718 --- /dev/null +++ b/repository/shape/compile/component_types_test.go @@ -0,0 +1,155 @@ +package compile + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + "github.com/viant/datly/repository/shape/plan" +) + +func TestResolveComponentNamespace(t *testing.T) { + dqlRoot := "/repo/dql" + source := "/repo/dql/platform/tvaffiliatestation/tvaffiliatestation.dql" + assert.Equal(t, "platform/acl/auth", resolveComponentNamespace("../acl/auth", source, dqlRoot)) + assert.Equal(t, "platform/acl/auth", resolveComponentNamespace("GET:/v1/api/platform/acl/auth", source, dqlRoot)) + assert.Equal(t, "platform/acl/auth", resolveComponentNamespace("v1/api/platform/acl/auth", source, dqlRoot)) +} + +func TestDQLToRouteNamespace(t *testing.T) { + ns, ok := dqlToRouteNamespace("/repo/dql/platform/tvaffiliatestation/tvaffiliatestation.dql") + require.True(t, ok) + assert.Equal(t, "platform/tvaffiliatestation/tvaffiliatestation", ns) +} + +func TestSourceRoots_CustomLayout(t *testing.T) { + layout := compilePathLayout{ + dqlMarker: "/sqlsrc/", + routesRelative: "config/routes", + } + platformRoot, routesRoot, dqlRoot, ok := sourceRootsWithLayout("/repo/sqlsrc/platform/agency/agency.dql", layout) + require.True(t, ok) + assert.Equal(t, "/repo", filepath.ToSlash(platformRoot)) + assert.Equal(t, "/repo/config/routes", filepath.ToSlash(routesRoot)) + assert.Equal(t, "/repo/sqlsrc", filepath.ToSlash(dqlRoot)) + + ns, ok := dqlToRouteNamespaceWithLayout("/repo/sqlsrc/platform/agency/agency.dql", layout) + require.True(t, ok) + assert.Equal(t, "platform/agency/agency", ns) +} + +func TestAppendComponentTypes(t *testing.T) { + temp := t.TempDir() + dqlDir := filepath.Join(temp, "dql", "platform", "tvaffiliatestation") + routesDir := filepath.Join(temp, "repo", "dev", "Datly", "routes", "platform", "acl") + require.NoError(t, os.MkdirAll(dqlDir, 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(routesDir, "auth"), 0o755)) + require.NoError(t, os.MkdirAll(routesDir, 0o755)) + sourcePath := filepath.Join(dqlDir, "tvaffiliatestation.dql") + require.NoError(t, os.WriteFile(sourcePath, []byte("SELECT 1"), 0o644)) + + authYAML := `Resource: + Types: + - Name: Input + DataType: "*Input" + Package: acl/auth + ModulePath: github.vianttech.com/viant/platform/pkg/platform/acl/auth + Parameters: + - In: + Kind: component + Name: GET:/v1/api/platform/acl/user +Routes: + - Handler: + OutputType: acl/auth.Output +` + userYAML := `Resource: + Types: + - Name: UserView + DataType: "struct{Id int;}" + Package: acl + ModulePath: github.vianttech.com/viant/platform/pkg/platform/acl +` + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "auth", "auth.yaml"), []byte(authYAML), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "user.yaml"), []byte(userYAML), 0o644)) + + result := &plan.Result{ + States: []*plan.State{ + {Name: "Auth", Kind: "component", In: "../acl/auth"}, + }, + } + appendComponentTypes(&shape.Source{Path: sourcePath, DQL: "#set($Auth = $component<../acl/auth>())"}, result) + require.Len(t, result.Types, 2) + names := map[string]bool{} + for _, item := range result.Types { + names[item.Name] = true + } + assert.True(t, names["Input"]) + assert.True(t, names["UserView"]) + assert.Equal(t, "*Output", result.States[0].DataType) +} + +func TestAppendComponentTypes_MissingComponentRoute(t *testing.T) { + temp := t.TempDir() + dqlDir := filepath.Join(temp, "dql", "platform", "sample") + require.NoError(t, os.MkdirAll(dqlDir, 0o755)) + sourcePath := filepath.Join(dqlDir, "sample.dql") + dql := "#set($Auth = $component<../acl/missing>())\nSELECT 1" + require.NoError(t, os.WriteFile(sourcePath, []byte(dql), 0o644)) + result := &plan.Result{ + States: []*plan.State{{Name: "Auth", Kind: "component", In: "../acl/missing"}}, + } + diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: dql}, result) + require.NotEmpty(t, diags) + assert.Equal(t, "DQL-COMP-ROUTE-MISSING", diags[0].Code) + assert.GreaterOrEqual(t, diags[0].Span.Start.Line, 1) + assert.GreaterOrEqual(t, diags[0].Span.Start.Char, 1) +} + +func TestAppendComponentTypes_TypeCollisionEmitsDiagnostic(t *testing.T) { + temp := t.TempDir() + dqlDir := filepath.Join(temp, "dql", "platform", "tvaffiliatestation") + routesDir := filepath.Join(temp, "repo", "dev", "Datly", "routes", "platform", "acl") + require.NoError(t, os.MkdirAll(dqlDir, 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(routesDir, "auth"), 0o755)) + sourcePath := filepath.Join(dqlDir, "tvaffiliatestation.dql") + require.NoError(t, os.WriteFile(sourcePath, []byte("SELECT 1"), 0o644)) + + authYAML := `Resource: + Types: + - Name: Input + DataType: "*Input" + Package: acl/auth + ModulePath: github.vianttech.com/viant/platform/pkg/platform/acl/auth +` + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "auth", "auth.yaml"), []byte(authYAML), 0o644)) + + result := &plan.Result{ + States: []*plan.State{ + {Name: "Auth", Kind: "component", In: "../acl/auth"}, + }, + Types: []*plan.Type{ + { + Name: "Input", + DataType: "*Input", + Package: "campaign/patch", + ModulePath: "github.vianttech.com/viant/platform/pkg/platform/campaign/patch", + }, + }, + } + diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: "#set($Auth = $component<../acl/auth>())"}, result) + require.NotEmpty(t, diags) + var found bool + for _, item := range diags { + if item != nil && item.Code == dqldiag.CodeCompTypeCollision { + found = true + break + } + } + assert.True(t, found) + require.Len(t, result.Types, 1) + assert.Equal(t, "campaign/patch", result.Types[0].Package) +} diff --git a/repository/shape/compile/dml/compiler.go b/repository/shape/compile/dml/compiler.go new file mode 100644 index 000000000..8b6838fda --- /dev/null +++ b/repository/shape/compile/dml/compiler.go @@ -0,0 +1,13 @@ +package dml + +import ( + "github.com/viant/datly/repository/shape/compile/pipeline" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" + "github.com/viant/datly/repository/shape/plan" +) + +// Compile builds an exec-oriented view and validates DML statements. +func Compile(sourceName, sqlText string, statements dqlstmt.Statements) (*plan.View, []*dqlshape.Diagnostic) { + return pipeline.BuildExec(sourceName, sqlText, statements) +} diff --git a/repository/shape/compile/dml/compiler_test.go b/repository/shape/compile/dml/compiler_test.go new file mode 100644 index 000000000..7c1d907f9 --- /dev/null +++ b/repository/shape/compile/dml/compiler_test.go @@ -0,0 +1,25 @@ +package dml + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" +) + +func TestCompile_Insert(t *testing.T) { + sqlText := "INSERT INTO ORDERS(id) VALUES (1)" + view, diags := Compile("orders_exec", sqlText, dqlstmt.New(sqlText)) + require.NotNil(t, view) + assert.Equal(t, "ORDERS", view.Table) + assert.Empty(t, diags) +} + +func TestCompile_ServiceMissingArg(t *testing.T) { + sqlText := "$sql.Insert($rec)" + _, diags := Compile("orders_exec", sqlText, dqlstmt.New(sqlText)) + require.NotEmpty(t, diags) + assert.Equal(t, dqldiag.CodeDMLServiceArg, diags[0].Code) +} diff --git a/repository/shape/compile/enrich.go b/repository/shape/compile/enrich.go new file mode 100644 index 000000000..08b80570c --- /dev/null +++ b/repository/shape/compile/enrich.go @@ -0,0 +1,756 @@ +package compile + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/compile/pipeline" + "github.com/viant/datly/repository/shape/plan" + "gopkg.in/yaml.v3" +) + +var ( + ruleHeaderExpr = regexp.MustCompile(`(?s)^\s*/\*\s*(\{.*?\})\s*\*/`) + embedExpr = regexp.MustCompile(`(?is)\$\{\s*embed:\s*([^}]+)\}`) + fromTableExpr = regexp.MustCompile(`(?is)\bfrom\s+([a-zA-Z_$][a-zA-Z0-9_$.{}/]*)`) + summaryJoinExpr = regexp.MustCompile(`(?is)\bjoin\s*\((.*?)\)\s*summary\s+on\s+1\s*=\s*1`) + joinEmbedExpr = regexp.MustCompile(`(?is)\bjoin\s*\(\s*\$\{\s*embed:\s*([^}]+)\}\s*\)\s*(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*)`) + joinBodyExpr = regexp.MustCompile(`(?is)\bjoin\s*\((.*?)\)\s*(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*)\s+on\b`) +) + +type ruleSettings struct { + Connector string `json:"Connector"` + Name string `json:"Name"` + Type string `json:"Type"` + Method string `json:"Method"` + URI string `json:"URI"` +} + +func applySourceParityEnrichment(result *plan.Result, source *shape.Source) { + applySourceParityEnrichmentWithLayout(result, source, defaultCompilePathLayout()) +} + +func applySourceParityEnrichmentWithLayout(result *plan.Result, source *shape.Source, layout compilePathLayout) { + if result == nil || len(result.Views) == 0 { + return + } + settings := extractRuleSettings(source) + legacyViews := loadLegacyRouteViewAttrsWithLayout(source, settings, layout) + baseDir := sourceSQLBaseDir(source) + module := sourceModuleWithLayout(source, layout) + sourceName := pipeline.SanitizeName(source.Name) + joinEmbedRefs := map[string]string{} + joinSubqueryBodies := map[string]string{} + if len(result.Views) > 0 && result.Views[0] != nil { + sqlForJoinExtract := result.Views[0].SQL + if source != nil && strings.TrimSpace(source.DQL) != "" { + sqlForJoinExtract = source.DQL + } + joinEmbedRefs = extractJoinEmbedRefs(sqlForJoinExtract) + joinSubqueryBodies = extractJoinSubqueryBodies(sqlForJoinExtract) + } + for idx, item := range result.Views { + if item == nil { + continue + } + if legacy, ok := lookupLegacyRouteViewAttr(legacyViews, item.Name); ok { + if legacy.Mode != "" { + item.Mode = legacy.Mode + } + if legacy.Module != "" { + item.Module = legacy.Module + } + if legacy.AllowNulls != nil { + value := *legacy.AllowNulls + item.AllowNulls = &value + } + if legacy.SelectorNamespace != "" { + item.SelectorNamespace = legacy.SelectorNamespace + } + if legacy.SelectorNoLimit != nil { + value := *legacy.SelectorNoLimit + item.SelectorNoLimit = &value + } + if legacy.SchemaType != "" { + item.SchemaType = legacy.SchemaType + } + if legacy.Cardinality != "" { + item.Cardinality = legacy.Cardinality + } + if legacy.HasSummary != nil && *legacy.HasSummary && strings.TrimSpace(item.Summary) == "" { + item.Summary = "legacy-summary" + } + } + if item.SQLURI == "" && baseDir != "" { + item.SQLURI = baseDir + "/" + item.Name + ".sql" + } + if item.Module == "" { + item.Module = module + } + if item.SelectorNamespace == "" { + item.SelectorNamespace = defaultSelectorNamespace(item.Name) + } + if item.SchemaType == "" { + item.SchemaType = defaultSchemaType(item.Name, settings, idx == 0) + } + if shouldInferTable(item) { + candidateSQL := item.SQL + if strings.TrimSpace(candidateSQL) == "" { + candidateSQL = item.Table + } + if table := inferTableFromSQL(candidateSQL, source); table != "" { + item.Table = table + } + } + if strings.HasPrefix(strings.TrimSpace(item.Table), "(") || normalizedTemplatePlaceholderTable(strings.TrimSpace(item.Table)) { + if ref, ok := joinEmbedRefs[item.Name]; ok { + if table := inferTableFromEmbedRef(source, ref); table != "" { + item.Table = table + } + } + if body, ok := joinSubqueryBodies[item.Name]; ok { + if table := inferTableFromSQL(body, source); table != "" { + item.Table = table + } + } + if table := inferTableFromSiblingSQL(item.Name, source); table != "" { + item.Table = table + } + } + if item.Connector == "" && settings.Connector != "" { + item.Connector = settings.Connector + } + if item.Connector == "" && source != nil && strings.TrimSpace(source.Connector) != "" { + item.Connector = strings.TrimSpace(source.Connector) + } + if item.Connector == "" { + item.Connector = inferConnector(item, source) + } + if item.Summary == "" { + item.Summary = extractSummarySQL(item.SQL) + if item.Summary == "" && source != nil { + item.Summary = extractSummarySQL(source.DQL) + } + } + } + if source != nil && strings.TrimSpace(source.Path) != "" { + normalizeRootViewName(result, sourceName, settings) + } +} + +type legacyRouteViewAttr struct { + Name string + Mode string + Module string + AllowNulls *bool + SelectorNamespace string + SelectorNoLimit *bool + SchemaType string + Cardinality string + HasSummary *bool +} + +func loadLegacyRouteViewAttrs(source *shape.Source, settings *ruleSettings) []legacyRouteViewAttr { + return loadLegacyRouteViewAttrsWithLayout(source, settings, defaultCompilePathLayout()) +} + +func loadLegacyRouteViewAttrsWithLayout(source *shape.Source, settings *ruleSettings, layout compilePathLayout) []legacyRouteViewAttr { + if source == nil || strings.TrimSpace(source.Path) == "" { + return nil + } + platformRoot, relativeDir, stem, ok := platformPathParts(source.Path, layout) + if !ok { + return nil + } + typeExpr := "" + if settings != nil { + typeExpr = strings.TrimSpace(settings.Type) + } + typeExpr = strings.Trim(typeExpr, `"'`) + typeExpr = strings.TrimSuffix(typeExpr, ".Handler") + typeStem := "" + if typeExpr != "" { + typeStem = filepath.Base(filepath.FromSlash(typeExpr)) + } + routesRoot := joinRelativePath(platformRoot, layout.routesRelative) + routesBase := filepath.Join(routesRoot, filepath.FromSlash(relativeDir)) + candidates := legacyRouteYAMLCandidates(routesBase, stem, typeStem) + for _, candidate := range candidates { + if attrs := parseLegacyRouteViewAttrs(candidate); len(attrs) > 0 { + return attrs + } + } + return nil +} + +func parseLegacyRouteViewAttrs(path string) []legacyRouteViewAttr { + data, err := os.ReadFile(path) + if err != nil { + return nil + } + var payload struct { + Resource struct { + Views []struct { + Name string `yaml:"Name"` + Mode string `yaml:"Mode"` + Module string `yaml:"Module"` + AllowNulls *bool `yaml:"AllowNulls"` + Selector struct { + Namespace string `yaml:"Namespace"` + NoLimit *bool `yaml:"NoLimit"` + } `yaml:"Selector"` + Template struct { + Summary *struct{} `yaml:"Summary"` + } `yaml:"Template"` + Schema struct { + Cardinality string `yaml:"Cardinality"` + DataType string `yaml:"DataType"` + Name string `yaml:"Name"` + } `yaml:"Schema"` + } `yaml:"Views"` + } `yaml:"Resource"` + } + if err = yaml.Unmarshal(data, &payload); err != nil { + return nil + } + result := make([]legacyRouteViewAttr, 0, len(payload.Resource.Views)) + for _, item := range payload.Resource.Views { + cardinality := strings.TrimSpace(item.Schema.Cardinality) + if cardinality != "" { + cardinality = strings.ToLower(cardinality) + } + result = append(result, legacyRouteViewAttr{ + Name: strings.TrimSpace(item.Name), + Mode: strings.TrimSpace(item.Mode), + Module: strings.TrimSpace(item.Module), + AllowNulls: item.AllowNulls, + SelectorNamespace: strings.TrimSpace(item.Selector.Namespace), + SelectorNoLimit: item.Selector.NoLimit, + SchemaType: firstNonEmptyString(strings.TrimSpace(item.Schema.DataType), strings.TrimSpace(item.Schema.Name)), + Cardinality: cardinality, + HasSummary: func() *bool { + if item.Template.Summary == nil { + return nil + } + value := true + return &value + }(), + }) + } + return result +} + +func lookupLegacyRouteViewAttr(items []legacyRouteViewAttr, name string) (legacyRouteViewAttr, bool) { + name = strings.TrimSpace(name) + if name == "" { + return legacyRouteViewAttr{}, false + } + for _, item := range items { + if strings.EqualFold(strings.TrimSpace(item.Name), name) { + return item, true + } + } + return legacyRouteViewAttr{}, false +} + +func firstNonEmptyString(values ...string) string { + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + return value + } + } + return "" +} + +func extractSummarySQL(sqlText string) string { + sqlText = strings.TrimSpace(sqlText) + if sqlText == "" || !strings.Contains(sqlText, "$View.") { + return "" + } + matches := summaryJoinExpr.FindStringSubmatch(sqlText) + if len(matches) < 2 { + return "" + } + return strings.TrimSpace(matches[1]) +} + +func extractRuleSettings(source *shape.Source) *ruleSettings { + if source == nil || strings.TrimSpace(source.DQL) == "" { + return &ruleSettings{} + } + matches := ruleHeaderExpr.FindStringSubmatch(source.DQL) + if len(matches) < 2 { + return &ruleSettings{} + } + rawJSON := strings.TrimSpace(matches[1]) + ret := &ruleSettings{} + _ = json.Unmarshal([]byte(rawJSON), ret) + return ret +} + +func sourceSQLBaseDir(source *shape.Source) string { + if source == nil { + return "" + } + path := strings.TrimSpace(source.Path) + if path == "" { + return "" + } + base := strings.TrimSpace(filepath.Base(path)) + if base == "" { + return "" + } + stem := strings.TrimSpace(strings.TrimSuffix(base, filepath.Ext(base))) + if stem == "" || stem == "." || stem == string(filepath.Separator) { + return "" + } + return stem +} + +func sourceModule(source *shape.Source) string { + return sourceModuleWithLayout(source, defaultCompilePathLayout()) +} + +func sourceModuleWithLayout(source *shape.Source, layout compilePathLayout) string { + if source == nil || strings.TrimSpace(source.Path) == "" { + return "" + } + normalized := filepath.ToSlash(source.Path) + marker := layout.dqlMarker + if marker == "" { + marker = defaultCompilePathLayout().dqlMarker + } + idx := strings.Index(normalized, marker) + if idx == -1 { + return "" + } + relative := strings.TrimPrefix(normalized[idx+len(marker):], "/") + dir := strings.TrimSpace(filepath.ToSlash(filepath.Dir(relative))) + if dir == "." || dir == "/" { + return "" + } + return dir +} + +func defaultSelectorNamespace(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "" + } + var b strings.Builder + for i := 0; i < len(name); i++ { + ch := name[i] + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') { + b.WriteByte(byte(strings.ToLower(string(ch))[0])) + } + } + value := b.String() + switch { + case len(value) >= 2: + return value[:2] + case len(value) == 1: + return value + default: + return "" + } +} + +func defaultSchemaType(name string, settings *ruleSettings, root bool) string { + if root && settings != nil && strings.TrimSpace(settings.Name) != "" { + return "*" + strings.TrimSpace(settings.Name) + "View" + } + name = strings.TrimSpace(name) + if name == "" { + return "" + } + return "*" + toExportedTypeName(name) + "View" +} + +func toExportedTypeName(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "" + } + parts := strings.FieldsFunc(name, func(r rune) bool { + return r == '_' || r == '-' || r == ' ' || r == '.' + }) + if len(parts) == 0 { + return "" + } + var b strings.Builder + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + b.WriteString(strings.ToUpper(part[:1])) + if len(part) > 1 { + b.WriteString(part[1:]) + } + } + return b.String() +} + +func shouldInferTable(item *plan.View) bool { + if item == nil { + return false + } + name := strings.TrimSpace(item.Name) + table := strings.TrimSpace(item.Table) + if table == "" { + return true + } + if strings.HasPrefix(table, "(") { + return true + } + if normalizedTemplatePlaceholderTable(table) { + return true + } + return strings.EqualFold(name, table) +} + +func normalizedTemplatePlaceholderTable(table string) bool { + if table == "" { + return false + } + parts := strings.Split(table, ".") + if len(parts) < 3 { + return false + } + for i := 0; i < len(parts)-1; i++ { + part := strings.TrimSpace(parts[i]) + if part == "" { + return false + } + for _, ch := range part { + if ch < '0' || ch > '9' { + return false + } + } + } + return true +} + +func inferTableFromSQL(sqlText string, source *shape.Source) string { + sqlText = strings.TrimSpace(sqlText) + if sqlText == "" { + return "" + } + if expr := topLevelFromExpr(sqlText); expr != "" { + if table := tableFromFromExpr(expr, source); table != "" { + return table + } + } + if table := pipeline.InferTableFromSQL(sqlText); table != "" { + if !strings.EqualFold(table, "DQLView") { + return table + } + } + cleaned := embedExpr.ReplaceAllString(sqlText, " ") + match := fromTableExpr.FindStringSubmatch(cleaned) + if len(match) >= 2 { + return strings.Trim(match[1], "`\"") + } + if table := inferFromEmbeddedSQL(sqlText, source); table != "" { + return table + } + return "" +} + +func inferFromEmbeddedSQL(sqlText string, source *shape.Source) string { + matches := embedExpr.FindStringSubmatch(sqlText) + if len(matches) < 2 { + return "" + } + ref := strings.TrimSpace(matches[1]) + ref = strings.Trim(ref, `"'`) + if ref == "" { + return "" + } + resolved := resolveEmbedPath(source, ref) + if resolved == "" { + return "" + } + embedded, err := os.ReadFile(resolved) + if err != nil { + return "" + } + queryNode, _, err := pipeline.ParseSelectWithDiagnostic(string(embedded)) + if err != nil || queryNode == nil { + fallback := fromTableExpr.FindStringSubmatch(string(embedded)) + if len(fallback) < 2 { + return "" + } + return strings.Trim(fallback[1], "`\"") + } + _, table, err := pipeline.InferRoot(queryNode, "") + if err != nil || strings.TrimSpace(table) == "" { + return "" + } + if strings.EqualFold(strings.TrimSpace(table), "DQLView") { + return "" + } + return strings.Trim(table, "`\"") +} + +func resolveEmbedPath(source *shape.Source, ref string) string { + if filepath.IsAbs(ref) { + return ref + } + if source == nil || strings.TrimSpace(source.Path) == "" { + return "" + } + base := source.Path + if fi, err := os.Stat(base); err == nil && fi.IsDir() { + return filepath.Clean(filepath.Join(base, ref)) + } + return filepath.Clean(filepath.Join(filepath.Dir(base), ref)) +} + +func inferTableFromSiblingSQL(viewName string, source *shape.Source) string { + viewName = strings.TrimSpace(viewName) + if viewName == "" || source == nil || strings.TrimSpace(source.Path) == "" { + return "" + } + sibling := filepath.Join(filepath.Dir(source.Path), viewName+".sql") + data, err := os.ReadFile(sibling) + if err != nil { + sibling = filepath.Join(filepath.Dir(source.Path), strings.ToLower(viewName)+".sql") + data, err = os.ReadFile(sibling) + } + if err != nil { + return "" + } + return inferTableFromSQL(string(data), source) +} + +func inferTableFromEmbedRef(source *shape.Source, ref string) string { + ref = strings.Trim(strings.TrimSpace(ref), `"'`) + if ref == "" { + return "" + } + resolved := resolveEmbedPath(source, ref) + if resolved == "" { + return "" + } + data, err := os.ReadFile(resolved) + if err != nil { + return "" + } + return pipeline.InferTableFromSQL(string(data)) +} + +func topLevelFromExpr(sqlText string) string { + lower := strings.ToLower(sqlText) + depth := 0 + inSingle := false + inDouble := false + inBacktick := false + for i := 0; i < len(sqlText); i++ { + ch := sqlText[i] + switch ch { + case '\'': + if !inDouble && !inBacktick { + inSingle = !inSingle + } + case '"': + if !inSingle && !inBacktick { + inDouble = !inDouble + } + case '`': + if !inSingle && !inDouble { + inBacktick = !inBacktick + } + case '(': + if !inSingle && !inDouble && !inBacktick { + depth++ + } + case ')': + if !inSingle && !inDouble && !inBacktick && depth > 0 { + depth-- + } + } + if inSingle || inDouble || inBacktick || depth != 0 { + continue + } + if i+6 > len(sqlText) { + break + } + if lower[i:i+4] != "from" { + continue + } + if i > 0 { + prev := lower[i-1] + if (prev >= 'a' && prev <= 'z') || (prev >= '0' && prev <= '9') || prev == '_' { + continue + } + } + j := i + 4 + for j < len(sqlText) && (sqlText[j] == ' ' || sqlText[j] == '\n' || sqlText[j] == '\t' || sqlText[j] == '\r') { + j++ + } + if j >= len(sqlText) { + return "" + } + if sqlText[j] == '(' { + start := j + d := 0 + for ; j < len(sqlText); j++ { + if sqlText[j] == '(' { + d++ + } else if sqlText[j] == ')' { + d-- + if d == 0 { + j++ + break + } + } + } + for j < len(sqlText) && (sqlText[j] == ' ' || sqlText[j] == '\n' || sqlText[j] == '\t' || sqlText[j] == '\r') { + j++ + } + for j < len(sqlText) { + c := sqlText[j] + if !(c == '_' || c == '.' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + break + } + j++ + } + return strings.TrimSpace(sqlText[start:j]) + } + start := j + for j < len(sqlText) { + c := sqlText[j] + if !(c == '_' || c == '.' || c == '/' || c == '{' || c == '}' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '$') { + break + } + j++ + } + return strings.TrimSpace(sqlText[start:j]) + } + return "" +} + +func tableFromFromExpr(fromExpr string, source *shape.Source) string { + fromExpr = strings.TrimSpace(fromExpr) + if fromExpr == "" { + return "" + } + if strings.HasPrefix(fromExpr, "(") { + if table := inferFromEmbeddedSQL(fromExpr, source); table != "" { + return table + } + inner := fromExpr + if idx := strings.LastIndex(inner, ")"); idx > 0 { + inner = strings.TrimSpace(inner[1:idx]) + } + return inferTableFromSQL(inner, source) + } + return strings.Trim(fromExpr, "`\"") +} + +func inferConnector(item *plan.View, source *shape.Source) string { + if item == nil { + return "" + } + path := "" + if source != nil { + path = strings.ToLower(strings.ReplaceAll(source.Path, "\\", "/")) + } + table := strings.ToUpper(strings.TrimSpace(item.Table)) + switch { + case strings.Contains(path, "/dql/system/"): + return "system" + case strings.HasPrefix(table, "CI_") || strings.Contains(table, ".CI_"): + return "ci_ads" + case strings.Contains(path, "/dql/ui/"): + return "sitemgmt" + case strings.Contains(table, "SITE"): + return "sitemgmt" + default: + return "" + } +} + +func normalizeRootViewName(result *plan.Result, sourceName string, settings *ruleSettings) { + if result == nil || len(result.Views) == 0 { + return + } + root := result.Views[0] + if root == nil { + return + } + desired := sourceName + if desired == "" { + return + } + _ = settings + current := strings.TrimSpace(root.Name) + if current == "" { + root.Name = desired + root.Path = desired + root.Holder = desired + return + } + if strings.EqualFold(current, desired) { + return + } + suspicious := map[string]bool{ + "and": true, "or": true, "status": true, "value": true, "watching": true, + } + if !suspicious[strings.ToLower(current)] { + return + } + if result.ViewsByName != nil { + delete(result.ViewsByName, root.Name) + } else { + result.ViewsByName = map[string]*plan.View{} + } + root.Name = desired + root.Path = desired + root.Holder = desired + result.ViewsByName[root.Name] = root +} + +func extractJoinEmbedRefs(sqlText string) map[string]string { + result := map[string]string{} + if strings.TrimSpace(sqlText) == "" { + return result + } + for _, m := range joinEmbedExpr.FindAllStringSubmatch(sqlText, -1) { + if len(m) < 3 { + continue + } + ref := strings.TrimSpace(m[1]) + alias := strings.TrimSpace(m[2]) + if ref == "" || alias == "" { + continue + } + result[alias] = ref + } + return result +} + +func extractJoinSubqueryBodies(sqlText string) map[string]string { + result := map[string]string{} + if strings.TrimSpace(sqlText) == "" { + return result + } + for _, m := range joinBodyExpr.FindAllStringSubmatch(sqlText, -1) { + if len(m) < 3 { + continue + } + body := strings.TrimSpace(m[1]) + alias := strings.TrimSpace(m[2]) + if body == "" || alias == "" { + continue + } + result[alias] = body + } + return result +} diff --git a/repository/shape/compile/enrich_test.go b/repository/shape/compile/enrich_test.go new file mode 100644 index 000000000..1c532d398 --- /dev/null +++ b/repository/shape/compile/enrich_test.go @@ -0,0 +1,172 @@ +package compile + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" +) + +func TestApplySourceParityEnrichment_RuleConnectorAndSQLURI(t *testing.T) { + source := &shape.Source{ + Path: "/repo/dql/platform/timezone/timezone.dql", + DQL: `/* {"Connector":"ci_ads"} */ SELECT * FROM CI_TIME_ZONE t`, + } + result := &plan.Result{ + Views: []*plan.View{ + {Name: "timezone", Table: "timezone", SQL: "SELECT * FROM CI_TIME_ZONE t"}, + }, + } + + applySourceParityEnrichment(result, source) + + require.Equal(t, "ci_ads", result.Views[0].Connector) + require.Equal(t, "timezone/timezone.sql", result.Views[0].SQLURI) + require.Equal(t, "CI_TIME_ZONE", result.Views[0].Table) +} + +func TestApplySourceParityEnrichment_InferTableFromSubquery(t *testing.T) { + source := &shape.Source{ + Path: "/repo/dql/platform/advertiser/advertiser.dql", + DQL: `SELECT x.* FROM (SELECT a.* FROM CI_ADVERTISER a) x`, + } + result := &plan.Result{ + Views: []*plan.View{ + {Name: "advertiser", Table: "advertiser", SQL: `SELECT x.* FROM (SELECT a.* FROM CI_ADVERTISER a) x`}, + }, + } + + applySourceParityEnrichment(result, source) + + require.Equal(t, "CI_ADVERTISER", result.Views[0].Table) + require.Equal(t, "advertiser/advertiser.sql", result.Views[0].SQLURI) +} + +func TestApplySourceParityEnrichment_InferTableFromEmbed(t *testing.T) { + tempDir := t.TempDir() + dqlDir := filepath.Join(tempDir, "dql", "platform", "timezone") + require.NoError(t, os.MkdirAll(dqlDir, 0o755)) + embedded := filepath.Join(dqlDir, "timezone.sql") + require.NoError(t, os.WriteFile(embedded, []byte(`SELECT tz.ID FROM CI_TIME_ZONE tz`), 0o644)) + source := &shape.Source{ + Path: filepath.Join(dqlDir, "timezone.dql"), + DQL: `SELECT timezone.* FROM (${embed: timezone.sql}) timezone`, + } + result := &plan.Result{ + Views: []*plan.View{ + {Name: "timezone", Table: "timezone", SQL: `SELECT timezone.* FROM (${embed: timezone.sql}) timezone`}, + }, + } + + applySourceParityEnrichment(result, source) + + require.Equal(t, "CI_TIME_ZONE", result.Views[0].Table) + require.Equal(t, "timezone/timezone.sql", result.Views[0].SQLURI) +} + +func TestTopLevelFromExpr_IgnoresNestedFrom(t *testing.T) { + sqlText := `SELECT a.*, EXISTS(SELECT 1 FROM CI_ENTITY_WATCHLIST w WHERE w.ENTITY_ID = a.ID) AS watching FROM (SELECT x.* FROM CI_ADVERTISER x) a` + require.Equal(t, "(SELECT x.* FROM CI_ADVERTISER x) a", topLevelFromExpr(sqlText)) +} + +func TestInferConnector(t *testing.T) { + require.Equal(t, "system", inferConnector(&plan.View{Table: "session"}, &shape.Source{Path: "/repo/dql/system/session/session.dql"})) + require.Equal(t, "ci_ads", inferConnector(&plan.View{Table: "CI_ADVERTISER"}, &shape.Source{Path: "/repo/dql/platform/advertiser/advertiser.dql"})) + require.Equal(t, "sitemgmt", inferConnector(&plan.View{Table: "SITE_MAP"}, &shape.Source{Path: "/repo/dql/ui/agency/detail/campaign.dql"})) +} + +func TestExtractSummarySQL(t *testing.T) { + sqlText := `SELECT b.* FROM CI_BROWSER b +JOIN ( + SELECT COUNT(1) AS CNT + FROM ($View.browser.SQL) t +) summary ON 1=1` + require.Contains(t, extractSummarySQL(sqlText), "COUNT(1)") +} + +func TestInferTableFromSQL_PreservesTemplateQualifiedTable(t *testing.T) { + sqlText := `SELECT SITE_ID FROM ${sitemgmt_project}.${sitemgmt_dataset}.SITE_LIST_MATCH slm` + require.Equal(t, "${sitemgmt_project}.${sitemgmt_dataset}.SITE_LIST_MATCH", inferTableFromSQL(sqlText, nil)) +} + +func TestShouldInferTable_NormalizedTemplatePlaceholderTable(t *testing.T) { + require.True(t, shouldInferTable(&plan.View{Name: "match", Table: "1.1.SITE_LIST_MATCH"})) + require.False(t, shouldInferTable(&plan.View{Name: "match", Table: "SITE_LIST_MATCH"})) +} + +func TestInferTableFromSQL_PathLikeTable(t *testing.T) { + sqlText := `SELECT user_id FROM session/attributes WHERE user_id = 1` + require.Equal(t, "session/attributes", inferTableFromSQL(sqlText, nil)) +} + +func TestApplySourceParityEnrichment_InferTableFromSiblingSQLOnPlaceholderTable(t *testing.T) { + tempDir := t.TempDir() + dqlDir := filepath.Join(tempDir, "dql", "platform", "sitelist") + require.NoError(t, os.MkdirAll(dqlDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dqlDir, "match.sql"), []byte(`SELECT SITE_ID FROM ${sitemgmt_project}.${sitemgmt_dataset}.SITE_LIST_MATCH slm`), 0o644)) + source := &shape.Source{ + Path: filepath.Join(dqlDir, "match.dql"), + DQL: `SELECT 1`, + } + result := &plan.Result{ + Views: []*plan.View{ + {Name: "match", Table: "1.1.SITE_LIST_MATCH"}, + }, + } + + applySourceParityEnrichment(result, source) + + require.Equal(t, "${sitemgmt_project}.${sitemgmt_dataset}.SITE_LIST_MATCH", result.Views[0].Table) +} + +func TestExtractJoinSubqueryBodies(t *testing.T) { + sqlText := `SELECT sl.* FROM SITE_LIST sl +JOIN ( + SELECT SITE_ID, SITE_LIST_ID FROM ${sitemgmt_project}.${sitemgmt_dataset}.SITE_LIST_MATCH +) match ON match.SITE_LIST_ID = sl.ID +JOIN ( + ${embed: match_rules.sql} + ${predicate.Builder().CombineOr($predicate.FilterGroup(1, "AND")).Build("WHERE")} +) matchRules ON matchRules.SITE_LIST_ID = sl.ID` + bodies := extractJoinSubqueryBodies(sqlText) + require.Contains(t, bodies, "match") + require.Contains(t, bodies["match"], "SITE_LIST_MATCH") + require.Contains(t, bodies, "matchRules") + require.Contains(t, bodies["matchRules"], "${embed: match_rules.sql}") +} + +func TestApplySourceParityEnrichment_Metadata(t *testing.T) { + source := &shape.Source{ + Path: "/repo/dql/platform/tvaffiliatestation/tvaffiliatestation.dql", + DQL: `/* {"Name":"TvAffiliateStation"} */ +SELECT use_connector(tvAffiliateStation, 'ci_ads'), + allow_nulls(tvAffiliateStation), + set_limit(tvAffiliateStation, 0) +FROM CI_TV_AFFILIATE_STATION tvAffiliateStation +JOIN ( + SELECT COUNT(1) AS CNT FROM ($View.tvAffiliateStation.SQL) t +) summary ON 1=1`, + } + result := &plan.Result{ + Views: []*plan.View{ + {Name: "tvAffiliateStation", Table: "CI_TV_AFFILIATE_STATION", SQL: "SELECT * FROM CI_TV_AFFILIATE_STATION tvAffiliateStation"}, + }, + } + hints := extractViewHints(source.DQL) + applyViewHints(result, hints) + applySourceParityEnrichment(result, source) + + require.Len(t, result.Views, 1) + actual := result.Views[0] + require.NotNil(t, actual.AllowNulls) + require.True(t, *actual.AllowNulls) + require.NotNil(t, actual.SelectorNoLimit) + require.True(t, *actual.SelectorNoLimit) + require.Equal(t, "tv", actual.SelectorNamespace) + require.Equal(t, "platform/tvaffiliatestation", actual.Module) + require.Equal(t, "*TvAffiliateStationView", actual.SchemaType) + require.NotEmpty(t, actual.Summary) +} diff --git a/repository/shape/compile/hints.go b/repository/shape/compile/hints.go new file mode 100644 index 000000000..93a6bfb3d --- /dev/null +++ b/repository/shape/compile/hints.go @@ -0,0 +1,185 @@ +package compile + +import ( + "reflect" + "regexp" + "strconv" + "strings" + + "github.com/viant/datly/repository/shape/plan" +) + +var ( + useConnectorExpr = regexp.MustCompile(`(?i)use_connector\s*\(\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*,\s*(?:'([a-zA-Z_][a-zA-Z0-9_]*)'|"([a-zA-Z_][a-zA-Z0-9_]*)"|([a-zA-Z_][a-zA-Z0-9_]*))\s*\)`) + allowNullsExpr = regexp.MustCompile(`(?i)allow_nulls\s*\(\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\)`) + setLimitExpr = regexp.MustCompile(`(?i)set_limit\s*\(\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*,\s*(-?[0-9]+)\s*\)`) +) + +type viewHint struct { + Connector string + AllowNulls *bool + NoLimit *bool +} + +func extractViewHints(dql string) map[string]viewHint { + result := map[string]viewHint{} + for _, match := range useConnectorExpr.FindAllStringSubmatch(dql, -1) { + if len(match) < 5 { + continue + } + alias := strings.TrimSpace(match[1]) + connector := strings.TrimSpace(match[2]) + if connector == "" { + connector = strings.TrimSpace(match[3]) + } + if connector == "" { + connector = strings.TrimSpace(match[4]) + } + if alias == "" || connector == "" { + continue + } + hint := result[alias] + hint.Connector = connector + result[alias] = hint + } + for _, match := range allowNullsExpr.FindAllStringSubmatch(dql, -1) { + if len(match) < 2 { + continue + } + alias := strings.TrimSpace(match[1]) + if alias == "" { + continue + } + hint := result[alias] + value := true + hint.AllowNulls = &value + result[alias] = hint + } + for _, match := range setLimitExpr.FindAllStringSubmatch(dql, -1) { + if len(match) < 3 { + continue + } + alias := strings.TrimSpace(match[1]) + limitRaw := strings.TrimSpace(match[2]) + if alias == "" || limitRaw == "" { + continue + } + limit, err := strconv.Atoi(limitRaw) + if err != nil { + continue + } + hint := result[alias] + noLimit := limit == 0 + hint.NoLimit = &noLimit + result[alias] = hint + } + return result +} + +func appendRelationViews(result *plan.Result, root *plan.View, hints map[string]viewHint) { + if result == nil || root == nil || len(root.Relations) == 0 { + return + } + for _, relation := range root.Relations { + if relation == nil { + continue + } + name := strings.TrimSpace(relation.Ref) + if name == "" { + name = strings.TrimSpace(relation.Name) + } + if name == "" { + continue + } + if len(relation.On) == 0 { + continue + } + if _, exists := result.ViewsByName[name]; exists { + continue + } + table := strings.TrimSpace(relation.Table) + if table == "" { + table = name + } + table = normalizeRelationTable(table) + view := &plan.View{ + Path: name, + Holder: name, + Name: name, + Table: table, + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + } + result.Views = append(result.Views, view) + result.ViewsByName[name] = view + } +} + +func applyViewHints(result *plan.Result, hints map[string]viewHint) { + if result == nil || len(result.Views) == 0 { + return + } + if len(hints) == 0 { + return + } + for _, item := range result.Views { + if item == nil { + continue + } + for _, key := range []string{item.Name, item.Holder} { + key = strings.TrimSpace(key) + if key == "" { + continue + } + hint, ok := hints[key] + if !ok { + continue + } + if item.Connector == "" && hint.Connector != "" { + item.Connector = hint.Connector + } + if item.AllowNulls == nil && hint.AllowNulls != nil { + value := *hint.AllowNulls + item.AllowNulls = &value + } + if item.SelectorNoLimit == nil && hint.NoLimit != nil { + value := *hint.NoLimit + item.SelectorNoLimit = &value + } + } + } +} + +func normalizeRelationTable(table string) string { + table = strings.TrimSpace(table) + if table == "" { + return table + } + lower := strings.ToLower(table) + fromIdx := strings.Index(lower, " from ") + if fromIdx == -1 { + return table + } + tail := strings.TrimSpace(table[fromIdx+6:]) + if tail == "" { + return table + } + stop := len(tail) + for i := 0; i < len(tail); i++ { + switch tail[i] { + case ' ', '\t', '\n', '\r', ')': + stop = i + i = len(tail) + } + } + if stop == 0 { + return table + } + normalized := strings.TrimSpace(tail[:stop]) + normalized = strings.Trim(normalized, "`\"") + if normalized == "" { + return table + } + return normalized +} diff --git a/repository/shape/compile/hints_test.go b/repository/shape/compile/hints_test.go new file mode 100644 index 000000000..768f82863 --- /dev/null +++ b/repository/shape/compile/hints_test.go @@ -0,0 +1,43 @@ +package compile + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape/plan" +) + +func TestExtractViewHints_WithQuotedConnector(t *testing.T) { + dql := "SELECT use_connector(match, 'bq_sitemgmt_match'), use_connector(site, \"ci_ads\"), allow_nulls(match), set_limit(match, 0)" + hints := extractViewHints(dql) + require.Len(t, hints, 2) + assert.Equal(t, "bq_sitemgmt_match", hints["match"].Connector) + assert.Equal(t, "ci_ads", hints["site"].Connector) + require.NotNil(t, hints["match"].AllowNulls) + assert.True(t, *hints["match"].AllowNulls) + require.NotNil(t, hints["match"].NoLimit) + assert.True(t, *hints["match"].NoLimit) +} + +func TestApplyViewHints_Metadata(t *testing.T) { + trueValue := true + result := &plan.Result{ + Views: []*plan.View{ + {Name: "match", Table: "MATCH"}, + }, + } + applyViewHints(result, map[string]viewHint{ + "match": { + Connector: "ci_ads", + AllowNulls: &trueValue, + NoLimit: &trueValue, + }, + }) + require.Len(t, result.Views, 1) + assert.Equal(t, "ci_ads", result.Views[0].Connector) + require.NotNil(t, result.Views[0].AllowNulls) + assert.True(t, *result.Views[0].AllowNulls) + require.NotNil(t, result.Views[0].SelectorNoLimit) + assert.True(t, *result.Views[0].SelectorNoLimit) +} diff --git a/repository/shape/compile/legacy_adapter.go b/repository/shape/compile/legacy_adapter.go new file mode 100644 index 000000000..d26be3c85 --- /dev/null +++ b/repository/shape/compile/legacy_adapter.go @@ -0,0 +1,655 @@ +package compile + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" + "gopkg.in/yaml.v3" +) + +func resolveGeneratedCompanionDQL(source *shape.Source) string { + if source == nil || strings.TrimSpace(source.Path) == "" { + return "" + } + settings := extractRuleSettings(source) + typeExpr := strings.TrimSpace(settings.Type) + if typeExpr == "" { + return "" + } + typeExpr = strings.TrimSuffix(typeExpr, ".Handler") + typeExpr = strings.Trim(typeExpr, `"'`) + if typeExpr == "" { + return "" + } + dir := filepath.Dir(source.Path) + baseTypePath := filepath.FromSlash(typeExpr) + stem := filepath.Base(baseTypePath) + candidates := []string{ + filepath.Join(dir, "gen", baseTypePath+".dql"), + filepath.Join(dir, "gen", baseTypePath+".sql"), + filepath.Join(dir, "gen", stem+".dql"), + filepath.Join(dir, "gen", stem+".sql"), + } + for _, candidate := range candidates { + data, err := os.ReadFile(candidate) + if err != nil { + continue + } + content := strings.TrimSpace(string(data)) + if content != "" { + return content + } + } + return "" +} + +func resolveLegacyRouteViews(source *shape.Source) []*plan.View { + return resolveLegacyRouteViewsWithLayout(source, defaultCompilePathLayout()) +} + +func resolveLegacyRouteViewsWithLayout(source *shape.Source, layout compilePathLayout) []*plan.View { + if source == nil || strings.TrimSpace(source.Path) == "" { + return nil + } + platformRoot, relativeDir, stem, ok := platformPathParts(source.Path, layout) + if !ok { + return nil + } + settings := extractRuleSettings(source) + typeExpr := strings.TrimSpace(settings.Type) + typeExpr = strings.Trim(typeExpr, `"'`) + typeExpr = strings.TrimSuffix(typeExpr, ".Handler") + typeStem := "" + if typeExpr != "" { + typeStem = filepath.Base(filepath.FromSlash(typeExpr)) + } + routesRoot := joinRelativePath(platformRoot, layout.routesRelative) + routesBase := filepath.Join(routesRoot, filepath.FromSlash(relativeDir)) + legacyMeta := []legacyViewMeta(nil) + for _, candidateYAML := range legacyRouteYAMLCandidates(routesBase, stem, typeStem) { + legacyMeta = loadLegacyRouteViewMeta(candidateYAML) + if len(legacyMeta) > 0 { + break + } + } + searchDirs := []string{ + filepath.Join(routesBase, typeStem, stem), + filepath.Join(routesBase, typeStem), + filepath.Join(routesBase, stem, stem), + filepath.Join(routesBase, stem), + routesBase, + } + var sqlFiles []string + for _, dir := range searchDirs { + entries, err := os.ReadDir(dir) + if err != nil { + continue + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(strings.ToLower(entry.Name()), ".sql") { + continue + } + sqlFiles = append(sqlFiles, filepath.Join(dir, entry.Name())) + } + if len(sqlFiles) > 0 { + break + } + } + if len(sqlFiles) == 0 { + return nil + } + sort.Strings(sqlFiles) + result := make([]*plan.View, 0, len(sqlFiles)) + rootIndex := -1 + for _, sqlFile := range sqlFiles { + name := strings.TrimSuffix(filepath.Base(sqlFile), filepath.Ext(sqlFile)) + if name == "" { + continue + } + data, err := os.ReadFile(sqlFile) + if err != nil { + continue + } + sqlText := string(data) + table := "" + if name != stem { + table = inferTableFromSQL(sqlText, source) + } + connector := strings.TrimSpace(settings.Connector) + if connector == "" { + connector = strings.TrimSpace(source.Connector) + } + if connector == "" { + connector = inferConnector(&plan.View{Table: table}, source) + } + viewItem := &plan.View{ + Path: name, + Holder: name, + Name: name, + Table: table, + SQL: sqlText, + SQLURI: filepath.ToSlash(filepath.Join(stem, name+".sql")), + Connector: connector, + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + } + if meta, ok := lookupLegacyViewMeta(legacyMeta, name); ok { + if strings.TrimSpace(meta.Table) != "" { + viewItem.Table = strings.TrimSpace(meta.Table) + } + if strings.TrimSpace(meta.Connector) != "" { + viewItem.Connector = strings.TrimSpace(meta.Connector) + } + if strings.TrimSpace(meta.SQLURI) != "" { + viewItem.SQLURI = strings.TrimSpace(meta.SQLURI) + } + } + if name == stem { + rootIndex = len(result) + } + result = append(result, viewItem) + } + if len(result) == 0 { + return nil + } + if rootIndex > 0 { + root := result[rootIndex] + copy(result[1:rootIndex+1], result[0:rootIndex]) + result[0] = root + } + if result[0].Name != stem { + rootConnector := result[0].Connector + result = append([]*plan.View{{ + Path: stem, + Holder: stem, + Name: stem, + Table: "", + SQLURI: filepath.ToSlash(filepath.Join(stem, stem+".sql")), + Connector: rootConnector, + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + }}, result...) + } + result[0].Table = "" + result[0].Name = stem + result[0].Holder = stem + result[0].Path = stem + if meta, ok := lookupLegacyViewMeta(legacyMeta, stem); ok { + if strings.TrimSpace(meta.Table) != "" { + result[0].Table = strings.TrimSpace(meta.Table) + } + if strings.TrimSpace(meta.Connector) != "" { + result[0].Connector = strings.TrimSpace(meta.Connector) + } + if strings.TrimSpace(meta.SQLURI) != "" { + result[0].SQLURI = strings.TrimSpace(meta.SQLURI) + } + } + if result[0].SQLURI == "" { + result[0].SQLURI = filepath.ToSlash(filepath.Join(stem, stem+".sql")) + } + return result +} + +type legacyViewMeta struct { + Name string + Table string + Connector string + SQLURI string +} + +func loadLegacyRouteViewMeta(yamlPath string) []legacyViewMeta { + data, err := os.ReadFile(yamlPath) + if err != nil { + return nil + } + var payload struct { + Resource struct { + Views []struct { + Name string `yaml:"Name"` + Table string `yaml:"Table"` + Connector struct { + Ref string `yaml:"Ref"` + } `yaml:"Connector"` + Template struct { + SourceURL string `yaml:"SourceURL"` + } `yaml:"Template"` + } `yaml:"Views"` + } `yaml:"Resource"` + } + if err = yaml.Unmarshal(data, &payload); err != nil { + return nil + } + result := make([]legacyViewMeta, 0, len(payload.Resource.Views)) + for _, item := range payload.Resource.Views { + result = append(result, legacyViewMeta{ + Name: strings.TrimSpace(item.Name), + Table: strings.TrimSpace(item.Table), + Connector: strings.TrimSpace(item.Connector.Ref), + SQLURI: strings.TrimSpace(item.Template.SourceURL), + }) + } + return result +} + +func lookupLegacyViewMeta(items []legacyViewMeta, name string) (legacyViewMeta, bool) { + name = strings.TrimSpace(name) + if name == "" { + return legacyViewMeta{}, false + } + for _, item := range items { + if strings.EqualFold(strings.TrimSpace(item.Name), name) { + return item, true + } + } + return legacyViewMeta{}, false +} + +func resolveLegacyRouteStates(source *shape.Source) []*plan.State { + return resolveLegacyRouteStatesWithLayout(source, defaultCompilePathLayout()) +} + +func resolveLegacyRouteStatesWithLayout(source *shape.Source, layout compilePathLayout) []*plan.State { + if source == nil || strings.TrimSpace(source.Path) == "" { + return nil + } + platformRoot, relativeDir, stem, ok := platformPathParts(source.Path, layout) + if !ok { + return nil + } + settings := extractRuleSettings(source) + typeExpr := strings.TrimSpace(settings.Type) + typeExpr = strings.Trim(typeExpr, `"'`) + typeExpr = strings.TrimSuffix(typeExpr, ".Handler") + typeStem := "" + if typeExpr != "" { + typeStem = filepath.Base(filepath.FromSlash(typeExpr)) + } + routesRoot := joinRelativePath(platformRoot, layout.routesRelative) + routesBase := filepath.Join(routesRoot, filepath.FromSlash(relativeDir)) + yamlCandidates := legacyRouteYAMLCandidates(routesBase, stem, typeStem) + var payload struct { + Resource struct { + Parameters []struct { + Name string `yaml:"Name"` + URI string `yaml:"URI"` + Value string `yaml:"Value"` + Required *bool `yaml:"Required"` + Cacheable *bool `yaml:"Cacheable"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` + Predicates []struct { + Group int `yaml:"Group"` + Name string `yaml:"Name"` + Ensure bool `yaml:"Ensure"` + Args []string `yaml:"Args"` + } `yaml:"Predicates"` + } `yaml:"Parameters"` + Views []struct { + Name string `yaml:"Name"` + Selector struct { + LimitParameter struct { + Name string `yaml:"Name"` + Cacheable *bool `yaml:"Cacheable"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` + } `yaml:"LimitParameter"` + OffsetParameter struct { + Name string `yaml:"Name"` + Cacheable *bool `yaml:"Cacheable"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` + } `yaml:"OffsetParameter"` + PageParameter struct { + Name string `yaml:"Name"` + Cacheable *bool `yaml:"Cacheable"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` + } `yaml:"PageParameter"` + FieldsParameter struct { + Name string `yaml:"Name"` + Cacheable *bool `yaml:"Cacheable"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` + } `yaml:"FieldsParameter"` + OrderByParameter struct { + Name string `yaml:"Name"` + Cacheable *bool `yaml:"Cacheable"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` + } `yaml:"OrderByParameter"` + } `yaml:"Selector"` + } `yaml:"Views"` + } `yaml:"Resource"` + } + loaded := false + for _, candidate := range yamlCandidates { + data, err := os.ReadFile(candidate) + if err != nil { + continue + } + if err = yaml.Unmarshal(data, &payload); err != nil { + continue + } + loaded = true + break + } + if !loaded || len(payload.Resource.Parameters) == 0 { + return nil + } + result := make([]*plan.State, 0, len(payload.Resource.Parameters)) + for _, item := range payload.Resource.Parameters { + stateItem := &plan.State{ + Name: strings.TrimSpace(item.Name), + Path: strings.TrimSpace(item.Name), + Kind: strings.TrimSpace(item.In.Kind), + In: strings.TrimSpace(item.In.Name), + URI: strings.TrimSpace(item.URI), + Value: strings.TrimSpace(item.Value), + Required: item.Required, + Cacheable: item.Cacheable, + } + for _, predicate := range item.Predicates { + stateItem.Predicates = append(stateItem.Predicates, &plan.StatePredicate{ + Group: predicate.Group, + Name: strings.TrimSpace(predicate.Name), + Ensure: predicate.Ensure, + Arguments: append([]string{}, predicate.Args...), + }) + } + result = append(result, stateItem) + } + seen := map[string]bool{} + for _, item := range result { + if item == nil { + continue + } + key := strings.ToLower(strings.TrimSpace(item.Name)) + "|" + strings.ToLower(strings.TrimSpace(item.Kind)) + "|" + strings.ToLower(strings.TrimSpace(item.In)) + seen[key] = true + } + for _, viewItem := range payload.Resource.Views { + selectorName := strings.TrimSpace(viewItem.Name) + for _, param := range []struct { + Name string + Cacheable *bool + InKind string + InName string + }{ + { + Name: strings.TrimSpace(viewItem.Selector.LimitParameter.Name), + Cacheable: viewItem.Selector.LimitParameter.Cacheable, + InKind: strings.TrimSpace(viewItem.Selector.LimitParameter.In.Kind), + InName: strings.TrimSpace(viewItem.Selector.LimitParameter.In.Name), + }, + { + Name: strings.TrimSpace(viewItem.Selector.OffsetParameter.Name), + Cacheable: viewItem.Selector.OffsetParameter.Cacheable, + InKind: strings.TrimSpace(viewItem.Selector.OffsetParameter.In.Kind), + InName: strings.TrimSpace(viewItem.Selector.OffsetParameter.In.Name), + }, + { + Name: strings.TrimSpace(viewItem.Selector.PageParameter.Name), + Cacheable: viewItem.Selector.PageParameter.Cacheable, + InKind: strings.TrimSpace(viewItem.Selector.PageParameter.In.Kind), + InName: strings.TrimSpace(viewItem.Selector.PageParameter.In.Name), + }, + { + Name: strings.TrimSpace(viewItem.Selector.FieldsParameter.Name), + Cacheable: viewItem.Selector.FieldsParameter.Cacheable, + InKind: strings.TrimSpace(viewItem.Selector.FieldsParameter.In.Kind), + InName: strings.TrimSpace(viewItem.Selector.FieldsParameter.In.Name), + }, + { + Name: strings.TrimSpace(viewItem.Selector.OrderByParameter.Name), + Cacheable: viewItem.Selector.OrderByParameter.Cacheable, + InKind: strings.TrimSpace(viewItem.Selector.OrderByParameter.In.Kind), + InName: strings.TrimSpace(viewItem.Selector.OrderByParameter.In.Name), + }, + } { + if param.Name == "" { + continue + } + kind := firstNonEmptyString(strings.ToLower(param.InKind), "query") + inName := firstNonEmptyString(param.InName, strings.ToLower(param.Name)) + key := strings.ToLower(param.Name) + "|" + kind + "|" + strings.ToLower(inName) + if seen[key] { + continue + } + item := &plan.State{ + Name: param.Name, + Path: param.Name, + Kind: kind, + In: inName, + QuerySelector: selectorName, + Cacheable: param.Cacheable, + } + result = append(result, item) + seen[key] = true + } + } + return result +} + +func resolveLegacyRouteTypes(source *shape.Source) []*plan.Type { + return resolveLegacyRouteTypesWithLayout(source, defaultCompilePathLayout()) +} + +func resolveLegacyRouteTypesWithLayout(source *shape.Source, layout compilePathLayout) []*plan.Type { + if source == nil || strings.TrimSpace(source.Path) == "" { + return nil + } + platformRoot, relativeDir, stem, ok := platformPathParts(source.Path, layout) + if !ok { + return nil + } + settings := extractRuleSettings(source) + typeExpr := strings.TrimSpace(settings.Type) + typeExpr = strings.Trim(typeExpr, `"'`) + typeExpr = strings.TrimSuffix(typeExpr, ".Handler") + typeStem := "" + if typeExpr != "" { + typeStem = filepath.Base(filepath.FromSlash(typeExpr)) + } + routesRoot := joinRelativePath(platformRoot, layout.routesRelative) + routesBase := filepath.Join(routesRoot, filepath.FromSlash(relativeDir)) + yamlCandidates := legacyRouteYAMLCandidates(routesBase, stem, typeStem) + var payload struct { + Resource struct { + Types []struct { + Name string `yaml:"Name"` + Alias string `yaml:"Alias"` + DataType string `yaml:"DataType"` + Cardinality string `yaml:"Cardinality"` + Package string `yaml:"Package"` + ModulePath string `yaml:"ModulePath"` + } `yaml:"Types"` + } `yaml:"Resource"` + } + loaded := false + for _, candidate := range yamlCandidates { + data, err := os.ReadFile(candidate) + if err != nil { + continue + } + if err = yaml.Unmarshal(data, &payload); err != nil { + continue + } + loaded = true + break + } + if !loaded || len(payload.Resource.Types) == 0 { + return nil + } + result := make([]*plan.Type, 0, len(payload.Resource.Types)) + seen := map[string]bool{} + for _, item := range payload.Resource.Types { + name := strings.TrimSpace(item.Name) + if name == "" { + continue + } + key := strings.ToLower(name) + if seen[key] { + continue + } + seen[key] = true + result = append(result, &plan.Type{ + Name: name, + Alias: strings.TrimSpace(item.Alias), + DataType: strings.TrimSpace(item.DataType), + Cardinality: strings.TrimSpace(item.Cardinality), + Package: strings.TrimSpace(item.Package), + ModulePath: strings.TrimSpace(item.ModulePath), + }) + } + return result +} + +func mergeLegacyRouteStates(result *plan.Result, source *shape.Source) { + mergeLegacyRouteStatesWithLayout(result, source, defaultCompilePathLayout()) +} + +func mergeLegacyRouteStatesWithLayout(result *plan.Result, source *shape.Source, layout compilePathLayout) { + if result == nil { + return + } + legacy := resolveLegacyRouteStatesWithLayout(source, layout) + if len(legacy) == 0 { + return + } + existing := map[string]bool{} + for _, item := range result.States { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + key := strings.ToLower(strings.TrimSpace(item.Name)) + "|" + strings.ToLower(strings.TrimSpace(item.Kind)) + "|" + strings.ToLower(strings.TrimSpace(item.In)) + existing[key] = true + } + for _, item := range legacy { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + key := strings.ToLower(strings.TrimSpace(item.Name)) + "|" + strings.ToLower(strings.TrimSpace(item.Kind)) + "|" + strings.ToLower(strings.TrimSpace(item.In)) + if existing[key] { + continue + } + result.States = append(result.States, item) + existing[key] = true + } +} + +func mergeLegacyRouteTypes(result *plan.Result, source *shape.Source) { + mergeLegacyRouteTypesWithLayout(result, source, defaultCompilePathLayout()) +} + +func mergeLegacyRouteTypesWithLayout(result *plan.Result, source *shape.Source, layout compilePathLayout) { + if result == nil { + return + } + legacy := resolveLegacyRouteTypesWithLayout(source, layout) + if len(legacy) == 0 { + return + } + existing := map[string]bool{} + for _, item := range result.Types { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + existing[strings.ToLower(strings.TrimSpace(item.Name))] = true + } + for _, item := range legacy { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + key := strings.ToLower(strings.TrimSpace(item.Name)) + if existing[key] { + continue + } + result.Types = append(result.Types, item) + existing[key] = true + } +} + +func legacyRouteYAMLCandidates(routesBase, stem, typeStem string) []string { + stemFileVariants := routeStemAlternatives(stem) + stemDirVariants := routeStemAlternatives(stem) + typeVariants := routeStemAlternatives(typeStem) + var result []string + seen := map[string]bool{} + appendCandidate := func(path string) { + path = filepath.Clean(path) + if path == "." || path == "" || seen[path] { + return + } + seen[path] = true + result = append(result, path) + } + for _, fileStem := range stemFileVariants { + appendCandidate(filepath.Join(routesBase, fileStem+".yaml")) + for _, dirStem := range stemDirVariants { + appendCandidate(filepath.Join(routesBase, dirStem, fileStem+".yaml")) + } + for _, itemTypeStem := range typeVariants { + if strings.TrimSpace(itemTypeStem) == "" { + continue + } + appendCandidate(filepath.Join(routesBase, itemTypeStem, fileStem+".yaml")) + } + } + return result +} + +func routeStemAlternatives(value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + alts := []string{value} + dashed := strings.ReplaceAll(value, "_", "-") + if dashed != value { + alts = append(alts, dashed) + } + return alts +} + +func platformPathParts(sourcePath string, layout compilePathLayout) (platformRoot, relativeDir, stem string, ok bool) { + sourcePath = filepath.Clean(strings.TrimSpace(sourcePath)) + if sourcePath == "" { + return "", "", "", false + } + normalized := filepath.ToSlash(sourcePath) + marker := layout.dqlMarker + if marker == "" { + marker = defaultCompilePathLayout().dqlMarker + } + idx := strings.Index(normalized, marker) + if idx == -1 { + return "", "", "", false + } + platformRoot = sourcePath[:idx] + relative := strings.TrimPrefix(normalized[idx+len(marker):], "/") + relativeDir = filepath.Dir(relative) + stem = strings.TrimSuffix(filepath.Base(sourcePath), filepath.Ext(sourcePath)) + if strings.TrimSpace(stem) == "" { + return "", "", "", false + } + return platformRoot, relativeDir, stem, true +} diff --git a/repository/shape/compile/pathlayout.go b/repository/shape/compile/pathlayout.go new file mode 100644 index 000000000..a1bc081e2 --- /dev/null +++ b/repository/shape/compile/pathlayout.go @@ -0,0 +1,67 @@ +package compile + +import ( + "path/filepath" + "strings" + + "github.com/viant/datly/repository/shape" +) + +type compilePathLayout struct { + dqlMarker string + routesRelative string +} + +func defaultCompilePathLayout() compilePathLayout { + return compilePathLayout{ + dqlMarker: "/dql/", + routesRelative: "repo/dev/Datly/routes", + } +} + +func newCompilePathLayout(opts *shape.CompileOptions) compilePathLayout { + ret := defaultCompilePathLayout() + if opts == nil { + return ret + } + if marker := normalizeDQLMarker(opts.DQLPathMarker); marker != "" { + ret.dqlMarker = marker + } + if rel := normalizeRoutesRelative(opts.RoutesRelativePath); rel != "" { + ret.routesRelative = rel + } + return ret +} + +func normalizeDQLMarker(input string) string { + input = strings.TrimSpace(strings.ReplaceAll(input, "\\", "/")) + if input == "" { + return "" + } + input = strings.Trim(input, "/") + if input == "" { + return "" + } + return "/" + input + "/" +} + +func normalizeRoutesRelative(input string) string { + input = strings.TrimSpace(strings.ReplaceAll(input, "\\", "/")) + input = strings.Trim(input, "/") + if input == "" { + return "" + } + return input +} + +func joinRelativePath(base string, rel string) string { + rel = normalizeRoutesRelative(rel) + if rel == "" { + return base + } + parts := strings.Split(rel, "/") + args := make([]string, 0, len(parts)+1) + args = append(args, base) + args = append(args, parts...) + return filepath.Join(args...) +} diff --git a/repository/shape/compile/pipeline/diag.go b/repository/shape/compile/pipeline/diag.go new file mode 100644 index 000000000..45bf78568 --- /dev/null +++ b/repository/shape/compile/pipeline/diag.go @@ -0,0 +1,47 @@ +package pipeline + +import ( + "unicode/utf8" + + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" +) + +func StatementSpan(sqlText string, stmt *dqlstmt.Statement) dqlshape.Span { + if stmt == nil { + return pointSpan(sqlText, 0) + } + return pointSpan(sqlText, stmt.Start) +} + +func pointSpan(text string, offset int) dqlshape.Span { + start := positionAt(text, offset) + end := start + return dqlshape.Span{Start: start, End: end} +} + +func positionAt(text string, offset int) dqlshape.Position { + if offset < 0 { + offset = 0 + } + if offset > len(text) { + offset = len(text) + } + line := 1 + char := 1 + index := 0 + for index < offset { + r, width := utf8.DecodeRuneInString(text[index:]) + if width <= 0 { + break + } + index += width + if r == '\n' { + line++ + char = 1 + } else { + char++ + } + } + return dqlshape.Position{Offset: offset, Line: line, Char: char} +} diff --git a/repository/shape/compile/pipeline/exec.go b/repository/shape/compile/pipeline/exec.go new file mode 100644 index 000000000..6bf144888 --- /dev/null +++ b/repository/shape/compile/pipeline/exec.go @@ -0,0 +1,109 @@ +package pipeline + +import ( + "reflect" + "strings" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/sqlparser" +) + +func BuildExec(sourceName, sqlText string, statements dqlstmt.Statements) (*plan.View, []*dqlshape.Diagnostic) { + name := SanitizeName(sourceName) + if name == "" { + name = "DQLView" + } + tables := statements.DMLTables(sqlText) + table := name + if len(tables) > 0 { + table = tables[0] + } + fieldType := reflect.TypeOf([]map[string]interface{}{}) + elementType := reflect.TypeOf(map[string]interface{}{}) + view := &plan.View{ + Path: name, + Holder: name, + Name: name, + Mode: "SQLExec", + Table: table, + SQL: sqlText, + Cardinality: "many", + FieldType: fieldType, + ElementType: elementType, + } + return view, ValidateExecStatements(sqlText, statements) +} + +func ValidateExecStatements(sqlText string, statements dqlstmt.Statements) []*dqlshape.Diagnostic { + var result []*dqlshape.Diagnostic + for _, stmt := range statements { + if stmt == nil || !stmt.IsExec { + continue + } + body := strings.TrimSpace(sqlText[stmt.Start:stmt.End]) + if body == "" { + continue + } + lower := strings.ToLower(body) + span := StatementSpan(sqlText, stmt) + switch { + case stmt.Kind == dqlstmt.KindService: + if firstQuoted(body) == "" { + result = append(result, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDMLServiceArg, + Severity: dqlshape.SeverityError, + Message: "service DML call is missing quoted table argument", + Hint: "use $sql.Insert(\"TABLE\", ...) or $sql.Update(\"TABLE\", ...)", + Span: span, + }) + } + case strings.HasPrefix(lower, "insert"): + if _, err := sqlparser.ParseInsert(body); err != nil { + result = append(result, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDMLInsert, + Severity: dqlshape.SeverityError, + Message: strings.TrimSpace(err.Error()), + Hint: "fix INSERT statement syntax", + Span: span, + }) + } + case strings.HasPrefix(lower, "update"): + if _, err := sqlparser.ParseUpdate(body); err != nil { + result = append(result, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDMLUpdate, + Severity: dqlshape.SeverityError, + Message: strings.TrimSpace(err.Error()), + Hint: "fix UPDATE statement syntax", + Span: span, + }) + } + case strings.HasPrefix(lower, "delete"): + if _, err := sqlparser.ParseDelete(body); err != nil { + result = append(result, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDMLDelete, + Severity: dqlshape.SeverityError, + Message: strings.TrimSpace(err.Error()), + Hint: "fix DELETE statement syntax", + Span: span, + }) + } + } + } + return result +} + +func firstQuoted(input string) string { + index := strings.Index(input, `"`) + if index == -1 { + return "" + } + tail := input[index+1:] + end := strings.Index(tail, `"`) + if end == -1 { + return "" + } + return strings.TrimSpace(tail[:end]) +} diff --git a/repository/shape/compile/pipeline/exec_test.go b/repository/shape/compile/pipeline/exec_test.go new file mode 100644 index 000000000..8c70fc74d --- /dev/null +++ b/repository/shape/compile/pipeline/exec_test.go @@ -0,0 +1,26 @@ +package pipeline + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" +) + +func TestBuildExec(t *testing.T) { + sqlText := "INSERT INTO ORDERS(id) VALUES (1)" + view, diags := BuildExec("orders_exec", sqlText, dqlstmt.New(sqlText)) + require.NotNil(t, view) + assert.Equal(t, "ORDERS", view.Table) + assert.Equal(t, "many", view.Cardinality) + assert.Empty(t, diags) +} + +func TestValidateExecStatements_ServiceArg(t *testing.T) { + sqlText := "$sql.Insert($rec)" + diags := ValidateExecStatements(sqlText, dqlstmt.New(sqlText)) + require.NotEmpty(t, diags) + assert.Equal(t, dqldiag.CodeDMLServiceArg, diags[0].Code) +} diff --git a/repository/shape/compile/pipeline/infer.go b/repository/shape/compile/pipeline/infer.go new file mode 100644 index 000000000..c19bcb79b --- /dev/null +++ b/repository/shape/compile/pipeline/infer.go @@ -0,0 +1,227 @@ +package pipeline + +import ( + "fmt" + "reflect" + "regexp" + "strings" + + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/query" +) + +var nonWord = regexp.MustCompile(`[^a-zA-Z0-9_]+`) + +func InferRoot(queryNode *query.Select, fallback string) (string, string, error) { + name := SanitizeName(fallback) + if name == "" { + name = "DQLView" + } + if queryNode == nil { + return name, name, nil + } + if alias := SanitizeName(queryNode.From.Alias); alias != "" { + name = alias + } + table := "" + if queryNode.From.X != nil { + table = strings.TrimSpace(sqlparser.Stringify(queryNode.From.X)) + } + if name == "" || name == SanitizeName(fallback) { + if subAlias := inferSubqueryAlias(table); subAlias != "" { + name = subAlias + } + } + if table == "" || strings.HasPrefix(table, "(") { + if inferred := inferSubqueryTable(table); inferred != "" { + table = inferred + } else { + table = name + } + } + if name == "" { + return "", "", fmt.Errorf("shape compile: failed to infer view name") + } + return name, table, nil +} + +func inferSubqueryAlias(fromExpr string) string { + fromExpr = strings.TrimSpace(fromExpr) + if fromExpr == "" || !strings.HasPrefix(fromExpr, "(") { + return "" + } + depth := 0 + closeIdx := -1 + for i := 0; i < len(fromExpr); i++ { + switch fromExpr[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + closeIdx = i + i = len(fromExpr) + } + } + } + if closeIdx == -1 || closeIdx+1 >= len(fromExpr) { + return "" + } + rest := strings.TrimSpace(fromExpr[closeIdx+1:]) + restLower := strings.ToLower(rest) + if strings.HasPrefix(restLower, "as ") { + rest = strings.TrimSpace(rest[3:]) + } + if rest == "" { + return "" + } + end := 0 + for end < len(rest) { + c := rest[end] + if !(c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (end > 0 && c >= '0' && c <= '9')) { + break + } + end++ + } + if end == 0 { + return "" + } + return SanitizeName(rest[:end]) +} + +func inferSubqueryTable(fromExpr string) string { + inner, ok := extractSubqueryBody(fromExpr) + if !ok { + return "" + } + normalized := normalizeParserSQL(inner) + queryNode, _, err := ParseSelectWithDiagnostic(normalized) + if err != nil || queryNode == nil { + return "" + } + _, table, err := InferRoot(queryNode, "") + if err != nil { + return "" + } + table = strings.TrimSpace(strings.Trim(table, "`\"")) + if strings.EqualFold(table, "DQLView") { + return "" + } + return table +} + +func extractSubqueryBody(fromExpr string) (string, bool) { + fromExpr = strings.TrimSpace(fromExpr) + if !strings.HasPrefix(fromExpr, "(") { + return "", false + } + depth := 0 + for i := 0; i < len(fromExpr); i++ { + switch fromExpr[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + if i <= 1 { + return "", false + } + return strings.TrimSpace(fromExpr[1:i]), true + } + } + } + return "", false +} + +func InferProjectionType(queryNode *query.Select) (reflect.Type, reflect.Type, string) { + if queryNode == nil || len(queryNode.List) == 0 || queryNode.List.IsStarExpr() { + return reflect.TypeOf([]map[string]interface{}{}), reflect.TypeOf(map[string]interface{}{}), "many" + } + fields := make([]reflect.StructField, 0, len(queryNode.List)) + used := map[string]int{} + for index, item := range queryNode.List { + column := sqlparser.NewColumn(item) + columnName := strings.TrimSpace(column.Identity()) + if columnName == "" { + columnName = fmt.Sprintf("col_%d", index+1) + } + fieldName := ExportedName(columnName) + if fieldName == "" { + fieldName = fmt.Sprintf("Col%d", index+1) + } + if count := used[fieldName]; count > 0 { + fieldName = fmt.Sprintf("%s%d", fieldName, count+1) + } + used[fieldName]++ + + typ := parseColumnType(column.Type) + fields = append(fields, reflect.StructField{ + Name: fieldName, + Type: typ, + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s,omitempty" sqlx:"name=%s"`, strings.ToLower(fieldName), columnName)), + }) + } + element := reflect.StructOf(fields) + return reflect.SliceOf(element), element, "many" +} + +func SanitizeName(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if value == strings.ToUpper(value) { + value = strings.ToLower(value) + } + value = nonWord.ReplaceAllString(value, "_") + value = strings.Trim(value, "_") + if value == "" { + return "" + } + if value[0] >= '0' && value[0] <= '9' { + value = "V_" + value + } + return value +} + +func ExportedName(value string) string { + value = nonWord.ReplaceAllString(strings.TrimSpace(value), "_") + value = strings.Trim(value, "_") + if value == "" { + return "" + } + parts := strings.Split(strings.ToLower(value), "_") + for i, item := range parts { + if item == "" { + continue + } + parts[i] = strings.ToUpper(item[:1]) + item[1:] + } + name := strings.Join(parts, "") + if name == "" { + return "" + } + if name[0] >= '0' && name[0] <= '9' { + name = "N" + name + } + return name +} + +func parseColumnType(dataType string) reflect.Type { + switch strings.ToLower(strings.TrimSpace(dataType)) { + case "", "string", "text", "varchar", "char", "uuid", "json", "jsonb": + return reflect.TypeOf("") + case "bool", "boolean": + return reflect.TypeOf(false) + case "int", "int32", "smallint", "integer": + return reflect.TypeOf(int(0)) + case "int64", "bigint": + return reflect.TypeOf(int64(0)) + case "float", "float32", "real": + return reflect.TypeOf(float32(0)) + case "float64", "double", "numeric", "decimal": + return reflect.TypeOf(float64(0)) + default: + return reflect.TypeOf("") + } +} diff --git a/repository/shape/compile/pipeline/infer_test.go b/repository/shape/compile/pipeline/infer_test.go new file mode 100644 index 000000000..748fcded4 --- /dev/null +++ b/repository/shape/compile/pipeline/infer_test.go @@ -0,0 +1,42 @@ +package pipeline + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/viant/sqlparser" +) + +func TestInferSubqueryAlias(t *testing.T) { + assert.Equal(t, "session", inferSubqueryAlias("(SELECT * FROM session) session JOIN (SELECT * FROM attr) attribute ON attribute.id = session.id")) + assert.Equal(t, "x", inferSubqueryAlias("(SELECT 1) AS x")) + assert.Equal(t, "publisherglobaloverride", inferSubqueryAlias(`( + SELECT MIN(g.BUSINESS_MODEL_ID) AS BUSINESS_MODEL_ID + FROM CI_GLOBAL_PUBLISHER_OVERRIDE g +) publisherglobaloverride`)) + assert.Equal(t, "", inferSubqueryAlias("orders o")) +} + +func TestSanitizeName_AllCapsToLower(t *testing.T) { + assert.Equal(t, "value", SanitizeName("VALUE")) + assert.Equal(t, "status", SanitizeName("STATUS")) +} + +func TestInferSubqueryTable(t *testing.T) { + assert.Equal(t, "CI_ADVERTISER", inferSubqueryTable("(SELECT a.* FROM CI_ADVERTISER a) advertiser")) + assert.Equal(t, "", inferSubqueryTable("orders o")) +} + +func TestInferRoot_SubqueryFrom(t *testing.T) { + queryNode, err := sqlparser.ParseQuery(`SELECT advertiser.* FROM (SELECT a.* FROM CI_ADVERTISER a) advertiser`) + assert.NoError(t, err) + name, table, err := InferRoot(queryNode, "fallback") + assert.NoError(t, err) + assert.Equal(t, "advertiser", name) + assert.Equal(t, "CI_ADVERTISER", table) +} + +func TestInferTableFromSQL_ResolvesTopLevelFrom(t *testing.T) { + sqlText := `SELECT a.*, EXISTS(SELECT 1 FROM CI_ENTITY_WATCHLIST w WHERE w.ENTITY_ID = a.ID) AS watching FROM (SELECT x.* FROM CI_ADVERTISER x) a` + assert.Equal(t, "CI_ADVERTISER", InferTableFromSQL(sqlText)) +} diff --git a/repository/shape/compile/pipeline/parse.go b/repository/shape/compile/pipeline/parse.go new file mode 100644 index 000000000..c897454a4 --- /dev/null +++ b/repository/shape/compile/pipeline/parse.go @@ -0,0 +1,62 @@ +package pipeline + +import ( + "strings" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/parsly" + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/query" +) + +func ParseSelectWithDiagnostic(sqlText string) (*query.Select, *dqlshape.Diagnostic, error) { + sqlText = trimLeadingBlockComments(sqlText) + var diagnostic *dqlshape.Diagnostic + onError := func(err error, cur *parsly.Cursor, _ interface{}) error { + offset := 0 + if cur != nil { + offset = cur.Pos + } + if offset < 0 { + offset = 0 + } + diagnostic = &dqlshape.Diagnostic{ + Code: dqldiag.CodeParseSyntax, + Severity: dqlshape.SeverityError, + Message: strings.TrimSpace(err.Error()), + Hint: "check SQL syntax near the reported location", + Span: pointSpan(sqlText, offset), + } + return err + } + result, err := sqlparser.ParseQuery(sqlText, sqlparser.WithErrorHandler(onError)) + if err != nil { + if diagnostic == nil { + diagnostic = &dqlshape.Diagnostic{ + Code: dqldiag.CodeParseSyntax, + Severity: dqlshape.SeverityError, + Message: strings.TrimSpace(err.Error()), + Hint: "check SQL syntax near the reported location", + Span: pointSpan(sqlText, 0), + } + } + return nil, diagnostic, err + } + if result == nil { + return nil, nil, nil + } + return result, nil, nil +} + +func trimLeadingBlockComments(sqlText string) string { + remaining := strings.TrimLeft(sqlText, " \t\r\n") + for strings.HasPrefix(remaining, "/*") { + end := strings.Index(remaining, "*/") + if end == -1 { + return remaining + } + remaining = strings.TrimLeft(remaining[end+2:], " \t\r\n") + } + return remaining +} diff --git a/repository/shape/compile/pipeline/parse_test.go b/repository/shape/compile/pipeline/parse_test.go new file mode 100644 index 000000000..69292fc8a --- /dev/null +++ b/repository/shape/compile/pipeline/parse_test.go @@ -0,0 +1,35 @@ +package pipeline + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" +) + +func TestParseSelectWithDiagnostic_OK(t *testing.T) { + queryNode, diag, err := ParseSelectWithDiagnostic("SELECT id FROM orders o") + require.NoError(t, err) + require.Nil(t, diag) + require.NotNil(t, queryNode) + assert.Equal(t, "o", queryNode.From.Alias) +} + +func TestParseSelectWithDiagnostic_Syntax(t *testing.T) { + queryNode, diag, err := ParseSelectWithDiagnostic("SELECT id FROM orders WHERE (") + require.Error(t, err) + require.Nil(t, queryNode) + require.NotNil(t, diag) + assert.Equal(t, dqldiag.CodeParseSyntax, diag.Code) + assert.Equal(t, 1, diag.Span.Start.Line) + assert.Greater(t, diag.Span.Start.Char, 1) +} + +func TestParseSelectWithDiagnostic_LeadingBlockComment(t *testing.T) { + queryNode, diag, err := ParseSelectWithDiagnostic("/* {\"URI\":\"/x\"} */\nSELECT id FROM orders o") + require.NoError(t, err) + require.Nil(t, diag) + require.NotNil(t, queryNode) + assert.Equal(t, "o", queryNode.From.Alias) +} diff --git a/repository/shape/compile/pipeline/policy.go b/repository/shape/compile/pipeline/policy.go new file mode 100644 index 000000000..432bcd6fc --- /dev/null +++ b/repository/shape/compile/pipeline/policy.go @@ -0,0 +1,28 @@ +package pipeline + +import dqlstmt "github.com/viant/datly/repository/shape/dql/statement" + +type Decision struct { + HasRead bool + HasExec bool + HasUnknown bool +} + +func Classify(statements dqlstmt.Statements) Decision { + var ret Decision + for _, stmt := range statements { + if stmt == nil { + continue + } + if stmt.Kind == dqlstmt.KindExec || stmt.Kind == dqlstmt.KindService { + ret.HasExec = true + continue + } + if stmt.Kind == dqlstmt.KindRead { + ret.HasRead = true + continue + } + ret.HasUnknown = true + } + return ret +} diff --git a/repository/shape/compile/pipeline/policy_test.go b/repository/shape/compile/pipeline/policy_test.go new file mode 100644 index 000000000..2ff7d3077 --- /dev/null +++ b/repository/shape/compile/pipeline/policy_test.go @@ -0,0 +1,36 @@ +package pipeline + +import ( + "testing" + + "github.com/stretchr/testify/assert" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" +) + +func TestClassify_ReadOnly(t *testing.T) { + decision := Classify(dqlstmt.New("SELECT id FROM orders")) + assert.True(t, decision.HasRead) + assert.False(t, decision.HasExec) + assert.False(t, decision.HasUnknown) +} + +func TestClassify_ExecOnly(t *testing.T) { + decision := Classify(dqlstmt.New("UPDATE orders SET id = 1")) + assert.False(t, decision.HasRead) + assert.True(t, decision.HasExec) + assert.False(t, decision.HasUnknown) +} + +func TestClassify_Mixed(t *testing.T) { + decision := Classify(dqlstmt.New("SELECT id FROM orders\nUPDATE orders SET id = 1")) + assert.True(t, decision.HasRead) + assert.True(t, decision.HasExec) + assert.False(t, decision.HasUnknown) +} + +func TestClassify_UnknownTemplateOnly(t *testing.T) { + decision := Classify(dqlstmt.New("$Foo.Bar($x)")) + assert.False(t, decision.HasRead) + assert.False(t, decision.HasExec) + assert.True(t, decision.HasUnknown) +} diff --git a/repository/shape/compile/pipeline/read.go b/repository/shape/compile/pipeline/read.go new file mode 100644 index 000000000..c52f9cc28 --- /dev/null +++ b/repository/shape/compile/pipeline/read.go @@ -0,0 +1,199 @@ +package pipeline + +import ( + "reflect" + "regexp" + "strings" + + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/sqlparser/query" +) + +var ( + criteriaBindingExpr = regexp.MustCompile(`(?i)\$criteria\.AppendBinding\([^)]*\)`) + selectorExpr = regexp.MustCompile(`\$\{?([a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*)\}?`) + veltyExpr = regexp.MustCompile(`\$\{[^}]+\}`) + fromTableSimpleExpr = regexp.MustCompile(`(?is)\bfrom\s+([a-zA-Z_][a-zA-Z0-9_$.]*)(?:\s+(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*))?`) + braceExpr = regexp.MustCompile(`[{}]`) +) + +func BuildRead(sourceName, sqlText string) (*plan.View, []*dqlshape.Diagnostic, error) { + parserSQL := normalizeParserSQL(sqlText) + queryNode, parseDiag, err := ParseSelectWithDiagnostic(parserSQL) + if err != nil && parserSQL != sqlText { + if rawNode, _, rawErr := ParseSelectWithDiagnostic(sqlText); rawErr == nil && isUsableQuery(rawNode) { + queryNode = rawNode + parserSQL = sqlText + parseDiag = nil + err = nil + } + } + if err == nil && needsFallbackParse(sqlText, queryNode) { + fallbackSQL := normalizeParserSQL(sqlText) + if fallbackNode, _, fallbackErr := ParseSelectWithDiagnostic(fallbackSQL); fallbackErr == nil && isUsableQuery(fallbackNode) { + queryNode = fallbackNode + parserSQL = fallbackSQL + parseDiag = nil + err = nil + } + } + if hasTemplateSignals(sqlText) && (err != nil || parseDiag != nil) { + if parseDiag != nil { + parseDiag.Severity = dqlshape.SeverityWarning + } + var diags []*dqlshape.Diagnostic + if parseDiag != nil { + diags = append(diags, parseDiag) + } + return buildLooseRead(sourceName, sqlText), diags, nil + } + var diags []*dqlshape.Diagnostic + if parseDiag != nil { + diags = append(diags, parseDiag) + } + if err != nil { + if hasTemplateSignals(sqlText) { + if parseDiag != nil { + parseDiag.Severity = dqlshape.SeverityWarning + } + return buildLooseRead(sourceName, sqlText), diags, nil + } + return nil, diags, nil + } + relations, relationDiags := ExtractJoinRelations(parserSQL, queryNode) + diags = append(diags, relationDiags...) + name, table, inferErr := InferRoot(queryNode, sourceName) + if inferErr != nil { + return nil, nil, inferErr + } + fallback := SanitizeName(sourceName) + if name == fallback && table == fallback { + if derived := inferRootFromRelations(relations); derived != "" { + name = derived + table = derived + } + } + fieldType, elementType, cardinality := InferProjectionType(queryNode) + if fieldType == nil || elementType == nil { + fieldType = reflect.TypeOf([]map[string]interface{}{}) + elementType = reflect.TypeOf(map[string]interface{}{}) + cardinality = "many" + } + view := &plan.View{ + Path: name, + Holder: name, + Name: name, + Mode: "SQLQuery", + Table: table, + SQL: sqlText, + Cardinality: cardinality, + FieldType: fieldType, + ElementType: elementType, + Relations: relations, + } + return view, diags, nil +} + +func buildLooseRead(sourceName, sqlText string) *plan.View { + name, table := inferLooseRoot(sourceName, sqlText) + fieldType := reflect.TypeOf([]map[string]interface{}{}) + elementType := reflect.TypeOf(map[string]interface{}{}) + return &plan.View{ + Path: name, + Holder: name, + Name: name, + Mode: "SQLQuery", + Table: table, + SQL: sqlText, + Cardinality: "many", + FieldType: fieldType, + ElementType: elementType, + } +} + +func inferLooseRoot(sourceName, sqlText string) (string, string) { + name := SanitizeName(sourceName) + if name == "" { + name = "DQLView" + } + if matches := fromTableSimpleExpr.FindStringSubmatch(sqlText); len(matches) > 1 { + table := strings.Trim(matches[1], "`\"") + return name, table + } + return name, name +} + +func hasTemplateSignals(sqlText string) bool { + lower := strings.ToLower(sqlText) + return strings.Contains(lower, "#if(") || strings.Contains(lower, "#elseif(") || strings.Contains(lower, "#else") || + strings.Contains(lower, "#end") || strings.Contains(lower, "${") || strings.Contains(lower, "$unsafe.") || + strings.Contains(lower, "$view.") || strings.Contains(lower, "$predicate.") +} + +func isUsableQuery(queryNode *query.Select) bool { + return queryNode != nil && queryNode.From.X != nil +} + +func needsFallbackParse(rawSQL string, queryNode *query.Select) bool { + if !isUsableQuery(queryNode) { + return true + } + lower := strings.ToLower(rawSQL) + if strings.Contains(lower, " join ") && len(queryNode.Joins) == 0 { + return true + } + return false +} + +func normalizeParserSQL(sqlText string) string { + if sqlText == "" { + return sqlText + } + normalized := criteriaBindingExpr.ReplaceAllString(sqlText, "1") + normalized = veltyExpr.ReplaceAllStringFunc(normalized, func(match string) string { + if strings.Contains(match, "sql.Insert") || strings.Contains(match, "sql.Update") || strings.Contains(match, "Nop") { + return match + } + lower := strings.ToLower(match) + if strings.Contains(lower, `build("where")`) || strings.Contains(lower, "build('where')") { + return " WHERE 1 " + } + if strings.Contains(lower, `build("and")`) || strings.Contains(lower, "build('and')") { + return " AND 1 " + } + return "1" + }) + normalized = selectorExpr.ReplaceAllStringFunc(normalized, func(match string) string { + lower := match + if len(match) > 0 && match[0] == '$' { + lower = match[1:] + } + lower = braceExpr.ReplaceAllString(lower, "") + switch lower { + case "sql.Insert", "sql.Update", "Nop": + return match + default: + return "1" + } + }) + return normalized +} + +func inferRootFromRelations(relations []*plan.Relation) string { + for _, relation := range relations { + if relation == nil { + continue + } + for _, link := range relation.On { + if link == nil { + continue + } + name := SanitizeName(link.ParentNamespace) + if name != "" { + return name + } + } + } + return "" +} diff --git a/repository/shape/compile/pipeline/read_test.go b/repository/shape/compile/pipeline/read_test.go new file mode 100644 index 000000000..0c82d72a8 --- /dev/null +++ b/repository/shape/compile/pipeline/read_test.go @@ -0,0 +1,65 @@ +package pipeline + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/query" +) + +func TestBuildRead(t *testing.T) { + view, diags, err := BuildRead("orders_report", "SELECT o.id, i.sku FROM orders o JOIN items i ON o.id = i.order_id") + require.NoError(t, err) + require.NotNil(t, view) + assert.Equal(t, "o", view.Name) + assert.Equal(t, "orders", view.Table) + assert.Equal(t, "many", view.Cardinality) + require.Len(t, view.Relations, 1) + assert.Equal(t, "i", view.Relations[0].Ref) + assert.Empty(t, diags) +} + +func TestBuildRead_SubqueryJoin_UsesParentNamespaceAsRoot(t *testing.T) { + sqlText := `SELECT session.* +FROM (SELECT * FROM session WHERE user_id = $criteria.AppendBinding($Unsafe.Jwt.UserID)) session +JOIN (SELECT * FROM session/attributes) attribute ON attribute.user_id = session.user_id` + view, _, err := BuildRead("system/session", sqlText) + require.NoError(t, err) + require.NotNil(t, view) + assert.Equal(t, "session", view.Name) + assert.Equal(t, "session", view.Table) + require.NotEmpty(t, view.Relations) + assert.Equal(t, "attribute", view.Relations[0].Ref) +} + +func TestNormalizeParserSQL(t *testing.T) { + input := "SELECT * FROM session WHERE user_id = $criteria.AppendBinding($Unsafe.Jwt.UserID) AND x = $Jwt.UserID" + actual := normalizeParserSQL(input) + assert.NotContains(t, actual, "$criteria.AppendBinding") + assert.NotContains(t, actual, "$Jwt.UserID") + assert.Contains(t, actual, "user_id = 1") +} + +func TestNormalizeParserSQL_VeltyBlockExpression(t *testing.T) { + input := `SELECT b.* FROM CI_BROWSER b ${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")} AND b.ARCHIVED = 0` + actual := normalizeParserSQL(input) + assert.NotContains(t, actual, "${predicate.Builder()") + assert.Contains(t, actual, "SELECT b.* FROM CI_BROWSER b WHERE 1 AND b.ARCHIVED = 0") +} + +func TestNeedsFallbackParse(t *testing.T) { + assert.True(t, needsFallbackParse("SELECT * FROM t JOIN x ON t.id = x.id", &query.Select{})) + assert.False(t, needsFallbackParse("SELECT * FROM t", &query.Select{From: query.From{X: expr.NewSelector("t")}})) +} + +func TestBuildRead_FallbackWhenInitialParseFails(t *testing.T) { + sqlText := `SELECT b.* FROM CI_BROWSER b ${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")} AND b.ARCHIVED = 0` + view, diags, err := BuildRead("browser", sqlText) + require.NoError(t, err) + require.NotNil(t, view) + assert.Equal(t, "b", view.Name) + assert.Equal(t, "CI_BROWSER", view.Table) + assert.Empty(t, diags) +} diff --git a/repository/shape/compile/pipeline/relation.go b/repository/shape/compile/pipeline/relation.go new file mode 100644 index 000000000..722ff5167 --- /dev/null +++ b/repository/shape/compile/pipeline/relation.go @@ -0,0 +1,329 @@ +package pipeline + +import ( + "fmt" + "regexp" + "strings" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/node" + "github.com/viant/sqlparser/query" +) + +var joinSelectorEqExpr = regexp.MustCompile(`(?i)([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)`) + +func ExtractJoinRelations(raw string, queryNode *query.Select) ([]*plan.Relation, []*dqlshape.Diagnostic) { + if queryNode == nil || len(queryNode.Joins) == 0 { + return nil, nil + } + rootAlias := rootNamespace(queryNode) + var relations []*plan.Relation + var diagnostics []*dqlshape.Diagnostic + + for idx, join := range queryNode.Joins { + if join == nil { + continue + } + offset := relationOffset(raw, join) + span := pointSpan(raw, offset) + ref, table := relationRef(join, idx+1) + relation := &plan.Relation{ + Name: ref, + Holder: ExportedName(ref), + Ref: ref, + Table: table, + Kind: strings.TrimSpace(join.Kind), + Raw: strings.TrimSpace(join.Raw), + } + if relation.Holder == "" { + relation.Holder = fmt.Sprintf("Rel%d", idx+1) + } + if join.On == nil || join.On.X == nil { + diagnostics = append(diagnostics, &dqlshape.Diagnostic{ + Code: dqldiag.CodeRelMissingON, + Severity: dqlshape.SeverityWarning, + Message: "join is missing ON condition", + Hint: "use explicit ON condition to derive relation links", + Span: span, + }) + relation.Warnings = append(relation.Warnings, "missing ON condition") + relations = append(relations, relation) + continue + } + pairs := collectJoinPairs(join.On.X) + if len(pairs) == 0 { + onExpr := strings.TrimSpace(sqlparser.Stringify(join.On.X)) + if shouldFallbackToRawJoinPairs(onExpr) { + pairs = collectJoinPairsFromRaw(onExpr) + } + } + if len(pairs) == 0 { + diagnostics = append(diagnostics, &dqlshape.Diagnostic{ + Code: dqldiag.CodeRelUnsupported, + Severity: dqlshape.SeverityWarning, + Message: "join ON condition could not be translated into relation links", + Hint: "use equality predicates between concrete columns, e.g. a.id = b.ref_id", + Span: span, + }) + relation.Warnings = append(relation.Warnings, "unsupported ON predicate") + relations = append(relations, relation) + continue + } + for _, pair := range pairs { + link, warning := orientJoinPair(pair, rootAlias, ref) + if warning != "" { + diagnostics = append(diagnostics, &dqlshape.Diagnostic{ + Code: dqldiag.CodeRelAmbiguous, + Severity: dqlshape.SeverityWarning, + Message: warning, + Hint: "use explicit aliases so one side belongs to root and the other to joined table", + Span: span, + }) + relation.Warnings = append(relation.Warnings, warning) + } + if link == nil { + continue + } + relation.On = append(relation.On, link) + } + if len(relation.On) == 0 { + diagnostics = append(diagnostics, &dqlshape.Diagnostic{ + Code: dqldiag.CodeRelNoLinks, + Severity: dqlshape.SeverityWarning, + Message: "join ON condition does not expose extractable column links", + Hint: "ensure both sides of '=' are concrete column references", + Span: span, + }) + relation.Warnings = append(relation.Warnings, "no extractable links") + } + relations = append(relations, relation) + } + return relations, diagnostics +} + +func collectJoinPairsFromRaw(input string) []joinPair { + input = strings.TrimSpace(input) + if input == "" { + return nil + } + var result []joinPair + for _, m := range joinSelectorEqExpr.FindAllStringSubmatch(input, -1) { + if len(m) < 5 { + continue + } + left := strings.TrimSpace(m[1] + "." + m[2]) + right := strings.TrimSpace(m[3] + "." + m[4]) + if left == "" || right == "" { + continue + } + result = append(result, joinPair{left: left, right: right}) + } + return result +} + +func shouldFallbackToRawJoinPairs(input string) bool { + input = strings.TrimSpace(strings.ToLower(input)) + if input == "" { + return false + } + // Restrict raw fallback to simple selector equality text to avoid brittle extraction + // for quoted identifiers, function calls, casts, and richer predicates. + bannedFragments := []string{ + "`", "\"", "'", "(", ")", "::", " collate ", " case ", " when ", " then ", " else ", " end ", + " coalesce", " cast", " concat", " substr", " lower", " upper", " trim", + } + for _, fragment := range bannedFragments { + if strings.Contains(input, fragment) { + return false + } + } + return true +} + +type joinPair struct { + left string + right string +} + +func collectJoinPairs(n node.Node) []joinPair { + switch actual := n.(type) { + case *expr.Binary: + op := strings.ToUpper(strings.TrimSpace(actual.Op)) + if op == "AND" || op == "OR" { + left := collectJoinPairs(actual.X) + right := collectJoinPairs(actual.Y) + return append(left, right...) + } + if op != "=" { + return nil + } + left := selectorName(actual.X) + right := selectorName(actual.Y) + if left == "" || right == "" { + return nil + } + return []joinPair{{left: left, right: right}} + case *expr.Parenthesis: + return collectJoinPairs(actual.X) + default: + return nil + } +} + +func selectorName(n node.Node) string { + switch actual := n.(type) { + case *expr.Selector: + return strings.TrimSpace(sqlparser.Stringify(actual)) + case *expr.Parenthesis: + return selectorName(actual.X) + default: + return "" + } +} + +func orientJoinPair(pair joinPair, rootAlias, refAlias string) (*plan.RelationLink, string) { + leftNS, leftCol := splitSelector(pair.left) + rightNS, rightCol := splitSelector(pair.right) + if leftCol == "" || rightCol == "" { + return nil, "" + } + switch { + case leftNS == rootAlias && (rightNS == refAlias || rightNS == ""): + return &plan.RelationLink{ + ParentNamespace: leftNS, + ParentColumn: leftCol, + RefNamespace: firstNonEmpty(rightNS, refAlias), + RefColumn: rightCol, + Expression: pair.left + "=" + pair.right, + }, "" + case rightNS == rootAlias && (leftNS == refAlias || leftNS == ""): + return &plan.RelationLink{ + ParentNamespace: rightNS, + ParentColumn: rightCol, + RefNamespace: firstNonEmpty(leftNS, refAlias), + RefColumn: leftCol, + Expression: pair.left + "=" + pair.right, + }, "" + case leftNS == "" && rightNS == "": + return &plan.RelationLink{ + ParentNamespace: rootAlias, + ParentColumn: leftCol, + RefNamespace: refAlias, + RefColumn: rightCol, + Expression: pair.left + "=" + pair.right, + }, "join columns lack namespaces, relation orientation was inferred" + case leftNS == refAlias: + parentNS := rightNS + if parentNS == "" { + parentNS = rootAlias + } + return &plan.RelationLink{ + ParentNamespace: parentNS, + ParentColumn: rightCol, + RefNamespace: leftNS, + RefColumn: leftCol, + Expression: pair.left + "=" + pair.right, + }, "" + case rightNS == refAlias: + parentNS := leftNS + if parentNS == "" { + parentNS = rootAlias + } + return &plan.RelationLink{ + ParentNamespace: parentNS, + ParentColumn: leftCol, + RefNamespace: rightNS, + RefColumn: rightCol, + Expression: pair.left + "=" + pair.right, + }, "" + default: + return nil, fmt.Sprintf("ambiguous join link %q cannot be oriented between root=%q and ref=%q", pair.left+"="+pair.right, rootAlias, refAlias) + } +} + +func relationOffset(raw string, join *query.Join) int { + if strings.TrimSpace(raw) == "" { + return 0 + } + if join != nil && join.On != nil && join.On.X != nil { + if onExpr := strings.TrimSpace(sqlparser.Stringify(join.On.X)); onExpr != "" { + if idx := strings.Index(strings.ToLower(raw), strings.ToLower(onExpr)); idx >= 0 { + return idx + } + } + } + if join != nil && strings.TrimSpace(join.Raw) != "" { + if idx := strings.Index(strings.ToLower(raw), strings.ToLower(strings.TrimSpace(join.Raw))); idx >= 0 { + return idx + } + } + return 0 +} + +func rootNamespace(queryNode *query.Select) string { + if queryNode == nil { + return "" + } + if alias := strings.TrimSpace(queryNode.From.Alias); alias != "" { + return alias + } + if queryNode.From.X == nil { + return "" + } + root := strings.TrimSpace(sqlparser.Stringify(queryNode.From.X)) + root = strings.Trim(root, "`\"") + if root == "" { + return "" + } + if idx := strings.LastIndex(root, "."); idx != -1 { + root = root[idx+1:] + } + return root +} + +func relationRef(join *query.Join, ordinal int) (string, string) { + if join == nil { + return fmt.Sprintf("join_%d", ordinal), "" + } + ref := strings.TrimSpace(join.Alias) + table := "" + if join.With != nil { + table = strings.TrimSpace(sqlparser.Stringify(join.With)) + } + if ref == "" { + ref = table + if idx := strings.LastIndex(ref, "."); idx != -1 { + ref = ref[idx+1:] + } + } + ref = SanitizeName(strings.Trim(ref, "`\"")) + if ref == "" { + ref = fmt.Sprintf("join_%d", ordinal) + } + return ref, table +} + +func splitSelector(selector string) (string, string) { + selector = strings.TrimSpace(selector) + if selector == "" { + return "", "" + } + selector = strings.Trim(selector, "`\"") + if idx := strings.Index(selector, "."); idx != -1 { + return strings.Trim(selector[:idx], "`\""), strings.Trim(selector[idx+1:], "`\"") + } + return "", selector +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} diff --git a/repository/shape/compile/pipeline/relation_test.go b/repository/shape/compile/pipeline/relation_test.go new file mode 100644 index 000000000..62f5c9d3a --- /dev/null +++ b/repository/shape/compile/pipeline/relation_test.go @@ -0,0 +1,89 @@ +package pipeline + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + "github.com/viant/sqlparser" +) + +func TestExtractJoinRelations(t *testing.T) { + sqlText := "SELECT o.id FROM orders o JOIN order_items i ON o.id = i.order_id" + queryNode, err := sqlparser.ParseQuery(sqlText) + require.NoError(t, err) + relations, diags := ExtractJoinRelations(sqlText, queryNode) + require.Len(t, relations, 1) + assert.Equal(t, "i", relations[0].Ref) + require.Len(t, relations[0].On, 1) + assert.Equal(t, "id", relations[0].On[0].ParentColumn) + assert.Equal(t, "order_id", relations[0].On[0].RefColumn) + assert.Empty(t, diags) +} + +func TestExtractJoinRelations_UnsupportedPredicate(t *testing.T) { + sqlText := "SELECT o.id FROM orders o JOIN order_items i ON o.id > i.order_id" + queryNode, err := sqlparser.ParseQuery(sqlText) + require.NoError(t, err) + _, diags := ExtractJoinRelations(sqlText, queryNode) + require.NotEmpty(t, diags) + assert.Equal(t, dqldiag.CodeRelUnsupported, diags[0].Code) +} + +func TestExtractJoinRelations_WithAndLiteral(t *testing.T) { + sqlText := "SELECT t.id FROM taxonomy t LEFT JOIN provider p ON p.id = t.provider_id AND 1=1" + queryNode, err := sqlparser.ParseQuery(sqlText) + require.NoError(t, err) + relations, diags := ExtractJoinRelations(sqlText, queryNode) + require.Len(t, relations, 1) + require.Len(t, relations[0].On, 1) + assert.Equal(t, "provider_id", relations[0].On[0].ParentColumn) + assert.Equal(t, "id", relations[0].On[0].RefColumn) + assert.Empty(t, diags) +} + +func TestExtractJoinRelations_NonRootParentChain(t *testing.T) { + sqlText := "SELECT sl.id FROM site_list sl JOIN site_list_match m ON m.site_list_id = sl.id JOIN ci_site s ON s.id = m.site_id JOIN ci_publisher p ON p.id = s.publisher_id" + queryNode, err := sqlparser.ParseQuery(sqlText) + require.NoError(t, err) + relations, diags := ExtractJoinRelations(sqlText, queryNode) + require.Len(t, relations, 3) + + require.Len(t, relations[0].On, 1) + assert.Equal(t, "sl", relations[0].On[0].ParentNamespace) + assert.Equal(t, "id", relations[0].On[0].ParentColumn) + assert.Equal(t, "m", relations[0].On[0].RefNamespace) + assert.Equal(t, "site_list_id", relations[0].On[0].RefColumn) + + require.Len(t, relations[1].On, 1) + assert.Equal(t, "m", relations[1].On[0].ParentNamespace) + assert.Equal(t, "site_id", relations[1].On[0].ParentColumn) + assert.Equal(t, "s", relations[1].On[0].RefNamespace) + assert.Equal(t, "id", relations[1].On[0].RefColumn) + + require.Len(t, relations[2].On, 1) + assert.Equal(t, "s", relations[2].On[0].ParentNamespace) + assert.Equal(t, "publisher_id", relations[2].On[0].ParentColumn) + assert.Equal(t, "p", relations[2].On[0].RefNamespace) + assert.Equal(t, "id", relations[2].On[0].RefColumn) + assert.Empty(t, diags) +} + +func TestExtractJoinRelations_DoesNotFallbackForComplexRawPredicate(t *testing.T) { + sqlText := "SELECT o.id FROM orders o JOIN order_items i ON COALESCE(o.id, 0) = i.order_id" + queryNode, err := sqlparser.ParseQuery(sqlText) + require.NoError(t, err) + relations, diags := ExtractJoinRelations(sqlText, queryNode) + require.Len(t, relations, 1) + assert.Empty(t, relations[0].On) + require.NotEmpty(t, diags) + assert.Equal(t, dqldiag.CodeRelUnsupported, diags[0].Code) +} + +func TestShouldFallbackToRawJoinPairs(t *testing.T) { + assert.True(t, shouldFallbackToRawJoinPairs("o.id = i.order_id")) + assert.False(t, shouldFallbackToRawJoinPairs("COALESCE(o.id, 0) = i.order_id")) + assert.False(t, shouldFallbackToRawJoinPairs("`o`.`id` = `i`.`order_id`")) + assert.False(t, shouldFallbackToRawJoinPairs(`"o"."id" = "i"."order_id"`)) +} diff --git a/repository/shape/compile/pipeline/table.go b/repository/shape/compile/pipeline/table.go new file mode 100644 index 000000000..5888aeaec --- /dev/null +++ b/repository/shape/compile/pipeline/table.go @@ -0,0 +1,21 @@ +package pipeline + +import "strings" + +// InferTableFromSQL infers root table from SQL text using parser-first strategy. +func InferTableFromSQL(sqlText string) string { + sqlText = strings.TrimSpace(sqlText) + if sqlText == "" { + return "" + } + normalized := normalizeParserSQL(sqlText) + queryNode, _, err := ParseSelectWithDiagnostic(normalized) + if err != nil || queryNode == nil { + return "" + } + _, table, err := InferRoot(queryNode, "") + if err != nil { + return "" + } + return strings.TrimSpace(strings.Trim(table, "`\"")) +} diff --git a/repository/shape/compile/policy.go b/repository/shape/compile/policy.go new file mode 100644 index 000000000..1bd02ca63 --- /dev/null +++ b/repository/shape/compile/policy.go @@ -0,0 +1,48 @@ +package compile + +import ( + "strings" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" +) + +func hasEscalationWarnings(diags []*dqlshape.Diagnostic) bool { + for _, item := range diags { + if item == nil { + continue + } + if item.Severity != dqlshape.SeverityWarning { + continue + } + if strings.HasPrefix(item.Code, dqldiag.PrefixRel) || strings.HasPrefix(item.Code, dqldiag.PrefixSQLI) { + return true + } + } + return false +} + +func hasErrorDiagnostics(diags []*dqlshape.Diagnostic) bool { + for _, item := range diags { + if item == nil { + continue + } + if item.Severity == dqlshape.SeverityError { + return true + } + } + return false +} + +func filterEscalationDiagnostics(diags []*dqlshape.Diagnostic) []*dqlshape.Diagnostic { + var result []*dqlshape.Diagnostic + for _, item := range diags { + if item == nil { + continue + } + if strings.HasPrefix(item.Code, dqldiag.PrefixRel) || strings.HasPrefix(item.Code, dqldiag.PrefixSQLI) { + result = append(result, item) + } + } + return result +} diff --git a/repository/shape/compile/policy_test.go b/repository/shape/compile/policy_test.go new file mode 100644 index 000000000..a15e9f882 --- /dev/null +++ b/repository/shape/compile/policy_test.go @@ -0,0 +1,40 @@ +package compile + +import ( + "testing" + + "github.com/stretchr/testify/assert" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" +) + +func TestPolicy_HasEscalationWarnings(t *testing.T) { + diags := []*dqlshape.Diagnostic{ + {Code: dqldiag.CodeRelAmbiguous, Severity: dqlshape.SeverityWarning}, + } + assert.True(t, hasEscalationWarnings(diags)) + assert.False(t, hasEscalationWarnings([]*dqlshape.Diagnostic{ + {Code: dqldiag.CodeViewMissingSQL, Severity: dqlshape.SeverityWarning}, + })) +} + +func TestPolicy_HasErrorDiagnostics(t *testing.T) { + assert.True(t, hasErrorDiagnostics([]*dqlshape.Diagnostic{ + {Code: dqldiag.CodeParseSyntax, Severity: dqlshape.SeverityError}, + })) + assert.False(t, hasErrorDiagnostics([]*dqlshape.Diagnostic{ + {Code: dqldiag.CodeRelAmbiguous, Severity: dqlshape.SeverityWarning}, + })) +} + +func TestPolicy_FilterEscalationDiagnostics(t *testing.T) { + diags := []*dqlshape.Diagnostic{ + {Code: dqldiag.CodeViewMissingSQL, Severity: dqlshape.SeverityWarning}, + {Code: dqldiag.CodeSQLIRawSelector, Severity: dqlshape.SeverityWarning}, + {Code: dqldiag.CodeRelNoLinks, Severity: dqlshape.SeverityWarning}, + } + filtered := filterEscalationDiagnostics(diags) + assert.Len(t, filtered, 2) + assert.Equal(t, dqldiag.CodeSQLIRawSelector, filtered[0].Code) + assert.Equal(t, dqldiag.CodeRelNoLinks, filtered[1].Code) +} diff --git a/repository/shape/compile/preprocess_handler.go b/repository/shape/compile/preprocess_handler.go new file mode 100644 index 000000000..bea319f78 --- /dev/null +++ b/repository/shape/compile/preprocess_handler.go @@ -0,0 +1,150 @@ +package compile + +import ( + "os" + "path/filepath" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/compile/pipeline" + dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" + "github.com/viant/datly/repository/shape/plan" +) + +type handlerPreprocessResult struct { + Pre *dqlpre.Result + Statements dqlstmt.Statements + Decision pipeline.Decision + LegacyViews []*plan.View + EffectiveSource *shape.Source + ForceLegacyContract bool +} + +func buildHandlerIfNeeded(source *shape.Source, pre *dqlpre.Result, statements dqlstmt.Statements, decision pipeline.Decision, layout compilePathLayout) *handlerPreprocessResult { + ret := &handlerPreprocessResult{ + Pre: pre, + Statements: statements, + Decision: decision, + EffectiveSource: source, + } + if source == nil { + return ret + } + unknownOnly := decision.HasUnknown && !decision.HasRead && !decision.HasExec + if !unknownOnly && !isHandlerSignal(source) { + return ret + } + if buildHandlerFromContractIfNeeded(ret, source, layout) { + return ret + } + if buildGeneratedFallbackIfNeeded(ret, source, layout) { + return ret + } + return ret +} + +func buildHandlerFromContractIfNeeded(ret *handlerPreprocessResult, source *shape.Source, layout compilePathLayout) bool { + if ret == nil || source == nil { + return false + } + return buildLegacyRouteFallbackIfNeeded(ret, source, layout) +} + +func buildGeneratedFallbackIfNeeded(ret *handlerPreprocessResult, source *shape.Source, layout compilePathLayout) bool { + if ret == nil || source == nil { + return false + } + if alternate := resolveGeneratedLegacySource(source); alternate != nil { + if buildLegacyRouteFallbackIfNeeded(ret, alternate, layout) { + return true + } + } + generated := strings.TrimSpace(resolveGeneratedCompanionDQL(source)) + if generated == "" { + return false + } + candidate := dqlpre.Prepare(generated) + if strings.TrimSpace(candidate.SQL) == "" { + return false + } + candidateStatements := dqlstmt.New(candidate.SQL) + candidateDecision := pipeline.Classify(candidateStatements) + if !candidateDecision.HasRead && !candidateDecision.HasExec { + return false + } + ret.Pre = candidate + ret.Statements = candidateStatements + ret.Decision = candidateDecision + return true +} + +func buildLegacyRouteFallbackIfNeeded(ret *handlerPreprocessResult, source *shape.Source, layout compilePathLayout) bool { + if ret == nil || source == nil { + return false + } + legacyFallbackViews := resolveLegacyRouteViewsWithLayout(source, layout) + if len(legacyFallbackViews) == 0 { + return false + } + ret.LegacyViews = legacyFallbackViews + ret.EffectiveSource = source + ret.ForceLegacyContract = true + return true +} + +func resolveGeneratedLegacySource(source *shape.Source) *shape.Source { + if source == nil || strings.TrimSpace(source.Path) == "" { + return nil + } + path := filepath.Clean(source.Path) + normalized := filepath.ToSlash(path) + genIdx := strings.Index(normalized, "/gen/") + if genIdx == -1 { + return nil + } + prefix := normalized[:genIdx] + suffix := strings.TrimPrefix(normalized[genIdx+len("/gen/"):], "/") + parts := strings.Split(suffix, "/") + if len(parts) < 2 { + return nil + } + fileName := parts[len(parts)-1] + stem := strings.TrimSuffix(fileName, filepath.Ext(fileName)) + candidates := []string{ + filepath.FromSlash(prefix + "/" + fileName), + filepath.FromSlash(prefix + "/" + stem + ".sql"), + filepath.FromSlash(prefix + "/" + stem + ".dql"), + } + for _, candidate := range candidates { + data, err := os.ReadFile(candidate) + if err != nil { + continue + } + clone := *source + clone.Path = candidate + clone.DQL = string(data) + return &clone + } + return nil +} + +func isHandlerSignal(source *shape.Source) bool { + if source == nil { + return false + } + settings := extractRuleSettings(source) + if settings != nil { + if strings.TrimSpace(settings.Type) != "" { + return true + } + if method := strings.TrimSpace(strings.ToUpper(settings.Method)); method != "" && method != "GET" { + return true + } + if strings.Contains(strings.ToLower(strings.TrimSpace(settings.URI)), "/proxy") { + return true + } + } + raw := strings.ToLower(strings.TrimSpace(source.DQL)) + return strings.Contains(raw, "$nop(") || strings.Contains(raw, "$proxy(") +} diff --git a/repository/shape/compile/preprocess_handler_test.go b/repository/shape/compile/preprocess_handler_test.go new file mode 100644 index 000000000..7d4e57829 --- /dev/null +++ b/repository/shape/compile/preprocess_handler_test.go @@ -0,0 +1,143 @@ +package compile + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/compile/pipeline" + dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" +) + +func TestIsHandlerSignal(t *testing.T) { + assert.True(t, isHandlerSignal(&shape.Source{DQL: `/* {"Type":"campaign/patch.Handler"} */`})) + assert.True(t, isHandlerSignal(&shape.Source{DQL: `$Nop($Data)`})) + assert.True(t, isHandlerSignal(&shape.Source{DQL: `$Proxy($Data)`})) + assert.False(t, isHandlerSignal(&shape.Source{DQL: `SELECT id FROM proxy_audit`})) + assert.False(t, isHandlerSignal(&shape.Source{DQL: `/* proxy disabled */ SELECT 1`})) + assert.False(t, isHandlerSignal(&shape.Source{DQL: `SELECT 1`})) +} + +func TestBuildHandlerFromContractIfNeeded_LegacyFallbackViews(t *testing.T) { + tempDir := t.TempDir() + sourcePath := filepath.Join(tempDir, "dql", "platform", "campaign", "post.dql") + require.NoError(t, os.MkdirAll(filepath.Dir(sourcePath), 0o755)) + dql := `/* {"Type":"campaign/patch.Handler","Connector":"ci_ads"} */` + require.NoError(t, os.WriteFile(sourcePath, []byte(dql), 0o644)) + + routeDir := filepath.Join(tempDir, "repo", "dev", "Datly", "routes", "platform", "campaign", "patch", "post") + require.NoError(t, os.MkdirAll(routeDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(routeDir, "post.sql"), []byte(`SELECT 1`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(routeDir, "CurCampaign.sql"), []byte(`SELECT * FROM CI_CAMPAIGN`), 0o644)) + + source := &shape.Source{Path: sourcePath, DQL: dql} + pre := dqlpre.Prepare(source.DQL) + statements := dqlstmt.New(pre.SQL) + decision := pipeline.Classify(statements) + result := &handlerPreprocessResult{Pre: pre, Statements: statements, Decision: decision, EffectiveSource: source} + applied := buildHandlerFromContractIfNeeded(result, source, defaultCompilePathLayout()) + require.True(t, applied) + require.NotNil(t, result) + require.NotEmpty(t, result.LegacyViews) + assert.Equal(t, "post", result.LegacyViews[0].Name) +} + +func TestBuildGeneratedFallbackIfNeeded_GeneratedCompanion(t *testing.T) { + tempDir := t.TempDir() + dqlPath := filepath.Join(tempDir, "platform", "adorder", "patch.dql") + require.NoError(t, os.MkdirAll(filepath.Join(filepath.Dir(dqlPath), "gen", "adorder"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(dqlPath), "gen", "adorder", "patch.dql"), []byte("SELECT o.id FROM ORDERS o"), 0o644)) + source := &shape.Source{ + Name: "patch", + Path: dqlPath, + DQL: `/* {"Type":"adorder/patch.Handler"} */`, + } + pre := dqlpre.Prepare(source.DQL) + statements := dqlstmt.New(pre.SQL) + decision := pipeline.Classify(statements) + result := &handlerPreprocessResult{Pre: pre, Statements: statements, Decision: decision, EffectiveSource: source} + applied := buildGeneratedFallbackIfNeeded(result, source, defaultCompilePathLayout()) + require.True(t, applied) + require.NotNil(t, result) + assert.Empty(t, result.LegacyViews) + assert.Contains(t, result.Pre.SQL, "SELECT o.id FROM ORDERS o") + assert.True(t, result.Decision.HasRead) +} + +func TestResolveGeneratedLegacySource(t *testing.T) { + tempDir := t.TempDir() + genPath := filepath.Join(tempDir, "dql", "system", "session", "gen", "session", "patch.dql") + require.NoError(t, os.MkdirAll(filepath.Dir(genPath), 0o755)) + require.NoError(t, os.WriteFile(genPath, []byte(`/* {"Method":"PATCH","URI":"/v1/api/system/session"} */`), 0o644)) + legacySQL := filepath.Join(tempDir, "dql", "system", "session", "patch.sql") + require.NoError(t, os.MkdirAll(filepath.Dir(legacySQL), 0o755)) + require.NoError(t, os.WriteFile(legacySQL, []byte(`/* {"Type":"session/patch.Handler"} */`), 0o644)) + + source := &shape.Source{Path: genPath, DQL: `/* {"Method":"PATCH","URI":"/v1/api/system/session"} */`} + actual := resolveGeneratedLegacySource(source) + require.NotNil(t, actual) + assert.Equal(t, legacySQL, actual.Path) + assert.Contains(t, actual.DQL, `"Type":"session/patch.Handler"`) +} + +func TestBuildGeneratedFallbackIfNeeded_GeneratedLegacyRoute(t *testing.T) { + tempDir := t.TempDir() + genPath := filepath.Join(tempDir, "dql", "system", "session", "gen", "session", "patch.dql") + require.NoError(t, os.MkdirAll(filepath.Dir(genPath), 0o755)) + require.NoError(t, os.WriteFile(genPath, []byte(`/* {"Method":"PATCH","URI":"/v1/api/system/session"} */`), 0o644)) + legacySQL := filepath.Join(tempDir, "dql", "system", "session", "patch.sql") + require.NoError(t, os.MkdirAll(filepath.Dir(legacySQL), 0o755)) + require.NoError(t, os.WriteFile(legacySQL, []byte(`/* {"Type":"session/patch.Handler","Connector":"system"} */`), 0o644)) + + routesDir := filepath.Join(tempDir, "repo", "dev", "Datly", "routes", "system", "session", "patch") + require.NoError(t, os.MkdirAll(routesDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(routesDir), "patch.yaml"), []byte(`Resource: + Views: + - Name: patch + Mode: SQLExec + Connector: + Ref: system + Template: + SourceURL: patch/patch.sql + Parameters: + - Name: Session + In: + Kind: body + Name: data + Types: + - Name: Input + DataType: "*Input" + Package: session/patch +`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "patch.sql"), []byte(`$Nop($Unsafe.Session)`), 0o644)) + + source := &shape.Source{Path: genPath, DQL: `/* {"Method":"PATCH","URI":"/v1/api/system/session"} */`} + pre := dqlpre.Prepare(source.DQL) + statements := dqlstmt.New(pre.SQL) + decision := pipeline.Classify(statements) + result := &handlerPreprocessResult{Pre: pre, Statements: statements, Decision: decision, EffectiveSource: source} + applied := buildGeneratedFallbackIfNeeded(result, source, defaultCompilePathLayout()) + require.True(t, applied) + require.NotNil(t, result) + require.True(t, result.ForceLegacyContract) + require.NotNil(t, result.EffectiveSource) + assert.Equal(t, legacySQL, result.EffectiveSource.Path) + require.NotEmpty(t, result.LegacyViews) + assert.Equal(t, "patch", result.LegacyViews[0].Name) +} + +func TestBuildLegacyRouteFallbackIfNeeded_NoLegacyRoute(t *testing.T) { + source := &shape.Source{Path: filepath.Join(t.TempDir(), "dql", "x", "y", "z.dql"), DQL: `SELECT 1`} + pre := dqlpre.Prepare(source.DQL) + statements := dqlstmt.New(pre.SQL) + decision := pipeline.Classify(statements) + result := &handlerPreprocessResult{Pre: pre, Statements: statements, Decision: decision, EffectiveSource: source} + applied := buildLegacyRouteFallbackIfNeeded(result, source, defaultCompilePathLayout()) + assert.False(t, applied) + assert.Empty(t, result.LegacyViews) + assert.False(t, result.ForceLegacyContract) +} diff --git a/repository/shape/compile/span.go b/repository/shape/compile/span.go new file mode 100644 index 000000000..154ff9b2f --- /dev/null +++ b/repository/shape/compile/span.go @@ -0,0 +1,10 @@ +package compile + +import ( + dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" +) + +func relationSpan(raw string, offset int) dqlshape.Span { + return dqlpre.PointSpan(raw, offset) +} diff --git a/repository/shape/compile/statedecl.go b/repository/shape/compile/statedecl.go new file mode 100644 index 000000000..eab76c647 --- /dev/null +++ b/repository/shape/compile/statedecl.go @@ -0,0 +1,223 @@ +package compile + +import ( + "strconv" + "strings" + + "github.com/viant/datly/repository/shape/plan" +) + +func appendDeclaredStates(rawDQL string, result *plan.Result) { + if result == nil || strings.TrimSpace(rawDQL) == "" { + return + } + seen := map[string]bool{} + for _, block := range extractSetBlocks(rawDQL) { + holder, kind, location, tail, ok := parseSetDeclarationBody(block.Body) + if !ok { + continue + } + if kind == "view" || kind == "data_view" { + continue + } + key := declaredStateKey(holder, kind, location) + if seen[key] { + continue + } + state := &plan.State{ + Path: holder, + Name: holder, + Kind: kind, + In: location, + } + switch strings.ToLower(kind) { + case "query": + required := false + state.Required = &required + case "header": + required := true + state.Required = &required + } + applyDeclaredStateOptions(state, tail) + result.States = append(result.States, state) + seen[key] = true + } +} + +func declaredStateKey(name, kind, in string) string { + return strings.ToLower(strings.TrimSpace(name)) + "|" + + strings.ToLower(strings.TrimSpace(kind)) + "|" + + strings.ToLower(strings.TrimSpace(in)) +} + +func applyDeclaredStateOptions(state *plan.State, tail string) { + if state == nil || strings.TrimSpace(tail) == "" { + return + } + cursor := newOptionCursor(tail) + for cursor.next() { + name, args := cursor.option() + switch { + case strings.EqualFold(name, "WithURI"): + if len(args) == 1 { + state.URI = trimQuote(args[0]) + } + case strings.EqualFold(name, "Optional"): + required := false + state.Required = &required + case strings.EqualFold(name, "Required"): + required := true + state.Required = &required + case strings.EqualFold(name, "Cacheable"): + if len(args) == 1 { + if value, err := strconv.ParseBool(strings.TrimSpace(trimQuote(args[0]))); err == nil { + state.Cacheable = &value + } + } + case strings.EqualFold(name, "QuerySelector"): + if len(args) == 1 { + state.QuerySelector = trimQuote(args[0]) + if state.Cacheable == nil { + cacheable := false + state.Cacheable = &cacheable + } + } + case strings.EqualFold(name, "WithPredicate"), strings.EqualFold(name, "Predicate"): + appendStatePredicate(state, args, false) + case strings.EqualFold(name, "EnsurePredicate"): + appendStatePredicate(state, args, true) + case strings.EqualFold(name, "When"): + if len(args) == 1 { + state.When = trimQuote(args[0]) + } + case strings.EqualFold(name, "Scope"): + if len(args) == 1 { + state.Scope = trimQuote(args[0]) + } + case strings.EqualFold(name, "WithType"): + if len(args) == 1 { + state.DataType = trimQuote(args[0]) + } + case strings.EqualFold(name, "Value"): + if len(args) == 1 { + state.Value = trimQuote(args[0]) + } + case strings.EqualFold(name, "Async"): + state.Async = true + } + } +} + +func appendStatePredicate(state *plan.State, args []string, ensure bool) { + if state == nil || len(args) == 0 { + return + } + group := 0 + nameIdx := 0 + if len(args) >= 2 { + if parsed, err := strconv.Atoi(strings.TrimSpace(trimQuote(args[0]))); err == nil { + group = parsed + nameIdx = 1 + } + } + if len(args) <= nameIdx { + return + } + predicate := &plan.StatePredicate{ + Group: group, + Name: trimQuote(args[nameIdx]), + Ensure: ensure, + Arguments: []string{}, + } + for _, arg := range args[nameIdx+1:] { + predicate.Arguments = append(predicate.Arguments, trimQuote(arg)) + } + state.Predicates = append(state.Predicates, predicate) +} + +type optionCursor struct { + raw string + cursor int + name string + args []string +} + +func newOptionCursor(raw string) *optionCursor { + return &optionCursor{raw: raw} +} + +func (o *optionCursor) next() bool { + o.name = "" + o.args = nil + for o.cursor < len(o.raw) && (o.raw[o.cursor] == ' ' || o.raw[o.cursor] == '\n' || o.raw[o.cursor] == '\t' || o.raw[o.cursor] == '\r') { + o.cursor++ + } + if o.cursor >= len(o.raw) || o.raw[o.cursor] != '.' { + return false + } + o.cursor++ + start := o.cursor + for o.cursor < len(o.raw) { + ch := o.raw[o.cursor] + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' { + o.cursor++ + continue + } + break + } + if o.cursor == start { + return false + } + o.name = strings.TrimSpace(o.raw[start:o.cursor]) + for o.cursor < len(o.raw) && (o.raw[o.cursor] == ' ' || o.raw[o.cursor] == '\n' || o.raw[o.cursor] == '\t' || o.raw[o.cursor] == '\r') { + o.cursor++ + } + if o.cursor >= len(o.raw) || o.raw[o.cursor] != '(' { + return false + } + groupStart := o.cursor + depth := 0 + inSingle := false + inDouble := false + escape := false + for o.cursor < len(o.raw) { + ch := o.raw[o.cursor] + if escape { + escape = false + o.cursor++ + continue + } + switch ch { + case '\\': + escape = true + case '\'': + if !inDouble { + inSingle = !inSingle + } + case '"': + if !inSingle { + inDouble = !inDouble + } + case '(': + if !inSingle && !inDouble { + depth++ + } + case ')': + if !inSingle && !inDouble { + depth-- + if depth == 0 { + o.cursor++ + content := o.raw[groupStart+1 : o.cursor-1] + o.args = splitArgs(content) + return true + } + } + } + o.cursor++ + } + return false +} + +func (o *optionCursor) option() (string, []string) { + return o.name, o.args +} diff --git a/repository/shape/compile/statedecl_test.go b/repository/shape/compile/statedecl_test.go new file mode 100644 index 000000000..a538e7c3a --- /dev/null +++ b/repository/shape/compile/statedecl_test.go @@ -0,0 +1,71 @@ +package compile + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape/plan" +) + +func TestAppendDeclaredStates(t *testing.T) { + dql := ` +#set($_ = $Jwt(header/Authorization).WithCodec(JwtClaim).WithStatusCode(401)) +#set($_ = $Name(query/name).WithPredicate(0,'contains','sl','NAME').Optional()) +#set($_ = $Fields<[]string>(query/fields).QuerySelector(site_list)) +#set($_ = $Meta(output/summary)) +SELECT id FROM SITE_LIST sl` + result := &plan.Result{} + appendDeclaredStates(dql, result) + require.NotEmpty(t, result.States) + + byName := map[string]*plan.State{} + for _, item := range result.States { + if item != nil { + byName[item.Name] = item + } + } + require.NotNil(t, byName["Jwt"]) + assert.Equal(t, "header", byName["Jwt"].Kind) + require.NotNil(t, byName["Jwt"].Required) + assert.True(t, *byName["Jwt"].Required) + + require.NotNil(t, byName["Name"]) + assert.Equal(t, "query", byName["Name"].Kind) + require.NotNil(t, byName["Name"].Required) + assert.False(t, *byName["Name"].Required) + require.Len(t, byName["Name"].Predicates, 1) + assert.Equal(t, "contains", byName["Name"].Predicates[0].Name) + assert.Equal(t, 0, byName["Name"].Predicates[0].Group) + + require.NotNil(t, byName["Fields"]) + assert.Equal(t, "site_list", byName["Fields"].QuerySelector) + require.NotNil(t, byName["Fields"].Cacheable) + assert.False(t, *byName["Fields"].Cacheable) +} + +func TestAppendDeclaredStates_DuplicateDeclaration_FirstWins(t *testing.T) { + dql := ` +#set($_ = $Active(query/active).WithPredicate(0,'equal','tas','IS_TARGETABLE').Optional()) +#set($_ = $Active(query/active).WithPredicate(0,'equal','tas','ACTIVE').Optional()) +SELECT id FROM CI_TV_AFFILIATE_STATION tas` + result := &plan.Result{} + appendDeclaredStates(dql, result) + require.Len(t, result.States, 1) + require.Len(t, result.States[0].Predicates, 1) + assert.Equal(t, "Active", result.States[0].Name) + assert.Equal(t, "IS_TARGETABLE", result.States[0].Predicates[0].Arguments[1]) +} + +func TestAppendDeclaredStates_SupportsDefineDirective(t *testing.T) { + dql := ` +#define($_ = $Auth(header/Authorization).Required()) +SELECT id FROM USERS u` + result := &plan.Result{} + appendDeclaredStates(dql, result) + require.Len(t, result.States, 1) + assert.Equal(t, "Auth", result.States[0].Name) + assert.Equal(t, "header", result.States[0].Kind) + require.NotNil(t, result.States[0].Required) + assert.True(t, *result.States[0].Required) +} diff --git a/repository/shape/compile/typectx_defaults.go b/repository/shape/compile/typectx_defaults.go new file mode 100644 index 000000000..5bc0a9d9a --- /dev/null +++ b/repository/shape/compile/typectx_defaults.go @@ -0,0 +1,158 @@ +package compile + +import ( + "os" + "path" + "path/filepath" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/typectx" + "golang.org/x/mod/modfile" +) + +func applyTypeContextDefaults(ctx *typectx.Context, source *shape.Source, opts *shape.CompileOptions, layout compilePathLayout) *typectx.Context { + ret := cloneTypeContext(ctx) + if shouldInferTypeContext(opts) { + ret = mergeTypeContext(ret, inferDatlyGenTypeContext(source, layout)) + } + if opts != nil { + ret = ensureTypeContext(ret) + if ret != nil { + if value := strings.TrimSpace(opts.TypePackageDir); value != "" { + ret.PackageDir = value + } + if value := strings.TrimSpace(opts.TypePackageName); value != "" { + ret.PackageName = value + } + if value := strings.TrimSpace(opts.TypePackagePath); value != "" { + ret.PackagePath = value + } + } + } + return normalizeTypeContext(ret) +} + +func shouldInferTypeContext(opts *shape.CompileOptions) bool { + if opts == nil || opts.InferTypeContext == nil { + return true + } + return *opts.InferTypeContext +} + +func mergeTypeContext(dst *typectx.Context, src *typectx.Context) *typectx.Context { + if src == nil { + return dst + } + dst = ensureTypeContext(dst) + if strings.TrimSpace(dst.DefaultPackage) == "" { + dst.DefaultPackage = strings.TrimSpace(src.DefaultPackage) + } + if len(dst.Imports) == 0 && len(src.Imports) > 0 { + dst.Imports = append([]typectx.Import{}, src.Imports...) + } + if strings.TrimSpace(dst.PackageDir) == "" { + dst.PackageDir = strings.TrimSpace(src.PackageDir) + } + if strings.TrimSpace(dst.PackageName) == "" { + dst.PackageName = strings.TrimSpace(src.PackageName) + } + if strings.TrimSpace(dst.PackagePath) == "" { + dst.PackagePath = strings.TrimSpace(src.PackagePath) + } + return dst +} + +func inferDatlyGenTypeContext(source *shape.Source, layout compilePathLayout) *typectx.Context { + if source == nil { + return nil + } + sourcePath := strings.TrimSpace(source.Path) + if sourcePath == "" { + return nil + } + normalizedPath := filepath.ToSlash(filepath.Clean(sourcePath)) + idx := strings.Index(normalizedPath, layout.dqlMarker) + if idx == -1 { + return nil + } + projectRoot := filepath.FromSlash(strings.TrimSuffix(normalizedPath[:idx], "/")) + rel := strings.TrimPrefix(normalizedPath[idx+len(layout.dqlMarker):], "/") + if rel == "" { + return nil + } + routeDir := strings.Trim(path.Dir(rel), "/") + if routeDir == "." { + routeDir = "" + } + packageDir := "pkg" + if routeDir != "" { + packageDir = path.Join(packageDir, routeDir) + } + packageName := "main" + if routeDir != "" { + packageName = path.Base(routeDir) + } + packagePath := "" + if module := detectModulePath(projectRoot); module != "" { + packagePath = path.Join(module, packageDir) + } + return normalizeTypeContext(&typectx.Context{ + PackageDir: packageDir, + PackageName: packageName, + PackagePath: packagePath, + }) +} + +func detectModulePath(projectRoot string) string { + if strings.TrimSpace(projectRoot) == "" { + return "" + } + goModPath := filepath.Join(projectRoot, "go.mod") + data, err := os.ReadFile(goModPath) + if err != nil { + return "" + } + parsed, err := modfile.Parse(goModPath, data, nil) + if err != nil || parsed == nil || parsed.Module == nil { + return "" + } + return strings.TrimSpace(parsed.Module.Mod.Path) +} + +func ensureTypeContext(ctx *typectx.Context) *typectx.Context { + if ctx != nil { + return ctx + } + return &typectx.Context{} +} + +func cloneTypeContext(ctx *typectx.Context) *typectx.Context { + if ctx == nil { + return nil + } + ret := &typectx.Context{ + DefaultPackage: strings.TrimSpace(ctx.DefaultPackage), + PackageDir: strings.TrimSpace(ctx.PackageDir), + PackageName: strings.TrimSpace(ctx.PackageName), + PackagePath: strings.TrimSpace(ctx.PackagePath), + } + if len(ctx.Imports) > 0 { + ret.Imports = append([]typectx.Import{}, ctx.Imports...) + } + return ret +} + +func normalizeTypeContext(ctx *typectx.Context) *typectx.Context { + if ctx == nil { + return nil + } + if strings.TrimSpace(ctx.DefaultPackage) == "" && + len(ctx.Imports) == 0 && + strings.TrimSpace(ctx.PackageDir) == "" && + strings.TrimSpace(ctx.PackageName) == "" && + strings.TrimSpace(ctx.PackagePath) == "" { + return nil + } + return ctx +} diff --git a/repository/shape/compile/typectx_defaults_test.go b/repository/shape/compile/typectx_defaults_test.go new file mode 100644 index 000000000..4f0d01a36 --- /dev/null +++ b/repository/shape/compile/typectx_defaults_test.go @@ -0,0 +1,70 @@ +package compile + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/typectx" +) + +func TestApplyTypeContextDefaults_Matrix(t *testing.T) { + layout := defaultCompilePathLayout() + + projectDir := t.TempDir() + err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module github.vianttech.com/viant/platform\n\ngo 1.23\n"), 0o644) + require.NoError(t, err) + source := &shape.Source{ + Path: filepath.Join(projectDir, "dql", "platform", "taxonomy", "taxonomy.dql"), + } + + t.Run("inferred only", func(t *testing.T) { + got := applyTypeContextDefaults(nil, source, nil, layout) + require.NotNil(t, got) + require.Equal(t, "pkg/platform/taxonomy", got.PackageDir) + require.Equal(t, "taxonomy", got.PackageName) + require.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/taxonomy", got.PackagePath) + }) + + t.Run("directive context wins over inferred", func(t *testing.T) { + input := &typectx.Context{ + DefaultPackage: "github.com/acme/manual", + PackageDir: "pkg/manual", + PackageName: "manual", + PackagePath: "github.com/acme/manual", + } + got := applyTypeContextDefaults(input, source, nil, layout) + require.NotNil(t, got) + require.Equal(t, "pkg/manual", got.PackageDir) + require.Equal(t, "manual", got.PackageName) + require.Equal(t, "github.com/acme/manual", got.PackagePath) + require.Equal(t, "github.com/acme/manual", got.DefaultPackage) + }) + + t.Run("compile override wins over both", func(t *testing.T) { + input := &typectx.Context{ + PackageDir: "pkg/manual", + PackageName: "manual", + PackagePath: "github.com/acme/manual", + } + got := applyTypeContextDefaults(input, source, &shape.CompileOptions{ + TypePackageDir: "pkg/override", + TypePackageName: "override", + TypePackagePath: "github.com/acme/override", + }, layout) + require.NotNil(t, got) + require.Equal(t, "pkg/override", got.PackageDir) + require.Equal(t, "override", got.PackageName) + require.Equal(t, "github.com/acme/override", got.PackagePath) + }) + + t.Run("explicitly disable inference", func(t *testing.T) { + disabled := false + got := applyTypeContextDefaults(nil, source, &shape.CompileOptions{ + InferTypeContext: &disabled, + }, layout) + require.Nil(t, got) + }) +} diff --git a/repository/shape/compile/typectx_diagnostics.go b/repository/shape/compile/typectx_diagnostics.go new file mode 100644 index 000000000..36cc701f9 --- /dev/null +++ b/repository/shape/compile/typectx_diagnostics.go @@ -0,0 +1,37 @@ +package compile + +import ( + "fmt" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/typectx" +) + +func typeContextDiagnostics(ctx *typectx.Context, strict bool) []*dqlshape.Diagnostic { + issues := typectx.Validate(ctx) + if len(issues) == 0 { + return nil + } + severity := dqlshape.SeverityWarning + if strict { + severity = dqlshape.SeverityError + } + diags := make([]*dqlshape.Diagnostic, 0, len(issues)) + for _, issue := range issues { + if issue.Field == "" || issue.Message == "" { + continue + } + diags = append(diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeTypeCtxInvalid, + Severity: severity, + Message: fmt.Sprintf("type context %s: %s", issue.Field, issue.Message), + Hint: "set consistent TypeContext package fields or use compile overrides", + Span: dqlshape.Span{ + Start: dqlshape.Position{Line: 1, Char: 1}, + End: dqlshape.Position{Line: 1, Char: 1}, + }, + }) + } + return diags +} diff --git a/repository/shape/compile/viewdecl.go b/repository/shape/compile/viewdecl.go new file mode 100644 index 000000000..9e6c14c88 --- /dev/null +++ b/repository/shape/compile/viewdecl.go @@ -0,0 +1,107 @@ +package compile + +import ( + "fmt" + "strings" + + "github.com/viant/datly/repository/shape/compile/pipeline" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/parsly" + "github.com/viant/parsly/matcher" +) + +type declaredView struct { + Name string + SQL string + URI string + Connector string + Cardinality string + Tag string + Codec string + CodecArgs []string + HandlerName string + HandlerArgs []string + StatusCode *int + ErrorMessage string + QuerySelector string + CacheRef string + Limit *int + Cacheable *bool + When string + Scope string + DataType string + Of string + Value string + Async bool + Output bool + Predicates []declaredPredicate +} + +type declaredPredicate struct { + Name string + Source string + Ensure bool + Arguments []string +} + +const ( + vdWhitespaceToken = iota + vdSetToken + vdDefineToken + vdExprGroupToken + vdCommentToken + vdParamDeclToken + vdTypeToken + vdDotToken +) + +var ( + vdWhitespaceMatcher = parsly.NewToken(vdWhitespaceToken, "Whitespace", matcher.NewWhiteSpace()) + vdSetMatcher = parsly.NewToken(vdSetToken, "#set", matcher.NewFragment("#set")) + vdDefineMatcher = parsly.NewToken(vdDefineToken, "#define", matcher.NewFragment("#define")) + vdExprGroupMatcher = parsly.NewToken(vdExprGroupToken, "( ... )", matcher.NewBlock('(', ')', '\\')) + vdCommentMatcher = parsly.NewToken(vdCommentToken, "Comment", matcher.NewSeqBlock("/*", "*/")) + vdParamDeclMatcher = parsly.NewToken(vdParamDeclToken, "$_ = $", matcher.NewSpacedSet([]string{"$_ = $"})) + vdTypeMatcher = parsly.NewToken(vdTypeToken, "< ... >", matcher.NewSeqBlock("<", ">")) + vdDotMatcher = parsly.NewToken(vdDotToken, ".", matcher.NewByte('.')) +) + +func extractDeclaredViews(dql string) ([]*declaredView, []*dqlshape.Diagnostic) { + if strings.TrimSpace(dql) == "" { + return nil, nil + } + var views []*declaredView + var diags []*dqlshape.Diagnostic + for _, block := range extractSetBlocks(dql) { + holder, kind, location, tail, ok := parseSetDeclarationBody(block.Body) + if !ok { + continue + } + if kind != "view" && kind != "data_view" { + continue + } + sqlText := extractDeclarationSQL(tail) + if sqlText == "" { + diags = append(diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeViewMissingSQL, + Severity: dqlshape.SeverityWarning, + Message: fmt.Sprintf("view declaration %q has no inline SQL hint", location), + Hint: "use /* SELECT ... */ in declaration comment to derive an additional view", + Span: relationSpan(dql, block.Offset), + }) + continue + } + name := pipeline.SanitizeName(location) + if name == "" { + name = pipeline.SanitizeName(holder) + } + if name == "" { + continue + } + view := &declaredView{Name: name, SQL: strings.TrimSpace(sqlText)} + applyDeclaredViewOptions(view, tail, dql, block.Offset, &diags) + views = append(views, view) + } + return views, diags +} diff --git a/repository/shape/compile/viewdecl_append.go b/repository/shape/compile/viewdecl_append.go new file mode 100644 index 000000000..dabf0fd26 --- /dev/null +++ b/repository/shape/compile/viewdecl_append.go @@ -0,0 +1,155 @@ +package compile + +import ( + "reflect" + "regexp" + "strings" + + "github.com/viant/datly/repository/shape/compile/pipeline" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/sqlparser" +) + +var summaryParentRefExpr = regexp.MustCompile(`(?i)\$View\.([a-zA-Z_][a-zA-Z0-9_]*)\.SQL\b`) + +func appendDeclaredViews(rawDQL string, result *plan.Result) { + if result == nil { + return + } + declared, diags := extractDeclaredViews(rawDQL) + if len(diags) > 0 { + result.Diagnostics = append(result.Diagnostics, diags...) + } + for _, item := range declared { + if item == nil || strings.TrimSpace(item.Name) == "" || strings.TrimSpace(item.SQL) == "" { + continue + } + if parent := lookupSummaryParentView(result, item.SQL); parent != nil { + if strings.TrimSpace(parent.Summary) == "" { + parent.Summary = strings.TrimSpace(item.SQL) + } + continue + } + if _, exists := result.ViewsByName[item.Name]; exists { + continue + } + view := &plan.View{ + Path: item.Name, + Holder: item.Name, + Name: item.Name, + Table: item.Name, + SQL: item.SQL, + SQLURI: item.URI, + Connector: item.Connector, + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + Declaration: buildViewDeclaration(item), + } + if item.Cardinality != "" { + view.Cardinality = item.Cardinality + } + if queryNode, err := sqlparser.ParseQuery(item.SQL); err == nil && queryNode != nil { + if inferredName, inferredTable, err := pipeline.InferRoot(queryNode, item.Name); err == nil { + view.Name = inferredName + view.Holder = inferredName + view.Path = inferredName + view.Table = inferredTable + } + if fType, eType, card := pipeline.InferProjectionType(queryNode); fType != nil && eType != nil { + view.FieldType = fType + view.ElementType = eType + if item.Cardinality == "" { + view.Cardinality = card + } + } + } + result.Views = append(result.Views, view) + result.ViewsByName[view.Name] = view + } +} + +func lookupSummaryParentView(result *plan.Result, sqlText string) *plan.View { + if result == nil || strings.TrimSpace(sqlText) == "" { + return nil + } + matches := summaryParentRefExpr.FindStringSubmatch(sqlText) + if len(matches) < 2 { + return nil + } + parent := strings.TrimSpace(matches[1]) + if parent == "" { + return nil + } + if view, ok := result.ViewsByName[parent]; ok && view != nil { + return view + } + sanitized := pipeline.SanitizeName(parent) + if sanitized != "" { + if view, ok := result.ViewsByName[sanitized]; ok && view != nil { + return view + } + } + for name, view := range result.ViewsByName { + if view == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(name), parent) || (sanitized != "" && strings.EqualFold(strings.TrimSpace(name), sanitized)) { + return view + } + } + for _, view := range result.Views { + if view == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(view.Name), parent) || (sanitized != "" && strings.EqualFold(strings.TrimSpace(view.Name), sanitized)) { + return view + } + } + return nil +} + +func buildViewDeclaration(item *declaredView) *plan.ViewDeclaration { + if item == nil { + return nil + } + ret := &plan.ViewDeclaration{ + Tag: item.Tag, + Codec: item.Codec, + CodecArgs: append([]string{}, item.CodecArgs...), + HandlerName: item.HandlerName, + HandlerArgs: append([]string{}, item.HandlerArgs...), + StatusCode: item.StatusCode, + ErrorMessage: item.ErrorMessage, + QuerySelector: item.QuerySelector, + CacheRef: item.CacheRef, + Limit: item.Limit, + Cacheable: item.Cacheable, + When: item.When, + Scope: item.Scope, + DataType: item.DataType, + Of: item.Of, + Value: item.Value, + Async: item.Async, + Output: item.Output, + } + if len(item.Predicates) > 0 { + ret.Predicates = make([]*plan.ViewPredicate, 0, len(item.Predicates)) + for _, predicate := range item.Predicates { + ret.Predicates = append(ret.Predicates, &plan.ViewPredicate{ + Name: predicate.Name, + Source: predicate.Source, + Ensure: predicate.Ensure, + Arguments: append([]string{}, predicate.Arguments...), + }) + } + } + if ret.Tag == "" && ret.Codec == "" && len(ret.CodecArgs) == 0 && ret.HandlerName == "" && + len(ret.HandlerArgs) == 0 && ret.StatusCode == nil && ret.ErrorMessage == "" && + ret.QuerySelector == "" && ret.CacheRef == "" && ret.Limit == nil && ret.Cacheable == nil && + ret.When == "" && ret.Scope == "" && ret.DataType == "" && ret.Of == "" && ret.Value == "" && + !ret.Async && !ret.Output && len(ret.Predicates) == 0 { + return nil + } + return ret +} diff --git a/repository/shape/compile/viewdecl_options.go b/repository/shape/compile/viewdecl_options.go new file mode 100644 index 000000000..dd8ea2fba --- /dev/null +++ b/repository/shape/compile/viewdecl_options.go @@ -0,0 +1,382 @@ +package compile + +import ( + "fmt" + "strconv" + "strings" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/parsly" +) + +func extractDeclarationSQL(fragment string) string { + cursor := parsly.NewCursor("", []byte(fragment), 0) + for cursor.Pos < cursor.InputSize { + match := cursor.MatchAfterOptional(vdWhitespaceMatcher, vdCommentMatcher) + if match.Code == vdCommentToken { + text := match.Text(cursor) + if len(text) < 4 { + return "" + } + return normalizeHintSQL(text[2 : len(text)-2]) + } + cursor.Pos++ + } + return "" +} + +func normalizeHintSQL(body string) string { + body = strings.TrimSpace(body) + if body == "" { + return "" + } + if strings.HasPrefix(body, "{") { + if closeIdx := strings.Index(body, "}"); closeIdx != -1 { + body = strings.TrimSpace(body[closeIdx+1:]) + } + } + if body == "" { + return "" + } + switch body[0] { + case '?': + body = strings.TrimSpace(body[1:]) + case '!': + body = strings.TrimSpace(body[1:]) + if strings.HasPrefix(body, "!") { + body = strings.TrimSpace(body[1:]) + } + if len(body) >= 3 { + var status int + if _, err := fmt.Sscanf(body[:3], "%d", &status); err == nil { + body = strings.TrimSpace(body[3:]) + } + } + } + return strings.TrimSpace(body) +} + +func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, diags *[]*dqlshape.Diagnostic) { + if view == nil || strings.TrimSpace(tail) == "" { + return + } + cursor := parsly.NewCursor("", []byte(tail), 0) + for cursor.Pos < cursor.InputSize { + _ = cursor.MatchOne(vdWhitespaceMatcher) + if cursor.MatchOne(vdDotMatcher).Code != vdDotToken { + cursor.Pos++ + continue + } + _ = cursor.MatchOne(vdWhitespaceMatcher) + name, ok := readIdentifier(cursor) + if !ok { + continue + } + _ = cursor.MatchOne(vdWhitespaceMatcher) + group := cursor.MatchOne(vdExprGroupMatcher) + if group.Code != vdExprGroupToken { + continue + } + content := group.Text(cursor) + if len(content) < 2 { + continue + } + args := splitArgs(content[1 : len(content)-1]) + switch { + case strings.EqualFold(name, "WithURI"): + if !expectArgs(view, name, args, 1, -1, dql, offset, diags) { + continue + } + view.URI = trimQuote(args[0]) + case strings.EqualFold(name, "WithConnector"), strings.EqualFold(name, "Connector"): + if !expectArgs(view, name, args, 1, -1, dql, offset, diags) { + continue + } + view.Connector = trimQuote(args[0]) + case strings.EqualFold(name, "Cardinality"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + card := strings.ToLower(strings.TrimSpace(trimQuote(args[0]))) + switch card { + case "one", "many": + view.Cardinality = card + default: + *diags = append(*diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeViewCardinality, + Severity: dqlshape.SeverityWarning, + Message: fmt.Sprintf("unsupported cardinality %q for declared view %q", args[0], view.Name), + Hint: "use Cardinality('one') or Cardinality('many')", + Span: relationSpan(dql, offset), + }) + } + case strings.EqualFold(name, "WithTag"), strings.EqualFold(name, "Tag"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + view.Tag = trimQuote(args[0]) + case strings.EqualFold(name, "WithCodec"), strings.EqualFold(name, "Codec"): + if !expectArgs(view, name, args, 1, -1, dql, offset, diags) { + continue + } + view.Codec = trimQuote(args[0]) + view.CodecArgs = nil + for _, arg := range args[1:] { + view.CodecArgs = append(view.CodecArgs, strings.TrimSpace(arg)) + } + case strings.EqualFold(name, "WithHandler"), strings.EqualFold(name, "Handler"): + if !expectArgs(view, name, args, 1, -1, dql, offset, diags) { + continue + } + view.HandlerName = trimQuote(args[0]) + view.HandlerArgs = nil + for _, arg := range args[1:] { + view.HandlerArgs = append(view.HandlerArgs, strings.TrimSpace(arg)) + } + case strings.EqualFold(name, "WithStatusCode"), strings.EqualFold(name, "StatusCode"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + statusCode, err := strconv.Atoi(strings.TrimSpace(trimQuote(args[0]))) + if err != nil { + *diags = append(*diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDeclOptionArgs, + Severity: dqlshape.SeverityWarning, + Message: fmt.Sprintf("invalid status code %q for declared view %q", args[0], view.Name), + Hint: "use numeric status code, e.g. StatusCode(400)", + Span: relationSpan(dql, offset), + }) + continue + } + view.StatusCode = &statusCode + case strings.EqualFold(name, "WithErrorMessage"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + view.ErrorMessage = trimQuote(args[0]) + case strings.EqualFold(name, "WithPredicate"), strings.EqualFold(name, "Predicate"): + if !expectArgs(view, name, args, 2, -1, dql, offset, diags) { + continue + } + view.Predicates = append(view.Predicates, declaredPredicate{ + Name: trimQuote(args[0]), + Source: trimQuote(args[1]), + Arguments: append([]string{}, args[2:]...), + }) + case strings.EqualFold(name, "EnsurePredicate"): + if !expectArgs(view, name, args, 2, -1, dql, offset, diags) { + continue + } + view.Predicates = append(view.Predicates, declaredPredicate{ + Name: trimQuote(args[0]), + Source: trimQuote(args[1]), + Ensure: true, + Arguments: append([]string{}, args[2:]...), + }) + case strings.EqualFold(name, "QuerySelector"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + view.QuerySelector = trimQuote(args[0]) + if !isAllowedQuerySelector(strings.ToLower(view.Name)) { + *diags = append(*diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDeclQuerySelector, + Severity: dqlshape.SeverityWarning, + Message: fmt.Sprintf("query selector %q can only be used with limit, offset, page, fields, orderby", view.QuerySelector), + Hint: "use QuerySelector on declarations named limit/offset/page/fields/orderby", + Span: relationSpan(dql, offset), + }) + } + case strings.EqualFold(name, "WithCache"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + view.CacheRef = trimQuote(args[0]) + case strings.EqualFold(name, "WithLimit"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + limit, err := strconv.Atoi(strings.TrimSpace(trimQuote(args[0]))) + if err != nil { + appendOptionArgDiagnostic(view, name, fmt.Sprintf("invalid integer limit %q", args[0]), dql, offset, diags) + continue + } + view.Limit = &limit + case strings.EqualFold(name, "Cacheable"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + value, err := strconv.ParseBool(strings.TrimSpace(trimQuote(args[0]))) + if err != nil { + appendOptionArgDiagnostic(view, name, fmt.Sprintf("invalid bool cacheable %q", args[0]), dql, offset, diags) + continue + } + view.Cacheable = &value + case strings.EqualFold(name, "When"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + view.When = trimQuote(args[0]) + case strings.EqualFold(name, "Scope"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + view.Scope = trimQuote(args[0]) + case strings.EqualFold(name, "WithType"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + view.DataType = trimQuote(args[0]) + case strings.EqualFold(name, "Of"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + view.Of = trimQuote(args[0]) + case strings.EqualFold(name, "Value"): + if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + continue + } + view.Value = trimQuote(args[0]) + case strings.EqualFold(name, "Async"): + if !expectArgs(view, name, args, 0, 0, dql, offset, diags) { + continue + } + view.Async = true + case strings.EqualFold(name, "Output"): + if !expectArgs(view, name, args, 0, 0, dql, offset, diags) { + continue + } + view.Output = true + } + } +} + +func splitArgs(raw string) []string { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + var result []string + var current strings.Builder + inSingle := false + inDouble := false + escape := false + parens := 0 + brackets := 0 + braces := 0 + for i := 0; i < len(raw); i++ { + ch := raw[i] + if escape { + current.WriteByte(ch) + escape = false + continue + } + switch ch { + case '\\': + current.WriteByte(ch) + escape = true + case '\'': + if !inDouble { + inSingle = !inSingle + } + current.WriteByte(ch) + case '"': + if !inSingle { + inDouble = !inDouble + } + current.WriteByte(ch) + case '(': + if !inSingle && !inDouble { + parens++ + } + current.WriteByte(ch) + case ')': + if !inSingle && !inDouble && parens > 0 { + parens-- + } + current.WriteByte(ch) + case '[': + if !inSingle && !inDouble { + brackets++ + } + current.WriteByte(ch) + case ']': + if !inSingle && !inDouble && brackets > 0 { + brackets-- + } + current.WriteByte(ch) + case '{': + if !inSingle && !inDouble { + braces++ + } + current.WriteByte(ch) + case '}': + if !inSingle && !inDouble && braces > 0 { + braces-- + } + current.WriteByte(ch) + case ',': + if inSingle || inDouble || parens > 0 || brackets > 0 || braces > 0 { + current.WriteByte(ch) + continue + } + part := strings.TrimSpace(current.String()) + if part != "" { + result = append(result, part) + } + current.Reset() + default: + current.WriteByte(ch) + } + } + if tail := strings.TrimSpace(current.String()); tail != "" { + result = append(result, tail) + } + return result +} + +func trimQuote(v string) string { + v = strings.TrimSpace(v) + if len(v) >= 2 { + if (v[0] == '\'' && v[len(v)-1] == '\'') || (v[0] == '"' && v[len(v)-1] == '"') { + return v[1 : len(v)-1] + } + } + return v +} + +func expectArgs(view *declaredView, option string, args []string, min, max int, dql string, offset int, diags *[]*dqlshape.Diagnostic) bool { + if len(args) < min { + appendOptionArgDiagnostic(view, option, fmt.Sprintf("expected at least %d args, got %d", min, len(args)), dql, offset, diags) + return false + } + if max >= 0 && len(args) > max { + appendOptionArgDiagnostic(view, option, fmt.Sprintf("expected at most %d args, got %d", max, len(args)), dql, offset, diags) + return false + } + return true +} + +func appendOptionArgDiagnostic(view *declaredView, option, detail, dql string, offset int, diags *[]*dqlshape.Diagnostic) { + viewName := "" + if view != nil { + viewName = view.Name + } + *diags = append(*diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDeclOptionArgs, + Severity: dqlshape.SeverityWarning, + Message: fmt.Sprintf("invalid %s declaration for view %q: %s", option, viewName, detail), + Hint: "check option arity and argument formatting", + Span: relationSpan(dql, offset), + }) +} + +func isAllowedQuerySelector(name string) bool { + switch strings.ToLower(strings.TrimSpace(name)) { + case "limit", "offset", "page", "fields", "orderby": + return true + default: + return false + } +} diff --git a/repository/shape/compile/viewdecl_parity_test.go b/repository/shape/compile/viewdecl_parity_test.go new file mode 100644 index 000000000..03d455521 --- /dev/null +++ b/repository/shape/compile/viewdecl_parity_test.go @@ -0,0 +1,66 @@ +package compile + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" +) + +func TestViewDecl_ParityFixtures(t *testing.T) { + testCases := []struct { + name string + viewName string + tail string + expectDiag string + expectTag string + expectCodec string + expectHandler string + expectPreds int + }{ + { + name: "tag/codec/handler", + viewName: "limit", + tail: ".WithTag('json:\"id\"').WithCodec(AsJSON).WithHandler('Build')", + expectTag: `json:"id"`, + expectCodec: "AsJSON", + expectHandler: "Build", + }, + { + name: "status arg validation", + viewName: "limit", + tail: ".WithStatusCode('x')", + expectDiag: dqldiag.CodeDeclOptionArgs, + }, + { + name: "query selector validation", + viewName: "customer_id", + tail: ".QuerySelector('items')", + expectDiag: dqldiag.CodeDeclQuerySelector, + }, + { + name: "predicate forms", + viewName: "limit", + tail: ".WithPredicate('ByID','id=?',1).EnsurePredicate('Tenant','tenant=?',2)", + expectPreds: 2, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + view := &declaredView{Name: testCase.viewName} + var diags []*dqlshape.Diagnostic + applyDeclaredViewOptions(view, testCase.tail, "SELECT 1", 0, &diags) + if testCase.expectDiag != "" { + require.NotEmpty(t, diags) + assert.Equal(t, testCase.expectDiag, diags[0].Code) + return + } + assert.Equal(t, testCase.expectTag, view.Tag) + assert.Equal(t, testCase.expectCodec, view.Codec) + assert.Equal(t, testCase.expectHandler, view.HandlerName) + assert.Len(t, view.Predicates, testCase.expectPreds) + }) + } +} diff --git a/repository/shape/compile/viewdecl_parse.go b/repository/shape/compile/viewdecl_parse.go new file mode 100644 index 000000000..51fd45a9d --- /dev/null +++ b/repository/shape/compile/viewdecl_parse.go @@ -0,0 +1,90 @@ +package compile + +import ( + "strings" + "unicode" + + "github.com/viant/parsly" +) + +type setBlock struct { + Offset int + Body string +} + +func extractSetBlocks(dql string) []setBlock { + cursor := parsly.NewCursor("", []byte(dql), 0) + var result []setBlock + for cursor.Pos < cursor.InputSize { + matched := cursor.MatchAfterOptional(vdWhitespaceMatcher, vdSetMatcher, vdDefineMatcher) + if matched.Code != vdSetToken && matched.Code != vdDefineToken { + cursor.Pos++ + continue + } + offset := cursor.Pos - len(matched.Text(cursor)) + group := cursor.MatchAfterOptional(vdWhitespaceMatcher, vdExprGroupMatcher) + if group.Code != vdExprGroupToken { + continue + } + body := group.Text(cursor) + if len(body) < 2 { + continue + } + result = append(result, setBlock{ + Offset: offset, + Body: body[1 : len(body)-1], + }) + } + return result +} + +func parseSetDeclarationBody(body string) (holder, kind, location, tail string, ok bool) { + cursor := parsly.NewCursor("", []byte(body), 0) + if cursor.MatchAfterOptional(vdWhitespaceMatcher, vdParamDeclMatcher).Code != vdParamDeclToken { + return "", "", "", "", false + } + id, matched := readIdentifier(cursor) + if !matched { + return "", "", "", "", false + } + holder = id + _ = cursor.MatchOne(vdWhitespaceMatcher) + _ = cursor.MatchOne(vdTypeMatcher) + _ = cursor.MatchOne(vdWhitespaceMatcher) + kindLoc := cursor.MatchOne(vdExprGroupMatcher) + if kindLoc.Code != vdExprGroupToken { + return "", "", "", "", false + } + inGroup := kindLoc.Text(cursor) + if len(inGroup) < 2 { + return "", "", "", "", false + } + raw := strings.TrimSpace(inGroup[1 : len(inGroup)-1]) + slash := strings.Index(raw, "/") + if slash == -1 { + return "", "", "", "", false + } + kind = strings.ToLower(strings.TrimSpace(raw[:slash])) + location = strings.TrimSpace(raw[slash+1:]) + tail = strings.TrimSpace(string(cursor.Input[cursor.Pos:])) + return holder, kind, location, tail, true +} + +func readIdentifier(cursor *parsly.Cursor) (string, bool) { + if cursor.Pos >= cursor.InputSize { + return "", false + } + start := cursor.Pos + for cursor.Pos < cursor.InputSize { + ch := rune(cursor.Input[cursor.Pos]) + if ch == '_' || ch == '$' || unicode.IsLetter(ch) || unicode.IsDigit(ch) { + cursor.Pos++ + continue + } + break + } + if cursor.Pos == start { + return "", false + } + return string(cursor.Input[start:cursor.Pos]), true +} diff --git a/repository/shape/compile/viewdecl_test.go b/repository/shape/compile/viewdecl_test.go new file mode 100644 index 000000000..0136c64a1 --- /dev/null +++ b/repository/shape/compile/viewdecl_test.go @@ -0,0 +1,187 @@ +package compile + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/plan" +) + +func TestViewDecl_ExtractSetBlocks(t *testing.T) { + dql := "#set($_ = $Extra(view/extra_view) /* SELECT id FROM EXTRA e */)\n" + + "#define($_ = $Extra2(view/extra_view_2) /* SELECT id FROM EXTRA2 e */)\n" + + "SELECT id FROM ORDERS o" + blocks := extractSetBlocks(dql) + require.Len(t, blocks, 2) + assert.Contains(t, blocks[0].Body, "$Extra") + assert.Contains(t, blocks[1].Body, "$Extra2") +} + +func TestViewDecl_ParseSetDeclarationBody(t *testing.T) { + holder, kind, location, tail, ok := parseSetDeclarationBody("$_ = $Extra(view/extra_view).WithURI('/x')") + require.True(t, ok) + assert.Equal(t, "Extra", holder) + assert.Equal(t, "view", kind) + assert.Equal(t, "extra_view", location) + assert.Contains(t, tail, ".WithURI('/x')") +} + +func TestViewDecl_ApplyOptions_InvalidCardinality(t *testing.T) { + view := &declaredView{Name: "extra"} + var diags []*dqlshape.Diagnostic + applyDeclaredViewOptions(view, ".Cardinality('few')", "SELECT 1", 0, &diags) + require.NotEmpty(t, diags) + assert.Equal(t, dqldiag.CodeViewCardinality, diags[0].Code) +} + +func TestViewDecl_AppendDeclaredViews(t *testing.T) { + dql := "#set($_ = $Extra(view/extra_view).WithURI('/x') /* SELECT code FROM EXTRA e */)" + result := &plan.Result{ + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + } + appendDeclaredViews(dql, result) + require.NotEmpty(t, result.Views) + found := false + for _, item := range result.Views { + if item != nil && item.SQLURI == "/x" { + found = true + break + } + } + assert.True(t, found) +} + +func TestViewDecl_ApplyOptions_Extended(t *testing.T) { + view := &declaredView{Name: "limit"} + var diags []*dqlshape.Diagnostic + tail := ".WithTag('json:\"id\"').WithCodec(AsJSON,'x').WithHandler('Build',a,b)." + + "WithStatusCode(422).WithErrorMessage('bad req').WithPredicate('ByID','id = ?', 101)." + + "EnsurePredicate('Tenant','tenant_id = ?', 7).QuerySelector('qs').WithCache('c1').WithLimit(10)." + + "Cacheable(true).When('x > 1').Scope('team').WithType('[]Order').Of('list').Value('abc').Async().Output()" + applyDeclaredViewOptions(view, tail, "SELECT 1", 0, &diags) + + require.Empty(t, diags) + assert.Equal(t, `json:"id"`, view.Tag) + assert.Equal(t, "AsJSON", view.Codec) + require.Len(t, view.CodecArgs, 1) + assert.Equal(t, "'x'", view.CodecArgs[0]) + assert.Equal(t, "Build", view.HandlerName) + require.Len(t, view.HandlerArgs, 2) + assert.Equal(t, "a", view.HandlerArgs[0]) + assert.Equal(t, "b", view.HandlerArgs[1]) + require.NotNil(t, view.StatusCode) + assert.Equal(t, 422, *view.StatusCode) + assert.Equal(t, "bad req", view.ErrorMessage) + require.Len(t, view.Predicates, 2) + assert.Equal(t, "ByID", view.Predicates[0].Name) + assert.False(t, view.Predicates[0].Ensure) + assert.Equal(t, "Tenant", view.Predicates[1].Name) + assert.True(t, view.Predicates[1].Ensure) + assert.Equal(t, "qs", view.QuerySelector) + assert.Equal(t, "c1", view.CacheRef) + require.NotNil(t, view.Limit) + assert.Equal(t, 10, *view.Limit) + require.NotNil(t, view.Cacheable) + assert.True(t, *view.Cacheable) + assert.Equal(t, "x > 1", view.When) + assert.Equal(t, "team", view.Scope) + assert.Equal(t, "[]Order", view.DataType) + assert.Equal(t, "list", view.Of) + assert.Equal(t, "abc", view.Value) + assert.True(t, view.Async) + assert.True(t, view.Output) +} + +func TestViewDecl_ApplyOptions_QuerySelectorValidation(t *testing.T) { + view := &declaredView{Name: "customer_id"} + var diags []*dqlshape.Diagnostic + applyDeclaredViewOptions(view, ".QuerySelector('q')", "SELECT 1", 0, &diags) + require.NotEmpty(t, diags) + assert.Equal(t, dqldiag.CodeDeclQuerySelector, diags[0].Code) +} + +func TestViewDecl_SplitArgs_Nested(t *testing.T) { + args := splitArgs(`'a', fn(1,2), {'k': [1,2]}, "x,y"`) + require.Len(t, args, 4) + assert.Equal(t, "'a'", args[0]) + assert.Equal(t, "fn(1,2)", args[1]) + assert.Equal(t, "{'k': [1,2]}", args[2]) + assert.Equal(t, `"x,y"`, args[3]) +} + +func TestViewDecl_AppendDeclaredViews_ExtendedDeclarationMetadata(t *testing.T) { + dql := "#set($_ = $limit(view/limit).WithTag('json:\"id\"').WithCodec(AsJSON).WithHandler('Build',a)." + + "WithStatusCode(409).WithErrorMessage('conflict').WithPredicate('ByID','id=?',1)." + + "EnsurePredicate('Tenant','tenant=?',2).QuerySelector('items').WithCache('c1').WithLimit(5)." + + "Cacheable(false).When('x').Scope('s').WithType('Order').Of('o').Value('v').Async().Output() /* SELECT id FROM EXTRA e */)" + result := &plan.Result{ + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + } + appendDeclaredViews(dql, result) + require.NotEmpty(t, result.Views) + var target *plan.View + for _, item := range result.Views { + if item != nil && item.Name == "e" { + target = item + break + } + } + require.NotNil(t, target) + require.NotNil(t, target.Declaration) + assert.Equal(t, `json:"id"`, target.Declaration.Tag) + assert.Equal(t, "AsJSON", target.Declaration.Codec) + assert.Equal(t, "Build", target.Declaration.HandlerName) + require.NotNil(t, target.Declaration.StatusCode) + assert.Equal(t, 409, *target.Declaration.StatusCode) + assert.Equal(t, "conflict", target.Declaration.ErrorMessage) + assert.Equal(t, "items", target.Declaration.QuerySelector) + assert.Equal(t, "c1", target.Declaration.CacheRef) + require.NotNil(t, target.Declaration.Limit) + assert.Equal(t, 5, *target.Declaration.Limit) + require.NotNil(t, target.Declaration.Cacheable) + assert.False(t, *target.Declaration.Cacheable) + assert.Equal(t, "x", target.Declaration.When) + assert.Equal(t, "s", target.Declaration.Scope) + assert.Equal(t, "Order", target.Declaration.DataType) + assert.Equal(t, "o", target.Declaration.Of) + assert.Equal(t, "v", target.Declaration.Value) + assert.True(t, target.Declaration.Async) + assert.True(t, target.Declaration.Output) + require.Len(t, target.Declaration.Predicates, 2) +} + +func TestViewDecl_AppendDeclaredViews_AttachSummaryFromMetaViewSQL(t *testing.T) { + root := &plan.View{Name: "Browser", Path: "Browser", Holder: "Browser"} + result := &plan.Result{ + Views: []*plan.View{root}, + ViewsByName: map[string]*plan.View{"Browser": root}, + ByPath: map[string]*plan.Field{}, + } + dql := "#set($_ = $Summary(view/summary) /* SELECT COUNT(1) CNT FROM ($View.browser.SQL) t */)" + + appendDeclaredViews(dql, result) + + require.Len(t, result.Views, 1) + require.NotNil(t, root) + assert.Contains(t, root.Summary, "COUNT(1)") + assert.Contains(t, root.Summary, "$View.browser.SQL") +} + +func TestViewDecl_AppendDeclaredViews_MetaViewSQL_NoParentFallbackToView(t *testing.T) { + result := &plan.Result{ + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + } + dql := "#set($_ = $Summary(view/summary) /* SELECT COUNT(1) CNT FROM ($View.browser.SQL) t */)" + + appendDeclaredViews(dql, result) + + require.Len(t, result.Views, 1) + assert.Empty(t, result.Views[0].Summary) + assert.NotEmpty(t, result.Views[0].Name) +} diff --git a/repository/shape/dql_engine_test.go b/repository/shape/dql_engine_test.go index fafe3f67f..529748ea9 100644 --- a/repository/shape/dql_engine_test.go +++ b/repository/shape/dql_engine_test.go @@ -40,3 +40,27 @@ func TestEngine_LoadDQLComponent(t *testing.T) { assert.Equal(t, "/v1/api/reports/orders", component.Name) assert.Equal(t, "t", component.RootView) } + +func TestEngine_LoadDQLComponent_DeclarationMetadata(t *testing.T) { + engine := shape.New( + shape.WithCompiler(shapeCompile.New()), + shape.WithLoader(shapeLoad.New()), + shape.WithName("/v1/api/reports/orders"), + ) + dql := ` +#set($_ = $limit(view/limit).WithPredicate('ByID','id = ?', 1).QuerySelector('items') /* SELECT id FROM ORDERS o */) +SELECT id FROM ORDERS t` + artifact, err := engine.LoadDQLComponent(context.Background(), dql) + require.NoError(t, err) + require.NotNil(t, artifact) + component, ok := artifact.Component.(*shapeLoad.Component) + require.True(t, ok) + require.NotNil(t, component.Declarations) + require.NotNil(t, component.QuerySelectors) + require.NotNil(t, component.Predicates) + assert.Equal(t, []string{"o"}, component.QuerySelectors["items"]) + require.NotNil(t, component.Declarations["o"]) + assert.Equal(t, "items", component.Declarations["o"].QuerySelector) + require.NotEmpty(t, component.Predicates["o"]) + assert.Equal(t, "ByID", component.Predicates["o"][0].Name) +} diff --git a/repository/shape/engine_compile_options_test.go b/repository/shape/engine_compile_options_test.go new file mode 100644 index 000000000..28de1a8ff --- /dev/null +++ b/repository/shape/engine_compile_options_test.go @@ -0,0 +1,77 @@ +package shape + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type captureCompiler struct { + last CompileOptions +} + +func (c *captureCompiler) Compile(_ context.Context, source *Source, opts ...CompileOption) (*PlanResult, error) { + compiled := &CompileOptions{} + for _, opt := range opts { + if opt != nil { + opt(compiled) + } + } + c.last = *compiled + return &PlanResult{Source: source}, nil +} + +func TestEngine_Compile_UsesLegacyParityDefaults(t *testing.T) { + compiler := &captureCompiler{} + engine := New(WithCompiler(compiler)) + + _, err := engine.compile(context.Background(), &Source{Name: "orders", DQL: "SELECT 1"}) + require.NoError(t, err) + assert.False(t, compiler.last.Strict) + assert.Equal(t, CompileProfileCompat, compiler.last.Profile) + assert.Equal(t, CompileMixedModeExecWins, compiler.last.MixedMode) + assert.Equal(t, CompileUnknownNonReadWarn, compiler.last.UnknownNonReadMode) + assert.Equal(t, CompileColumnDiscoveryAuto, compiler.last.ColumnDiscoveryMode) +} + +func TestEngine_Compile_ForwardsCustomDefaults(t *testing.T) { + compiler := &captureCompiler{} + engine := New( + WithCompiler(compiler), + WithStrict(true), + WithCompileProfileDefault(CompileProfileStrict), + WithMixedModeDefault(CompileMixedModeReadWins), + WithUnknownNonReadModeDefault(CompileUnknownNonReadError), + WithColumnDiscoveryModeDefault(CompileColumnDiscoveryOff), + ) + + _, err := engine.compile(context.Background(), &Source{Name: "orders", DQL: "SELECT 1"}) + require.NoError(t, err) + assert.True(t, compiler.last.Strict) + assert.Equal(t, CompileProfileStrict, compiler.last.Profile) + assert.Equal(t, CompileMixedModeReadWins, compiler.last.MixedMode) + assert.Equal(t, CompileUnknownNonReadError, compiler.last.UnknownNonReadMode) + assert.Equal(t, CompileColumnDiscoveryOff, compiler.last.ColumnDiscoveryMode) +} + +func TestEngine_Compile_LegacyDefaultsOption(t *testing.T) { + compiler := &captureCompiler{} + engine := New( + WithCompiler(compiler), + WithStrict(true), + WithCompileProfileDefault(CompileProfileStrict), + WithMixedModeDefault(CompileMixedModeReadWins), + WithUnknownNonReadModeDefault(CompileUnknownNonReadError), + WithLegacyTranslatorDefaults(), + ) + + _, err := engine.compile(context.Background(), &Source{Name: "orders", DQL: "SELECT 1"}) + require.NoError(t, err) + assert.False(t, compiler.last.Strict) + assert.Equal(t, CompileProfileCompat, compiler.last.Profile) + assert.Equal(t, CompileMixedModeExecWins, compiler.last.MixedMode) + assert.Equal(t, CompileUnknownNonReadWarn, compiler.last.UnknownNonReadMode) + assert.Equal(t, CompileColumnDiscoveryAuto, compiler.last.ColumnDiscoveryMode) +} diff --git a/repository/shape/load/loader.go b/repository/shape/load/loader.go index 149117d25..55528601b 100644 --- a/repository/shape/load/loader.go +++ b/repository/shape/load/loader.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/viant/datly/repository/shape" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/repository/shape/typectx" shapevalidate "github.com/viant/datly/repository/shape/validate" @@ -87,6 +88,28 @@ func buildComponent(source *shape.Source, pResult *plan.Result) *Component { continue } ret.Views = append(ret.Views, aView.Name) + if aView.Declaration != nil { + if ret.Declarations == nil { + ret.Declarations = map[string]*plan.ViewDeclaration{} + } + ret.Declarations[aView.Name] = aView.Declaration + if selector := strings.TrimSpace(aView.Declaration.QuerySelector); selector != "" { + if ret.QuerySelectors == nil { + ret.QuerySelectors = map[string][]string{} + } + ret.QuerySelectors[selector] = append(ret.QuerySelectors[selector], aView.Name) + } + if len(aView.Declaration.Predicates) > 0 { + if ret.Predicates == nil { + ret.Predicates = map[string][]*plan.ViewPredicate{} + } + ret.Predicates[aView.Name] = append(ret.Predicates[aView.Name], aView.Declaration.Predicates...) + } + } + if len(aView.Relations) > 0 { + ret.Relations = append(ret.Relations, aView.Relations...) + ret.ViewRelations = append(ret.ViewRelations, toViewRelations(aView.Relations)...) + } } rootView := pickRootView(pResult.Views) if rootView != nil { @@ -117,6 +140,8 @@ func buildComponent(source *shape.Source, pResult *plan.Result) *Component { } } ret.TypeContext = cloneTypeContext(pResult.TypeContext) + ret.Directives = cloneDirectives(pResult.Directives) + ret.ColumnsDiscovery = pResult.ColumnsDiscovery return ret } @@ -126,6 +151,9 @@ func cloneTypeContext(input *typectx.Context) *typectx.Context { } ret := &typectx.Context{ DefaultPackage: strings.TrimSpace(input.DefaultPackage), + PackageDir: strings.TrimSpace(input.PackageDir), + PackageName: strings.TrimSpace(input.PackageName), + PackagePath: strings.TrimSpace(input.PackagePath), } for _, item := range input.Imports { pkg := strings.TrimSpace(item.Package) @@ -137,7 +165,38 @@ func cloneTypeContext(input *typectx.Context) *typectx.Context { Package: pkg, }) } - if ret.DefaultPackage == "" && len(ret.Imports) == 0 { + if ret.DefaultPackage == "" && + len(ret.Imports) == 0 && + ret.PackageDir == "" && + ret.PackageName == "" && + ret.PackagePath == "" { + return nil + } + return ret +} + +func cloneDirectives(input *dqlshape.Directives) *dqlshape.Directives { + if input == nil { + return nil + } + ret := &dqlshape.Directives{ + Meta: strings.TrimSpace(input.Meta), + DefaultConnector: strings.TrimSpace(input.DefaultConnector), + } + if input.Cache != nil { + ret.Cache = &dqlshape.CacheDirective{ + Enabled: input.Cache.Enabled, + TTL: strings.TrimSpace(input.Cache.TTL), + } + } + if input.MCP != nil { + ret.MCP = &dqlshape.MCPDirective{ + Name: strings.TrimSpace(input.MCP.Name), + Description: strings.TrimSpace(input.MCP.Description), + DescriptionPath: strings.TrimSpace(input.MCP.DescriptionPath), + } + } + if ret.Meta == "" && ret.DefaultConnector == "" && ret.Cache == nil && ret.MCP == nil { return nil } return ret @@ -178,7 +237,16 @@ func materializeView(item *plan.View) (*view.View, error) { } schema := newSchema(schemaType, item.Cardinality) - opts := []view.Option{view.WithSchema(schema), view.WithMode(view.ModeQuery)} + mode := view.ModeQuery + switch strings.TrimSpace(item.Mode) { + case string(view.ModeExec): + mode = view.ModeExec + case string(view.ModeHandler): + mode = view.ModeHandler + case string(view.ModeQuery): + mode = view.ModeQuery + } + opts := []view.Option{view.WithSchema(schema), view.WithMode(mode)} if item.Connector != "" { opts = append(opts, view.WithConnectorRef(item.Connector)) @@ -186,6 +254,13 @@ func materializeView(item *plan.View) (*view.View, error) { if item.SQL != "" || item.SQLURI != "" { tmpl := view.NewTemplate(item.SQL) tmpl.SourceURL = item.SQLURI + if strings.TrimSpace(item.Summary) != "" { + tmpl.Summary = &view.TemplateSummary{ + Name: "Summary", + Source: item.Summary, + Kind: view.MetaKindRecord, + } + } opts = append(opts, view.WithTemplate(tmpl)) } if item.CacheRef != "" { @@ -203,6 +278,27 @@ func materializeView(item *plan.View) (*view.View, error) { return nil, err } aView.Ref = item.Ref + aView.Module = item.Module + aView.AllowNulls = item.AllowNulls + if strings.TrimSpace(item.SelectorNamespace) != "" || item.SelectorNoLimit != nil { + if aView.Selector == nil { + aView.Selector = &view.Config{} + } + if strings.TrimSpace(item.SelectorNamespace) != "" { + aView.Selector.Namespace = strings.TrimSpace(item.SelectorNamespace) + } + if item.SelectorNoLimit != nil { + aView.Selector.NoLimit = *item.SelectorNoLimit + } + } + if aView.Schema != nil && strings.TrimSpace(item.SchemaType) != "" { + if aView.Schema.DataType == "" { + aView.Schema.DataType = strings.TrimSpace(item.SchemaType) + } + if aView.Schema.Name == "" { + aView.Schema.Name = strings.Trim(strings.TrimSpace(item.SchemaType), "*") + } + } return aView, nil } @@ -216,6 +312,53 @@ func bestSchemaType(item *plan.View) reflect.Type { return nil } +func toViewRelations(input []*plan.Relation) []*view.Relation { + if len(input) == 0 { + return nil + } + result := make([]*view.Relation, 0, len(input)) + for _, item := range input { + if item == nil { + continue + } + relation := &view.Relation{ + Name: item.Name, + Holder: item.Holder, + On: toViewLinks(item.On, true), + Of: view.NewReferenceView( + toViewLinks(item.On, false), + view.NewView(item.Ref, item.Table), + ), + } + result = append(result, relation) + } + return result +} + +func toViewLinks(input []*plan.RelationLink, parent bool) view.Links { + if len(input) == 0 { + return nil + } + result := make(view.Links, 0, len(input)) + for _, item := range input { + if item == nil { + continue + } + link := &view.Link{} + if parent { + link.Field = item.ParentField + link.Namespace = item.ParentNamespace + link.Column = item.ParentColumn + } else { + link.Field = item.RefField + link.Namespace = item.RefNamespace + link.Column = item.RefColumn + } + result = append(result, link) + } + return result +} + func newSchema(rType reflect.Type, cardinality string) *state.Schema { if cardinality == "many" && rType.Kind() != reflect.Slice { return state.NewSchema(rType, state.WithMany()) diff --git a/repository/shape/load/loader_test.go b/repository/shape/load/loader_test.go index aab074ba6..e2d45d3ee 100644 --- a/repository/shape/load/loader_test.go +++ b/repository/shape/load/loader_test.go @@ -3,11 +3,13 @@ package load import ( "context" "embed" + "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/viant/datly/repository/shape" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/repository/shape/scan" "github.com/viant/datly/repository/shape/typectx" @@ -74,6 +76,47 @@ func TestLoader_LoadViews_InvalidPlanType(t *testing.T) { assert.Contains(t, err.Error(), "unsupported plan type") } +func TestLoader_LoadViews_Metadata(t *testing.T) { + noLimit := true + allowNulls := true + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "meta"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "items", + Table: "ITEMS", + Module: "platform/items", + AllowNulls: &allowNulls, + SelectorNamespace: "it", + SelectorNoLimit: &noLimit, + SchemaType: "*ItemView", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM ITEMS", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + loader := New() + artifacts, err := loader.LoadViews(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifacts) + require.Len(t, artifacts.Views, 1) + actual := artifacts.Views[0] + assert.Equal(t, "platform/items", actual.Module) + require.NotNil(t, actual.AllowNulls) + assert.True(t, *actual.AllowNulls) + require.NotNil(t, actual.Selector) + assert.Equal(t, "it", actual.Selector.Namespace) + assert.True(t, actual.Selector.NoLimit) + require.NotNil(t, actual.Schema) + assert.Equal(t, "*ItemView", actual.Schema.DataType) +} + func TestLoader_LoadComponent(t *testing.T) { scanner := scan.New() scanned, err := scanner.Scan(context.Background(), &shape.Source{Name: "/v1/api/report", Struct: &reportSource{}}) @@ -84,12 +127,26 @@ func TestLoader_LoadComponent(t *testing.T) { require.NoError(t, err) actualPlan, ok := planned.Plan.(*plan.Result) require.True(t, ok) + actualPlan.ColumnsDiscovery = true actualPlan.TypeContext = &typectx.Context{ DefaultPackage: "mdp/performance", Imports: []typectx.Import{ {Alias: "perf", Package: "github.com/acme/mdp/performance"}, }, } + actualPlan.Directives = &dqlshape.Directives{ + Meta: "docs/report.md", + DefaultConnector: "analytics", + Cache: &dqlshape.CacheDirective{ + Enabled: true, + TTL: "5m", + }, + MCP: &dqlshape.MCPDirective{ + Name: "report.list", + Description: "List report rows", + DescriptionPath: "docs/mcp/report.md", + }, + } loader := New() artifact, err := loader.LoadComponent(context.Background(), planned) @@ -113,4 +170,69 @@ func TestLoader_LoadComponent(t *testing.T) { assert.Equal(t, "mdp/performance", component.TypeContext.DefaultPackage) require.Len(t, component.TypeContext.Imports, 1) assert.Equal(t, "perf", component.TypeContext.Imports[0].Alias) + require.NotNil(t, component.Directives) + assert.Equal(t, "docs/report.md", component.Directives.Meta) + assert.Equal(t, "analytics", component.Directives.DefaultConnector) + require.NotNil(t, component.Directives.Cache) + assert.True(t, component.Directives.Cache.Enabled) + assert.Equal(t, "5m", component.Directives.Cache.TTL) + require.NotNil(t, component.Directives.MCP) + assert.Equal(t, "report.list", component.Directives.MCP.Name) + assert.True(t, component.ColumnsDiscovery) +} + +func TestLoader_LoadComponent_RelationFieldsPreserved(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/report"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "Rows", + Name: "rows", + Table: "REPORT", + Cardinality: "many", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + Relations: []*plan.Relation{ + { + Name: "detail", + Holder: "Detail", + Ref: "detail", + Table: "REPORT_DETAIL", + On: []*plan.RelationLink{ + { + ParentField: "ReportID", + ParentNamespace: "rows", + ParentColumn: "REPORT_ID", + RefField: "ID", + RefNamespace: "detail", + RefColumn: "ID", + }, + }, + }, + }, + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := artifact.Component.(*Component) + require.True(t, ok) + require.Len(t, component.ViewRelations, 1) + require.Len(t, component.ViewRelations[0].On, 1) + require.Len(t, component.ViewRelations[0].Of.On, 1) + + parent := component.ViewRelations[0].On[0] + ref := component.ViewRelations[0].Of.On[0] + assert.Equal(t, "ReportID", parent.Field) + assert.Equal(t, "rows", parent.Namespace) + assert.Equal(t, "REPORT_ID", parent.Column) + assert.Equal(t, "ID", ref.Field) + assert.Equal(t, "detail", ref.Namespace) + assert.Equal(t, "ID", ref.Column) } diff --git a/repository/shape/load/model.go b/repository/shape/load/model.go index 8f5d384d6..6459f57a4 100644 --- a/repository/shape/load/model.go +++ b/repository/shape/load/model.go @@ -1,17 +1,26 @@ package load import "github.com/viant/datly/repository/shape/plan" +import dqlshape "github.com/viant/datly/repository/shape/dql/shape" import "github.com/viant/datly/repository/shape/typectx" +import "github.com/viant/datly/view" // Component is a shape-loaded runtime-neutral component artifact. // It intentionally avoids repository package coupling to keep shape/load reusable. type Component struct { - Name string - URI string - Method string - RootView string - Views []string - TypeContext *typectx.Context + Name string + URI string + Method string + RootView string + Views []string + Relations []*plan.Relation + ViewRelations []*view.Relation + Declarations map[string]*plan.ViewDeclaration + QuerySelectors map[string][]string + Predicates map[string][]*plan.ViewPredicate + TypeContext *typectx.Context + Directives *dqlshape.Directives + ColumnsDiscovery bool Input []*plan.State Output []*plan.State diff --git a/repository/shape/model.go b/repository/shape/model.go index f71fd5c28..4e0bde7b1 100644 --- a/repository/shape/model.go +++ b/repository/shape/model.go @@ -19,6 +19,8 @@ const ( // Source represents the caller-provided shape source. type Source struct { Name string + Path string + Connector string Struct any Type reflect.Type TypeName string diff --git a/repository/shape/normalize/sql.go b/repository/shape/normalize/sql.go new file mode 100644 index 000000000..945840dd9 --- /dev/null +++ b/repository/shape/normalize/sql.go @@ -0,0 +1,56 @@ +package normalize + +import ( + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/node" + "github.com/viant/sqlparser/query" + "github.com/viant/tagly/format/text" +) + +type mapper map[string]string + +func (m mapper) Map(name string) string { + ret, ok := m[name] + if ok { + return ret + } + return name +} + +func SQL(input string, generated bool, option func() sqlparser.Option) string { + if !generated { + return input + } + sqlQuery, err := sqlparser.ParseQuery(input, option()) + if err != nil { + return input + } + ns := mapper{} + if sqlQuery.From.Alias != "" { + ns[sqlQuery.From.Alias] = normalizeName(sqlQuery.From.Alias) + } + for _, join := range sqlQuery.Joins { + ns[join.Alias] = normalizeName(join.Alias) + } + + sqlparser.Traverse(sqlQuery, func(n node.Node) bool { + switch actual := n.(type) { + case *expr.Selector: + actual.Name = ns.Map(actual.Name) + case *query.Join: + actual.Alias = ns.Map(actual.Alias) + case *query.Item: + actual.Alias = ns.Map(actual.Alias) + case *query.From: + actual.Alias = ns.Map(actual.Alias) + } + return true + }) + return sqlparser.Stringify(sqlQuery) +} + +func normalizeName(k string) string { + caseFormat := text.DetectCaseFormat(k) + return caseFormat.Format(k, text.CaseFormatUpperCamel) +} diff --git a/repository/shape/normalize/sql_test.go b/repository/shape/normalize/sql_test.go new file mode 100644 index 000000000..aaba9af7d --- /dev/null +++ b/repository/shape/normalize/sql_test.go @@ -0,0 +1,66 @@ +package normalize + +import ( + "testing" + + "github.com/stretchr/testify/require" + legacy "github.com/viant/datly/cmd/options" + "github.com/viant/sqlparser" +) + +func parserOption() sqlparser.Option { + return sqlparser.WithErrorHandler(nil) +} + +func TestSQL_ParityWithLegacyNormalizer(t *testing.T) { + type normalizeCase struct { + Name string + Generated bool + SQL string + } + cases := []normalizeCase{ + { + Name: "skip normalization when not generated", + Generated: false, + SQL: "SELECT a.id FROM users a JOIN orders b ON a.id = b.user_id", + }, + { + Name: "invalid sql returns input", + Generated: true, + SQL: "SELECT * FROM (", + }, + { + Name: "normalize from and join aliases in selectors and alias nodes", + Generated: true, + SQL: "SELECT a.id, b.user_id FROM users a JOIN orders b ON a.id = b.user_id", + }, + { + Name: "keep alias that is already normalized", + Generated: true, + SQL: "SELECT UserAlias.id FROM users UserAlias", + }, + { + Name: "normalize snake_case alias", + Generated: true, + SQL: "SELECT order_item.id FROM users order_item", + }, + } + for _, testCase := range cases { + t.Run(testCase.Name, func(t *testing.T) { + expected := (&legacy.Rule{Generated: testCase.Generated}).NormalizeSQL(testCase.SQL, parserOption) + actual := SQL(testCase.SQL, testCase.Generated, parserOption) + require.Equal(t, expected, actual) + }) + } +} + +func TestMapper_Map(t *testing.T) { + m := mapper{"a": "A"} + require.Equal(t, "A", m.Map("a")) + require.Equal(t, "b", m.Map("b")) +} + +func TestNormalizeName(t *testing.T) { + require.Equal(t, "UserAlias", normalizeName("user_alias")) + require.Equal(t, "UserAlias", normalizeName("UserAlias")) +} diff --git a/repository/shape/options.go b/repository/shape/options.go index 05b0a7748..27b970fae 100644 --- a/repository/shape/options.go +++ b/repository/shape/options.go @@ -2,14 +2,18 @@ package shape // Options stores shape facade dependencies and behavior flags. type Options struct { - Mode Mode - Strict bool - Name string - Scanner Scanner - Planner Planner - Loader Loader - Compiler DQLCompiler - Runtime RuntimeRegistrar + Mode Mode + Strict bool + Name string + Scanner Scanner + Planner Planner + Loader Loader + Compiler DQLCompiler + Runtime RuntimeRegistrar + CompileProfile CompileProfile + CompileMixedMode CompileMixedMode + UnknownNonReadMode CompileUnknownNonReadMode + ColumnDiscoveryMode CompileColumnDiscoveryMode } // Option mutates Options. @@ -17,7 +21,12 @@ type Option func(*Options) // NewOptions builds Options from varargs. func NewOptions(opts ...Option) *Options { - ret := &Options{} + ret := &Options{ + CompileProfile: CompileProfileCompat, + CompileMixedMode: CompileMixedModeExecWins, + UnknownNonReadMode: CompileUnknownNonReadWarn, + ColumnDiscoveryMode: CompileColumnDiscoveryAuto, + } for _, opt := range opts { opt(ret) } @@ -71,3 +80,161 @@ func WithRuntime(runtime RuntimeRegistrar) Option { o.Runtime = runtime } } + +// WithCompileProfileDefault sets default compiler profile used by Engine DQL compile path. +func WithCompileProfileDefault(profile CompileProfile) Option { + return func(o *Options) { + o.CompileProfile = profile + } +} + +// WithMixedModeDefault sets default compiler mixed read/exec mode used by Engine DQL compile path. +func WithMixedModeDefault(mode CompileMixedMode) Option { + return func(o *Options) { + o.CompileMixedMode = mode + } +} + +// WithUnknownNonReadModeDefault sets default unknown non-read mode used by Engine DQL compile path. +func WithUnknownNonReadModeDefault(mode CompileUnknownNonReadMode) Option { + return func(o *Options) { + o.UnknownNonReadMode = mode + } +} + +// WithColumnDiscoveryModeDefault sets default column discovery policy used by Engine DQL compile path. +func WithColumnDiscoveryModeDefault(mode CompileColumnDiscoveryMode) Option { + return func(o *Options) { + o.ColumnDiscoveryMode = mode + } +} + +// WithLegacyTranslatorDefaults configures Engine compile defaults to legacy-compatible behavior. +func WithLegacyTranslatorDefaults() Option { + return func(o *Options) { + o.Strict = false + o.CompileProfile = CompileProfileCompat + o.CompileMixedMode = CompileMixedModeExecWins + o.UnknownNonReadMode = CompileUnknownNonReadWarn + o.ColumnDiscoveryMode = CompileColumnDiscoveryAuto + } +} + +func WithCompileStrict(strict bool) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.Strict = strict + } +} + +func WithMixedMode(mode CompileMixedMode) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.MixedMode = mode + } +} + +func WithUnknownNonReadMode(mode CompileUnknownNonReadMode) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.UnknownNonReadMode = mode + } +} + +func WithCompileProfile(profile CompileProfile) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.Profile = profile + } +} + +func WithColumnDiscoveryMode(mode CompileColumnDiscoveryMode) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.ColumnDiscoveryMode = mode + } +} + +// WithDQLPathMarker overrides the path marker used to locate platform root from source path. +// Default is "/dql/". +func WithDQLPathMarker(marker string) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.DQLPathMarker = marker + } +} + +// WithRoutesRelativePath overrides routes path relative to detected platform root. +// Default is "repo/dev/Datly/routes". +func WithRoutesRelativePath(path string) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.RoutesRelativePath = path + } +} + +// WithTypeContextPackageDir sets default type-context package directory (for xgen parity). +func WithTypeContextPackageDir(dir string) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.TypePackageDir = dir + } +} + +// WithTypeContextPackageName sets default type-context package name (for xgen parity). +func WithTypeContextPackageName(name string) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.TypePackageName = name + } +} + +// WithTypeContextPackagePath sets default type-context package import path (for xgen parity). +func WithTypeContextPackagePath(path string) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.TypePackagePath = path + } +} + +// WithTypeContextPackageDefaults sets package dir/name/path in one call. +func WithTypeContextPackageDefaults(dir, name, path string) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.TypePackageDir = dir + o.TypePackageName = name + o.TypePackagePath = path + } +} + +// WithInferTypeContextDefaults enables/disables source-path based type context defaults. +func WithInferTypeContextDefaults(enabled bool) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.InferTypeContext = &enabled + } +} diff --git a/repository/shape/parity_test.go b/repository/shape/parity_test.go index 713bfd311..725dbe631 100644 --- a/repository/shape/parity_test.go +++ b/repository/shape/parity_test.go @@ -31,6 +31,15 @@ type paritySource struct { Rows []parityRow `view:"rows,table=REPORT,connector=dev" sql:"uri=scan/testdata/report.sql"` } +type parityJoinRow struct { + ReportID int `source:"REPORT_ID"` +} + +type parityJoinSource struct { + parityEmbedded + Rows []parityJoinRow `view:"rows,table=REPORT,connector=dev" sql:"uri=scan/testdata/report.sql" on:"ReportID:rows.REPORT_ID=ID:detail.ID"` +} + func TestEngineParity_StructPipeline(t *testing.T) { source := &paritySource{} scanner := shapeScan.New() @@ -65,3 +74,35 @@ func TestEngineParity_StructPipeline(t *testing.T) { assert.Equal(t, mv.Schema.Cardinality, ev.Schema.Cardinality) assert.Equal(t, reflect.TypeOf(mv.Schema.CompType()), reflect.TypeOf(ev.Schema.CompType())) } + +func TestEngineParity_Component_SourceTagFieldJoin(t *testing.T) { + source := &parityJoinSource{} + scanner := shapeScan.New() + planner := shapePlan.New() + loader := shapeLoad.New() + + engine := shape.New( + shape.WithName("/v1/api/parity"), + shape.WithScanner(scanner), + shape.WithPlanner(planner), + shape.WithLoader(loader), + ) + artifact, err := engine.LoadComponent(context.Background(), source) + require.NoError(t, err) + require.NotNil(t, artifact) + + component, ok := artifact.Component.(*shapeLoad.Component) + require.True(t, ok) + require.Len(t, component.ViewRelations, 1) + require.Len(t, component.ViewRelations[0].On, 1) + require.Len(t, component.ViewRelations[0].Of.On, 1) + + parent := component.ViewRelations[0].On[0] + ref := component.ViewRelations[0].Of.On[0] + assert.Equal(t, "ReportID", parent.Field) + assert.Equal(t, "rows", parent.Namespace) + assert.Equal(t, "REPORT_ID", parent.Column) + assert.Equal(t, "ID", ref.Field) + assert.Equal(t, "detail", ref.Namespace) + assert.Equal(t, "ID", ref.Column) +} diff --git a/repository/shape/plan/model.go b/repository/shape/plan/model.go index 8dacf2bbe..8935786a2 100644 --- a/repository/shape/plan/model.go +++ b/repository/shape/plan/model.go @@ -4,6 +4,7 @@ import ( "embed" "reflect" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/typectx" ) @@ -12,12 +13,26 @@ type Result struct { RootType reflect.Type EmbedFS *embed.FS - Fields []*Field - ByPath map[string]*Field - Views []*View - ViewsByName map[string]*View - States []*State - TypeContext *typectx.Context + Fields []*Field + ByPath map[string]*Field + Views []*View + ViewsByName map[string]*View + States []*State + Types []*Type + ColumnsDiscovery bool + TypeContext *typectx.Context + Directives *dqlshape.Directives + Diagnostics []*dqlshape.Diagnostic +} + +// Type is normalized type metadata collected during compile. +type Type struct { + Name string + Alias string + DataType string + Cardinality string + Package string + ModulePath string } // Field is a normalized projection of scanned field metadata. @@ -33,7 +48,9 @@ type View struct { Path string Name string Ref string + Mode string Table string + Module string Connector string CacheRef string Partitioner string @@ -42,31 +59,103 @@ type View struct { SQL string SQLURI string Summary string - Links []string + Relations []*Relation Holder string + AllowNulls *bool + SelectorNamespace string + SelectorNoLimit *bool + SchemaType string + ColumnsDiscovery bool + Cardinality string ElementType reflect.Type FieldType reflect.Type + Declaration *ViewDeclaration +} + +// ViewDeclaration captures declaration options used to derive a view from DQL directives. +type ViewDeclaration struct { + Tag string + Codec string + CodecArgs []string + HandlerName string + HandlerArgs []string + StatusCode *int + ErrorMessage string + QuerySelector string + CacheRef string + Limit *int + Cacheable *bool + When string + Scope string + DataType string + Of string + Value string + Async bool + Output bool + Predicates []*ViewPredicate +} + +// ViewPredicate captures WithPredicate / EnsurePredicate metadata. +type ViewPredicate struct { + Name string + Source string + Ensure bool + Arguments []string +} + +// Relation is normalized relation metadata extracted from DQL joins. +type Relation struct { + Name string + Holder string + Ref string + Table string + Kind string + Raw string + On []*RelationLink + Warnings []string +} + +// RelationLink represents one parent/ref join predicate. +type RelationLink struct { + ParentField string + ParentNamespace string + ParentColumn string + RefField string + RefNamespace string + RefColumn string + Expression string } // State is a normalized parameter field plan. type State struct { - Path string - Name string - Kind string - In string - When string - Scope string - DataType string - Required *bool - Async bool - Cacheable *bool - With string - URI string - ErrorCode int - ErrorMessage string + Path string + Name string + Kind string + In string + QuerySelector string + When string + Scope string + DataType string + Value string + Required *bool + Async bool + Cacheable *bool + With string + URI string + ErrorCode int + ErrorMessage string + Predicates []*StatePredicate TagType reflect.Type EffectiveType reflect.Type } + +// StatePredicate captures parameter predicate semantics from DQL declarations. +type StatePredicate struct { + Group int + Name string + Ensure bool + Arguments []string +} diff --git a/repository/shape/plan/planner.go b/repository/shape/plan/planner.go index ec66aea5a..2c3735dc9 100644 --- a/repository/shape/plan/planner.go +++ b/repository/shape/plan/planner.go @@ -86,7 +86,7 @@ func normalizeView(field *scan.Field) *View { result.SQLURI = tag.SQL.URI result.Summary = tag.SummarySQL.SQL if len(tag.LinkOn) > 0 { - result.Links = append(result.Links, tag.LinkOn...) + result.Relations = append(result.Relations, relationFromTagLinks(field.Name, tag.LinkOn)) } result.Ref = strings.TrimSpace(tag.TypeName) } @@ -101,6 +101,67 @@ func normalizeView(field *scan.Field) *View { return result } +func relationFromTagLinks(holder string, links []string) *Relation { + relation := &Relation{ + Name: strings.TrimSpace(holder), + Holder: strings.TrimSpace(holder), + Ref: strings.TrimSpace(holder), + } + for _, linkExpr := range links { + linkExpr = strings.TrimSpace(linkExpr) + if linkExpr == "" { + continue + } + left, right, ok := strings.Cut(linkExpr, "=") + if !ok { + continue + } + leftField, leftNS, leftCol := splitTagSelector(left) + rightField, rightNS, rightCol := splitTagSelector(right) + if leftCol == "" || rightCol == "" { + continue + } + relation.On = append(relation.On, &RelationLink{ + ParentField: leftField, + ParentNamespace: leftNS, + ParentColumn: leftCol, + RefField: rightField, + RefNamespace: rightNS, + RefColumn: rightCol, + Expression: strings.TrimSpace(left) + "=" + strings.TrimSpace(right), + }) + } + if relation.Ref == "" { + relation.Ref = "relation" + } + if relation.Holder == "" { + relation.Holder = relation.Ref + } + if relation.Name == "" { + relation.Name = relation.Holder + } + return relation +} + +func splitTagSelector(value string) (string, string, string) { + value = strings.TrimSpace(value) + value = strings.TrimSuffix(value, "(true)") + value = strings.TrimSuffix(value, "(false)") + field := "" + if idx := strings.Index(value, ":"); idx >= 0 { + field = strings.TrimSpace(value[:idx]) + value = value[idx+1:] + } + value = strings.Trim(value, "`\"") + if value == "" { + return field, "", "" + } + if idx := strings.Index(value, "."); idx >= 0 { + return field, strings.TrimSpace(value[:idx]), strings.TrimSpace(value[idx+1:]) + } + return field, "", strings.TrimSpace(value) +} + func normalizeState(field *scan.Field) *State { result := &State{Path: field.Path, TagType: field.Type} if field.StateTag == nil || field.StateTag.Parameter == nil { diff --git a/repository/shape/plan/planner_test.go b/repository/shape/plan/planner_test.go index 29bb1e792..7dc1edb1c 100644 --- a/repository/shape/plan/planner_test.go +++ b/repository/shape/plan/planner_test.go @@ -36,6 +36,18 @@ type reportSource struct { ID int `parameter:"id,kind=query,in=id"` } +type relationRow struct { + ID int +} + +type relationSource struct { + Rows []relationRow `view:"rows,table=REPORT" on:"rows.report_id=report.id"` +} + +type relationSourceWithFields struct { + Rows []relationRow `view:"rows,table=REPORT" on:"ReportID:rows.report_id=ID:report.id"` +} + func TestPlanner_Plan(t *testing.T) { scanner := scan.New() scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportSource{}}) @@ -78,6 +90,54 @@ func TestPlanner_Plan(t *testing.T) { assert.Equal(t, stateByPath["ID"].TagType, stateByPath["ID"].EffectiveType) } +func TestPlanner_Plan_LinkOnProducesStructuredRelations(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &relationSource{}}) + require.NoError(t, err) + + planner := New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + require.NotNil(t, planned) + + result, ok := planned.Plan.(*Result) + require.True(t, ok) + require.Len(t, result.Views, 1) + viewPlan := result.Views[0] + require.Len(t, viewPlan.Relations, 1) + relation := viewPlan.Relations[0] + require.Len(t, relation.On, 1) + assert.Equal(t, "rows", relation.On[0].ParentNamespace) + assert.Equal(t, "report_id", relation.On[0].ParentColumn) + assert.Equal(t, "report", relation.On[0].RefNamespace) + assert.Equal(t, "id", relation.On[0].RefColumn) +} + +func TestPlanner_Plan_LinkOnPreservesFieldSelectors(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &relationSourceWithFields{}}) + require.NoError(t, err) + + planner := New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + require.NotNil(t, planned) + + result, ok := planned.Plan.(*Result) + require.True(t, ok) + require.Len(t, result.Views, 1) + viewPlan := result.Views[0] + require.Len(t, viewPlan.Relations, 1) + relation := viewPlan.Relations[0] + require.Len(t, relation.On, 1) + assert.Equal(t, "ReportID", relation.On[0].ParentField) + assert.Equal(t, "rows", relation.On[0].ParentNamespace) + assert.Equal(t, "report_id", relation.On[0].ParentColumn) + assert.Equal(t, "ID", relation.On[0].RefField) + assert.Equal(t, "report", relation.On[0].RefNamespace) + assert.Equal(t, "id", relation.On[0].RefColumn) +} + func TestPlanner_Plan_InvalidDescriptors(t *testing.T) { planner := New() _, err := planner.Plan(context.Background(), &shape.ScanResult{Source: &shape.Source{Name: "x"}, Descriptors: "invalid"}) diff --git a/repository/shape/platform_parity_metadata_test.go b/repository/shape/platform_parity_metadata_test.go new file mode 100644 index 000000000..8ad4a6d8c --- /dev/null +++ b/repository/shape/platform_parity_metadata_test.go @@ -0,0 +1,77 @@ +package shape_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCompareMetadataParity(t *testing.T) { + trueValue := true + falseValue := false + + legacyMeta := &resourceMetaIR{ColumnsDiscovery: &trueValue} + shapeMeta := &resourceMetaIR{ColumnsDiscovery: &trueValue} + + legacyViews := []viewMetaIR{ + { + Name: "items", + Mode: "SQLQuery", + Module: "platform/items", + AllowNulls: &trueValue, + SelectorNamespace: "item", + SelectorNoLimit: &falseValue, + SchemaCardinality: "Many", + SchemaType: "*ItemView", + HasSummary: &trueValue, + }, + } + shapeViews := []viewMetaIR{ + { + Name: "items", + Mode: "SQLQuery", + Module: "platform/items", + AllowNulls: &trueValue, + SelectorNamespace: "item", + SelectorNoLimit: &falseValue, + SchemaCardinality: "Many", + SchemaType: "*ItemView", + HasSummary: &trueValue, + }, + } + + assert.Empty(t, compareMetadataParity(legacyMeta, shapeMeta, legacyViews, shapeViews)) +} + +func TestCompareMetadataParity_DetectsMismatches(t *testing.T) { + trueValue := true + falseValue := false + + legacyMeta := &resourceMetaIR{ColumnsDiscovery: &trueValue} + shapeMeta := &resourceMetaIR{ColumnsDiscovery: &falseValue} + + legacyViews := []viewMetaIR{{ + Name: "items", + Mode: "SQLQuery", + Module: "platform/items", + AllowNulls: &trueValue, + SelectorNoLimit: &trueValue, + SchemaType: "*ItemView", + }} + shapeViews := []viewMetaIR{{ + Name: "items", + Mode: "SQLExec", + Module: "platform/items2", + AllowNulls: &falseValue, + SelectorNoLimit: &falseValue, + SchemaType: "*OtherView", + }} + + mismatches := compareMetadataParity(legacyMeta, shapeMeta, legacyViews, shapeViews) + assert.Contains(t, mismatches, "resource columnsDiscovery mismatch") + assert.Contains(t, mismatches, "view mode mismatch: items") + assert.Contains(t, mismatches, "view module mismatch: items") + assert.Contains(t, mismatches, "view allowNulls mismatch: items") + assert.Contains(t, mismatches, "view selector noLimit mismatch: items") + assert.Contains(t, mismatches, "view schema type mismatch: items") +} diff --git a/repository/shape/platform_parity_test.go b/repository/shape/platform_parity_test.go new file mode 100644 index 000000000..f6837b03c --- /dev/null +++ b/repository/shape/platform_parity_test.go @@ -0,0 +1,1478 @@ +package shape_test + +import ( + "context" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "testing" + + shape "github.com/viant/datly/repository/shape" + shapecompile "github.com/viant/datly/repository/shape/compile" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" + shapeload "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/view" + "gopkg.in/yaml.v3" +) + +type parityRule struct { + Mode string `yaml:"mode"` + Namespace string `yaml:"namespace"` + Source string `yaml:"source"` + Connector string `yaml:"connector,omitempty"` +} + +type legacyYAML struct { + ColumnsDiscovery *bool `yaml:"ColumnsDiscovery"` + TypeContext struct { + DefaultPackage string `yaml:"DefaultPackage"` + PackageDir string `yaml:"PackageDir"` + PackageName string `yaml:"PackageName"` + PackagePath string `yaml:"PackagePath"` + } `yaml:"TypeContext"` + Resource struct { + Views []struct { + Name string `yaml:"Name"` + Table string `yaml:"Table"` + Mode string `yaml:"Mode"` + Module string `yaml:"Module"` + AllowNulls *bool `yaml:"AllowNulls"` + Connector struct { + Ref string `yaml:"Ref"` + } `yaml:"Connector"` + Schema struct { + Cardinality string `yaml:"Cardinality"` + DataType string `yaml:"DataType"` + Name string `yaml:"Name"` + } `yaml:"Schema"` + Template struct { + SourceURL string `yaml:"SourceURL"` + Summary *struct { + Name string `yaml:"Name"` + Kind string `yaml:"Kind"` + } `yaml:"Summary"` + } `yaml:"Template"` + Selector struct { + Namespace string `yaml:"Namespace"` + NoLimit *bool `yaml:"NoLimit"` + LimitParameter selectorParam `yaml:"LimitParameter"` + OffsetParameter selectorParam `yaml:"OffsetParameter"` + PageParameter selectorParam `yaml:"PageParameter"` + FieldsParameter selectorParam `yaml:"FieldsParameter"` + OrderByParameter selectorParam `yaml:"OrderByParameter"` + } `yaml:"Selector"` + } `yaml:"Views"` + Parameters []struct { + Name string `yaml:"Name"` + URI string `yaml:"URI"` + Value string `yaml:"Value"` + Required *bool `yaml:"Required"` + Cacheable *bool `yaml:"Cacheable"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` + Predicates []struct { + Group int `yaml:"Group"` + Name string `yaml:"Name"` + Ensure bool `yaml:"Ensure"` + Args []string `yaml:"Args"` + } `yaml:"Predicates"` + } `yaml:"Parameters"` + Types []struct { + Name string `yaml:"Name"` + Alias string `yaml:"Alias"` + DataType string `yaml:"DataType"` + Cardinality string `yaml:"Cardinality"` + Package string `yaml:"Package"` + ModulePath string `yaml:"ModulePath"` + } `yaml:"Types"` + } `yaml:"Resource"` + Routes []struct { + Method string `yaml:"Method"` + URI string `yaml:"URI"` + View struct { + Ref string `yaml:"Ref"` + } `yaml:"View"` + } `yaml:"Routes"` +} + +type viewIR struct { + Name string `yaml:"name"` + Table string `yaml:"table"` + Connector string `yaml:"connector,omitempty"` + SQLURI string `yaml:"sqlUri,omitempty"` +} + +type routeIR struct { + Method string `yaml:"method,omitempty"` + URI string `yaml:"uri,omitempty"` + View string `yaml:"view,omitempty"` +} + +type resourceMetaIR struct { + ColumnsDiscovery *bool `yaml:"columnsDiscovery,omitempty"` +} + +type viewMetaIR struct { + Name string `yaml:"name"` + Mode string `yaml:"mode,omitempty"` + Module string `yaml:"module,omitempty"` + AllowNulls *bool `yaml:"allowNulls,omitempty"` + SelectorNamespace string `yaml:"selectorNamespace,omitempty"` + SelectorNoLimit *bool `yaml:"selectorNoLimit,omitempty"` + SchemaCardinality string `yaml:"schemaCardinality,omitempty"` + SchemaType string `yaml:"schemaType,omitempty"` + HasSummary *bool `yaml:"hasSummary,omitempty"` +} + +type parityOutput struct { + Namespace string `yaml:"namespace"` + Source string `yaml:"source"` + LegacyYAML string `yaml:"legacyYaml"` + LegacyMeta *resourceMetaIR `yaml:"legacyMeta,omitempty"` + LegacyViews []viewIR `yaml:"legacyViews,omitempty"` + LegacyViewMeta []viewMetaIR `yaml:"legacyViewMeta,omitempty"` + LegacyParams []paramIR `yaml:"legacyParams,omitempty"` + LegacyRoutes []routeIR `yaml:"legacyRoutes,omitempty"` + LegacyTypes []typeIR `yaml:"legacyTypes,omitempty"` + LegacyTypeCtx *typeCtxIR `yaml:"legacyTypeContext,omitempty"` + ShapeMeta *resourceMetaIR `yaml:"shapeMeta,omitempty"` + ShapeViews []viewIR `yaml:"shapeViews,omitempty"` + ShapeViewMeta []viewMetaIR `yaml:"shapeViewMeta,omitempty"` + ShapeParams []paramIR `yaml:"shapeParams,omitempty"` + ShapeTypes []typeIR `yaml:"shapeTypes,omitempty"` + ShapeTypeCtx *typeCtxIR `yaml:"shapeTypeContext,omitempty"` + ShapeDiags []string `yaml:"shapeDiagnostics,omitempty"` + Mismatches []string `yaml:"mismatches,omitempty"` + CompileFailed bool `yaml:"compileFailed,omitempty"` + RawDiagnostics []*dqlshape.Diagnostic `yaml:"-"` +} + +type parityReport struct { + Total int `yaml:"total"` + Compared int `yaml:"compared"` + WithDiff int `yaml:"withDiff"` + MissingYAML int `yaml:"missingYaml"` + Failures int `yaml:"failures"` + TopIssues []string `yaml:"topIssues,omitempty"` +} + +type selectorParam struct { + Name string `yaml:"Name"` + Cacheable *bool `yaml:"Cacheable"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` +} + +type paramIR struct { + Name string `yaml:"name"` + Kind string `yaml:"kind,omitempty"` + In string `yaml:"in,omitempty"` + Required *bool `yaml:"required,omitempty"` + Cacheable *bool `yaml:"cacheable,omitempty"` + URI string `yaml:"uri,omitempty"` + Value string `yaml:"value,omitempty"` + QuerySelector string `yaml:"querySelector,omitempty"` + Predicates []string `yaml:"predicates,omitempty"` +} + +type typeIR struct { + Name string `yaml:"name"` + Alias string `yaml:"alias,omitempty"` + DataType string `yaml:"dataType,omitempty"` + Cardinality string `yaml:"cardinality,omitempty"` + Package string `yaml:"package,omitempty"` + ModulePath string `yaml:"modulePath,omitempty"` +} + +type typeCtxIR struct { + DefaultPackage string `yaml:"defaultPackage,omitempty"` + PackageDir string `yaml:"packageDir,omitempty"` + PackageName string `yaml:"packageName,omitempty"` + PackagePath string `yaml:"packagePath,omitempty"` +} + +type parityEntryEval struct { + Output parityOutput + SourceReadable bool + MissingLegacyYAML bool +} + +func TestPlatform_DQLToRoute_ParityIR_SmokeHandlers(t *testing.T) { + platformRoot := os.Getenv("PLATFORM_ROOT") + if platformRoot == "" { + platformRoot = "/Users/awitas/go/src/github.vianttech.com/viant/platform" + } + rulesRoot := filepath.Join(platformRoot, "e2e", "rule") + routesRoot := filepath.Join(platformRoot, "repo", "dev", "Datly", "routes") + if _, err := os.Stat(rulesRoot); err != nil { + if os.Getenv("PLATFORM_PARITY_SMOKE_REQUIRED") == "1" { + t.Fatalf("platform rules not found at %s", rulesRoot) + } + t.Skipf("platform rules not found at %s", rulesRoot) + } + entries, err := collectRuleMappings(rulesRoot) + if err != nil { + t.Fatalf("collect mappings: %v", err) + } + if len(entries) == 0 { + t.Fatalf("no dql->route mappings found under %s", rulesRoot) + } + entryBySource := map[string]parityRule{} + for _, entry := range entries { + entryBySource[entry.Source] = entry + } + highRiskHandlers := collectSmokeHandlerSources(entries, routesRoot) + if len(highRiskHandlers) < 5 { + t.Fatalf("smoke handler discovery returned too few sources: %d", len(highRiskHandlers)) + } + + compiler := shapecompile.New() + for _, source := range highRiskHandlers { + entry, ok := entryBySource[source] + if !ok { + t.Fatalf("smoke source not found in rule mappings: %s", source) + } + eval := evaluateParityEntry(platformRoot, routesRoot, entry, compiler) + if !eval.SourceReadable { + t.Fatalf("unable to read source for smoke source: %s", source) + } + if eval.MissingLegacyYAML { + t.Fatalf("missing legacy yaml for smoke source: %s", source) + } + out := eval.Output + if out.CompileFailed { + t.Fatalf("shape compile failed for %s: %v", source, out.ShapeDiags) + } + if len(out.Mismatches) > 0 { + t.Fatalf("parity mismatches for %s: %v", source, out.Mismatches) + } + } +} + +func TestPlatform_DQLToRoute_ParityIR(t *testing.T) { + platformRoot := os.Getenv("PLATFORM_ROOT") + if platformRoot == "" { + platformRoot = "/Users/awitas/go/src/github.vianttech.com/viant/platform" + } + rulesRoot := filepath.Join(platformRoot, "e2e", "rule") + routesRoot := filepath.Join(platformRoot, "repo", "dev", "Datly", "routes") + if _, err := os.Stat(rulesRoot); err != nil { + t.Skipf("platform rules not found at %s", rulesRoot) + } + entries, err := collectRuleMappings(rulesRoot) + if err != nil { + t.Fatalf("collect mappings: %v", err) + } + if len(entries) == 0 { + t.Fatalf("no dql->route mappings found under %s", rulesRoot) + } + targetSource := strings.TrimSpace(os.Getenv("PLATFORM_PARITY_SOURCE")) + runAll := strings.EqualFold(targetSource, "all") || targetSource == "*" || strings.EqualFold(strings.TrimSpace(os.Getenv("PLATFORM_PARITY_ALL")), "1") + if targetSource == "" && !runAll { + t.Skip("set PLATFORM_PARITY_SOURCE to run transient platform parity check") + } + if !runAll { + var filtered []parityRule + for _, entry := range entries { + if entry.Source == targetSource { + filtered = append(filtered, entry) + } + } + if len(filtered) == 0 { + t.Fatalf("target source not found in rules: %s", targetSource) + } + entries = filtered + } + + compiler := shapecompile.New() + report := parityReport{Total: len(entries)} + issueCounts := map[string]int{} + + for _, entry := range entries { + eval := evaluateParityEntry(platformRoot, routesRoot, entry, compiler) + if !eval.SourceReadable { + continue + } + if eval.MissingLegacyYAML { + report.MissingYAML++ + continue + } + report.Compared++ + out := eval.Output + routeYAMLPath := out.LegacyYAML + if out.CompileFailed { + issueCounts["shape compile failed"]++ + report.Failures++ + writeIRFile(routeYAMLPath+".shape.ir.yaml", out) + report.WithDiff++ + continue + } + if len(out.Mismatches) > 0 { + report.WithDiff++ + for _, m := range out.Mismatches { + issueCounts[m]++ + } + } + writeIRFile(routeYAMLPath+".shape.ir.yaml", out) + } + + report.TopIssues = topIssues(issueCounts, 10) + reportPath := filepath.Join(routesRoot, "_shape_parity_report.yaml") + writeYAML(reportPath, report) + t.Logf("parity report: %s", reportPath) + t.Logf("total=%d compared=%d withDiff=%d missingYaml=%d failures=%d", report.Total, report.Compared, report.WithDiff, report.MissingYAML, report.Failures) +} + +func collectSmokeHandlerSources(entries []parityRule, routesRoot string) []string { + excluded := map[string]bool{} + var result []string + for _, entry := range entries { + source := strings.TrimSpace(entry.Source) + if !isHandlerLikeSource(source) { + continue + } + if excluded[source] { + continue + } + routeYAMLPath := filepath.Join(routesRoot, entry.Namespace, routeYAMLName(source)) + if _, err := os.Stat(routeYAMLPath); err != nil { + continue + } + result = append(result, source) + } + sort.Strings(result) + return dedupe(result) +} + +func isHandlerLikeSource(source string) bool { + source = strings.ToLower(strings.TrimSpace(source)) + if source == "" { + return false + } + if strings.Contains(source, "/gen/") && (strings.HasSuffix(source, ".dql") || strings.HasSuffix(source, ".sql")) { + return true + } + return strings.HasSuffix(source, "/patch.dql") || + strings.HasSuffix(source, "/patch.sql") || + strings.HasSuffix(source, "/post.dql") || + strings.HasSuffix(source, "/post.sql") || + strings.HasSuffix(source, "/put.dql") || + strings.HasSuffix(source, "/put.sql") || + strings.HasSuffix(source, "/delete.dql") || + strings.HasSuffix(source, "/delete.sql") || + strings.HasSuffix(source, "/upload.dql") || + strings.HasSuffix(source, "/upload.sql") || + strings.HasSuffix(source, "/export.dql") || + strings.HasSuffix(source, "/export.sql") || + strings.HasSuffix(source, "/action.dql") || + strings.HasSuffix(source, "/action.sql") +} + +func evaluateParityEntry(platformRoot, routesRoot string, entry parityRule, compiler *shapecompile.DQLCompiler) parityEntryEval { + sourcePath := filepath.Join(platformRoot, entry.Source) + routeYAMLPath, _ := resolveLegacyRouteYAMLPath(routesRoot, entry.Namespace, entry.Source) + if routeYAMLPath == "" { + routeYAMLPath = filepath.Join(routesRoot, entry.Namespace, routeYAMLName(entry.Source)) + } + out := parityEntryEval{Output: parityOutput{ + Namespace: entry.Namespace, + Source: entry.Source, + LegacyYAML: routeYAMLPath, + }} + sourceBytes, readErr := os.ReadFile(sourcePath) + if readErr != nil { + return out + } + out.SourceReadable = true + legacyBytes, legacyErr := os.ReadFile(routeYAMLPath) + if legacyErr != nil { + out.MissingLegacyYAML = true + return out + } + sourceName := strings.TrimSuffix(filepath.Base(sourcePath), filepath.Ext(sourcePath)) + if sourceName == "" { + sourceName = entry.Namespace + } + + var legacy legacyYAML + if err := yaml.Unmarshal(legacyBytes, &legacy); err == nil { + out.Output.LegacyMeta = &resourceMetaIR{ColumnsDiscovery: legacy.ColumnsDiscovery} + out.Output.LegacyViews = make([]viewIR, 0, len(legacy.Resource.Views)) + out.Output.LegacyViewMeta = make([]viewMetaIR, 0, len(legacy.Resource.Views)) + for _, v := range legacy.Resource.Views { + out.Output.LegacyViews = append(out.Output.LegacyViews, viewIR{ + Name: v.Name, + Table: v.Table, + Connector: v.Connector.Ref, + SQLURI: v.Template.SourceURL, + }) + var hasSummary *bool + if v.Template.Summary != nil { + value := true + hasSummary = &value + } + out.Output.LegacyViewMeta = append(out.Output.LegacyViewMeta, viewMetaIR{ + Name: strings.TrimSpace(v.Name), + Mode: strings.TrimSpace(v.Mode), + Module: strings.TrimSpace(v.Module), + AllowNulls: v.AllowNulls, + SelectorNamespace: strings.TrimSpace(v.Selector.Namespace), + SelectorNoLimit: v.Selector.NoLimit, + SchemaCardinality: strings.TrimSpace(v.Schema.Cardinality), + SchemaType: firstNonEmpty(strings.TrimSpace(v.Schema.DataType), strings.TrimSpace(v.Schema.Name)), + HasSummary: hasSummary, + }) + } + for _, r := range legacy.Routes { + out.Output.LegacyRoutes = append(out.Output.LegacyRoutes, routeIR{ + Method: r.Method, + URI: r.URI, + View: r.View.Ref, + }) + } + out.Output.LegacyTypeCtx = normalizeTypeContextIR( + legacy.TypeContext.DefaultPackage, + legacy.TypeContext.PackageDir, + legacy.TypeContext.PackageName, + legacy.TypeContext.PackagePath, + ) + out.Output.LegacyParams = normalizeLegacyParams(legacy) + out.Output.LegacyTypes = normalizeLegacyTypes(legacy) + } + + planResult, compileErr := compiler.Compile(context.Background(), &shape.Source{ + Name: sourceName, + Path: sourcePath, + Connector: entry.Connector, + DQL: string(sourceBytes), + }) + if compileErr != nil { + out.Output.CompileFailed = true + if cErr, ok := compileErr.(*shapecompile.CompileError); ok { + out.Output.RawDiagnostics = cErr.Diagnostics + for _, d := range cErr.Diagnostics { + if d == nil { + continue + } + out.Output.ShapeDiags = append(out.Output.ShapeDiags, d.Error()) + } + } else { + out.Output.ShapeDiags = append(out.Output.ShapeDiags, compileErr.Error()) + } + out.Output.Mismatches = append(out.Output.Mismatches, "shape compile failed") + return out + } + + planned, _ := planResult.Plan.(*plan.Result) + if planned != nil { + out.Output.ShapeMeta = &resourceMetaIR{} + if sourcePath != "" { + value := true + out.Output.ShapeMeta.ColumnsDiscovery = &value + } + out.Output.ShapeViews = make([]viewIR, 0, len(planned.Views)) + out.Output.ShapeViewMeta = make([]viewMetaIR, 0, len(planned.Views)) + for _, v := range planned.Views { + if v == nil { + continue + } + out.Output.ShapeViews = append(out.Output.ShapeViews, viewIR{ + Name: v.Name, + Table: v.Table, + Connector: v.Connector, + SQLURI: v.SQLURI, + }) + var hasSummary *bool + if strings.TrimSpace(v.Summary) != "" { + value := true + hasSummary = &value + } + out.Output.ShapeViewMeta = append(out.Output.ShapeViewMeta, viewMetaIR{ + Name: strings.TrimSpace(v.Name), + Mode: inferShapeViewMode(v.SQL), + Module: strings.TrimSpace(v.Module), + AllowNulls: v.AllowNulls, + SelectorNamespace: strings.TrimSpace(v.SelectorNamespace), + SelectorNoLimit: v.SelectorNoLimit, + SchemaCardinality: normalizeCardinality(strings.TrimSpace(v.Cardinality)), + SchemaType: strings.TrimSpace(v.SchemaType), + HasSummary: hasSummary, + }) + } + for _, d := range planned.Diagnostics { + if d == nil { + continue + } + out.Output.ShapeDiags = append(out.Output.ShapeDiags, d.Error()) + } + loader := shapeload.New() + if artifacts, err := loader.LoadViews(context.Background(), planResult); err == nil && artifacts != nil && artifacts.Resource != nil { + mergeShapeViewMetadata(out.Output.ShapeViewMeta, artifacts.Resource.Views) + } + out.Output.ShapeParams = normalizeShapeParams(planned) + out.Output.ShapeTypes = normalizeShapeTypes(planned, sourcePath) + if planned.TypeContext != nil { + out.Output.ShapeTypeCtx = normalizeTypeContextIR( + planned.TypeContext.DefaultPackage, + planned.TypeContext.PackageDir, + planned.TypeContext.PackageName, + planned.TypeContext.PackagePath, + ) + } + } + + out.Output.Mismatches = compareParity(out.Output.LegacyViews, out.Output.ShapeViews) + out.Output.Mismatches = append(out.Output.Mismatches, compareMetadataParity(out.Output.LegacyMeta, out.Output.ShapeMeta, out.Output.LegacyViewMeta, out.Output.ShapeViewMeta)...) + out.Output.Mismatches = append(out.Output.Mismatches, compareParamParity(out.Output.LegacyParams, out.Output.ShapeParams)...) + out.Output.Mismatches = append(out.Output.Mismatches, compareTypeParity(out.Output.LegacyTypes, out.Output.ShapeTypes)...) + out.Output.Mismatches = append(out.Output.Mismatches, compareTypeContextParity(out.Output.LegacyTypeCtx, out.Output.ShapeTypeCtx)...) + out.Output.Mismatches = dedupe(out.Output.Mismatches) + return out +} + +func resolveLegacyRouteYAMLPath(routesRoot, namespace, source string) (string, bool) { + candidates := legacyRouteYAMLCandidatePaths(routesRoot, namespace, source) + for _, candidate := range candidates { + if _, err := os.Stat(candidate); err == nil { + return candidate, true + } + } + return "", false +} + +func legacyRouteYAMLCandidatePaths(routesRoot, namespace, source string) []string { + namespace = strings.Trim(strings.TrimSpace(namespace), "/") + stem := strings.TrimSuffix(filepath.Base(strings.TrimSpace(source)), filepath.Ext(strings.TrimSpace(source))) + if stem == "" { + stem = "route" + } + fileName := stem + ".yaml" + nsPath := filepath.FromSlash(namespace) + leaf := filepath.Base(nsPath) + parent := filepath.Dir(nsPath) + + appendUnique := func(items *[]string, seen map[string]bool, path string) { + path = filepath.Clean(path) + if path == "." || path == "" || seen[path] { + return + } + seen[path] = true + *items = append(*items, path) + } + + seen := map[string]bool{} + result := make([]string, 0, 8) + appendUnique(&result, seen, filepath.Join(routesRoot, nsPath, fileName)) + appendUnique(&result, seen, filepath.Join(routesRoot, nsPath, stem, fileName)) + if leaf != "" && leaf != "." { + appendUnique(&result, seen, filepath.Join(routesRoot, nsPath, leaf+".yaml")) + } + if parent != "" && parent != "." { + appendUnique(&result, seen, filepath.Join(routesRoot, parent, fileName)) + appendUnique(&result, seen, filepath.Join(routesRoot, parent, stem, fileName)) + parentLeaf := filepath.Base(parent) + if parentLeaf != "" && parentLeaf != "." { + appendUnique(&result, seen, filepath.Join(routesRoot, parent, parentLeaf+".yaml")) + } + } + if strings.Contains(strings.ToLower(source), "/gen/") { + appendUnique(&result, seen, filepath.Join(routesRoot, nsPath, "patch", "patch.yaml")) + } + return result +} + +func collectRuleMappings(rulesRoot string) ([]parityRule, error) { + var files []string + if err := filepath.WalkDir(rulesRoot, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".yaml") { + return nil + } + files = append(files, path) + return nil + }); err != nil { + return nil, err + } + re := regexp.MustCompile(`\$appPath/bin/datly\s+(gen|translate)\s+.*-u=([^\s]+)\s+-s='([^']+)'(.*)`) + seen := map[string]bool{} + var result []parityRule + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + continue + } + lines := strings.Split(string(data), "\n") + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + m := re.FindStringSubmatch(line) + if len(m) < 4 { + continue + } + src := strings.TrimSpace(m[3]) + if !(strings.HasSuffix(src, ".dql") || strings.HasSuffix(src, ".sql")) { + continue + } + connector := inferRuleConnector("") + if len(m) >= 5 { + connector = inferRuleConnector(m[4]) + } + key := m[2] + "|" + src + if seen[key] { + continue + } + seen[key] = true + result = append(result, parityRule{ + Mode: strings.TrimSpace(m[1]), + Namespace: strings.TrimSpace(m[2]), + Source: src, + Connector: connector, + }) + } + } + sort.Slice(result, func(i, j int) bool { + if result[i].Namespace == result[j].Namespace { + return result[i].Source < result[j].Source + } + return result[i].Namespace < result[j].Namespace + }) + return result, nil +} + +func inferRuleConnector(tail string) string { + lower := strings.ToLower(tail) + switch { + case strings.Contains(lower, "$optionsaero"): + return "system" + case strings.Contains(lower, "$optionssitemgmt"): + return "sitemgmt" + case strings.Contains(lower, "$options"): + return "ci_ads" + default: + return "" + } +} + +func routeYAMLName(source string) string { + base := filepath.Base(source) + ext := filepath.Ext(base) + return strings.TrimSuffix(base, ext) + ".yaml" +} + +func compareParity(legacy, shapeViews []viewIR) []string { + var result []string + if len(legacy) != len(shapeViews) { + result = append(result, "view count mismatch") + } + legacyByName := map[string]viewIR{} + for _, v := range legacy { + legacyByName[strings.ToLower(v.Name)] = v + } + for _, s := range shapeViews { + l, ok := legacyByName[strings.ToLower(s.Name)] + if !ok { + result = append(result, "missing view in legacy: "+s.Name) + continue + } + if l.Table != "" && s.Table != "" && !strings.EqualFold(l.Table, s.Table) { + result = append(result, "table mismatch: "+s.Name) + } + if l.Connector != "" && s.Connector == "" { + result = append(result, "connector missing in shape: "+s.Name) + } + if l.Connector != "" && s.Connector != "" && !strings.EqualFold(strings.TrimSpace(l.Connector), strings.TrimSpace(s.Connector)) { + result = append(result, "connector mismatch: "+s.Name) + } + if l.SQLURI != "" && s.SQLURI == "" { + result = append(result, "sql uri missing in shape: "+s.Name) + } + if l.SQLURI != "" && s.SQLURI != "" && !equalSQLURI(l.SQLURI, s.SQLURI) { + result = append(result, "sql uri mismatch: "+s.Name) + } + } + return dedupe(result) +} + +func equalSQLURI(legacy, shape string) bool { + normalize := func(v string) string { + v = strings.ReplaceAll(strings.TrimSpace(v), "\\", "/") + return strings.TrimPrefix(v, "./") + } + return strings.EqualFold(normalize(legacy), normalize(shape)) +} + +func normalizeCardinality(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "one": + return "One" + case "many": + return "Many" + default: + return strings.TrimSpace(value) + } +} + +func inferShapeViewMode(sql string) string { + sql = strings.TrimSpace(sql) + if sql == "" { + return "" + } + statements := dqlstmt.New(sql) + hasRead := false + hasExec := false + for _, item := range statements { + if item == nil { + continue + } + switch item.Kind { + case dqlstmt.KindRead: + hasRead = true + case dqlstmt.KindExec: + hasExec = true + } + } + switch { + case hasRead && !hasExec: + return "SQLQuery" + case hasExec && !hasRead: + return "SQLExec" + case hasRead && hasExec: + return "SQLExec" + } + stmt := strings.ToLower(sql) + if strings.HasPrefix(stmt, "select") || strings.HasPrefix(stmt, "with") { + return "SQLQuery" + } + return "" +} + +func mergeShapeViewMetadata(meta []viewMetaIR, views view.Views) { + if len(meta) == 0 || len(views) == 0 { + return + } + index := map[string]int{} + for i, item := range meta { + index[strings.ToLower(strings.TrimSpace(item.Name))] = i + } + for _, candidate := range views { + if candidate == nil { + continue + } + key := strings.ToLower(strings.TrimSpace(candidate.Name)) + pos, ok := index[key] + if !ok { + continue + } + if mode := strings.TrimSpace(string(candidate.Mode)); mode != "" { + meta[pos].Mode = mode + } + if meta[pos].Module == "" { + meta[pos].Module = strings.TrimSpace(candidate.Module) + } + if meta[pos].AllowNulls == nil { + meta[pos].AllowNulls = candidate.AllowNulls + } + if candidate.Selector != nil { + if meta[pos].SelectorNamespace == "" { + meta[pos].SelectorNamespace = strings.TrimSpace(candidate.Selector.Namespace) + } + if meta[pos].SelectorNoLimit == nil { + meta[pos].SelectorNoLimit = &candidate.Selector.NoLimit + } + } + if candidate.Schema != nil { + if meta[pos].SchemaCardinality == "" { + meta[pos].SchemaCardinality = strings.TrimSpace(string(candidate.Schema.Cardinality)) + } + if meta[pos].SchemaType == "" { + meta[pos].SchemaType = firstNonEmpty(strings.TrimSpace(candidate.Schema.DataType), strings.TrimSpace(candidate.Schema.Name)) + } + } + if candidate.Template != nil && candidate.Template.Summary != nil { + value := true + meta[pos].HasSummary = &value + } + } +} + +func compareMetadataParity(legacyMeta, shapeMeta *resourceMetaIR, legacyViews, shapeViews []viewMetaIR) []string { + var result []string + if legacyMeta != nil && legacyMeta.ColumnsDiscovery != nil { + if shapeMeta == nil || shapeMeta.ColumnsDiscovery == nil { + result = append(result, "resource columnsDiscovery missing in shape") + } else if *legacyMeta.ColumnsDiscovery != *shapeMeta.ColumnsDiscovery { + result = append(result, "resource columnsDiscovery mismatch") + } + } + legacyByName := map[string]viewMetaIR{} + for _, item := range legacyViews { + legacyByName[strings.ToLower(strings.TrimSpace(item.Name))] = item + } + for _, shapeItem := range shapeViews { + key := strings.ToLower(strings.TrimSpace(shapeItem.Name)) + legacyItem, ok := legacyByName[key] + if !ok { + continue + } + if legacyItem.Mode != "" { + if shapeItem.Mode == "" { + result = append(result, "view mode missing in shape: "+shapeItem.Name) + } else if !strings.EqualFold(legacyItem.Mode, shapeItem.Mode) { + result = append(result, "view mode mismatch: "+shapeItem.Name) + } + } + if legacyItem.Module != "" { + if shapeItem.Module == "" { + result = append(result, "view module missing in shape: "+shapeItem.Name) + } else if !strings.EqualFold(strings.TrimSpace(legacyItem.Module), strings.TrimSpace(shapeItem.Module)) { + result = append(result, "view module mismatch: "+shapeItem.Name) + } + } + if legacyItem.AllowNulls != nil { + if shapeItem.AllowNulls == nil { + result = append(result, "view allowNulls missing in shape: "+shapeItem.Name) + } else if *legacyItem.AllowNulls != *shapeItem.AllowNulls { + result = append(result, "view allowNulls mismatch: "+shapeItem.Name) + } + } + if legacyItem.SelectorNamespace != "" { + if shapeItem.SelectorNamespace == "" { + result = append(result, "view selector namespace missing in shape: "+shapeItem.Name) + } else if !strings.EqualFold(strings.TrimSpace(legacyItem.SelectorNamespace), strings.TrimSpace(shapeItem.SelectorNamespace)) { + result = append(result, "view selector namespace mismatch: "+shapeItem.Name) + } + } + if legacyItem.SelectorNoLimit != nil { + if shapeItem.SelectorNoLimit == nil { + result = append(result, "view selector noLimit missing in shape: "+shapeItem.Name) + } else if *legacyItem.SelectorNoLimit != *shapeItem.SelectorNoLimit { + result = append(result, "view selector noLimit mismatch: "+shapeItem.Name) + } + } + if legacyItem.SchemaCardinality != "" { + if shapeItem.SchemaCardinality == "" { + result = append(result, "view schema cardinality missing in shape: "+shapeItem.Name) + } else if !strings.EqualFold(strings.TrimSpace(legacyItem.SchemaCardinality), strings.TrimSpace(shapeItem.SchemaCardinality)) { + result = append(result, "view schema cardinality mismatch: "+shapeItem.Name) + } + } + if legacyItem.SchemaType != "" { + if shapeItem.SchemaType == "" { + result = append(result, "view schema type missing in shape: "+shapeItem.Name) + } else if !strings.EqualFold(strings.TrimSpace(legacyItem.SchemaType), strings.TrimSpace(shapeItem.SchemaType)) { + result = append(result, "view schema type mismatch: "+shapeItem.Name) + } + } + if legacyItem.HasSummary != nil { + if shapeItem.HasSummary == nil { + result = append(result, "view template summary missing in shape: "+shapeItem.Name) + } else if *legacyItem.HasSummary != *shapeItem.HasSummary { + result = append(result, "view template summary mismatch: "+shapeItem.Name) + } + } + } + return dedupe(result) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + return value + } + } + return "" +} + +func normalizeLegacyParams(legacy legacyYAML) []paramIR { + querySelectors := map[string]string{} + querySelectorCacheable := map[string]*bool{} + querySelectorIn := map[string]string{} + for _, v := range legacy.Resource.Views { + viewName := strings.TrimSpace(v.Name) + for _, param := range []selectorParam{v.Selector.LimitParameter, v.Selector.OffsetParameter, v.Selector.PageParameter, v.Selector.FieldsParameter, v.Selector.OrderByParameter} { + name := strings.TrimSpace(param.Name) + if name == "" || viewName == "" { + continue + } + querySelectors[strings.ToLower(name)] = viewName + querySelectorIn[strings.ToLower(name)] = strings.TrimSpace(param.In.Name) + if param.Cacheable != nil { + value := *param.Cacheable + querySelectorCacheable[strings.ToLower(name)] = &value + } + } + } + result := make([]paramIR, 0, len(legacy.Resource.Parameters)) + seen := map[string]bool{} + for _, p := range legacy.Resource.Parameters { + name := strings.TrimSpace(p.Name) + item := paramIR{ + Name: name, + Kind: strings.TrimSpace(p.In.Kind), + In: strings.TrimSpace(p.In.Name), + Required: p.Required, + Cacheable: p.Cacheable, + URI: strings.TrimSpace(p.URI), + Value: strings.TrimSpace(p.Value), + } + if selector, ok := querySelectors[strings.ToLower(name)]; ok { + item.QuerySelector = selector + if item.Cacheable == nil { + item.Cacheable = querySelectorCacheable[strings.ToLower(name)] + } + } + for _, pred := range p.Predicates { + item.Predicates = append(item.Predicates, normalizePredicateSig(pred.Group, pred.Name, pred.Ensure, pred.Args)) + } + sort.Strings(item.Predicates) + result = append(result, item) + seen[strings.ToLower(name)] = true + } + for key, selector := range querySelectors { + if seen[key] { + continue + } + name := strings.TrimSpace(key) + if name == "" { + continue + } + legacyName := name + for _, v := range legacy.Resource.Views { + for _, param := range []selectorParam{v.Selector.LimitParameter, v.Selector.OffsetParameter, v.Selector.PageParameter, v.Selector.FieldsParameter, v.Selector.OrderByParameter} { + if strings.EqualFold(strings.TrimSpace(param.Name), key) { + legacyName = strings.TrimSpace(param.Name) + break + } + } + } + result = append(result, paramIR{ + Name: legacyName, + Kind: "query", + In: strings.TrimSpace(querySelectorIn[key]), + QuerySelector: selector, + Cacheable: querySelectorCacheable[key], + }) + } + sort.Slice(result, func(i, j int) bool { + if strings.EqualFold(result[i].Name, result[j].Name) { + if strings.EqualFold(result[i].Kind, result[j].Kind) { + return strings.ToLower(result[i].In) < strings.ToLower(result[j].In) + } + return strings.ToLower(result[i].Kind) < strings.ToLower(result[j].Kind) + } + return strings.ToLower(result[i].Name) < strings.ToLower(result[j].Name) + }) + return result +} + +func normalizeLegacyTypes(legacy legacyYAML) []typeIR { + if len(legacy.Resource.Types) == 0 { + return nil + } + result := make([]typeIR, 0, len(legacy.Resource.Types)) + seen := map[string]bool{} + for _, item := range legacy.Resource.Types { + name := strings.TrimSpace(item.Name) + if name == "" { + continue + } + key := strings.ToLower(name) + if seen[key] { + continue + } + seen[key] = true + result = append(result, typeIR{ + Name: name, + Alias: strings.TrimSpace(item.Alias), + DataType: normalizeTypeSignature(item.DataType), + Cardinality: normalizeCardinality(strings.TrimSpace(item.Cardinality)), + Package: strings.TrimSpace(item.Package), + ModulePath: strings.TrimSpace(item.ModulePath), + }) + } + sort.Slice(result, func(i, j int) bool { + return strings.ToLower(result[i].Name) < strings.ToLower(result[j].Name) + }) + return result +} + +func normalizeShapeTypes(planned *plan.Result, sourcePath string) []typeIR { + if planned == nil { + return nil + } + modulePrefix := inferModulePrefix(sourcePath) + typeImportByAlias, typeImportByPkg := typeImports(planned) + byName := map[string]typeIR{} + + register := func(item typeIR, overwrite bool) { + name := strings.TrimSpace(item.Name) + if name == "" { + return + } + key := strings.ToLower(name) + if existing, ok := byName[key]; ok { + if (overwrite || existing.DataType == "") && item.DataType != "" { + existing.DataType = item.DataType + } + if (overwrite || existing.Cardinality == "") && item.Cardinality != "" { + existing.Cardinality = item.Cardinality + } + if (overwrite || existing.Package == "") && item.Package != "" { + existing.Package = item.Package + } + if (overwrite || existing.ModulePath == "") && item.ModulePath != "" { + existing.ModulePath = item.ModulePath + } + if overwrite && item.Alias != "" { + existing.Alias = item.Alias + } + byName[key] = existing + return + } + byName[key] = item + } + + for _, item := range planned.Views { + if item == nil { + continue + } + dataType := strings.TrimSpace(item.SchemaType) + name := typeNameFromDataType(dataType) + if name == "" && item.ElementType != nil { + name = strings.TrimSpace(item.ElementType.Name()) + if dataType == "" && name != "" { + dataType = "*" + name + } + } + if name == "" { + continue + } + pkg := packageFromDataType(dataType) + modulePath := "" + if strings.TrimSpace(item.Module) != "" && modulePrefix != "" { + modulePath = modulePrefix + strings.Trim(strings.TrimSpace(item.Module), "/") + } + if modulePath == "" && pkg != "" { + modulePath = firstNonEmpty(typeImportByAlias[strings.ToLower(pkg)], typeImportByPkg[strings.ToLower(pkg)]) + } + register(typeIR{ + Name: name, + DataType: normalizeTypeSignature(dataType), + Cardinality: normalizeCardinality(strings.TrimSpace(item.Cardinality)), + Package: pkg, + ModulePath: modulePath, + }, false) + } + + for _, item := range planned.States { + if item == nil || strings.TrimSpace(item.DataType) == "" { + continue + } + dataType := strings.TrimSpace(item.DataType) + name := typeNameFromDataType(dataType) + if name == "" { + continue + } + pkg := packageFromDataType(dataType) + modulePath := firstNonEmpty(typeImportByAlias[strings.ToLower(pkg)], typeImportByPkg[strings.ToLower(pkg)]) + register(typeIR{ + Name: name, + DataType: normalizeTypeSignature(dataType), + Package: pkg, + ModulePath: modulePath, + }, false) + } + for _, item := range planned.Types { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + register(typeIR{ + Name: strings.TrimSpace(item.Name), + Alias: strings.TrimSpace(item.Alias), + DataType: normalizeTypeSignature(item.DataType), + Cardinality: normalizeCardinality(strings.TrimSpace(item.Cardinality)), + Package: strings.TrimSpace(item.Package), + ModulePath: strings.TrimSpace(item.ModulePath), + }, true) + } + + result := make([]typeIR, 0, len(byName)) + for _, item := range byName { + result = append(result, item) + } + sort.Slice(result, func(i, j int) bool { + return strings.ToLower(result[i].Name) < strings.ToLower(result[j].Name) + }) + return result +} + +func compareTypeParity(legacy, shapeTypes []typeIR) []string { + var result []string + if len(legacy) == 0 { + return nil + } + shapeByName := map[string]typeIR{} + for _, item := range shapeTypes { + shapeByName[strings.ToLower(strings.TrimSpace(item.Name))] = item + } + for _, legacyType := range legacy { + key := strings.ToLower(strings.TrimSpace(legacyType.Name)) + shapeType, ok := shapeByName[key] + if !ok { + result = append(result, "missing type in shape: "+legacyType.Name) + continue + } + if legacyType.DataType != "" && shapeType.DataType != "" && legacyType.DataType != shapeType.DataType { + result = append(result, "type dataType mismatch: "+legacyType.Name) + } + if legacyType.Cardinality != "" && shapeType.Cardinality != "" && !strings.EqualFold(legacyType.Cardinality, shapeType.Cardinality) { + result = append(result, "type cardinality mismatch: "+legacyType.Name) + } + if legacyType.Package != "" && shapeType.Package != "" && !strings.EqualFold(legacyType.Package, shapeType.Package) { + result = append(result, "type package mismatch: "+legacyType.Name) + } + if legacyType.ModulePath != "" && shapeType.ModulePath != "" && !strings.EqualFold(legacyType.ModulePath, shapeType.ModulePath) { + result = append(result, "type module path mismatch: "+legacyType.Name) + } + if legacyType.Alias != "" && shapeType.Alias != "" && !strings.EqualFold(legacyType.Alias, shapeType.Alias) { + result = append(result, "type alias mismatch: "+legacyType.Name) + } + } + return dedupe(result) +} + +func normalizeTypeContextIR(defaultPackage, packageDir, packageName, packagePath string) *typeCtxIR { + ret := &typeCtxIR{ + DefaultPackage: strings.TrimSpace(defaultPackage), + PackageDir: strings.TrimSpace(packageDir), + PackageName: strings.TrimSpace(packageName), + PackagePath: strings.TrimSpace(packagePath), + } + if ret.DefaultPackage == "" && ret.PackageDir == "" && ret.PackageName == "" && ret.PackagePath == "" { + return nil + } + return ret +} + +func compareTypeContextParity(legacy, shape *typeCtxIR) []string { + if legacy == nil { + return nil + } + if shape == nil { + return []string{"missing type context in shape"} + } + var result []string + if legacy.DefaultPackage != "" && shape.DefaultPackage != "" && !strings.EqualFold(legacy.DefaultPackage, shape.DefaultPackage) { + result = append(result, "type context default package mismatch") + } + if legacy.PackageDir != "" && shape.PackageDir != "" && !strings.EqualFold(legacy.PackageDir, shape.PackageDir) { + result = append(result, "type context package dir mismatch") + } + if legacy.PackageName != "" && shape.PackageName != "" && !strings.EqualFold(legacy.PackageName, shape.PackageName) { + result = append(result, "type context package name mismatch") + } + if legacy.PackagePath != "" && shape.PackagePath != "" && !strings.EqualFold(legacy.PackagePath, shape.PackagePath) { + result = append(result, "type context package path mismatch") + } + return dedupe(result) +} + +func normalizeTypeSignature(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + parts := strings.Fields(value) + return strings.Join(parts, " ") +} + +func typeNameFromDataType(dataType string) string { + dataType = strings.TrimSpace(dataType) + if dataType == "" { + return "" + } + dataType = strings.TrimLeft(dataType, "*[]") + if dataType == "" { + return "" + } + if idx := strings.LastIndex(dataType, "."); idx != -1 { + dataType = dataType[idx+1:] + } + if idx := strings.Index(dataType, "{"); idx != -1 { + dataType = dataType[:idx] + } + return strings.TrimSpace(dataType) +} + +func packageFromDataType(dataType string) string { + dataType = strings.TrimSpace(dataType) + dataType = strings.TrimLeft(dataType, "*[]") + if idx := strings.LastIndex(dataType, "."); idx != -1 { + return strings.TrimSpace(dataType[:idx]) + } + return "" +} + +func inferModulePrefix(sourcePath string) string { + normalized := filepath.ToSlash(strings.TrimSpace(sourcePath)) + if normalized == "" { + return "" + } + const marker = "/src/" + idx := strings.Index(normalized, marker) + if idx == -1 { + return "" + } + root := normalized[idx+len(marker):] + if slash := strings.Index(root, "/dql/"); slash != -1 { + root = root[:slash] + } + root = strings.Trim(root, "/") + if root == "" { + return "" + } + return root + "/pkg/" +} + +func typeImports(planned *plan.Result) (map[string]string, map[string]string) { + byAlias := map[string]string{} + byPkg := map[string]string{} + if planned == nil || planned.TypeContext == nil { + return byAlias, byPkg + } + appendPkg := func(pkg string) { + pkg = strings.TrimSpace(pkg) + if pkg == "" { + return + } + base := pkg + if idx := strings.LastIndex(base, "/"); idx != -1 { + base = base[idx+1:] + } + base = strings.ToLower(strings.TrimSpace(base)) + if base != "" { + byPkg[base] = pkg + } + } + if packagePath := strings.TrimSpace(planned.TypeContext.PackagePath); packagePath != "" { + appendPkg(packagePath) + if pkgName := strings.ToLower(strings.TrimSpace(planned.TypeContext.PackageName)); pkgName != "" { + byAlias[pkgName] = packagePath + byPkg[pkgName] = packagePath + } + } + appendPkg(planned.TypeContext.DefaultPackage) + for _, item := range planned.TypeContext.Imports { + pkg := strings.TrimSpace(item.Package) + if pkg == "" { + continue + } + if alias := strings.ToLower(strings.TrimSpace(item.Alias)); alias != "" { + byAlias[alias] = pkg + } + appendPkg(pkg) + } + return byAlias, byPkg +} + +func normalizeShapeParams(planned *plan.Result) []paramIR { + if planned == nil || len(planned.States) == 0 { + return nil + } + result := make([]paramIR, 0, len(planned.States)) + for _, s := range planned.States { + if s == nil { + continue + } + item := paramIR{ + Name: strings.TrimSpace(s.Name), + Kind: strings.TrimSpace(s.Kind), + In: strings.TrimSpace(s.In), + Required: s.Required, + Cacheable: s.Cacheable, + URI: strings.TrimSpace(s.URI), + Value: strings.TrimSpace(s.Value), + QuerySelector: strings.TrimSpace(s.QuerySelector), + } + for _, pred := range s.Predicates { + if pred == nil { + continue + } + item.Predicates = append(item.Predicates, normalizePredicateSig(pred.Group, pred.Name, pred.Ensure, pred.Arguments)) + } + sort.Strings(item.Predicates) + result = append(result, item) + } + sort.Slice(result, func(i, j int) bool { + if strings.EqualFold(result[i].Name, result[j].Name) { + if strings.EqualFold(result[i].Kind, result[j].Kind) { + return strings.ToLower(result[i].In) < strings.ToLower(result[j].In) + } + return strings.ToLower(result[i].Kind) < strings.ToLower(result[j].Kind) + } + return strings.ToLower(result[i].Name) < strings.ToLower(result[j].Name) + }) + return result +} + +func normalizePredicateSig(group int, name string, ensure bool, args []string) string { + parts := make([]string, 0, len(args)) + for _, arg := range args { + parts = append(parts, strings.TrimSpace(arg)) + } + return strings.ToLower(strings.TrimSpace(name)) + "|" + strconv.Itoa(group) + "|" + strconv.FormatBool(ensure) + "|" + strings.Join(parts, ",") +} + +func compareParamParity(legacy, shapeParams []paramIR) []string { + var result []string + legacyByKey := map[string]paramIR{} + for _, item := range filterComparableParams(legacy) { + legacyByKey[paramKey(item)] = item + } + shapeByKey := map[string]paramIR{} + for _, item := range filterComparableParams(shapeParams) { + shapeByKey[paramKey(item)] = item + } + if len(legacyByKey) != len(shapeByKey) { + result = append(result, "parameter count mismatch") + } + for key, legacyItem := range legacyByKey { + shapeItem, ok := shapeByKey[key] + if !ok { + result = append(result, "missing parameter in shape: "+legacyItem.Name) + continue + } + if legacyItem.Required != nil && shapeItem.Required != nil && *legacyItem.Required != *shapeItem.Required { + result = append(result, "parameter required mismatch: "+legacyItem.Name) + } + if legacyItem.Cacheable != nil && shapeItem.Cacheable != nil && *legacyItem.Cacheable != *shapeItem.Cacheable { + result = append(result, "parameter cacheable mismatch: "+legacyItem.Name) + } + if legacyItem.QuerySelector != "" && !strings.EqualFold(legacyItem.QuerySelector, shapeItem.QuerySelector) { + result = append(result, "parameter query selector mismatch: "+legacyItem.Name) + } + if legacyItem.URI != "" && !strings.EqualFold(strings.TrimSpace(legacyItem.URI), strings.TrimSpace(shapeItem.URI)) { + result = append(result, "parameter uri mismatch: "+legacyItem.Name) + } + if len(legacyItem.Predicates) != len(shapeItem.Predicates) { + result = append(result, "parameter predicates count mismatch: "+legacyItem.Name) + continue + } + for i := range legacyItem.Predicates { + if legacyItem.Predicates[i] != shapeItem.Predicates[i] { + result = append(result, "parameter predicate mismatch: "+legacyItem.Name) + break + } + } + } + return dedupe(result) +} + +func paramKey(item paramIR) string { + kind := strings.ToLower(strings.TrimSpace(item.Kind)) + in := strings.ToLower(strings.TrimSpace(item.In)) + if kind == "component" { + in = normalizeComponentRef(in) + } + return strings.ToLower(strings.TrimSpace(item.Name)) + "|" + kind + "|" + in +} + +func normalizeComponentRef(in string) string { + in = strings.TrimSpace(strings.TrimPrefix(in, "get:")) + if in == "" { + return in + } + in = strings.TrimPrefix(in, "../") + in = strings.TrimPrefix(in, "./") + in = strings.TrimPrefix(in, "/") + if idx := strings.LastIndex(in, "/"); idx != -1 { + return in[idx+1:] + } + return in +} + +func filterComparableParams(items []paramIR) []paramIR { + if len(items) == 0 { + return nil + } + result := make([]paramIR, 0, len(items)) + for _, item := range items { + kind := strings.ToLower(strings.TrimSpace(item.Kind)) + switch kind { + case "output", "meta", "async": + continue + default: + result = append(result, item) + } + } + return result +} + +func dedupe(items []string) []string { + if len(items) == 0 { + return nil + } + seen := map[string]bool{} + var ret []string + for _, item := range items { + if item == "" || seen[item] { + continue + } + seen[item] = true + ret = append(ret, item) + } + sort.Strings(ret) + return ret +} + +func topIssues(counter map[string]int, limit int) []string { + type pair struct { + Issue string + Count int + } + var list []pair + for issue, count := range counter { + list = append(list, pair{Issue: issue, Count: count}) + } + sort.Slice(list, func(i, j int) bool { + if list[i].Count == list[j].Count { + return list[i].Issue < list[j].Issue + } + return list[i].Count > list[j].Count + }) + if len(list) > limit { + list = list[:limit] + } + var ret []string + for _, item := range list { + ret = append(ret, item.Issue) + } + return ret +} + +func writeIRFile(path string, v parityOutput) { + _ = os.MkdirAll(filepath.Dir(path), 0o755) + writeYAML(path, v) +} + +func writeYAML(path string, v interface{}) { + data, err := yaml.Marshal(v) + if err != nil { + return + } + _ = os.WriteFile(path, data, 0o644) +} diff --git a/repository/shape/platform_parity_types_test.go b/repository/shape/platform_parity_types_test.go new file mode 100644 index 000000000..76341f3f9 --- /dev/null +++ b/repository/shape/platform_parity_types_test.go @@ -0,0 +1,86 @@ +package shape_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" +) + +func TestNormalizeTypeSignature(t *testing.T) { + assert.Equal(t, "struct{ Id int; Name string }", normalizeTypeSignature(" struct{ Id int; Name string } ")) +} + +func TestTypeNameFromDataType(t *testing.T) { + assert.Equal(t, "TvAffiliateStationView", typeNameFromDataType("*tvaffiliatestation.TvAffiliateStationView")) + assert.Equal(t, "Output", typeNameFromDataType("*Output")) + assert.Equal(t, "struct", typeNameFromDataType("struct{Id int}")) +} + +func TestCompareTypeParity(t *testing.T) { + legacy := []typeIR{{ + Name: "TvAffiliateStationView", + DataType: "*tvaffiliatestation.TvAffiliateStationView", + Package: "tvaffiliatestation", + ModulePath: "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", + }} + shapeTypes := []typeIR{{ + Name: "TvAffiliateStationView", + DataType: "*tvaffiliatestation.TvAffiliateStationView", + Package: "tvaffiliatestation", + ModulePath: "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", + }} + assert.Empty(t, compareTypeParity(legacy, shapeTypes)) +} + +func TestNormalizeShapeTypes(t *testing.T) { + planned := &plan.Result{ + TypeContext: &typectx.Context{ + PackagePath: "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", + PackageName: "tvaffiliatestation", + }, + Views: []*plan.View{ + { + Name: "tvAffiliateStation", + Module: "platform/tvaffiliatestation", + SchemaType: "*tvaffiliatestation.TvAffiliateStationView", + Cardinality: "many", + }, + }, + } + actual := normalizeShapeTypes(planned, "/Users/awitas/go/src/github.vianttech.com/viant/platform/dql/platform/tvaffiliatestation/tvaffiliatestation.dql") + if assert.Len(t, actual, 1) { + assert.Equal(t, "TvAffiliateStationView", actual[0].Name) + assert.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", actual[0].ModulePath) + assert.Equal(t, "Many", actual[0].Cardinality) + } +} + +func TestTypeImports_UsesTypeContextPackagePath(t *testing.T) { + planned := &plan.Result{ + TypeContext: &typectx.Context{ + PackagePath: "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", + PackageName: "tvaffiliatestation", + }, + } + byAlias, byPkg := typeImports(planned) + assert.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", byAlias["tvaffiliatestation"]) + assert.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", byPkg["tvaffiliatestation"]) +} + +func TestCompareTypeContextParity(t *testing.T) { + legacy := &typeCtxIR{ + DefaultPackage: "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", + PackageDir: "pkg/platform/tvaffiliatestation", + PackageName: "tvaffiliatestation", + PackagePath: "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", + } + shape := &typeCtxIR{ + DefaultPackage: "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", + PackageDir: "pkg/platform/tvaffiliatestation", + PackageName: "tvaffiliatestation", + PackagePath: "github.vianttech.com/viant/platform/pkg/platform/tvaffiliatestation", + } + assert.Empty(t, compareTypeContextParity(legacy, shape)) +} diff --git a/repository/shape/shape.go b/repository/shape/shape.go index 570a63d5e..5f7f766d8 100644 --- a/repository/shape/shape.go +++ b/repository/shape/shape.go @@ -3,6 +3,11 @@ package shape import "context" type ( + CompileMixedMode string + CompileUnknownNonReadMode string + CompileProfile string + CompileColumnDiscoveryMode string + // Scanner discovers shape descriptors from Source. Scanner interface { Scan(ctx context.Context, source *Source, opts ...ScanOption) (*ScanResult, error) @@ -33,7 +38,19 @@ type ( ScanOptions struct{} PlanOptions struct{} LoadOptions struct{} - CompileOptions struct{} + CompileOptions struct { + Strict bool + Profile CompileProfile + MixedMode CompileMixedMode + UnknownNonReadMode CompileUnknownNonReadMode + ColumnDiscoveryMode CompileColumnDiscoveryMode + DQLPathMarker string + RoutesRelativePath string + TypePackageDir string + TypePackageName string + TypePackagePath string + InferTypeContext *bool + } ScanOption func(*ScanOptions) PlanOption func(*PlanOptions) @@ -41,6 +58,22 @@ type ( CompileOption func(*CompileOptions) ) +const ( + CompileMixedModeExecWins CompileMixedMode = "exec_wins" + CompileMixedModeReadWins CompileMixedMode = "read_wins" + CompileMixedModeErrorOnMixed CompileMixedMode = "error_on_mixed" + + CompileUnknownNonReadWarn CompileUnknownNonReadMode = "warn" + CompileUnknownNonReadError CompileUnknownNonReadMode = "error" + + CompileProfileCompat CompileProfile = "compat" + CompileProfileStrict CompileProfile = "strict" + + CompileColumnDiscoveryAuto CompileColumnDiscoveryMode = "auto" + CompileColumnDiscoveryOn CompileColumnDiscoveryMode = "on" + CompileColumnDiscoveryOff CompileColumnDiscoveryMode = "off" +) + // Engine is a thin facade over scan -> plan -> load pipeline. type Engine struct { options *Options @@ -139,7 +172,15 @@ func (e *Engine) compile(ctx context.Context, source *Source) (*PlanResult, erro if e.options.Compiler == nil { return nil, ErrCompilerNotConfigured } - return e.options.Compiler.Compile(ctx, source) + return e.options.Compiler.Compile( + ctx, + source, + WithCompileStrict(e.options.Strict), + WithCompileProfile(e.options.CompileProfile), + WithMixedMode(e.options.CompileMixedMode), + WithUnknownNonReadMode(e.options.UnknownNonReadMode), + WithColumnDiscoveryMode(e.options.ColumnDiscoveryMode), + ) } func (e *Engine) scanAndPlan(ctx context.Context, source *Source) (*PlanResult, error) { diff --git a/repository/shape/typectx/context.go b/repository/shape/typectx/context.go new file mode 100644 index 000000000..072e22b9f --- /dev/null +++ b/repository/shape/typectx/context.go @@ -0,0 +1,89 @@ +package typectx + +import ( + "path" + "strings" +) + +// ValidationIssue captures context consistency problems. +type ValidationIssue struct { + Field string + Message string +} + +// Normalize trims and canonicalizes context fields. +func Normalize(input *Context) *Context { + if input == nil { + return nil + } + ret := &Context{ + DefaultPackage: strings.TrimSpace(input.DefaultPackage), + PackageDir: cleanSlashes(strings.TrimSpace(input.PackageDir)), + PackageName: strings.TrimSpace(input.PackageName), + PackagePath: cleanSlashes(strings.TrimSpace(input.PackagePath)), + } + if ret.PackageName == "" { + if ret.PackagePath != "" { + ret.PackageName = path.Base(ret.PackagePath) + } else if ret.PackageDir != "" { + ret.PackageName = path.Base(ret.PackageDir) + } + } + if ret.DefaultPackage == "" && ret.PackagePath != "" { + ret.DefaultPackage = ret.PackagePath + } + for _, item := range input.Imports { + pkg := cleanSlashes(strings.TrimSpace(item.Package)) + if pkg == "" { + continue + } + alias := strings.TrimSpace(item.Alias) + if alias == "" { + alias = path.Base(pkg) + } + ret.Imports = append(ret.Imports, Import{ + Alias: alias, + Package: pkg, + }) + } + if ret.DefaultPackage == "" && + len(ret.Imports) == 0 && + ret.PackageDir == "" && + ret.PackageName == "" && + ret.PackagePath == "" { + return nil + } + return ret +} + +// Validate checks context consistency. +func Validate(ctx *Context) []ValidationIssue { + ctx = Normalize(ctx) + if ctx == nil { + return nil + } + var result []ValidationIssue + if strings.Contains(ctx.PackageName, "/") { + result = append(result, ValidationIssue{ + Field: "PackageName", + Message: "package name must not contain path separators", + }) + } + if ctx.PackagePath != "" && strings.Contains(ctx.PackagePath, ".") { + base := path.Base(ctx.PackagePath) + if ctx.PackageName != "" && base != ctx.PackageName { + result = append(result, ValidationIssue{ + Field: "PackagePath", + Message: "package path basename differs from package name", + }) + } + } + return result +} + +func cleanSlashes(value string) string { + value = strings.ReplaceAll(value, "\\", "/") + value = strings.TrimSpace(value) + value = strings.Trim(value, "/") + return value +} diff --git a/repository/shape/typectx/context_test.go b/repository/shape/typectx/context_test.go new file mode 100644 index 000000000..325709448 --- /dev/null +++ b/repository/shape/typectx/context_test.go @@ -0,0 +1,31 @@ +package typectx + +import "testing" + +func TestNormalize_FillsPackageFields(t *testing.T) { + ctx := Normalize(&Context{ + PackageDir: "pkg/platform/taxonomy", + PackagePath: "github.vianttech.com/viant/platform/pkg/platform/taxonomy", + }) + if ctx == nil { + t.Fatalf("expected normalized context") + } + if ctx.PackageName != "taxonomy" { + t.Fatalf("expected package name taxonomy, got %q", ctx.PackageName) + } + if ctx.DefaultPackage != "github.vianttech.com/viant/platform/pkg/platform/taxonomy" { + t.Fatalf("expected default package from package path, got %q", ctx.DefaultPackage) + } +} + +func TestValidate_DetectsInvalidPackageName(t *testing.T) { + issues := Validate(&Context{ + PackageName: "platform/taxonomy", + }) + if len(issues) == 0 { + t.Fatalf("expected validation issue") + } + if issues[0].Field != "PackageName" { + t.Fatalf("expected PackageName issue, got %q", issues[0].Field) + } +} diff --git a/repository/shape/typectx/model.go b/repository/shape/typectx/model.go index ae76febe5..acc03ca18 100644 --- a/repository/shape/typectx/model.go +++ b/repository/shape/typectx/model.go @@ -10,6 +10,9 @@ type Import struct { type Context struct { DefaultPackage string `json:",omitempty" yaml:",omitempty"` Imports []Import `json:",omitempty" yaml:",omitempty"` + PackageDir string `json:",omitempty" yaml:",omitempty"` + PackageName string `json:",omitempty" yaml:",omitempty"` + PackagePath string `json:",omitempty" yaml:",omitempty"` } // Provenance tracks where a resolved type came from. diff --git a/repository/shape/typectx/resolver.go b/repository/shape/typectx/resolver.go index daccf3b42..892c9ef6f 100644 --- a/repository/shape/typectx/resolver.go +++ b/repository/shape/typectx/resolver.go @@ -2,7 +2,6 @@ package typectx import ( "fmt" - "path" "sort" "strings" @@ -115,6 +114,9 @@ func (r *Resolver) aliasPackage(alias string) string { return item.Package } } + if r.context.PackageName != "" && r.context.PackagePath != "" && r.context.PackageName == alias { + return r.context.PackagePath + } return "" } @@ -177,6 +179,7 @@ func (r *Resolver) searchPackages() []scopedPackage { seen[pkg] = true result = append(result, scopedPackage{pkg: pkg, matchKind: matchKind}) } + appendPkg(r.context.PackagePath, "package_path") appendPkg(r.context.DefaultPackage, "default_package") for _, item := range r.context.Imports { appendPkg(item.Package, "import_package") @@ -237,30 +240,7 @@ func packageOf(key string) string { } func normalizeContext(input *Context) *Context { - if input == nil { - return nil - } - ret := &Context{ - DefaultPackage: strings.TrimSpace(input.DefaultPackage), - } - for _, item := range input.Imports { - pkg := strings.TrimSpace(item.Package) - if pkg == "" { - continue - } - alias := strings.TrimSpace(item.Alias) - if alias == "" { - alias = path.Base(pkg) - } - ret.Imports = append(ret.Imports, Import{ - Alias: alias, - Package: pkg, - }) - } - if ret.DefaultPackage == "" && len(ret.Imports) == 0 { - return nil - } - return ret + return Normalize(input) } func splitQualified(value string) (prefix string, name string, alias bool, qualified bool) { diff --git a/repository/shape/typectx/resolver_matrix_test.go b/repository/shape/typectx/resolver_matrix_test.go new file mode 100644 index 000000000..473ba368b --- /dev/null +++ b/repository/shape/typectx/resolver_matrix_test.go @@ -0,0 +1,86 @@ +package typectx + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/x" +) + +type matrixOrderDefault struct{} +type matrixOrderImport struct{} +type matrixOrderPkgPath struct{} +type matrixOrderAliasImport struct{} + +func TestResolver_ResolutionMatrix(t *testing.T) { + reg := x.NewRegistry() + reg.Register(x.NewType(reflect.TypeOf(matrixOrderDefault{}), x.WithPkgPath("github.com/acme/default"), x.WithName("Order"))) + reg.Register(x.NewType(reflect.TypeOf(matrixOrderImport{}), x.WithPkgPath("github.com/acme/imported"), x.WithName("ImportedOrder"))) + reg.Register(x.NewType(reflect.TypeOf(matrixOrderPkgPath{}), x.WithPkgPath("github.com/acme/pkgpath"), x.WithName("Order"))) + reg.Register(x.NewType(reflect.TypeOf(matrixOrderAliasImport{}), x.WithPkgPath("github.com/acme/alias/import"), x.WithName("Order"))) + + testCases := []struct { + name string + context *Context + expr string + wantKey string + ambiguous bool + }{ + { + name: "only default/imports", + context: &Context{ + DefaultPackage: "github.com/acme/default", + Imports: []Import{{Alias: "imp", Package: "github.com/acme/imported"}}, + }, + expr: "Order", + wantKey: "github.com/acme/default.Order", + }, + { + name: "only package triple", + context: &Context{ + PackagePath: "github.com/acme/pkgpath", + PackageName: "pkgpath", + PackageDir: "pkg/pkgpath", + }, + expr: "Order", + wantKey: "github.com/acme/pkgpath.Order", + }, + { + name: "default and package path conflict", + context: &Context{ + DefaultPackage: "github.com/acme/default", + PackagePath: "github.com/acme/pkgpath", + PackageName: "pkgpath", + }, + expr: "Order", + ambiguous: true, + }, + { + name: "alias import wins over package-name fallback", + context: &Context{ + PackagePath: "github.com/acme/pkgpath", + PackageName: "same", + Imports: []Import{{Alias: "same", Package: "github.com/acme/alias/import"}}, + }, + expr: "same.Order", + wantKey: "github.com/acme/alias/import.Order", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + resolver := NewResolver(reg, testCase.context) + key, err := resolver.Resolve(testCase.expr) + if testCase.ambiguous { + require.Error(t, err) + _, ok := err.(*AmbiguityError) + require.True(t, ok) + require.Empty(t, key) + return + } + require.NoError(t, err) + require.Equal(t, testCase.wantKey, key) + }) + } +} diff --git a/repository/shape/typectx/resolver_test.go b/repository/shape/typectx/resolver_test.go index f1e8e6761..632a785da 100644 --- a/repository/shape/typectx/resolver_test.go +++ b/repository/shape/typectx/resolver_test.go @@ -87,3 +87,28 @@ func TestResolver_ResolveWithProvenance(t *testing.T) { require.Equal(t, "/repo/mdp/performance/order.go", resolved.Provenance.File) require.Equal(t, "resource_type", resolved.Provenance.Kind) } + +func TestResolver_Resolve_Unqualified_PackagePath(t *testing.T) { + reg := x.NewRegistry() + reg.Register(x.NewType(reflect.TypeOf(resolveOrder{}), x.WithPkgPath("github.com/acme/mdp/performance"), x.WithName("Order"))) + resolver := NewResolver(reg, &Context{PackagePath: "github.com/acme/mdp/performance"}) + + resolved, err := resolver.ResolveWithProvenance("Order") + require.NoError(t, err) + require.NotNil(t, resolved) + require.Equal(t, "github.com/acme/mdp/performance.Order", resolved.ResolvedKey) + require.Equal(t, "package_path", resolved.MatchKind) +} + +func TestResolver_Resolve_Qualified_PackageNameFallback(t *testing.T) { + reg := x.NewRegistry() + reg.Register(x.NewType(reflect.TypeOf(resolveOrder{}), x.WithPkgPath("github.com/acme/mdp/performance"), x.WithName("Order"))) + resolver := NewResolver(reg, &Context{ + PackageName: "performance", + PackagePath: "github.com/acme/mdp/performance", + }) + + key, err := resolver.Resolve("performance.Order") + require.NoError(t, err) + require.Equal(t, "github.com/acme/mdp/performance.Order", key) +} diff --git a/repository/shape/xgen/generator.go b/repository/shape/xgen/generator.go index 89622576b..d0fc419a8 100644 --- a/repository/shape/xgen/generator.go +++ b/repository/shape/xgen/generator.go @@ -26,6 +26,7 @@ func GenerateFromDQLShape(doc *shape.Document, cfg *Config) (*Result, error) { if cfg == nil { cfg = &Config{} } + hydrateConfigFromTypeContext(doc, cfg) applyDefaults(cfg) projectDir, packageDir, err := resolvePaths(cfg.ProjectDir, cfg.PackageDir) if err != nil { @@ -131,10 +132,14 @@ func rewriteSafetyIssues(doc *shape.Document, cfg *Config, projectDir string) [] UseGOPATHFallback: policy.useGOPATH, }) var issues []string - for _, resolution := range doc.TypeResolutions { + for i := range doc.TypeResolutions { + resolution := &doc.TypeResolutions[i] if srcResolver != nil && strings.TrimSpace(resolution.Provenance.File) == "" { - pkg := firstNonEmpty(strings.TrimSpace(resolution.Provenance.Package), packageOfKey(resolution.ResolvedKey)) + pkg := inferResolutionPackage(*resolution, doc.TypeContext) name := typeNameFromKey(resolution.ResolvedKey) + if name == "" { + name = strings.TrimSpace(resolution.Expression) + } if pkg != "" && name != "" { if file, err := srcResolver.ResolveTypeFile(pkg, name); err == nil { resolution.Provenance.File = file @@ -144,7 +149,7 @@ func rewriteSafetyIssues(doc *shape.Document, cfg *Config, projectDir string) [] } } } - if issue := resolutionSafetyIssue(resolution, policy); issue != "" { + if issue := resolutionSafetyIssue(*resolution, policy); issue != "" { issues = append(issues, issue) } } @@ -152,6 +157,41 @@ func rewriteSafetyIssues(doc *shape.Document, cfg *Config, projectDir string) [] return uniqueStrings(issues) } +func hydrateConfigFromTypeContext(doc *shape.Document, cfg *Config) { + if doc == nil || cfg == nil || doc.TypeContext == nil { + return + } + if cfg.PackageDir == "" { + cfg.PackageDir = strings.TrimSpace(doc.TypeContext.PackageDir) + } + if cfg.PackageName == "" { + cfg.PackageName = strings.TrimSpace(doc.TypeContext.PackageName) + } + if cfg.PackagePath == "" { + cfg.PackagePath = strings.TrimSpace(doc.TypeContext.PackagePath) + } +} + +func inferResolutionPackage(resolution typectx.Resolution, ctx *typectx.Context) string { + pkg := strings.TrimSpace(resolution.Provenance.Package) + if pkg != "" { + return pkg + } + pkg = packageOfKey(resolution.ResolvedKey) + if pkg != "" { + return pkg + } + if ctx != nil { + if pkg = strings.TrimSpace(ctx.PackagePath); pkg != "" { + return pkg + } + if pkg = strings.TrimSpace(ctx.DefaultPackage); pkg != "" { + return pkg + } + } + return "" +} + func resolutionSafetyIssue(resolution typectx.Resolution, policy rewritePolicy) string { kind := strings.TrimSpace(strings.ToLower(resolution.Provenance.Kind)) if kind == "" { diff --git a/repository/shape/xgen/generator_test.go b/repository/shape/xgen/generator_test.go index 3315f148d..eb8f35b13 100644 --- a/repository/shape/xgen/generator_test.go +++ b/repository/shape/xgen/generator_test.go @@ -256,6 +256,181 @@ type DQLOrderView struct { Old string ` + "`json:\"old,omitempty\"`" + ` } +func TestGenerateFromDQLShape_UsesTypeContextPackageDefaults(t *testing.T) { + projectDir := t.TempDir() + if err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module example.com/demo\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatalf("write go.mod failed: %v", err) + } + doc := &dqlshape.Document{ + TypeContext: &typectx.Context{ + PackageDir: "pkg/platform/taxonomy", + PackageName: "taxonomy", + PackagePath: "example.com/demo/pkg/platform/taxonomy", + }, + Root: map[string]any{ + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "orders", + "ColumnsConfig": map[string]any{ + "ID": map[string]any{"Name": "ID", "DataType": "int"}, + }, + }, + }, + }, + }, + } + result, err := GenerateFromDQLShape(doc, &Config{ProjectDir: projectDir}) + if err != nil { + t.Fatalf("generate failed: %v", err) + } + if result == nil { + t.Fatalf("expected result") + } + if result.PackageName != "taxonomy" { + t.Fatalf("expected package name taxonomy, got %q", result.PackageName) + } + if result.PackagePath != "example.com/demo/pkg/platform/taxonomy" { + t.Fatalf("expected package path from type context, got %q", result.PackagePath) + } + if !strings.Contains(filepath.ToSlash(result.FilePath), "/pkg/platform/taxonomy/") { + t.Fatalf("expected file under type-context package dir, got %s", result.FilePath) + } +} + +func TestGenerateFromDQLShape_ProvenanceEnrichment_WithReplaceAndTypeContextPackagePath(t *testing.T) { + root := t.TempDir() + projectDir := filepath.Join(root, "project") + modelsDir := filepath.Join(root, "shared-models") + if err := os.MkdirAll(filepath.Join(projectDir, "internal", "gen"), 0o755); err != nil { + t.Fatalf("mkdir project failed: %v", err) + } + if err := os.MkdirAll(filepath.Join(modelsDir, "mdp"), 0o755); err != nil { + t.Fatalf("mkdir models failed: %v", err) + } + if err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module example.com/project\n\ngo 1.25\nreplace github.com/acme/models => ../shared-models\n"), 0o644); err != nil { + t.Fatalf("write project go.mod failed: %v", err) + } + if err := os.WriteFile(filepath.Join(modelsDir, "go.mod"), []byte("module github.com/acme/models\n\ngo 1.25\n"), 0o644); err != nil { + t.Fatalf("write models go.mod failed: %v", err) + } + if err := os.WriteFile(filepath.Join(modelsDir, "mdp", "types.go"), []byte("package mdp\ntype Order struct{}\n"), 0o644); err != nil { + t.Fatalf("write types.go failed: %v", err) + } + dest := filepath.Join(projectDir, "internal", "gen", "shapes_gen.go") + if err := os.WriteFile(dest, []byte("package gen\n"), 0o644); err != nil { + t.Fatalf("seed file failed: %v", err) + } + + doc := &dqlshape.Document{ + TypeContext: &typectx.Context{ + PackagePath: "github.com/acme/models/mdp", + }, + Root: map[string]any{ + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "orders", + "ColumnsConfig": map[string]any{ + "ID": map[string]any{"Name": "ID", "DataType": "int"}, + }, + }, + }, + }, + }, + TypeResolutions: []typectx.Resolution{ + { + Expression: "Order", + ResolvedKey: "Order", + Provenance: typectx.Provenance{ + Kind: "registry", + }, + }, + }, + } + + _, err := GenerateFromDQLShape(doc, &Config{ + ProjectDir: projectDir, + PackageDir: "internal/gen", + PackageName: "gen", + FileName: "shapes_gen.go", + AllowedSourceRoots: []string{modelsDir}, + }) + if err != nil { + t.Fatalf("expected provenance enrichment to allow rewrite, got: %v", err) + } +} + +func TestGenerateFromDQLShape_ProvenanceEnrichment_WithGOPATHFallback(t *testing.T) { + root := t.TempDir() + projectDir := filepath.Join(root, "project") + gopath := filepath.Join(root, "gopath") + modelsDir := filepath.Join(gopath, "src", "github.com", "legacy", "models") + if err := os.MkdirAll(filepath.Join(projectDir, "internal", "gen"), 0o755); err != nil { + t.Fatalf("mkdir project failed: %v", err) + } + if err := os.MkdirAll(modelsDir, 0o755); err != nil { + t.Fatalf("mkdir models failed: %v", err) + } + if err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module example.com/project\n\ngo 1.25\n"), 0o644); err != nil { + t.Fatalf("write project go.mod failed: %v", err) + } + if err := os.WriteFile(filepath.Join(modelsDir, "types.go"), []byte("package models\ntype Legacy struct{}\n"), 0o644); err != nil { + t.Fatalf("write types.go failed: %v", err) + } + dest := filepath.Join(projectDir, "internal", "gen", "shapes_gen.go") + if err := os.WriteFile(dest, []byte("package gen\n"), 0o644); err != nil { + t.Fatalf("seed file failed: %v", err) + } + + orig := os.Getenv("GOPATH") + if err := os.Setenv("GOPATH", gopath); err != nil { + t.Fatalf("set GOPATH failed: %v", err) + } + defer func() { _ = os.Setenv("GOPATH", orig) }() + + doc := &dqlshape.Document{ + TypeContext: &typectx.Context{ + PackagePath: "github.com/legacy/models", + }, + Root: map[string]any{ + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "legacy", + "ColumnsConfig": map[string]any{ + "ID": map[string]any{"Name": "ID", "DataType": "int"}, + }, + }, + }, + }, + }, + TypeResolutions: []typectx.Resolution{ + { + Expression: "Legacy", + ResolvedKey: "Legacy", + Provenance: typectx.Provenance{Kind: "registry"}, + }, + }, + } + _, err := GenerateFromDQLShape(doc, &Config{ + ProjectDir: projectDir, + PackageDir: "internal/gen", + PackageName: "gen", + FileName: "shapes_gen.go", + AllowedSourceRoots: []string{filepath.Join(gopath, "src")}, + UseGoModuleResolve: boolPtr(false), + UseGOPATHFallback: boolPtr(true), + }) + if err != nil { + t.Fatalf("expected GOPATH provenance enrichment to allow rewrite, got: %v", err) + } +} + +func boolPtr(value bool) *bool { + return &value +} + func KeepCustom() string { return "ok" } ` if err := os.WriteFile(dest, []byte(initial), 0o644); err != nil { diff --git a/service/executor/expand/evaluator.go b/service/executor/expand/evaluator.go index c733aca79..0c6dee479 100644 --- a/service/executor/expand/evaluator.go +++ b/service/executor/expand/evaluator.go @@ -35,7 +35,12 @@ type ( func WithCustomContexts(ctx ...*Variable) EvaluatorOption { return func(c *config) { - c.embededTypes = append(c.embededTypes, ctx...) + for _, item := range ctx { + if item == nil { + continue + } + c.embededTypes = append(c.embededTypes, item) + } } } @@ -47,7 +52,12 @@ func WithContext(ctx context.Context) EvaluatorOption { func WithVariable(namedVariable ...*NamedVariable) EvaluatorOption { return func(c *config) { - c.namedVariables = append(c.namedVariables, namedVariable...) + for _, item := range namedVariable { + if item == nil { + continue + } + c.namedVariables = append(c.namedVariables, item) + } } } @@ -65,6 +75,9 @@ func WithSetLiteral(setLiterals func(state *structology.State) error) EvaluatorO func WithTypeLookup(lookup xreflect.LookupType) EvaluatorOption { return func(c *config) { + if lookup == nil { + return + } c.typeLookup = lookup } } @@ -141,12 +154,18 @@ func NewEvaluator(template string, options ...EvaluatorOption) (*Evaluator, erro } for _, valueType := range aConfig.embededTypes { + if valueType == nil { + continue + } if err = evaluator.planner.EmbedVariable(valueType.Type); err != nil { return nil, err } } for _, variable := range aConfig.namedVariables { + if variable == nil { + continue + } if err = evaluator.planner.DefineVariable(variable.Name, variable.Type); err != nil { return nil, err } @@ -181,6 +200,9 @@ func NewEvaluator(template string, options ...EvaluatorOption) (*Evaluator, erro func createConfig(options []EvaluatorOption) *config { instance := newConfig() for _, option := range options { + if option == nil { + continue + } option(instance) } diff --git a/service/executor/expand/fn_new.go b/service/executor/expand/fn_new.go index 6f5b78286..6cbdf10ed 100644 --- a/service/executor/expand/fn_new.go +++ b/service/executor/expand/fn_new.go @@ -41,7 +41,7 @@ func (n *newer) NewResultType(call *expr.Call) (reflect.Type, error) { expression, ok := call.Args[0].(*expr.Literal) if !ok { - return nil, fmt.Errorf("expected arg to be type of %T but was %T", expression, call.Args[1]) + return nil, fmt.Errorf("expected arg to be type of %T but was %T", expression, call.Args[0]) } return types.LookupType(n.lookup, expression.Value) diff --git a/service/executor/expand/fn_printer.go b/service/executor/expand/fn_printer.go index 3eaabe2c9..620e7ef57 100644 --- a/service/executor/expand/fn_printer.go +++ b/service/executor/expand/fn_printer.go @@ -61,7 +61,7 @@ func (p *Printer) Println(args ...interface{}) string { func (p *Printer) Printf(format string, args ...interface{}) string { p.derefArgs(args) - fmt.Printf(p.Sprintf(format, args...)) + fmt.Print(p.Sprintf(format, args...)) return "" } @@ -107,12 +107,12 @@ func (p *Printer) Fatal(any interface{}, args ...interface{}) (string, error) { format, ok := any.(string) if ok { - return "", fmt.Errorf(p.Sprintf(format, args...)) + return "", fmt.Errorf("%s", p.Sprintf(format, args...)) } if err, ok := any.(error); ok { return "", err } - return "", fmt.Errorf(p.Sprintf("%+v", any)) + return "", fmt.Errorf("%s", p.Sprintf("%+v", any)) } // Fatalf fatal with formatting @@ -124,7 +124,7 @@ func (p *Printer) Fatalf(any interface{}, args ...interface{}) (string, error) { func (p *Printer) FatalfWithCode(code int, any interface{}, args ...interface{}) (string, error) { format, ok := any.(string) if ok { - return "", response.NewError(code, fmt.Sprintf(p.Sprintf(format, args...))) + return "", response.NewError(code, p.Sprintf(format, args...)) } if err, ok := any.(error); ok { return "", response.NewError(code, err.Error(), response.WithError(err)) diff --git a/service/jobs/service.go b/service/jobs/service.go index c2e8ac530..caaee6a19 100644 --- a/service/jobs/service.go +++ b/service/jobs/service.go @@ -2,6 +2,7 @@ package jobs import ( "context" + "errors" "fmt" "github.com/viant/datly/service/dbms" "github.com/viant/datly/service/reader" @@ -44,7 +45,7 @@ func (s *Service) matchFailedJob(matchKey string) (*async.Job, error) { if candidate.MatchKey == matchKey { var err error if candidate.Error != nil { - err = fmt.Errorf(*candidate.Error) + err = errors.New(*candidate.Error) } else { err = fmt.Errorf("job has status %s", candidate.Status) } diff --git a/service/session/state.go b/service/session/state.go index a629db3c9..5779250fc 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -471,13 +471,14 @@ func (s *Session) ensureValidValue(value interface{}, parameter *state.Parameter rawType = rawType.Elem() } - if rawType.Kind() != reflect.Struct { - break - } - if elem.Kind() == reflect.Interface && !elem.IsNil() { elem = elem.Elem() } + if rawType.Kind() != reflect.Struct { + value = elem.Interface() + valueType = reflect.TypeOf(value) + break + } if elem.Kind() == reflect.Ptr { value = elem.Interface() valueType = elem.Type() diff --git a/shared/combine.go b/shared/combine.go index b63329fa2..67cbbecca 100644 --- a/shared/combine.go +++ b/shared/combine.go @@ -7,7 +7,7 @@ func CombineErrors(header string, errors []error) error { return nil } - outputErr := fmt.Errorf(header) + outputErr := fmt.Errorf("%s", header) for _, err := range errors { outputErr = fmt.Errorf("%w; %v", outputErr, err.Error()) } diff --git a/utils/httputils/violation.go b/utils/httputils/violation.go index 7ff1c0a35..912c7f19c 100644 --- a/utils/httputils/violation.go +++ b/utils/httputils/violation.go @@ -50,7 +50,7 @@ func (v Violations) MergeErrors(errors []*response.Error) validator.Violations { aViolation := &validator.Violation{ Location: anError.View + "/" + anError.Parameter, Value: anError.Object, - Check: fmt.Sprint("%T", anError.Error()), + Check: fmt.Sprintf("%T", anError.Error()), Message: anError.Message, } ret = append(ret, aViolation) diff --git a/utils/types/types.go b/utils/types/types.go index dc29f1235..8e7ccd784 100644 --- a/utils/types/types.go +++ b/utils/types/types.go @@ -1,6 +1,7 @@ package types import ( + "fmt" "github.com/viant/sqlx/io" "github.com/viant/xreflect" "reflect" @@ -11,6 +12,9 @@ func LookupType(lookup xreflect.LookupType, typeName string, opts ...xreflect.Op if ok { return rType, nil } + if lookup == nil { + return nil, fmt.Errorf("type %q was not found and no lookup resolver is configured", typeName) + } return lookup(typeName, opts...) } diff --git a/view/tags/parameter_test.go b/view/tags/parameter_test.go index 6cb562220..aa27bb52e 100644 --- a/view/tags/parameter_test.go +++ b/view/tags/parameter_test.go @@ -24,7 +24,7 @@ func TestTag_updateParameter(t *testing.T) { { description: "async Parameter", tag: `parameter:"p1,kind=query,in=qp1,scope=async"`, - expect: &Parameter{Name: "p1", Kind: "query", In: "qp1", Scope: "myscope"}, + expect: &Parameter{Name: "p1", Kind: "query", In: "qp1", Scope: "async"}, }, } diff --git a/view/tags/view_test.go b/view/tags/view_test.go index e85c3f2d3..1127cf66c 100644 --- a/view/tags/view_test.go +++ b/view/tags/view_test.go @@ -29,7 +29,7 @@ func TestTag_updateView(t *testing.T) { description: "basic view", tag: `view:"foo,connector=dev" sql:"uri=testdata/foo.sql"`, expectView: &View{Name: "foo", Connector: "dev"}, - expectSQL: ViewSQL{SQL: "SELECT * FROM FOO"}, + expectSQL: ViewSQL{SQL: "SELECT * FROM FOO", URI: "testdata/foo.sql"}, expectTag: "foo,connector=dev", }, { diff --git a/view/view.go b/view/view.go index 9274c8053..b297489f1 100644 --- a/view/view.go +++ b/view/view.go @@ -1298,7 +1298,7 @@ func (v *View) markColumnsAsFilterable() error { for _, colName := range v.Selector.Constraints.Filterable { column, err := v._columns.Lookup(colName) if err != nil { - return fmt.Errorf("criteria column %v, on view has not been defined, %w", colName, v.Name, err) + return fmt.Errorf("criteria column %v on view %v has not been defined: %w", colName, v.Name, err) } column.Filterable = true } diff --git a/warmup/cache_test.go b/warmup/cache_test.go index e691c8357..408a4a31a 100644 --- a/warmup/cache_test.go +++ b/warmup/cache_test.go @@ -2,14 +2,13 @@ package warmup import ( "context" + "path" + "testing" + "github.com/stretchr/testify/assert" - "github.com/viant/afs" - "github.com/viant/datly/gateway/router" "github.com/viant/datly/internal/tests" "github.com/viant/datly/service/reader" "github.com/viant/datly/view" - "path" - "testing" ) func TestPopulateCache(t *testing.T) { @@ -59,14 +58,14 @@ func TestPopulateCache(t *testing.T) { resourcePath := path.Join("testdata", testCase.URL, "resource.yaml") - resource, err := router.NewResourceFromURL(context.TODO(), afs.New(), resourcePath, false) + resource, err := view.NewResourceFromURL(context.TODO(), resourcePath, nil, nil) if !assert.Nil(t, err, testCase.description) { continue } var views []*view.View - for _, route := range resource.Routes { - views = append(views, route.View) + for _, item := range resource.Views { + views = append(views, item) } inserted, err := PopulateCache(views) @@ -100,7 +99,7 @@ func checkIfCached(t *testing.T, cache *view.Cache, ctx context.Context, testCas builder := reader.NewBuilder() for _, cacheInput := range input { - build, err := builder.CacheSQL(aView, cacheInput.Selector) + build, err := builder.CacheSQL(ctx, aView, cacheInput.Selector) if err != nil { return err } @@ -116,7 +115,7 @@ func checkIfCached(t *testing.T, cache *view.Cache, ctx context.Context, testCas } if cacheInput.IndexMeta && aView.Template.Summary != nil { - metaIndex, err := builder.CacheMetaSQL(aView, cacheInput.Selector, &view.BatchData{ + metaIndex, err := builder.CacheMetaSQL(ctx, aView, cacheInput.Selector, &view.BatchData{ ValuesBatch: testCase.metaIndexed, Values: testCase.metaIndexed, }, nil, nil) From e7fcb5c34355196ed51603af6b4ddba91d046813 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 23 Feb 2026 09:42:13 -0800 Subject: [PATCH 126/279] Implemented near-full shape-engine parity with the legacy internal translator by expanding DQL compile/load (relations, handler/dml paths, diagnostics with line/char mapping, type-context defaults/resolution, declaration/settings directives, and metadata/type parity), and validated parity across platform routes with 0 mismatches in the all-sources sweep. Added explicit column-discovery policy controls (auto/on/off) with default auto behavior that requires discovery for SELECT * or missing concrete shape, preserves schema column order with append-only newly discovered columns, and fails compilation when discovery is required but disabled. --- cmd/command/translate_shape.go | 197 ++++++++ cmd/command/translate_shape_test.go | 30 ++ cmd/options/query_test.go | 34 ++ cmd/options/rule_engine_test.go | 21 + gateway/dql_bootstrap.go | 453 +++++++++++++++++++ gateway/dql_bootstrap_test.go | 122 +++++ internal/inference/join_test.go | 56 +++ internal/testutil/sqlnormalizer/cases.go | 43 ++ internal/translator/parser/sanitizer_test.go | 222 +++++++++ service/executor/expand/evaluator_test.go | 78 ++++ service/session/selector_injector_test.go | 89 ++++ 11 files changed, 1345 insertions(+) create mode 100644 cmd/command/translate_shape.go create mode 100644 cmd/command/translate_shape_test.go create mode 100644 cmd/options/query_test.go create mode 100644 cmd/options/rule_engine_test.go create mode 100644 gateway/dql_bootstrap.go create mode 100644 gateway/dql_bootstrap_test.go create mode 100644 internal/inference/join_test.go create mode 100644 internal/testutil/sqlnormalizer/cases.go create mode 100644 internal/translator/parser/sanitizer_test.go create mode 100644 service/executor/expand/evaluator_test.go create mode 100644 service/session/selector_injector_test.go diff --git a/cmd/command/translate_shape.go b/cmd/command/translate_shape.go new file mode 100644 index 000000000..109e99634 --- /dev/null +++ b/cmd/command/translate_shape.go @@ -0,0 +1,197 @@ +package command + +import ( + "context" + "encoding/json" + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/viant/afs/file" + "github.com/viant/afs/url" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/repository/shape" + shapeCompile "github.com/viant/datly/repository/shape/compile" + shapeLoad "github.com/viant/datly/repository/shape/load" + datlyservice "github.com/viant/datly/service" + "github.com/viant/datly/shared" + "github.com/viant/datly/view" + "gopkg.in/yaml.v3" +) + +func (s *Service) translateShape(ctx context.Context, opts *options.Options) error { + rule := opts.Rule() + compiler := shapeCompile.New() + loader := shapeLoad.New() + for rule.Index = 0; rule.Index < len(rule.Source); rule.Index++ { + sourceURL := rule.SourceURL() + _, name := url.Split(sourceURL, file.Scheme) + fmt.Printf("translating %v (shape)\n", name) + dql, err := rule.LoadSource(ctx, s.fs, sourceURL) + if err != nil { + return err + } + dql = strings.TrimSpace(dql) + if dql == "" { + return fmt.Errorf("source %s was empty", sourceURL) + } + shapeSource := &shape.Source{ + Name: strings.TrimSuffix(name, path.Ext(name)), + Path: url.Path(sourceURL), + DQL: dql, + Connector: strings.TrimSpace(rule.Connector), + } + planResult, err := compiler.Compile(ctx, shapeSource) + if err != nil { + return fmt.Errorf("failed to compile %s: %w", sourceURL, err) + } + componentArtifact, err := loader.LoadComponent(ctx, planResult) + if err != nil { + return fmt.Errorf("failed to load %s: %w", sourceURL, err) + } + component, ok := componentArtifact.Component.(*shapeLoad.Component) + if !ok || component == nil { + return fmt.Errorf("unexpected component artifact for %s", sourceURL) + } + if err = s.persistShapeRoute(ctx, opts, sourceURL, dql, componentArtifact.Resource, component); err != nil { + return err + } + } + paths := url.Join(opts.Repository().RepositoryURL, "Datly", "routes", "paths.yaml") + if ok, _ := s.fs.Exists(ctx, paths); ok { + _ = s.fs.Delete(ctx, paths) + } + return nil +} + +type shapeRuleFile struct { + Resource *view.Resource `yaml:"Resource,omitempty"` + Routes []*repository.Component `yaml:"Routes,omitempty"` + TypeContext any `yaml:"TypeContext,omitempty"` +} + +func (s *Service) persistShapeRoute(ctx context.Context, opts *options.Options, sourceURL, dql string, resource *view.Resource, component *shapeLoad.Component) error { + rule := opts.Rule() + routeYAML, routeRoot, relDir, stem, err := routePathForShape(rule, opts.Repository().RepositoryURL, sourceURL) + if err != nil { + return err + } + if resource != nil { + for _, item := range resource.Views { + if item == nil || item.Template == nil { + continue + } + if strings.TrimSpace(item.Template.Source) == "" { + continue + } + sqlRel := strings.TrimSpace(item.Template.SourceURL) + if sqlRel == "" { + sqlRel = path.Join(stem, item.Name+".sql") + } + sqlDest := path.Join(routeRoot, relDir, filepath.ToSlash(sqlRel)) + if err = s.fs.Upload(ctx, sqlDest, file.DefaultFileOsMode, strings.NewReader(item.Template.Source)); err != nil { + return fmt.Errorf("failed to persist sql %s: %w", sqlDest, err) + } + item.Template.SourceURL = sqlRel + } + } + rootView := "" + if component != nil { + rootView = strings.TrimSpace(component.RootView) + } + if rootView == "" && resource != nil && len(resource.Views) > 0 && resource.Views[0] != nil { + rootView = resource.Views[0].Name + } + method, uri := parseShapeRulePath(dql, rule.RuleName(), opts.Repository().APIPrefix) + route := &repository.Component{ + Path: contract.Path{ + Method: method, + URI: uri, + }, + Contract: contract.Contract{ + Service: serviceForMethod(method), + }, + View: &view.View{Reference: shared.Reference{Ref: rootView}}, + } + if component != nil { + route.TypeContext = component.TypeContext + if component.Directives != nil && component.Directives.MCP != nil { + route.Name = strings.TrimSpace(component.Directives.MCP.Name) + route.Description = strings.TrimSpace(component.Directives.MCP.Description) + route.DescriptionURI = strings.TrimSpace(component.Directives.MCP.DescriptionPath) + } + } + payload := &shapeRuleFile{ + Resource: resource, + Routes: []*repository.Component{route}, + } + if component != nil && component.TypeContext != nil { + payload.TypeContext = component.TypeContext + } + data, err := yaml.Marshal(payload) + if err != nil { + return err + } + if err = s.fs.Upload(ctx, routeYAML, file.DefaultFileOsMode, strings.NewReader(string(data))); err != nil { + return fmt.Errorf("failed to persist route yaml %s: %w", routeYAML, err) + } + return nil +} + +func routePathForShape(rule *options.Rule, repoURL, sourceURL string) (routeYAML string, routeRoot string, relDir string, stem string, err error) { + sourcePath := filepath.Clean(url.Path(sourceURL)) + basePath := filepath.Clean(rule.BaseRuleURL()) + relative, relErr := filepath.Rel(basePath, sourcePath) + if relErr != nil || strings.HasPrefix(relative, "..") { + relative = filepath.Base(sourcePath) + } + relative = filepath.ToSlash(relative) + relDir = filepath.ToSlash(path.Dir(relative)) + if relDir == "." { + relDir = "" + } + stem = strings.TrimSuffix(path.Base(relative), path.Ext(relative)) + routeRoot = url.Join(repoURL, "Datly", "routes") + routeYAML = url.Join(routeRoot, relDir, stem+".yaml") + return routeYAML, routeRoot, relDir, stem, nil +} + +type shapeRuleHeader struct { + Method string `json:"Method"` + URI string `json:"URI"` +} + +func parseShapeRulePath(dql, ruleName, apiPrefix string) (string, string) { + method := "GET" + uri := "/" + strings.Trim(strings.TrimSpace(ruleName), "/") + if prefix := strings.TrimSpace(apiPrefix); prefix != "" { + uri = strings.TrimRight(prefix, "/") + uri + } + start := strings.Index(dql, "/*") + end := strings.Index(dql, "*/") + if start != -1 && end > start+2 { + raw := strings.TrimSpace(dql[start+2 : end]) + if strings.HasPrefix(raw, "{") && strings.HasSuffix(raw, "}") { + header := &shapeRuleHeader{} + if err := json.Unmarshal([]byte(raw), header); err == nil { + if candidate := strings.TrimSpace(strings.ToUpper(header.Method)); candidate != "" { + method = candidate + } + if candidate := strings.TrimSpace(header.URI); candidate != "" { + uri = candidate + } + } + } + } + return method, uri +} + +func serviceForMethod(method string) datlyservice.Type { + if strings.EqualFold(method, "GET") { + return datlyservice.TypeReader + } + return datlyservice.TypeExecutor +} diff --git a/cmd/command/translate_shape_test.go b/cmd/command/translate_shape_test.go new file mode 100644 index 000000000..b76fac4eb --- /dev/null +++ b/cmd/command/translate_shape_test.go @@ -0,0 +1,30 @@ +package command + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/cmd/options" +) + +func TestParseShapeRulePath(t *testing.T) { + method, uri := parseShapeRulePath(`/* {"Method":"POST","URI":"/v1/api/orders"} */ SELECT 1`, "orders", "/v1/api") + assert.Equal(t, "POST", method) + assert.Equal(t, "/v1/api/orders", uri) + + method, uri = parseShapeRulePath(`SELECT 1`, "orders", "/v1/api") + assert.Equal(t, "GET", method) + assert.Equal(t, "/v1/api/orders", uri) +} + +func TestRoutePathForShape(t *testing.T) { + rule := &options.Rule{Project: "/repo", Source: []string{"/repo/dql/platform/campaign/post.dql"}} + routeYAML, routeRoot, relDir, stem, err := routePathForShape(rule, "/repo/dev", "/repo/dql/platform/campaign/post.dql") + require.NoError(t, err) + assert.Equal(t, "/repo/dev/Datly/routes/platform/campaign/post.yaml", routeYAML) + assert.Equal(t, "/repo/dev/Datly/routes", routeRoot) + assert.Equal(t, filepath.ToSlash("platform/campaign"), relDir) + assert.Equal(t, "post", stem) +} diff --git a/cmd/options/query_test.go b/cmd/options/query_test.go new file mode 100644 index 000000000..055d3915d --- /dev/null +++ b/cmd/options/query_test.go @@ -0,0 +1,34 @@ +package options + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/internal/testutil/sqlnormalizer" + "github.com/viant/sqlparser" +) + +func parserOption() sqlparser.Option { + return sqlparser.WithErrorHandler(nil) +} + +func TestRule_NormalizeSQL(t *testing.T) { + for _, testCase := range sqlnormalizer.Cases() { + t.Run(testCase.Name, func(t *testing.T) { + rule := &Rule{Generated: testCase.Generated} + actual := rule.NormalizeSQL(testCase.SQL, parserOption) + require.Equal(t, testCase.Expect, actual) + }) + } +} + +func TestMapper_Map(t *testing.T) { + m := mapper{"a": "A"} + require.Equal(t, "A", m.Map("a")) + require.Equal(t, "b", m.Map("b")) +} + +func TestNormalizeName(t *testing.T) { + require.Equal(t, "UserAlias", normalizeName("user_alias")) + require.Equal(t, "UserAlias", normalizeName("UserAlias")) +} diff --git a/cmd/options/rule_engine_test.go b/cmd/options/rule_engine_test.go new file mode 100644 index 000000000..bac95c6cb --- /dev/null +++ b/cmd/options/rule_engine_test.go @@ -0,0 +1,21 @@ +package options + +import "testing" + +func TestRule_EffectiveEngine(t *testing.T) { + testCases := []struct { + name string + engine string + want string + }{ + {name: "default", engine: "", want: EngineLegacy}, + {name: "shape", engine: "shape", want: EngineShape}, + {name: "invalid", engine: "other", want: EngineLegacy}, + } + for _, testCase := range testCases { + rule := &Rule{Engine: testCase.engine} + if got := rule.EffectiveEngine(); got != testCase.want { + t.Fatalf("%s: got %s, want %s", testCase.name, got, testCase.want) + } + } +} diff --git a/gateway/dql_bootstrap.go b/gateway/dql_bootstrap.go new file mode 100644 index 000000000..fe7d62dbc --- /dev/null +++ b/gateway/dql_bootstrap.go @@ -0,0 +1,453 @@ +package gateway + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/repository/shape" + shapeCompile "github.com/viant/datly/repository/shape/compile" + shapeLoad "github.com/viant/datly/repository/shape/load" + datlyservice "github.com/viant/datly/service" + "github.com/viant/datly/view" +) + +func (r *Service) applyDQLBootstrap(ctx context.Context, repo *repository.Service, cfg *DQLBootstrap) error { + if cfg == nil || len(cfg.Sources) == 0 { + return nil + } + sources, err := discoverDQLBootstrapSources(cfg.Sources, cfg.Exclude) + if err != nil { + return err + } + if len(sources) == 0 { + return fmt.Errorf("no DQL bootstrap sources matched") + } + compiler := shapeCompile.New() + loader := shapeLoad.New() + precedence := cfg.EffectivePrecedence() + var errors []error + for _, sourcePath := range sources { + component, err := compileBootstrapComponent(ctx, compiler, loader, repo, sourcePath, cfg, r.Config.APIPrefix) + if err != nil { + if cfg.ShouldFailFast() { + return err + } + errors = append(errors, err) + continue + } + exists, lookupErr := hasRepositoryProvider(ctx, repo, &component.Path) + if lookupErr != nil { + if cfg.ShouldFailFast() { + return lookupErr + } + errors = append(errors, lookupErr) + continue + } + if exists { + switch precedence { + case DQLBootstrapPrecedenceRoutesWins: + continue + case DQLBootstrapPrecedenceErrorOnMixed: + err = fmt.Errorf("DQL bootstrap conflict for %s:%s", component.Method, component.URI) + if cfg.ShouldFailFast() { + return err + } + errors = append(errors, err) + continue + } + } + repo.Register(component) + } + if len(errors) > 0 { + return fmt.Errorf("DQL bootstrap completed with %d errors: %w", len(errors), errors[0]) + } + return nil +} + +func compileBootstrapComponent(ctx context.Context, compiler *shapeCompile.DQLCompiler, loader *shapeLoad.Loader, repo *repository.Service, sourcePath string, cfg *DQLBootstrap, apiPrefix string) (*repository.Component, error) { + data, err := os.ReadFile(sourcePath) + if err != nil { + return nil, fmt.Errorf("failed to read DQL bootstrap source %s: %w", sourcePath, err) + } + dql := strings.TrimSpace(string(data)) + if dql == "" { + return nil, fmt.Errorf("empty DQL bootstrap source: %s", sourcePath) + } + sourceName := strings.TrimSuffix(filepath.Base(sourcePath), filepath.Ext(sourcePath)) + source := &shape.Source{ + Name: sourceName, + Path: sourcePath, + DQL: dql, + } + planResult, err := compiler.Compile(ctx, source, compileOptionsFromBootstrap(cfg)...) + if err != nil { + return nil, fmt.Errorf("failed to compile DQL bootstrap source %s: %w", sourcePath, err) + } + componentArtifact, err := loader.LoadComponent(ctx, planResult) + if err != nil { + return nil, fmt.Errorf("failed to load DQL bootstrap source %s: %w", sourcePath, err) + } + normalizeBootstrapInlineSQL(componentArtifact.Resource) + mergeBootstrapSharedResources(componentArtifact.Resource, repo) + loaded, ok := componentArtifact.Component.(*shapeLoad.Component) + if !ok || loaded == nil { + return nil, fmt.Errorf("unexpected shape component artifact for %s", sourcePath) + } + rootView := lookupRootView(componentArtifact.Resource, loaded.RootView) + if rootView == nil { + return nil, fmt.Errorf("missing root view %q for %s", loaded.RootView, sourcePath) + } + method, uri := resolvePathSettings(sourcePath, dql, apiPrefix) + componentModel := &repository.Component{ + Path: contract.Path{ + Method: method, + URI: uri, + }, + Contract: contract.Contract{ + Service: defaultServiceForMethod(method, rootView), + }, + View: rootView, + TypeContext: loaded.TypeContext, + } + loadOptions := []repository.Option{} + if repo != nil { + loadOptions = append(loadOptions, repository.WithResources(repo.Resources())) + loadOptions = append(loadOptions, repository.WithExtensions(repo.Extensions())) + } + components, err := repository.LoadComponentsFromMap(ctx, map[string]any{ + "Resource": componentArtifact.Resource, + "Components": []*repository.Component{componentModel}, + }, loadOptions...) + if err != nil { + return nil, fmt.Errorf("failed to materialize bootstrap component for %s: %w", sourcePath, err) + } + if err = components.Init(ctx); err != nil { + return nil, fmt.Errorf("failed to initialize bootstrap component for %s: %w", sourcePath, err) + } + if len(components.Components) == 0 || components.Components[0] == nil { + return nil, fmt.Errorf("empty initialized bootstrap component for %s", sourcePath) + } + return components.Components[0], nil +} + +func mergeBootstrapSharedResources(target *view.Resource, repo *repository.Service) { + if target == nil || repo == nil || repo.Resources() == nil { + return + } + if connectors, err := repo.Resources().Lookup(view.ResourceConnectors); err == nil && connectors != nil && connectors.Resource != nil { + target.MergeFrom(connectors.Resource, nil) + } + if constants, err := repo.Resources().Lookup(view.ResourceConstants); err == nil && constants != nil && constants.Resource != nil { + target.MergeFrom(constants.Resource, nil) + } +} + +func normalizeBootstrapInlineSQL(resource *view.Resource) { + if resource == nil { + return + } + for _, item := range resource.Views { + if item == nil || item.Template == nil { + continue + } + // DQL bootstrap compiles from in-memory source; keep SQL inline and avoid filesystem lookups. + item.Template.SourceURL = "" + } +} + +func defaultServiceForMethod(method string, rootView *view.View) datlyservice.Type { + if strings.EqualFold(method, "GET") { + return datlyservice.TypeReader + } + if rootView != nil && rootView.Mode == view.ModeQuery { + return datlyservice.TypeReader + } + return datlyservice.TypeExecutor +} + +func hasRepositoryProvider(ctx context.Context, repo *repository.Service, path *contract.Path) (bool, error) { + if repo == nil || repo.Registry() == nil || path == nil { + return false, nil + } + _, err := repo.Registry().LookupProvider(ctx, path) + if err != nil { + message := strings.ToLower(strings.TrimSpace(err.Error())) + if strings.Contains(message, "not found") { + return false, nil + } + return false, err + } + return true, nil +} + +func compileOptionsFromBootstrap(cfg *DQLBootstrap) []shape.CompileOption { + if cfg == nil { + return nil + } + var result []shape.CompileOption + switch strings.ToLower(strings.TrimSpace(cfg.CompileProfile)) { + case string(shape.CompileProfileStrict): + result = append(result, shape.WithCompileProfile(shape.CompileProfileStrict)) + case string(shape.CompileProfileCompat): + result = append(result, shape.WithCompileProfile(shape.CompileProfileCompat)) + } + switch strings.ToLower(strings.TrimSpace(cfg.MixedMode)) { + case string(shape.CompileMixedModeExecWins): + result = append(result, shape.WithMixedMode(shape.CompileMixedModeExecWins)) + case string(shape.CompileMixedModeReadWins): + result = append(result, shape.WithMixedMode(shape.CompileMixedModeReadWins)) + case string(shape.CompileMixedModeErrorOnMixed): + result = append(result, shape.WithMixedMode(shape.CompileMixedModeErrorOnMixed)) + } + switch strings.ToLower(strings.TrimSpace(cfg.UnknownNonReadMode)) { + case string(shape.CompileUnknownNonReadWarn): + result = append(result, shape.WithUnknownNonReadMode(shape.CompileUnknownNonReadWarn)) + case string(shape.CompileUnknownNonReadError): + result = append(result, shape.WithUnknownNonReadMode(shape.CompileUnknownNonReadError)) + } + switch strings.ToLower(strings.TrimSpace(cfg.ColumnDiscoveryMode)) { + case string(shape.CompileColumnDiscoveryAuto): + result = append(result, shape.WithColumnDiscoveryMode(shape.CompileColumnDiscoveryAuto)) + case string(shape.CompileColumnDiscoveryOn): + result = append(result, shape.WithColumnDiscoveryMode(shape.CompileColumnDiscoveryOn)) + case string(shape.CompileColumnDiscoveryOff): + result = append(result, shape.WithColumnDiscoveryMode(shape.CompileColumnDiscoveryOff)) + } + if marker := strings.TrimSpace(cfg.DQLPathMarker); marker != "" { + result = append(result, shape.WithDQLPathMarker(marker)) + } + if rel := strings.TrimSpace(cfg.RoutesRelativePath); rel != "" { + result = append(result, shape.WithRoutesRelativePath(rel)) + } + return result +} + +func discoverDQLBootstrapSources(includes, excludes []string) ([]string, error) { + seen := map[string]struct{}{} + var result []string + for _, include := range includes { + include = strings.TrimSpace(include) + if include == "" { + continue + } + expanded, err := expandBootstrapPattern(include) + if err != nil { + return nil, err + } + for _, candidate := range expanded { + if !isDQLSourceFile(candidate) { + continue + } + if matchesAnyPattern(candidate, excludes) { + continue + } + if _, ok := seen[candidate]; ok { + continue + } + seen[candidate] = struct{}{} + result = append(result, candidate) + } + } + sort.Strings(result) + return result, nil +} + +func expandBootstrapPattern(pattern string) ([]string, error) { + pattern = filepath.Clean(pattern) + if strings.Contains(pattern, "**") { + return expandDoubleStarPattern(pattern) + } + if hasGlobMeta(pattern) { + matches, err := filepath.Glob(pattern) + if err != nil { + return nil, err + } + return flattenPaths(matches) + } + return flattenPaths([]string{pattern}) +} + +func flattenPaths(items []string) ([]string, error) { + var result []string + for _, item := range items { + item = strings.TrimSpace(item) + if item == "" { + continue + } + info, err := os.Stat(item) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + if !info.IsDir() { + result = append(result, item) + continue + } + err = filepath.WalkDir(item, func(candidate string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + if isDQLSourceFile(candidate) { + result = append(result, candidate) + } + return nil + }) + if err != nil { + return nil, err + } + } + return result, nil +} + +func expandDoubleStarPattern(pattern string) ([]string, error) { + slash := filepath.ToSlash(pattern) + index := strings.Index(slash, "**") + root := strings.TrimSuffix(slash[:index], "/") + if root == "" { + root = "." + } + rootPath := filepath.FromSlash(root) + var result []string + err := filepath.WalkDir(rootPath, func(candidate string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + normalized := filepath.ToSlash(candidate) + if !globMatch(slash, normalized) { + return nil + } + result = append(result, candidate) + return nil + }) + return result, err +} + +func hasGlobMeta(pattern string) bool { + return strings.ContainsAny(pattern, "*?[") +} + +func matchesAnyPattern(candidate string, patterns []string) bool { + for _, pattern := range patterns { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } + if globMatch(filepath.ToSlash(pattern), filepath.ToSlash(candidate)) { + return true + } + } + return false +} + +func globMatch(pattern, candidate string) bool { + pattern = filepath.ToSlash(pattern) + candidate = filepath.ToSlash(candidate) + if strings.Contains(pattern, "**") { + return matchDoubleStar(strings.Split(pattern, "/"), strings.Split(candidate, "/")) + } + ok, _ := path.Match(pattern, candidate) + return ok +} + +func matchDoubleStar(pattern, candidate []string) bool { + if len(pattern) == 0 { + return len(candidate) == 0 + } + head := pattern[0] + if head == "**" { + if matchDoubleStar(pattern[1:], candidate) { + return true + } + if len(candidate) > 0 { + return matchDoubleStar(pattern, candidate[1:]) + } + return false + } + if len(candidate) == 0 { + return false + } + ok, _ := path.Match(head, candidate[0]) + if !ok { + return false + } + return matchDoubleStar(pattern[1:], candidate[1:]) +} + +func isDQLSourceFile(path string) bool { + ext := strings.ToLower(strings.TrimSpace(filepath.Ext(path))) + return ext == ".dql" || ext == ".sql" +} + +func lookupRootView(resource *view.Resource, root string) *view.View { + if resource == nil { + return nil + } + name := strings.TrimSpace(root) + if name != "" { + if candidate, _ := resource.View(name); candidate != nil { + return candidate + } + } + if len(resource.Views) > 0 { + return resource.Views[0] + } + return nil +} + +type bootstrapRuleSettings struct { + Method string `json:"Method"` + URI string `json:"URI"` +} + +func resolvePathSettings(sourcePath, dql, apiPrefix string) (string, string) { + method := "GET" + uri := "" + settings := parseBootstrapRuleSettings(dql) + if settings != nil { + if candidate := strings.TrimSpace(strings.ToUpper(settings.Method)); candidate != "" { + method = candidate + } + uri = strings.TrimSpace(settings.URI) + } + if uri == "" { + stem := strings.TrimSuffix(filepath.Base(sourcePath), filepath.Ext(sourcePath)) + uri = "/" + strings.Trim(stem, "/") + if prefix := strings.TrimSpace(apiPrefix); prefix != "" { + uri = strings.TrimRight(prefix, "/") + uri + } + } + return method, uri +} + +func parseBootstrapRuleSettings(dql string) *bootstrapRuleSettings { + start := strings.Index(dql, "/*") + end := strings.Index(dql, "*/") + if start == -1 || end == -1 || end <= start+2 { + return nil + } + raw := strings.TrimSpace(dql[start+2 : end]) + if !strings.HasPrefix(raw, "{") || !strings.HasSuffix(raw, "}") { + return nil + } + ret := &bootstrapRuleSettings{} + if err := json.Unmarshal([]byte(raw), ret); err != nil { + return nil + } + return ret +} diff --git a/gateway/dql_bootstrap_test.go b/gateway/dql_bootstrap_test.go new file mode 100644 index 000000000..b36714cd0 --- /dev/null +++ b/gateway/dql_bootstrap_test.go @@ -0,0 +1,122 @@ +package gateway + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/view" +) + +func TestConfigValidate_AllowsEmptyRouteURLWithDQLBootstrap(t *testing.T) { + cfg := &Config{ + ExposableConfig: ExposableConfig{ + DQLBootstrap: &DQLBootstrap{ + Sources: []string{"./testdata/*.dql"}, + }, + }, + } + require.NoError(t, cfg.Validate()) +} + +func TestConfigValidate_FailsWithoutRouteAndBootstrap(t *testing.T) { + cfg := &Config{} + require.ErrorContains(t, cfg.Validate(), "RouteURL was empty") +} + +func TestConfigValidate_FailsForEmptyBootstrapSources(t *testing.T) { + cfg := &Config{ + ExposableConfig: ExposableConfig{ + DQLBootstrap: &DQLBootstrap{}, + }, + } + require.ErrorContains(t, cfg.Validate(), "DQLBootstrap.Sources was empty") +} + +func TestDiscoverDQLBootstrapSources(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "sql", "nested"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "sql", "a.dql"), []byte("SELECT 1"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "sql", "nested", "b.sql"), []byte("SELECT 2"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "sql", "nested", "skip.dql"), []byte("SELECT 3"), 0o644)) + + sources, err := discoverDQLBootstrapSources( + []string{filepath.Join(root, "sql", "**", "*")}, + []string{filepath.Join(root, "sql", "**", "skip.dql")}, + ) + require.NoError(t, err) + require.Len(t, sources, 2) + assert.Contains(t, sources, filepath.Join(root, "sql", "a.dql")) + assert.Contains(t, sources, filepath.Join(root, "sql", "nested", "b.sql")) +} + +func TestResolvePathSettings(t *testing.T) { + method, uri := resolvePathSettings("/tmp/orders/get.dql", `/* {"Method":"POST","URI":"/v1/api/orders"} */ SELECT 1`, "/v1/api") + assert.Equal(t, "POST", method) + assert.Equal(t, "/v1/api/orders", uri) + + method, uri = resolvePathSettings("/tmp/orders/get.dql", `SELECT 1`, "/v1/api") + assert.Equal(t, "GET", method) + assert.Equal(t, "/v1/api/get", uri) +} + +func TestDQLBootstrapEffectivePrecedence(t *testing.T) { + assert.Equal(t, DQLBootstrapPrecedenceRoutesWins, (&DQLBootstrap{}).EffectivePrecedence()) + assert.Equal(t, DQLBootstrapPrecedenceDQLWins, (&DQLBootstrap{Precedence: "dql_wins"}).EffectivePrecedence()) + assert.Equal(t, DQLBootstrapPrecedenceRoutesWins, (&DQLBootstrap{Precedence: "unknown"}).EffectivePrecedence()) +} + +func TestApplyDQLBootstrap_Precedence(t *testing.T) { + ctx := context.Background() + repo, err := repository.New(ctx, repository.WithComponentURL(""), repository.WithNoPlugin()) + require.NoError(t, err) + + route := contract.Path{Method: "GET", URI: "/v1/api/test"} + repo.Register(&repository.Component{Path: route}) + connectors, err := repo.Resources().Lookup(view.ResourceConnectors) + require.NoError(t, err) + connectors.Connectors = append(connectors.Connectors, &view.Connector{ + Connection: view.Connection{ + DBConfig: view.DBConfig{ + Name: "test_conn", + Driver: "sqlite3", + DSN: "sqlite:./test.db", + }, + }, + }) + + root := t.TempDir() + source := filepath.Join(root, "test.dql") + require.NoError(t, os.WriteFile(source, []byte(`/* {"Method":"GET","URI":"/v1/api/test","Connector":"test_conn"} */ SELECT 1 AS id`), 0o644)) + srv := &Service{Config: &Config{ExposableConfig: ExposableConfig{APIPrefix: "/v1/api"}}} + + routesWins := &DQLBootstrap{ + Sources: []string{source}, + Precedence: DQLBootstrapPrecedenceRoutesWins, + } + require.NoError(t, srv.applyDQLBootstrap(ctx, repo, routesWins)) + provider, err := repo.Registry().LookupProvider(ctx, &route) + require.NoError(t, err) + require.NotNil(t, provider) + component, err := provider.Component(ctx) + require.NoError(t, err) + assert.Nil(t, component.View) + + dqlWins := &DQLBootstrap{ + Sources: []string{source}, + Precedence: DQLBootstrapPrecedenceDQLWins, + } + require.NoError(t, srv.applyDQLBootstrap(ctx, repo, dqlWins)) + provider, err = repo.Registry().LookupProvider(ctx, &route) + require.NoError(t, err) + require.NotNil(t, provider) + component, err = provider.Component(ctx) + require.NoError(t, err) + require.NotNil(t, component.View) + assert.Equal(t, "test", component.View.Name) +} diff --git a/internal/inference/join_test.go b/internal/inference/join_test.go new file mode 100644 index 000000000..ea30ddb95 --- /dev/null +++ b/internal/inference/join_test.go @@ -0,0 +1,56 @@ +package inference + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/sqlparser" +) + +func TestJoinRelationExtraction(t *testing.T) { + testCases := []struct { + description string + sql string + wantParent string + wantRelCol string + wantRefCol string + }{ + { + description: "simple join", + sql: "SELECT * FROM a a JOIN b b ON a.brand = b.b_brand", + wantParent: "a", + wantRelCol: "brand", + wantRefCol: "b_brand", + }, + { + description: "join with function on parent", + sql: "SELECT * FROM a a JOIN b b ON lower(a.brand) = b.b_brand", + wantParent: "a", + wantRelCol: "brand", + wantRefCol: "b_brand", + }, + { + description: "join with collate and multiple conditions", + sql: "SELECT * FROM a a JOIN b b ON " + + "a.brand COLLATE utf8mb4_bin = b.b_brand COLLATE utf8mb4_bin AND " + + "a.model COLLATE utf8mb4_bin = b.b_model COLLATE utf8mb4_bin", + wantParent: "a", + wantRelCol: "brand", + wantRefCol: "b_brand", + }, + } + + for _, testCase := range testCases { + q, err := sqlparser.ParseQuery(testCase.sql) + require.NoError(t, err, testCase.description) + require.NotEmpty(t, q.Joins, testCase.description) + + join := q.Joins[0] + parent := ParentAlias(join) + require.Equal(t, testCase.wantParent, parent, testCase.description) + + relCol, refCol := ExtractRelationColumns(join) + require.Equal(t, testCase.wantRelCol, relCol, testCase.description) + require.Equal(t, testCase.wantRefCol, refCol, testCase.description) + } +} diff --git a/internal/testutil/sqlnormalizer/cases.go b/internal/testutil/sqlnormalizer/cases.go new file mode 100644 index 000000000..73569e24a --- /dev/null +++ b/internal/testutil/sqlnormalizer/cases.go @@ -0,0 +1,43 @@ +package sqlnormalizer + +type Case struct { + Name string + Generated bool + SQL string + Expect string +} + +func Cases() []Case { + return []Case{ + { + Name: "skip normalization when not generated", + Generated: false, + SQL: "SELECT a.id FROM users a JOIN orders b ON a.id = b.user_id", + Expect: "SELECT a.id FROM users a JOIN orders b ON a.id = b.user_id", + }, + { + Name: "invalid sql returns input", + Generated: true, + SQL: "SELECT * FROM (", + Expect: "SELECT * FROM (", + }, + { + Name: "normalize from and join aliases in selectors and alias nodes", + Generated: true, + SQL: "SELECT a.id, b.user_id FROM users a JOIN orders b ON a.id = b.user_id", + Expect: "SELECT A.id, B.user_id FROM users A JOIN orders B ON A.id = B.user_id", + }, + { + Name: "keep alias that is already normalized", + Generated: true, + SQL: "SELECT UserAlias.id FROM users UserAlias", + Expect: "SELECT UserAlias.id FROM users UserAlias", + }, + { + Name: "normalize snake_case alias", + Generated: true, + SQL: "SELECT order_item.id FROM users order_item", + Expect: "SELECT OrderItem.id FROM users OrderItem", + }, + } +} diff --git a/internal/translator/parser/sanitizer_test.go b/internal/translator/parser/sanitizer_test.go new file mode 100644 index 000000000..38a9c8d1e --- /dev/null +++ b/internal/translator/parser/sanitizer_test.go @@ -0,0 +1,222 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/internal/inference" + "github.com/viant/datly/view/keywords" + "github.com/viant/velty/functions" +) + +func TestTemplate_Sanitize(t *testing.T) { + state := inference.State{} + tmpl, err := NewTemplate("#set($x = 1) SELECT * FROM t WHERE id = $x AND name = $Name", &state) + require.NoError(t, err) + actual := tmpl.Sanitize() + assert.Contains(t, actual, "#set($x = 1)") + assert.Contains(t, actual, "$criteria.AppendBinding($x)") + assert.Contains(t, actual, "$criteria.AppendBinding($Unsafe.Name)") +} + +func TestSanitize_SkipsFirstSetVariableOccurrence(t *testing.T) { + iter := newIterable(map[string]bool{"x": true}) + expr := &Expression{ + IsVariable: true, + OccurrenceIndex: 0, + Context: SetContext, + FullName: "$x", + Start: 0, + End: 2, + } + dst := []byte("$x") + actual, _ := sanitize(iter, expr, dst, 0, 0) + assert.Equal(t, "$x", string(actual)) +} + +func TestUnwrapBrackets(t *testing.T) { + raw, had := unwrapBrackets("${Foo}") + assert.Equal(t, "$Foo", raw) + assert.True(t, had) + + raw, had = unwrapBrackets("$Foo") + assert.Equal(t, "$Foo", raw) + assert.False(t, had) +} + +func TestSanitizeContent(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{Start: 0, End: 10} + assert.Equal(t, "$A", sanitizeContent(iter, expr, "$A")) + + iter = newIterable(nil) + parent := &Expression{Start: 0, End: 13, FullName: "$Fn($A, $B)"} + argA := &Expression{Start: 4, End: 6, FullName: "$A", Holder: "A"} + argB := &Expression{Start: 8, End: 10, FullName: "$B", Holder: "B"} + next := &Expression{Start: 20, End: 22, FullName: "$C", Holder: "C"} + iter.expressions = Expressions{argA, argB, next} + actual := sanitizeContent(iter, parent, parent.FullName) + assert.Equal(t, "$Fn($criteria.AppendBinding($Unsafe.A), $criteria.AppendBinding($Unsafe.B))", actual) +} + +func TestSanitizeParameter(t *testing.T) { + t.Run("standalone fn entry is preserved", func(t *testing.T) { + name := "TestStandaloneSanitize" + keywords.Add(name, functions.NewEntry(nil, &keywords.StandaloneFn{})) + iter := newIterable(nil) + expr := &Expression{Holder: name, FullName: "$" + name + "(1)"} + assert.Equal(t, "$"+name+"(1)", sanitizeParameter(expr, "$"+name+"(1)", iter, nil, 0)) + }) + + t.Run("set marker prefix preserved", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{Holder: "Value", Prefix: keywords.SetMarkerKey} + assert.Equal(t, "$Value", sanitizeParameter(expr, "$Value", iter, nil, 0)) + }) + + t.Run("namespace metadata preserved", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{ + Holder: "Any", + Entry: functions.NewEntry(nil, keywords.NewNamespace()), + } + assert.Equal(t, "$Any", sanitizeParameter(expr, "$Any", iter, nil, 0)) + }) + + t.Run("const parameter gets Unsafe prefix", func(t *testing.T) { + iter := newIterable(nil, inference.NewConstParameter("ConstX", 1)) + expr := &Expression{Holder: "ConstX"} + assert.Equal(t, "$Unsafe.ConstX", sanitizeParameter(expr, "$ConstX", iter, nil, 0)) + }) + + t.Run("func context with variable and Params prefix strips prefix", func(t *testing.T) { + iter := newIterable(map[string]bool{"X": true}) + expr := &Expression{Holder: "X", Prefix: keywords.ParamsKey, Context: FuncContext} + assert.Equal(t, "$X", sanitizeParameter(expr, "$Unsafe.X", iter, nil, 0)) + }) + + t.Run("func context with non variable and empty prefix adds Unsafe", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{Holder: "X", Prefix: "", Context: FuncContext} + assert.Equal(t, "$Unsafe.X", sanitizeParameter(expr, "$X", iter, nil, 0)) + }) + + t.Run("func context with variable and custom prefix keeps raw", func(t *testing.T) { + iter := newIterable(map[string]bool{"X": true}) + expr := &Expression{Holder: "X", Prefix: keywords.AndPrefix, Context: ForEachContext} + assert.Equal(t, "$X", sanitizeParameter(expr, "$X", iter, nil, 0)) + }) + + t.Run("func context with non variable and non empty prefix keeps raw", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{Holder: "X", Prefix: keywords.OrPrefix, Context: SetContext} + assert.Equal(t, "$X", sanitizeParameter(expr, "$X", iter, nil, 0)) + }) + + t.Run("func context with expression entry preserves raw", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{Holder: "X", Context: IfContext, Entry: functions.NewEntry(nil, nil)} + assert.Equal(t, "$X", sanitizeParameter(expr, "$X", iter, nil, 0)) + }) + + t.Run("append context variable with Params prefix strips prefix", func(t *testing.T) { + iter := newIterable(map[string]bool{"X": true}) + expr := &Expression{Holder: "X", Prefix: keywords.ParamsKey} + assert.Equal(t, "$X", sanitizeParameter(expr, "$Unsafe.X", iter, nil, 0)) + }) + + t.Run("append context variable placeholder", func(t *testing.T) { + iter := newIterable(map[string]bool{"X": true}) + expr := &Expression{Holder: "X"} + assert.Equal(t, "$criteria.AppendBinding($X)", sanitizeParameter(expr, "$X", iter, nil, 0)) + }) + + t.Run("append context params prefix preserved", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{Holder: "X", Prefix: keywords.ParamsKey} + assert.Equal(t, "$Unsafe.X", sanitizeParameter(expr, "$Unsafe.X", iter, nil, 0)) + }) + + t.Run("context metadata unexpand raw preserved", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{ + Holder: "Ctx", + Entry: functions.NewEntry(nil, keywords.NewContextMetadata("ctx", nil, true)), + } + assert.Equal(t, "$Ctx", sanitizeParameter(expr, "$Ctx", iter, nil, 0)) + }) + + t.Run("context metadata expandable becomes placeholder", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{ + Holder: "Ctx", + Entry: functions.NewEntry(nil, keywords.NewContextMetadata("ctx", nil, false)), + } + assert.Equal(t, "$criteria.AppendBinding($Ctx)", sanitizeParameter(expr, "$Ctx", iter, nil, 0)) + }) + + t.Run("non context metadata entry becomes placeholder", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{ + Holder: "Ctx", + Entry: functions.NewEntry(nil, struct{}{}), + } + assert.Equal(t, "$criteria.AppendBinding($Ctx)", sanitizeParameter(expr, "$Ctx", iter, nil, 0)) + }) + + t.Run("default path adds Unsafe and placeholder", func(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{Holder: "X"} + assert.Equal(t, "$criteria.AppendBinding($Unsafe.X)", sanitizeParameter(expr, "$X", iter, nil, 0)) + }) +} + +func TestSanitizeAsPlaceholder(t *testing.T) { + assert.Equal(t, "$criteria.AppendBinding($X)", sanitizeAsPlaceholder("$X")) +} + +func TestSanitize_WithBracketsWrapping(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{ + FullName: "${X}", + Holder: "X", + Start: 0, + End: 4, + } + dst := []byte("${X}") + actual, _ := sanitize(iter, expr, dst, 0, 0) + assert.Equal(t, "${criteria.AppendBinding($Unsafe.X)}", string(actual)) +} + +func TestSanitize_NoChangePathAndCursorOffset(t *testing.T) { + iter := newIterable(nil) + expr := &Expression{ + FullName: "$Unsafe.X", + Holder: "X", + Prefix: keywords.ParamsKey, + Start: 8, + End: 17, + } + dst := []byte("SELECT " + expr.FullName) + actual, offset := sanitize(iter, expr, dst, 0, 7) + assert.Equal(t, "SELECT $Unsafe.X", string(actual)) + assert.Equal(t, 0, offset) +} + +func newIterable(declared map[string]bool, params ...*inference.Parameter) *iterables { + if declared == nil { + declared = map[string]bool{} + } + state := inference.State{} + for _, param := range params { + if param != nil { + state.Append(param) + } + } + tmpl := &Template{ + Declared: declared, + State: &state, + } + return &iterables{expressionMatcher: &expressionMatcher{Template: tmpl}} +} diff --git a/service/executor/expand/evaluator_test.go b/service/executor/expand/evaluator_test.go new file mode 100644 index 000000000..b6e0a7e2d --- /dev/null +++ b/service/executor/expand/evaluator_test.go @@ -0,0 +1,78 @@ +package expand_test + +import ( + "testing" + + "github.com/viant/datly/service/executor/expand" +) + +func TestNewEvaluator_DefaultTypeLookup(t *testing.T) { + evaluator, err := expand.NewEvaluator(`#set($x = $New("int"))$x`) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if _, err := evaluator.Evaluate(nil); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} + +func TestNewEvaluator_WithNilTypeLookupOption(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fatalf("expected no panic, got %v", r) + } + }() + + evaluator, err := expand.NewEvaluator(`#set($x = $New("int"))$x`, expand.WithTypeLookup(nil)) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if _, err := evaluator.Evaluate(nil); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} + +func TestNewEvaluator_UnknownTypeReturnsError(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fatalf("expected no panic, got %v", r) + } + }() + + _, err := expand.NewEvaluator(`#set($x = $New("DefinitelyNotAType"))$x`, expand.WithTypeLookup(nil)) + if err == nil { + t.Fatalf("expected error") + } +} + +func TestNewEvaluator_WithNilNamedVariableOption(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fatalf("expected no panic, got %v", r) + } + }() + + evaluator, err := expand.NewEvaluator(`ok`, expand.WithVariable(nil)) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if _, err := evaluator.Evaluate(nil); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} + +func TestNewEvaluator_WithNilCustomContextOption(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fatalf("expected no panic, got %v", r) + } + }() + + evaluator, err := expand.NewEvaluator(`ok`, expand.WithCustomContexts(nil)) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if _, err := evaluator.Evaluate(nil); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} diff --git a/service/session/selector_injector_test.go b/service/session/selector_injector_test.go new file mode 100644 index 000000000..7f8275fb6 --- /dev/null +++ b/service/session/selector_injector_test.go @@ -0,0 +1,89 @@ +package session + +import ( + "context" + "net/http" + "reflect" + "testing" + + "github.com/viant/datly/repository" + "github.com/viant/datly/view" + vstate "github.com/viant/datly/view/state" + hstate "github.com/viant/xdatly/handler/state" +) + +func TestSessionBind_QuerySelectorOverride_PageComputesOffset(t *testing.T) { + ctx := context.Background() + + resource := view.NewResource(nil) + trueValue := true + aView := &view.View{ + Name: "v", + Mode: view.ModeQuery, + Selector: func() *view.Config { + cfg := view.QueryStateParameters.Clone() + cfg.Limit = 5 + cfg.Constraints = &view.Constraints{ + Criteria: true, + OrderBy: true, + Limit: true, + Offset: true, + Projection: true, + Page: &trueValue, + } + return cfg + }(), + } + aView.SetResource(resource) + aView.Template = &view.Template{Schema: vstate.NewSchema(reflect.TypeOf(struct{ Dummy int }{}))} + if err := aView.Template.Init(ctx, resource, aView); err != nil { + t.Fatalf("failed to init template: %v", err) + } + if err := aView.Selector.Init(ctx, resource, aView); err != nil { + t.Fatalf("failed to init selector: %v", err) + } + + component := &repository.Component{View: aView} + outputType, err := vstate.NewType( + vstate.WithSchema(vstate.NewSchema(reflect.TypeOf(struct{ X int }{}))), + vstate.WithResource(aView.Resource()), + ) + if err != nil { + t.Fatalf("failed to build component output type: %v", err) + } + component.Output.Type = *outputType + + sess := New(aView, WithComponent(component)) + var dest struct{} + + // request supplies different selector values; injected selector should take precedence + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/?_page=1&_limit=1", nil) + if err != nil { + t.Fatalf("failed to build request: %v", err) + } + + err = sess.Bind(ctx, &dest, hstate.WithQuerySelector(&hstate.NamedQuerySelector{ + Name: "v", + QuerySelector: hstate.QuerySelector{ + Page: 2, + }, + }), hstate.WithHttpRequest(req)) + if err != nil { + t.Fatalf("Bind() error: %v", err) + } + + if err := sess.SetViewState(ctx, aView); err != nil { + t.Fatalf("SetViewState() error: %v", err) + } + + selector := sess.State().Lookup(aView) + if selector.Page != 2 { + t.Fatalf("expected Page=2, got %d", selector.Page) + } + if selector.Limit != 5 { + t.Fatalf("expected Limit=5, got %d", selector.Limit) + } + if selector.Offset != 5 { + t.Fatalf("expected Offset=5, got %d", selector.Offset) + } +} From f16184b1403ce9f088c49aae53b7b1bb3af52538 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 24 Feb 2026 07:27:18 -0800 Subject: [PATCH 127/279] shape/compile: add type support helpers; refine preprocessing and type defaults; update tests --- go.mod | 12 +- go.sum | 6 + repository/shape/compile/compiler.go | 46 +-- repository/shape/compile/compiler_test.go | 74 ++--- repository/shape/compile/component_types.go | 75 ++++- .../shape/compile/component_types_test.go | 49 +++ repository/shape/compile/enrich.go | 301 ++++++------------ .../shape/compile/preprocess_handler.go | 39 +-- .../shape/compile/preprocess_handler_test.go | 52 +-- repository/shape/compile/strings_util.go | 13 + repository/shape/compile/type_support.go | 238 ++++++++++++++ repository/shape/compile/type_support_test.go | 70 ++++ repository/shape/compile/typectx_defaults.go | 102 +++++- .../shape/compile/typectx_defaults_test.go | 16 + repository/shape/platform_parity_test.go | 3 + warmup/cache_test.go | 5 + 16 files changed, 706 insertions(+), 395 deletions(-) create mode 100644 repository/shape/compile/strings_util.go create mode 100644 repository/shape/compile/type_support.go create mode 100644 repository/shape/compile/type_support_test.go diff --git a/go.mod b/go.mod index baaae6b4a..60d51b517 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,6 @@ module github.com/viant/datly go 1.25.0 -replace github.com/viant/velty => ../velty - -replace github.com/viant/x => ../x - -replace github.com/viant/sqlparser => ../sqlparser - require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible github.com/aws/aws-lambda-go v1.31.0 @@ -37,7 +31,7 @@ require ( github.com/viant/sqlx v0.21.0 github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 - github.com/viant/velty v0.2.1-0.20230927172116-ba56497b5c85 + github.com/viant/velty v0.4.0 github.com/viant/xreflect v0.7.3 github.com/viant/xunsafe v0.10.3 golang.org/x/mod v0.28.0 @@ -48,7 +42,7 @@ require ( require ( github.com/viant/govalidator v0.3.1 - github.com/viant/sqlparser v0.9.0 + github.com/viant/sqlparser v0.11.0 ) require ( @@ -59,7 +53,7 @@ require ( github.com/viant/mcp-protocol v0.9.0 github.com/viant/structology v0.8.0 github.com/viant/tagly v0.3.0 - github.com/viant/x v0.3.0 + github.com/viant/x v0.4.0 github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 github.com/viant/xdatly/handler v0.0.0-20251208172928-dd34b7f09fd5 diff --git a/go.sum b/go.sum index d1e092d55..5164a250c 100644 --- a/go.sum +++ b/go.sum @@ -1194,6 +1194,8 @@ github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= github.com/viant/pgo v0.11.0/go.mod h1:MFzHmkRFZlciugEgUvpl/3grK789PBSH4dUVSLOSo+Q= github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= +github.com/viant/sqlparser v0.11.0 h1:RVmAsEieZlnRO33DWWvDXJOTY+sXJGTymPaC1iWnkOc= +github.com/viant/sqlparser v0.11.0/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= @@ -1206,6 +1208,10 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI github.com/viant/toolbox v0.34.5/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/viant/toolbox v0.37.0 h1:+zwSdbQh6I6ZEyxokQJr+1gQKbLEw6erc+Av5dwKtLU= github.com/viant/toolbox v0.37.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/viant/velty v0.4.0 h1:eesQES/vCpcoPbM+gQLUBuLEL2sEO+A6s6lPpl8eKc4= +github.com/viant/velty v0.4.0/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= +github.com/viant/x v0.4.0 h1:n2xuxQdw4lYtMdi59IAQEZHPioNT9InENGGbapyz+P4= +github.com/viant/x v0.4.0/go.mod h1:1TvsnpZFqI9dYVzIkaSYJyJ/UkfxW7fnk0YFafWXrPg= github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a h1:7CLO2LjVnFgOwN0FL3Q4y5NrD7DpclS21AiW6tDLIc8= github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUBDzc4rWacOYvAq/TKrTV0agvMF0gwm2gaoHI= diff --git a/repository/shape/compile/compiler.go b/repository/shape/compile/compiler.go index db57701a6..d283d7c50 100644 --- a/repository/shape/compile/compiler.go +++ b/repository/shape/compile/compiler.go @@ -66,12 +66,7 @@ func (c *DQLCompiler) Compile(_ context.Context, source *shape.Source, opts ...s pre = prepared.Pre statements = prepared.Statements decision = prepared.Decision - legacyFallbackViews := prepared.LegacyViews - effectiveSource := source - if prepared.EffectiveSource != nil { - effectiveSource = prepared.EffectiveSource - } - if strings.TrimSpace(pre.SQL) == "" && len(legacyFallbackViews) == 0 { + if strings.TrimSpace(pre.SQL) == "" { allDiags = append(allDiags, &dqlshape.Diagnostic{ Code: dqldiag.CodeParseEmpty, Severity: dqlshape.SeverityError, @@ -87,11 +82,7 @@ func (c *DQLCompiler) Compile(_ context.Context, source *shape.Source, opts ...s var root *plan.View var compileDiags []*dqlshape.Diagnostic var err error - if len(legacyFallbackViews) > 0 { - root = legacyFallbackViews[0] - } else { - root, compileDiags, err = c.compileRoot(source.Name, pre.SQL, statements, decision, compileOptions.MixedMode, compileOptions.UnknownNonReadMode) - } + root, compileDiags, err = c.compileRoot(source.Name, pre.SQL, statements, decision, compileOptions.MixedMode, compileOptions.UnknownNonReadMode) if err != nil { return nil, err } @@ -102,18 +93,6 @@ func (c *DQLCompiler) Compile(_ context.Context, source *shape.Source, opts ...s } result := newPlanResult(root) - if len(legacyFallbackViews) > 1 { - for _, item := range legacyFallbackViews[1:] { - if item == nil || strings.TrimSpace(item.Name) == "" { - continue - } - if _, exists := result.ViewsByName[item.Name]; exists { - continue - } - result.Views = append(result.Views, item) - result.ViewsByName[item.Name] = item - } - } result.Diagnostics = allDiags result.TypeContext = pre.TypeCtx result.Directives = pre.Directives @@ -122,26 +101,11 @@ func (c *DQLCompiler) Compile(_ context.Context, source *shape.Source, opts ...s appendRelationViews(result, root, hints) appendDeclaredViews(source.DQL, result) appendDeclaredStates(source.DQL, result) - if prepared.ForceLegacyContract && len(legacyFallbackViews) > 0 { - if legacyStates := resolveLegacyRouteStatesWithLayout(effectiveSource, pathLayout); len(legacyStates) > 0 { - result.States = legacyStates - } - if legacyTypes := resolveLegacyRouteTypesWithLayout(effectiveSource, pathLayout); len(legacyTypes) > 0 { - result.Types = legacyTypes - } - } - result.Diagnostics = append(result.Diagnostics, appendComponentTypesWithLayout(effectiveSource, result, pathLayout)...) - mergeLegacyRouteStatesWithLayout(result, effectiveSource, pathLayout) - mergeLegacyRouteTypesWithLayout(result, effectiveSource, pathLayout) + _ = prepared applyViewHints(result, hints) - applySourceParityEnrichmentWithLayout(result, effectiveSource, pathLayout) + applySourceParityEnrichmentWithLayout(result, source, pathLayout) + applyLinkedTypeSupport(result, source) result.Diagnostics = append(result.Diagnostics, applyColumnDiscoveryPolicy(result, compileOptions)...) - if len(result.States) == 0 && len(legacyFallbackViews) > 0 { - result.States = resolveLegacyRouteStatesWithLayout(effectiveSource, pathLayout) - } - if len(result.Types) == 0 && len(legacyFallbackViews) > 0 { - result.Types = resolveLegacyRouteTypesWithLayout(effectiveSource, pathLayout) - } if enforceStrict && hasEscalationWarnings(result.Diagnostics) { return nil, &CompileError{Diagnostics: filterEscalationDiagnostics(result.Diagnostics)} diff --git a/repository/shape/compile/compiler_test.go b/repository/shape/compile/compiler_test.go index 63156250a..f14487850 100644 --- a/repository/shape/compile/compiler_test.go +++ b/repository/shape/compile/compiler_test.go @@ -4,7 +4,6 @@ import ( "context" "os" "path/filepath" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -60,8 +59,8 @@ SELECT id func TestDQLCompiler_Compile_PropagatesTypeContext(t *testing.T) { compiler := New() dql := ` -#settings($_ = $package('mdp/performance')) -#settings($_ = $import('perf', 'github.com/acme/mdp/performance')) +#package('mdp/performance') +#import('perf', 'github.com/acme/mdp/performance') SELECT id FROM ORDERS t` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) @@ -75,6 +74,26 @@ SELECT id FROM ORDERS t` assert.Equal(t, "perf", planned.TypeContext.Imports[0].Alias) } +func TestDQLCompiler_Compile_PropagatesImportedTypeContextWithModuleNormalization(t *testing.T) { + compiler := New() + projectDir := t.TempDir() + err := os.WriteFile(filepath.Join(projectDir, "go.mod"), []byte("module github.vianttech.com/viant/platform\n\ngo 1.23\n"), 0o644) + require.NoError(t, err) + source := &shape.Source{ + Name: "orders_report", + Path: filepath.Join(projectDir, "dql", "platform", "taxonomy", "get.dql"), + DQL: "#import('session','pkg/platform/system/session')\nSELECT id FROM ORDERS t", + } + res, err := compiler.Compile(context.Background(), source) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotNil(t, planned.TypeContext) + require.Len(t, planned.TypeContext.Imports, 1) + assert.Equal(t, "session", planned.TypeContext.Imports[0].Alias) + assert.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/system/session", planned.TypeContext.Imports[0].Package) +} + func TestDQLCompiler_Compile_PropagatesSpecialDirectives(t *testing.T) { compiler := New() dql := ` @@ -127,7 +146,7 @@ func TestDQLCompiler_Compile_ColumnDiscoveryOffFailsWhenRequired(t *testing.T) { func TestDQLCompiler_Compile_TypeContextValidationWarnsInCompat(t *testing.T) { compiler := New() dql := ` -#settings($_ = $package('github.com/acme/perf')) +#package('github.com/acme/perf') SELECT id FROM ORDERS t` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}, shape.WithTypeContextPackageName("bad/name")) require.NoError(t, err) @@ -196,7 +215,7 @@ func TestDQLCompiler_Compile_SyntaxError_RemapsAfterSanitize(t *testing.T) { func TestDQLCompiler_Compile_DirectiveOnly_HasLineAndChar(t *testing.T) { compiler := New() - _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "#settings($_ = $package('x'))"}) + _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "#package('x')"}) require.Error(t, err) compileErr, ok := err.(*CompileError) require.True(t, ok) @@ -211,7 +230,7 @@ func TestDQLCompiler_Compile_InvalidDirective_HasLineAndChar(t *testing.T) { compiler := New() _, err := compiler.Compile(context.Background(), &shape.Source{ Name: "orders_report", - DQL: "SELECT id FROM ORDERS t\n#settings($_ = $import('alias'))\nSELECT id FROM ORDERS t", + DQL: "SELECT id FROM ORDERS t\n#import('alias')\nSELECT id FROM ORDERS t", }) require.Error(t, err) compileErr, ok := err.(*CompileError) @@ -418,7 +437,7 @@ func TestDQLCompiler_Compile_DMLSyntaxError_HasLineAndChar(t *testing.T) { compiler := New() _, err := compiler.Compile(context.Background(), &shape.Source{ Name: "orders_exec", - DQL: "#settings($_ = $package('x'))\nINSERT INTO ORDERS(id VALUES (1)", + DQL: "#package('x')\nINSERT INTO ORDERS(id VALUES (1)", }) require.Error(t, err) compileErr, ok := err.(*CompileError) @@ -661,7 +680,7 @@ JOIN (SELECT * FROM session/attributes) attribute ON attribute.user_id = session assert.Equal(t, "system", related.Connector) } -func TestDQLCompiler_Compile_GeneratedHandler_NoBodyInput_UsesLegacyContractStates(t *testing.T) { +func TestDQLCompiler_Compile_GeneratedHandler_NoBodyInput_DoesNotLoadLegacyContractStates(t *testing.T) { tempDir := t.TempDir() genPath := filepath.Join(tempDir, "dql", "system", "upload", "gen", "upload", "delete.dql") require.NoError(t, os.MkdirAll(filepath.Dir(genPath), 0o755)) @@ -705,21 +724,10 @@ func TestDQLCompiler_Compile_GeneratedHandler_NoBodyInput_UsesLegacyContractStat assert.Equal(t, "SQLExec", planned.Views[0].Mode) assert.Equal(t, "system", planned.Views[0].Connector) - stateByName := map[string]*plan.State{} - for _, item := range planned.States { - if item == nil { - continue - } - stateByName[item.Name] = item - } - require.Contains(t, stateByName, "Method") - require.Contains(t, stateByName, "UploadId") - assert.Equal(t, "http_request", stateByName["Method"].Kind) - assert.Equal(t, "query", stateByName["UploadId"].Kind) - assert.NotContains(t, stateByName, "Body") + assert.Empty(t, planned.States) } -func TestDQLCompiler_Compile_HandlerLegacyTypes_PreferredOverComponentNameCollisions(t *testing.T) { +func TestDQLCompiler_Compile_HandlerLegacyTypes_NotLoadedFromLegacyRouteYAML(t *testing.T) { tempDir := t.TempDir() sourcePath := filepath.Join(tempDir, "dql", "platform", "campaign", "post.dql") require.NoError(t, os.MkdirAll(filepath.Dir(sourcePath), 0o755)) @@ -776,26 +784,10 @@ func TestDQLCompiler_Compile_HandlerLegacyTypes_PreferredOverComponentNameCollis planned, ok := res.Plan.(*plan.Result) require.True(t, ok) - typeByName := map[string]*plan.Type{} - for _, item := range planned.Types { - if item == nil || strings.TrimSpace(item.Name) == "" { - continue - } - typeByName[strings.ToLower(strings.TrimSpace(item.Name))] = item - } - - inputType, ok := typeByName["input"] - require.True(t, ok) - assert.Equal(t, "campaign/patch", inputType.Package) - assert.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/campaign/patch", inputType.ModulePath) - - handlerType, ok := typeByName["handler"] - require.True(t, ok) - assert.Equal(t, "campaign/patch", handlerType.Package) - assert.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/campaign/patch", handlerType.ModulePath) + assert.Empty(t, planned.Types) } -func TestDQLCompiler_Compile_CustomPathLayout_HandlerFallback(t *testing.T) { +func TestDQLCompiler_Compile_CustomPathLayout_NoLegacyHandlerFallback(t *testing.T) { tempDir := t.TempDir() sourcePath := filepath.Join(tempDir, "sqlsrc", "platform", "campaign", "post.dql") require.NoError(t, os.MkdirAll(filepath.Dir(sourcePath), 0o755)) @@ -825,7 +817,5 @@ func TestDQLCompiler_Compile_CustomPathLayout_HandlerFallback(t *testing.T) { require.True(t, ok) require.NotEmpty(t, planned.Views) assert.Equal(t, "post", planned.Views[0].Name) - assert.Equal(t, "SQLExec", planned.Views[0].Mode) - assert.Equal(t, "ci_ads", planned.Views[0].Connector) - assert.Contains(t, planned.Views[0].SQL, "$Nop(") + assert.NotContains(t, planned.Views[0].SQL, "$Nop(") } diff --git a/repository/shape/compile/component_types.go b/repository/shape/compile/component_types.go index 5c553bc90..2cd3d705e 100644 --- a/repository/shape/compile/component_types.go +++ b/repository/shape/compile/component_types.go @@ -39,6 +39,8 @@ func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, l visited: map[string]componentVisitState{}, outputByRoute: map[string]string{}, typesByName: map[string]*plan.Type{}, + payloadCache: map[string]routePayloadLookup{}, + reportedDiag: map[string]bool{}, } if strings.TrimSpace(sourceNamespace) != "" { collector.collect(sourceNamespace, relationSpan(source.DQL, 0), false) @@ -109,9 +111,19 @@ type componentCollector struct { visited map[string]componentVisitState outputByRoute map[string]string typesByName map[string]*plan.Type + payloadCache map[string]routePayloadLookup + reportedDiag map[string]bool diags []*dqlshape.Diagnostic } +type routePayloadLookup struct { + payload *routePayload + found bool + malformed bool + malformedAt string + detail string +} + func (c *componentCollector) collect(namespace string, span dqlshape.Span, required bool) (string, bool) { key := strings.ToLower(strings.TrimSpace(namespace)) if key == "" { @@ -132,10 +144,11 @@ func (c *componentCollector) collect(namespace string, span dqlshape.Span, requi } c.visited[key] = componentVisitActive - payload, ok := loadRoutePayload(c.routesRoot, namespace) + payload, ok := c.loadRoutePayload(namespace, span) if !ok { c.visited[key] = componentVisitDone - if required { + if required && !c.hasReported("missing:"+key) { + c.reportedDiag["missing:"+key] = true c.diags = append(c.diags, &dqlshape.Diagnostic{ Code: dqldiag.CodeCompRouteMissing, Severity: dqlshape.SeverityWarning, @@ -347,7 +360,13 @@ type routePayload struct { } func loadRoutePayload(routesRoot, namespace string) (*routePayload, bool) { + lookup := readRoutePayload(routesRoot, namespace) + return lookup.payload, lookup.found +} + +func readRoutePayload(routesRoot, namespace string) routePayloadLookup { candidates := routeYAMLCandidates(routesRoot, namespace) + lookup := routePayloadLookup{} for _, candidate := range candidates { data, err := os.ReadFile(candidate) if err != nil { @@ -355,11 +374,59 @@ func loadRoutePayload(routesRoot, namespace string) (*routePayload, bool) { } payload := &routePayload{} if err = yaml.Unmarshal(data, payload); err != nil { + if !lookup.malformed { + lookup.malformed = true + lookup.malformedAt = candidate + lookup.detail = strings.TrimSpace(err.Error()) + } continue } - return payload, true + lookup.payload = payload + lookup.found = true + lookup.malformed = false + lookup.malformedAt = "" + lookup.detail = "" + return lookup + } + return lookup +} + +func (c *componentCollector) loadRoutePayload(namespace string, span dqlshape.Span) (*routePayload, bool) { + key := strings.ToLower(strings.TrimSpace(namespace)) + if key == "" { + return nil, false + } + lookup, ok := c.payloadCache[key] + if !ok { + lookup = readRoutePayload(c.routesRoot, namespace) + c.payloadCache[key] = lookup + } + if lookup.malformed && !lookup.found && !c.hasReported("invalid:"+key) { + c.reportedDiag["invalid:"+key] = true + message := "component route YAML malformed: " + namespace + if strings.TrimSpace(lookup.malformedAt) != "" { + message += " (" + lookup.malformedAt + ")" + } + hint := "fix route YAML format" + if strings.TrimSpace(lookup.detail) != "" { + hint += ": " + lookup.detail + } + c.diags = append(c.diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeCompRouteInvalid, + Severity: dqlshape.SeverityWarning, + Message: message, + Hint: hint, + Span: span, + }) + } + return lookup.payload, lookup.found +} + +func (c *componentCollector) hasReported(key string) bool { + if c == nil || c.reportedDiag == nil { + return false } - return nil, false + return c.reportedDiag[key] } func routeOutputType(payload *routePayload) string { diff --git a/repository/shape/compile/component_types_test.go b/repository/shape/compile/component_types_test.go index 0e93ec718..51570a125 100644 --- a/repository/shape/compile/component_types_test.go +++ b/repository/shape/compile/component_types_test.go @@ -153,3 +153,52 @@ func TestAppendComponentTypes_TypeCollisionEmitsDiagnostic(t *testing.T) { require.Len(t, result.Types, 1) assert.Equal(t, "campaign/patch", result.Types[0].Package) } + +func TestAppendComponentTypes_InvalidRouteYAMLEmitsDiagnostic(t *testing.T) { + temp := t.TempDir() + dqlDir := filepath.Join(temp, "dql", "platform", "sample") + routesDir := filepath.Join(temp, "repo", "dev", "Datly", "routes", "platform", "acl") + require.NoError(t, os.MkdirAll(dqlDir, 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(routesDir, "auth"), 0o755)) + + sourcePath := filepath.Join(dqlDir, "sample.dql") + dql := "#set($Auth = $component<../acl/auth>())\nSELECT 1" + require.NoError(t, os.WriteFile(sourcePath, []byte(dql), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "auth", "auth.yaml"), []byte("Resource:\n Types: ["), 0o644)) + + result := &plan.Result{ + States: []*plan.State{{Name: "Auth", Kind: "component", In: "../acl/auth"}}, + } + diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: dql}, result) + require.NotEmpty(t, diags) + assert.Equal(t, dqldiag.CodeCompRouteInvalid, diags[0].Code) +} + +func TestAppendComponentTypes_InvalidRouteYAMLDedupedForRepeatedStates(t *testing.T) { + temp := t.TempDir() + dqlDir := filepath.Join(temp, "dql", "platform", "sample") + routesDir := filepath.Join(temp, "repo", "dev", "Datly", "routes", "platform", "acl") + require.NoError(t, os.MkdirAll(dqlDir, 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(routesDir, "auth"), 0o755)) + + sourcePath := filepath.Join(dqlDir, "sample.dql") + dql := "#set($Auth1 = $component<../acl/auth>())\n#set($Auth2 = $component<../acl/auth>())\nSELECT 1" + require.NoError(t, os.WriteFile(sourcePath, []byte(dql), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(routesDir, "auth", "auth.yaml"), []byte("Resource:\n Types: ["), 0o644)) + + result := &plan.Result{ + States: []*plan.State{ + {Name: "Auth1", Kind: "component", In: "../acl/auth"}, + {Name: "Auth2", Kind: "component", In: "../acl/auth"}, + }, + } + diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: dql}, result) + require.NotEmpty(t, diags) + invalidCount := 0 + for _, item := range diags { + if item != nil && item.Code == dqldiag.CodeCompRouteInvalid { + invalidCount++ + } + } + assert.Equal(t, 1, invalidCount) +} diff --git a/repository/shape/compile/enrich.go b/repository/shape/compile/enrich.go index 08b80570c..048064043 100644 --- a/repository/shape/compile/enrich.go +++ b/repository/shape/compile/enrich.go @@ -10,7 +10,6 @@ import ( "github.com/viant/datly/repository/shape" "github.com/viant/datly/repository/shape/compile/pipeline" "github.com/viant/datly/repository/shape/plan" - "gopkg.in/yaml.v3" ) var ( @@ -30,6 +29,16 @@ type ruleSettings struct { URI string `json:"URI"` } +type parityEnrichmentContext struct { + source *shape.Source + settings *ruleSettings + baseDir string + module string + sourceName string + joinEmbedRefs map[string]string + joinSubqueryBodies map[string]string +} + func applySourceParityEnrichment(result *plan.Result, source *shape.Source) { applySourceParityEnrichmentWithLayout(result, source, defaultCompilePathLayout()) } @@ -38,233 +47,114 @@ func applySourceParityEnrichmentWithLayout(result *plan.Result, source *shape.So if result == nil || len(result.Views) == 0 { return } - settings := extractRuleSettings(source) - legacyViews := loadLegacyRouteViewAttrsWithLayout(source, settings, layout) - baseDir := sourceSQLBaseDir(source) - module := sourceModuleWithLayout(source, layout) - sourceName := pipeline.SanitizeName(source.Name) - joinEmbedRefs := map[string]string{} - joinSubqueryBodies := map[string]string{} - if len(result.Views) > 0 && result.Views[0] != nil { - sqlForJoinExtract := result.Views[0].SQL - if source != nil && strings.TrimSpace(source.DQL) != "" { - sqlForJoinExtract = source.DQL - } - joinEmbedRefs = extractJoinEmbedRefs(sqlForJoinExtract) - joinSubqueryBodies = extractJoinSubqueryBodies(sqlForJoinExtract) - } + ctx := buildParityEnrichmentContext(result, source, layout) for idx, item := range result.Views { if item == nil { continue } - if legacy, ok := lookupLegacyRouteViewAttr(legacyViews, item.Name); ok { - if legacy.Mode != "" { - item.Mode = legacy.Mode - } - if legacy.Module != "" { - item.Module = legacy.Module - } - if legacy.AllowNulls != nil { - value := *legacy.AllowNulls - item.AllowNulls = &value - } - if legacy.SelectorNamespace != "" { - item.SelectorNamespace = legacy.SelectorNamespace - } - if legacy.SelectorNoLimit != nil { - value := *legacy.SelectorNoLimit - item.SelectorNoLimit = &value - } - if legacy.SchemaType != "" { - item.SchemaType = legacy.SchemaType - } - if legacy.Cardinality != "" { - item.Cardinality = legacy.Cardinality - } - if legacy.HasSummary != nil && *legacy.HasSummary && strings.TrimSpace(item.Summary) == "" { - item.Summary = "legacy-summary" - } - } - if item.SQLURI == "" && baseDir != "" { - item.SQLURI = baseDir + "/" + item.Name + ".sql" - } - if item.Module == "" { - item.Module = module - } - if item.SelectorNamespace == "" { - item.SelectorNamespace = defaultSelectorNamespace(item.Name) - } - if item.SchemaType == "" { - item.SchemaType = defaultSchemaType(item.Name, settings, idx == 0) - } - if shouldInferTable(item) { - candidateSQL := item.SQL - if strings.TrimSpace(candidateSQL) == "" { - candidateSQL = item.Table - } - if table := inferTableFromSQL(candidateSQL, source); table != "" { - item.Table = table - } - } - if strings.HasPrefix(strings.TrimSpace(item.Table), "(") || normalizedTemplatePlaceholderTable(strings.TrimSpace(item.Table)) { - if ref, ok := joinEmbedRefs[item.Name]; ok { - if table := inferTableFromEmbedRef(source, ref); table != "" { - item.Table = table - } - } - if body, ok := joinSubqueryBodies[item.Name]; ok { - if table := inferTableFromSQL(body, source); table != "" { - item.Table = table - } - } - if table := inferTableFromSiblingSQL(item.Name, source); table != "" { - item.Table = table - } - } - if item.Connector == "" && settings.Connector != "" { - item.Connector = settings.Connector - } - if item.Connector == "" && source != nil && strings.TrimSpace(source.Connector) != "" { - item.Connector = strings.TrimSpace(source.Connector) - } - if item.Connector == "" { - item.Connector = inferConnector(item, source) - } - if item.Summary == "" { - item.Summary = extractSummarySQL(item.SQL) - if item.Summary == "" && source != nil { - item.Summary = extractSummarySQL(source.DQL) - } - } + applyViewDefaults(item, idx == 0, ctx) + applyTableInference(item, ctx) + applyConnectorInference(item, ctx) + applySummaryInference(item, ctx) } if source != nil && strings.TrimSpace(source.Path) != "" { - normalizeRootViewName(result, sourceName, settings) + normalizeRootViewName(result, ctx.sourceName) } } -type legacyRouteViewAttr struct { - Name string - Mode string - Module string - AllowNulls *bool - SelectorNamespace string - SelectorNoLimit *bool - SchemaType string - Cardinality string - HasSummary *bool +func buildParityEnrichmentContext(result *plan.Result, source *shape.Source, layout compilePathLayout) *parityEnrichmentContext { + ctx := &parityEnrichmentContext{ + source: source, + settings: extractRuleSettings(source), + baseDir: sourceSQLBaseDir(source), + module: sourceModuleWithLayout(source, layout), + sourceName: pipeline.SanitizeName(source.Name), + joinEmbedRefs: map[string]string{}, + joinSubqueryBodies: map[string]string{}, + } + if len(result.Views) == 0 || result.Views[0] == nil { + return ctx + } + sqlForJoinExtract := result.Views[0].SQL + if source != nil && strings.TrimSpace(source.DQL) != "" { + sqlForJoinExtract = source.DQL + } + ctx.joinEmbedRefs = extractJoinEmbedRefs(sqlForJoinExtract) + ctx.joinSubqueryBodies = extractJoinSubqueryBodies(sqlForJoinExtract) + return ctx } -func loadLegacyRouteViewAttrs(source *shape.Source, settings *ruleSettings) []legacyRouteViewAttr { - return loadLegacyRouteViewAttrsWithLayout(source, settings, defaultCompilePathLayout()) +func applyViewDefaults(item *plan.View, root bool, ctx *parityEnrichmentContext) { + if item == nil || ctx == nil { + return + } + if item.SQLURI == "" && ctx.baseDir != "" { + item.SQLURI = ctx.baseDir + "/" + item.Name + ".sql" + } + if item.Module == "" { + item.Module = ctx.module + } + if item.SelectorNamespace == "" { + item.SelectorNamespace = defaultSelectorNamespace(item.Name) + } + if item.SchemaType == "" { + item.SchemaType = defaultSchemaType(item.Name, ctx.settings, root) + } } -func loadLegacyRouteViewAttrsWithLayout(source *shape.Source, settings *ruleSettings, layout compilePathLayout) []legacyRouteViewAttr { - if source == nil || strings.TrimSpace(source.Path) == "" { - return nil - } - platformRoot, relativeDir, stem, ok := platformPathParts(source.Path, layout) - if !ok { - return nil - } - typeExpr := "" - if settings != nil { - typeExpr = strings.TrimSpace(settings.Type) - } - typeExpr = strings.Trim(typeExpr, `"'`) - typeExpr = strings.TrimSuffix(typeExpr, ".Handler") - typeStem := "" - if typeExpr != "" { - typeStem = filepath.Base(filepath.FromSlash(typeExpr)) - } - routesRoot := joinRelativePath(platformRoot, layout.routesRelative) - routesBase := filepath.Join(routesRoot, filepath.FromSlash(relativeDir)) - candidates := legacyRouteYAMLCandidates(routesBase, stem, typeStem) - for _, candidate := range candidates { - if attrs := parseLegacyRouteViewAttrs(candidate); len(attrs) > 0 { - return attrs +func applyTableInference(item *plan.View, ctx *parityEnrichmentContext) { + if item == nil || ctx == nil { + return + } + if shouldInferTable(item) { + candidateSQL := item.SQL + if strings.TrimSpace(candidateSQL) == "" { + candidateSQL = item.Table + } + if table := inferTableFromSQL(candidateSQL, ctx.source); table != "" { + item.Table = table } } - return nil -} - -func parseLegacyRouteViewAttrs(path string) []legacyRouteViewAttr { - data, err := os.ReadFile(path) - if err != nil { - return nil - } - var payload struct { - Resource struct { - Views []struct { - Name string `yaml:"Name"` - Mode string `yaml:"Mode"` - Module string `yaml:"Module"` - AllowNulls *bool `yaml:"AllowNulls"` - Selector struct { - Namespace string `yaml:"Namespace"` - NoLimit *bool `yaml:"NoLimit"` - } `yaml:"Selector"` - Template struct { - Summary *struct{} `yaml:"Summary"` - } `yaml:"Template"` - Schema struct { - Cardinality string `yaml:"Cardinality"` - DataType string `yaml:"DataType"` - Name string `yaml:"Name"` - } `yaml:"Schema"` - } `yaml:"Views"` - } `yaml:"Resource"` - } - if err = yaml.Unmarshal(data, &payload); err != nil { - return nil - } - result := make([]legacyRouteViewAttr, 0, len(payload.Resource.Views)) - for _, item := range payload.Resource.Views { - cardinality := strings.TrimSpace(item.Schema.Cardinality) - if cardinality != "" { - cardinality = strings.ToLower(cardinality) + if strings.HasPrefix(strings.TrimSpace(item.Table), "(") || normalizedTemplatePlaceholderTable(strings.TrimSpace(item.Table)) { + if ref, ok := ctx.joinEmbedRefs[item.Name]; ok { + if table := inferTableFromEmbedRef(ctx.source, ref); table != "" { + item.Table = table + } + } + if body, ok := ctx.joinSubqueryBodies[item.Name]; ok { + if table := inferTableFromSQL(body, ctx.source); table != "" { + item.Table = table + } + } + if table := inferTableFromSiblingSQL(item.Name, ctx.source); table != "" { + item.Table = table } - result = append(result, legacyRouteViewAttr{ - Name: strings.TrimSpace(item.Name), - Mode: strings.TrimSpace(item.Mode), - Module: strings.TrimSpace(item.Module), - AllowNulls: item.AllowNulls, - SelectorNamespace: strings.TrimSpace(item.Selector.Namespace), - SelectorNoLimit: item.Selector.NoLimit, - SchemaType: firstNonEmptyString(strings.TrimSpace(item.Schema.DataType), strings.TrimSpace(item.Schema.Name)), - Cardinality: cardinality, - HasSummary: func() *bool { - if item.Template.Summary == nil { - return nil - } - value := true - return &value - }(), - }) } - return result } -func lookupLegacyRouteViewAttr(items []legacyRouteViewAttr, name string) (legacyRouteViewAttr, bool) { - name = strings.TrimSpace(name) - if name == "" { - return legacyRouteViewAttr{}, false +func applyConnectorInference(item *plan.View, ctx *parityEnrichmentContext) { + if item == nil || ctx == nil || item.Connector != "" { + return } - for _, item := range items { - if strings.EqualFold(strings.TrimSpace(item.Name), name) { - return item, true - } + if ctx.settings != nil && ctx.settings.Connector != "" { + item.Connector = ctx.settings.Connector + } + if item.Connector == "" && ctx.source != nil && strings.TrimSpace(ctx.source.Connector) != "" { + item.Connector = strings.TrimSpace(ctx.source.Connector) + } + if item.Connector == "" { + item.Connector = inferConnector(item, ctx.source) } - return legacyRouteViewAttr{}, false } -func firstNonEmptyString(values ...string) string { - for _, value := range values { - value = strings.TrimSpace(value) - if value != "" { - return value - } +func applySummaryInference(item *plan.View, ctx *parityEnrichmentContext) { + if item == nil || ctx == nil || item.Summary != "" { + return + } + item.Summary = extractSummarySQL(item.SQL) + if item.Summary == "" && ctx.source != nil { + item.Summary = extractSummarySQL(ctx.source.DQL) } - return "" } func extractSummarySQL(sqlText string) string { @@ -677,7 +567,7 @@ func inferConnector(item *plan.View, source *shape.Source) string { } } -func normalizeRootViewName(result *plan.Result, sourceName string, settings *ruleSettings) { +func normalizeRootViewName(result *plan.Result, sourceName string) { if result == nil || len(result.Views) == 0 { return } @@ -689,7 +579,6 @@ func normalizeRootViewName(result *plan.Result, sourceName string, settings *rul if desired == "" { return } - _ = settings current := strings.TrimSpace(root.Name) if current == "" { root.Name = desired diff --git a/repository/shape/compile/preprocess_handler.go b/repository/shape/compile/preprocess_handler.go index bea319f78..22f8c4dfe 100644 --- a/repository/shape/compile/preprocess_handler.go +++ b/repository/shape/compile/preprocess_handler.go @@ -9,16 +9,13 @@ import ( "github.com/viant/datly/repository/shape/compile/pipeline" dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" dqlstmt "github.com/viant/datly/repository/shape/dql/statement" - "github.com/viant/datly/repository/shape/plan" ) type handlerPreprocessResult struct { - Pre *dqlpre.Result - Statements dqlstmt.Statements - Decision pipeline.Decision - LegacyViews []*plan.View - EffectiveSource *shape.Source - ForceLegacyContract bool + Pre *dqlpre.Result + Statements dqlstmt.Statements + Decision pipeline.Decision + EffectiveSource *shape.Source } func buildHandlerIfNeeded(source *shape.Source, pre *dqlpre.Result, statements dqlstmt.Statements, decision pipeline.Decision, layout compilePathLayout) *handlerPreprocessResult { @@ -45,21 +42,17 @@ func buildHandlerIfNeeded(source *shape.Source, pre *dqlpre.Result, statements d } func buildHandlerFromContractIfNeeded(ret *handlerPreprocessResult, source *shape.Source, layout compilePathLayout) bool { - if ret == nil || source == nil { - return false - } - return buildLegacyRouteFallbackIfNeeded(ret, source, layout) + _ = ret + _ = source + _ = layout + return false } func buildGeneratedFallbackIfNeeded(ret *handlerPreprocessResult, source *shape.Source, layout compilePathLayout) bool { if ret == nil || source == nil { return false } - if alternate := resolveGeneratedLegacySource(source); alternate != nil { - if buildLegacyRouteFallbackIfNeeded(ret, alternate, layout) { - return true - } - } + _ = layout generated := strings.TrimSpace(resolveGeneratedCompanionDQL(source)) if generated == "" { return false @@ -79,20 +72,6 @@ func buildGeneratedFallbackIfNeeded(ret *handlerPreprocessResult, source *shape. return true } -func buildLegacyRouteFallbackIfNeeded(ret *handlerPreprocessResult, source *shape.Source, layout compilePathLayout) bool { - if ret == nil || source == nil { - return false - } - legacyFallbackViews := resolveLegacyRouteViewsWithLayout(source, layout) - if len(legacyFallbackViews) == 0 { - return false - } - ret.LegacyViews = legacyFallbackViews - ret.EffectiveSource = source - ret.ForceLegacyContract = true - return true -} - func resolveGeneratedLegacySource(source *shape.Source) *shape.Source { if source == nil || strings.TrimSpace(source.Path) == "" { return nil diff --git a/repository/shape/compile/preprocess_handler_test.go b/repository/shape/compile/preprocess_handler_test.go index 7d4e57829..1f8c8d08d 100644 --- a/repository/shape/compile/preprocess_handler_test.go +++ b/repository/shape/compile/preprocess_handler_test.go @@ -22,28 +22,21 @@ func TestIsHandlerSignal(t *testing.T) { assert.False(t, isHandlerSignal(&shape.Source{DQL: `SELECT 1`})) } -func TestBuildHandlerFromContractIfNeeded_LegacyFallbackViews(t *testing.T) { +func TestBuildHandlerFromContractIfNeeded_Disabled(t *testing.T) { tempDir := t.TempDir() sourcePath := filepath.Join(tempDir, "dql", "platform", "campaign", "post.dql") require.NoError(t, os.MkdirAll(filepath.Dir(sourcePath), 0o755)) dql := `/* {"Type":"campaign/patch.Handler","Connector":"ci_ads"} */` require.NoError(t, os.WriteFile(sourcePath, []byte(dql), 0o644)) - routeDir := filepath.Join(tempDir, "repo", "dev", "Datly", "routes", "platform", "campaign", "patch", "post") - require.NoError(t, os.MkdirAll(routeDir, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(routeDir, "post.sql"), []byte(`SELECT 1`), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(routeDir, "CurCampaign.sql"), []byte(`SELECT * FROM CI_CAMPAIGN`), 0o644)) - source := &shape.Source{Path: sourcePath, DQL: dql} pre := dqlpre.Prepare(source.DQL) statements := dqlstmt.New(pre.SQL) decision := pipeline.Classify(statements) result := &handlerPreprocessResult{Pre: pre, Statements: statements, Decision: decision, EffectiveSource: source} applied := buildHandlerFromContractIfNeeded(result, source, defaultCompilePathLayout()) - require.True(t, applied) + require.False(t, applied) require.NotNil(t, result) - require.NotEmpty(t, result.LegacyViews) - assert.Equal(t, "post", result.LegacyViews[0].Name) } func TestBuildGeneratedFallbackIfNeeded_GeneratedCompanion(t *testing.T) { @@ -63,7 +56,6 @@ func TestBuildGeneratedFallbackIfNeeded_GeneratedCompanion(t *testing.T) { applied := buildGeneratedFallbackIfNeeded(result, source, defaultCompilePathLayout()) require.True(t, applied) require.NotNil(t, result) - assert.Empty(t, result.LegacyViews) assert.Contains(t, result.Pre.SQL, "SELECT o.id FROM ORDERS o") assert.True(t, result.Decision.HasRead) } @@ -84,36 +76,11 @@ func TestResolveGeneratedLegacySource(t *testing.T) { assert.Contains(t, actual.DQL, `"Type":"session/patch.Handler"`) } -func TestBuildGeneratedFallbackIfNeeded_GeneratedLegacyRoute(t *testing.T) { +func TestBuildGeneratedFallbackIfNeeded_NoGeneratedCompanionWithoutTypeHeader(t *testing.T) { tempDir := t.TempDir() genPath := filepath.Join(tempDir, "dql", "system", "session", "gen", "session", "patch.dql") require.NoError(t, os.MkdirAll(filepath.Dir(genPath), 0o755)) require.NoError(t, os.WriteFile(genPath, []byte(`/* {"Method":"PATCH","URI":"/v1/api/system/session"} */`), 0o644)) - legacySQL := filepath.Join(tempDir, "dql", "system", "session", "patch.sql") - require.NoError(t, os.MkdirAll(filepath.Dir(legacySQL), 0o755)) - require.NoError(t, os.WriteFile(legacySQL, []byte(`/* {"Type":"session/patch.Handler","Connector":"system"} */`), 0o644)) - - routesDir := filepath.Join(tempDir, "repo", "dev", "Datly", "routes", "system", "session", "patch") - require.NoError(t, os.MkdirAll(routesDir, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(routesDir), "patch.yaml"), []byte(`Resource: - Views: - - Name: patch - Mode: SQLExec - Connector: - Ref: system - Template: - SourceURL: patch/patch.sql - Parameters: - - Name: Session - In: - Kind: body - Name: data - Types: - - Name: Input - DataType: "*Input" - Package: session/patch -`), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(routesDir, "patch.sql"), []byte(`$Nop($Unsafe.Session)`), 0o644)) source := &shape.Source{Path: genPath, DQL: `/* {"Method":"PATCH","URI":"/v1/api/system/session"} */`} pre := dqlpre.Prepare(source.DQL) @@ -121,23 +88,16 @@ func TestBuildGeneratedFallbackIfNeeded_GeneratedLegacyRoute(t *testing.T) { decision := pipeline.Classify(statements) result := &handlerPreprocessResult{Pre: pre, Statements: statements, Decision: decision, EffectiveSource: source} applied := buildGeneratedFallbackIfNeeded(result, source, defaultCompilePathLayout()) - require.True(t, applied) + require.False(t, applied) require.NotNil(t, result) - require.True(t, result.ForceLegacyContract) - require.NotNil(t, result.EffectiveSource) - assert.Equal(t, legacySQL, result.EffectiveSource.Path) - require.NotEmpty(t, result.LegacyViews) - assert.Equal(t, "patch", result.LegacyViews[0].Name) } -func TestBuildLegacyRouteFallbackIfNeeded_NoLegacyRoute(t *testing.T) { +func TestBuildGeneratedFallbackIfNeeded_NoGeneratedCompanion(t *testing.T) { source := &shape.Source{Path: filepath.Join(t.TempDir(), "dql", "x", "y", "z.dql"), DQL: `SELECT 1`} pre := dqlpre.Prepare(source.DQL) statements := dqlstmt.New(pre.SQL) decision := pipeline.Classify(statements) result := &handlerPreprocessResult{Pre: pre, Statements: statements, Decision: decision, EffectiveSource: source} - applied := buildLegacyRouteFallbackIfNeeded(result, source, defaultCompilePathLayout()) + applied := buildGeneratedFallbackIfNeeded(result, source, defaultCompilePathLayout()) assert.False(t, applied) - assert.Empty(t, result.LegacyViews) - assert.False(t, result.ForceLegacyContract) } diff --git a/repository/shape/compile/strings_util.go b/repository/shape/compile/strings_util.go new file mode 100644 index 000000000..5c18c9d8a --- /dev/null +++ b/repository/shape/compile/strings_util.go @@ -0,0 +1,13 @@ +package compile + +import "strings" + +func firstNonEmptyString(values ...string) string { + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + return value + } + } + return "" +} diff --git a/repository/shape/compile/type_support.go b/repository/shape/compile/type_support.go new file mode 100644 index 000000000..44a8b9bca --- /dev/null +++ b/repository/shape/compile/type_support.go @@ -0,0 +1,238 @@ +package compile + +import ( + "reflect" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/x" +) + +func applyLinkedTypeSupport(result *plan.Result, source *shape.Source) { + if result == nil || source == nil { + return + } + registry := source.EnsureTypeRegistry() + if registry == nil || len(registry.Keys()) == 0 { + return + } + resolver := typectx.NewResolver(registry, result.TypeContext) + rootTypeKey := resolveRootTypeKey(source, resolver, registry) + existing := existingTypesByName(result.Types) + + for idx, item := range result.Views { + if item == nil { + continue + } + resolvedKey := resolveViewTypeKey(item, idx == 0, rootTypeKey, resolver, registry) + if resolvedKey == "" { + continue + } + resolvedType := registry.Lookup(resolvedKey) + if resolvedType == nil || resolvedType.Type == nil { + continue + } + rType := unwrapResolvedType(resolvedType.Type) + if rType == nil { + continue + } + typeExpr, typePkg := schemaTypeExpression(rType, result.TypeContext) + if shouldSetSchemaType(item) && typeExpr != "" { + item.SchemaType = typeExpr + } + name := strings.TrimSpace(rType.Name()) + if name == "" { + continue + } + key := strings.ToLower(name) + if existing[key] { + continue + } + result.Types = append(result.Types, &plan.Type{ + Name: name, + DataType: typeExpr, + Cardinality: strings.TrimSpace(item.Cardinality), + Package: typePkg, + ModulePath: strings.TrimSpace(rType.PkgPath()), + }) + existing[key] = true + } +} + +func resolveRootTypeKey(source *shape.Source, resolver *typectx.Resolver, registry *x.Registry) string { + if source == nil || registry == nil { + return "" + } + if key := resolveTypeKey(strings.TrimSpace(source.TypeName), resolver, registry); key != "" { + return key + } + rType, err := source.ResolveRootType() + if err != nil || rType == nil { + return "" + } + return resolveTypeKey(x.NewType(rType).Key(), resolver, registry) +} + +func resolveViewTypeKey(item *plan.View, root bool, rootTypeKey string, resolver *typectx.Resolver, registry *x.Registry) string { + if item == nil || registry == nil { + return "" + } + candidates := make([]string, 0, 8) + seen := map[string]bool{} + appendCandidate := func(value string) { + value = strings.TrimSpace(value) + if value == "" { + return + } + if seen[value] { + return + } + seen[value] = true + candidates = append(candidates, value) + } + + if root && rootTypeKey != "" { + appendCandidate(rootTypeKey) + } + if item.Declaration != nil { + appendCandidate(item.Declaration.DataType) + appendCandidate(item.Declaration.Of) + } + appendCandidate(item.SchemaType) + name := toExportedTypeName(item.Name) + if name != "" { + appendCandidate(name + "View") + appendCandidate(name) + } + for _, candidate := range candidates { + if key := resolveTypeKey(candidate, resolver, registry); key != "" { + return key + } + } + return "" +} + +func resolveTypeKey(typeExpr string, resolver *typectx.Resolver, registry *x.Registry) string { + if registry == nil { + return "" + } + base := normalizeTypeLookupKey(typeExpr) + if base == "" { + return "" + } + if registry.Lookup(base) != nil { + return base + } + if resolver == nil { + return "" + } + resolved, err := resolver.Resolve(base) + if err != nil || resolved == "" { + return "" + } + if registry.Lookup(resolved) == nil { + return "" + } + return resolved +} + +func normalizeTypeLookupKey(typeExpr string) string { + value := strings.TrimSpace(typeExpr) + for { + switch { + case strings.HasPrefix(value, "*"): + value = strings.TrimPrefix(value, "*") + case strings.HasPrefix(value, "[]"): + value = strings.TrimPrefix(value, "[]") + default: + return strings.TrimSpace(value) + } + } +} + +func shouldSetSchemaType(item *plan.View) bool { + if item == nil { + return false + } + current := strings.TrimSpace(item.SchemaType) + if current == "" { + return true + } + expectedDefault := "*" + toExportedTypeName(item.Name) + "View" + return current == expectedDefault +} + +func existingTypesByName(input []*plan.Type) map[string]bool { + result := map[string]bool{} + for _, item := range input { + if item == nil { + continue + } + name := strings.ToLower(strings.TrimSpace(item.Name)) + if name == "" { + continue + } + result[name] = true + } + return result +} + +func schemaTypeExpression(rType reflect.Type, ctx *typectx.Context) (string, string) { + rType = unwrapResolvedType(rType) + if rType == nil { + return "", "" + } + typeName := strings.TrimSpace(rType.Name()) + if typeName == "" { + return "", "" + } + pkgPath := strings.TrimSpace(rType.PkgPath()) + if pkgPath == "" { + return "*" + typeName, "" + } + pkgAlias := packageAlias(pkgPath, ctx) + if pkgAlias == "" { + return "*" + typeName, "" + } + return "*" + pkgAlias + "." + typeName, pkgAlias +} + +func packageAlias(pkgPath string, ctx *typectx.Context) string { + pkgPath = strings.TrimSpace(pkgPath) + if pkgPath == "" { + return "" + } + if ctx != nil { + for _, item := range ctx.Imports { + if strings.TrimSpace(item.Package) != pkgPath { + continue + } + alias := strings.TrimSpace(item.Alias) + if alias != "" { + return alias + } + } + if strings.TrimSpace(ctx.PackagePath) == pkgPath && strings.TrimSpace(ctx.PackageName) != "" { + return strings.TrimSpace(ctx.PackageName) + } + } + index := strings.LastIndex(pkgPath, "/") + if index == -1 || index+1 >= len(pkgPath) { + return pkgPath + } + return pkgPath[index+1:] +} + +func unwrapResolvedType(rType reflect.Type) reflect.Type { + for rType != nil { + switch rType.Kind() { + case reflect.Ptr, reflect.Slice, reflect.Array: + rType = rType.Elem() + default: + return rType + } + } + return nil +} diff --git a/repository/shape/compile/type_support_test.go b/repository/shape/compile/type_support_test.go new file mode 100644 index 000000000..b3c376c55 --- /dev/null +++ b/repository/shape/compile/type_support_test.go @@ -0,0 +1,70 @@ +package compile + +import ( + "context" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/x" +) + +type linkedRootType struct { + ID int +} + +type OrdersView struct { + ID int +} + +func TestDQLCompiler_Compile_UsesLinkedRootTypeForSchemaType(t *testing.T) { + compiler := New() + source := &shape.Source{ + Name: "orders_report", + Type: reflect.TypeOf(linkedRootType{}), + TypeName: x.NewType(reflect.TypeOf(linkedRootType{})).Key(), + DQL: "SELECT t.id FROM ORDERS t", + } + + res, err := compiler.Compile(context.Background(), source) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Views) + assert.Equal(t, "*compile.linkedRootType", planned.Views[0].SchemaType) + require.NotEmpty(t, planned.Types) + assert.Equal(t, "linkedRootType", planned.Types[0].Name) + assert.Equal(t, "*compile.linkedRootType", planned.Types[0].DataType) +} + +func TestDQLCompiler_Compile_UsesLinkedRegistryTypeForNamedView(t *testing.T) { + compiler := New() + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(OrdersView{}))) + source := &shape.Source{ + Name: "orders", + TypeRegistry: registry, + DQL: "SELECT orders.id FROM ORDERS orders", + } + + res, err := compiler.Compile(context.Background(), source) + require.NoError(t, err) + planned, ok := res.Plan.(*plan.Result) + require.True(t, ok) + require.NotEmpty(t, planned.Views) + assert.Equal(t, "*compile.OrdersView", planned.Views[0].SchemaType) + + var found *plan.Type + for _, item := range planned.Types { + if item != nil && item.Name == "OrdersView" { + found = item + break + } + } + require.NotNil(t, found) + assert.Equal(t, "*compile.OrdersView", found.DataType) + assert.Equal(t, "compile", found.Package) +} diff --git a/repository/shape/compile/typectx_defaults.go b/repository/shape/compile/typectx_defaults.go index 5bc0a9d9a..5561d36a3 100644 --- a/repository/shape/compile/typectx_defaults.go +++ b/repository/shape/compile/typectx_defaults.go @@ -30,6 +30,7 @@ func applyTypeContextDefaults(ctx *typectx.Context, source *shape.Source, opts * } } } + ret = normalizeRelativeImports(ret, source, layout) return normalizeTypeContext(ret) } @@ -64,24 +65,11 @@ func mergeTypeContext(dst *typectx.Context, src *typectx.Context) *typectx.Conte } func inferDatlyGenTypeContext(source *shape.Source, layout compilePathLayout) *typectx.Context { - if source == nil { - return nil - } - sourcePath := strings.TrimSpace(source.Path) - if sourcePath == "" { - return nil - } - normalizedPath := filepath.ToSlash(filepath.Clean(sourcePath)) - idx := strings.Index(normalizedPath, layout.dqlMarker) - if idx == -1 { - return nil - } - projectRoot := filepath.FromSlash(strings.TrimSuffix(normalizedPath[:idx], "/")) - rel := strings.TrimPrefix(normalizedPath[idx+len(layout.dqlMarker):], "/") - if rel == "" { + parsed, ok := parseSourceLayout(source, layout) + if !ok { return nil } - routeDir := strings.Trim(path.Dir(rel), "/") + routeDir := strings.Trim(path.Dir(parsed.relativePath), "/") if routeDir == "." { routeDir = "" } @@ -94,7 +82,7 @@ func inferDatlyGenTypeContext(source *shape.Source, layout compilePathLayout) *t packageName = path.Base(routeDir) } packagePath := "" - if module := detectModulePath(projectRoot); module != "" { + if module := detectModulePath(parsed.projectRoot); module != "" { packagePath = path.Join(module, packageDir) } return normalizeTypeContext(&typectx.Context{ @@ -156,3 +144,83 @@ func normalizeTypeContext(ctx *typectx.Context) *typectx.Context { } return ctx } + +func normalizeRelativeImports(ctx *typectx.Context, source *shape.Source, layout compilePathLayout) *typectx.Context { + if ctx == nil || len(ctx.Imports) == 0 { + return ctx + } + modulePath := modulePathForSource(source, layout) + if modulePath == "" { + return ctx + } + for i, item := range ctx.Imports { + pkg := strings.TrimSpace(item.Package) + if pkg == "" { + continue + } + ctx.Imports[i].Package = normalizeImportPackage(pkg, modulePath) + } + return ctx +} + +func modulePathForSource(source *shape.Source, layout compilePathLayout) string { + parsed, ok := parseSourceLayout(source, layout) + if !ok { + return "" + } + return detectModulePath(parsed.projectRoot) +} + +func normalizeImportPackage(pkg, modulePath string) string { + pkg = strings.Trim(strings.ReplaceAll(strings.TrimSpace(pkg), "\\", "/"), "/") + if pkg == "" { + return "" + } + if !strings.Contains(pkg, "/") { + return pkg + } + if strings.HasPrefix(pkg, modulePath+"/") || pkg == modulePath { + return pkg + } + first := pkg + if index := strings.Index(first, "/"); index != -1 { + first = first[:index] + } + if strings.Contains(first, ".") { + return pkg + } + return path.Join(modulePath, pkg) +} + +type sourceLayout struct { + projectRoot string + relativePath string +} + +func parseSourceLayout(source *shape.Source, layout compilePathLayout) (*sourceLayout, bool) { + if source == nil { + return nil, false + } + sourcePath := strings.TrimSpace(source.Path) + if sourcePath == "" { + return nil, false + } + marker := strings.TrimSpace(layout.dqlMarker) + if marker == "" { + marker = defaultCompilePathLayout().dqlMarker + } + normalizedPath := filepath.ToSlash(filepath.Clean(sourcePath)) + idx := strings.Index(normalizedPath, marker) + if idx == -1 { + return nil, false + } + projectRoot := filepath.FromSlash(strings.TrimSuffix(normalizedPath[:idx], "/")) + relativePath := strings.TrimPrefix(normalizedPath[idx+len(marker):], "/") + if relativePath == "" { + return nil, false + } + return &sourceLayout{ + projectRoot: projectRoot, + relativePath: relativePath, + }, true +} diff --git a/repository/shape/compile/typectx_defaults_test.go b/repository/shape/compile/typectx_defaults_test.go index 4f0d01a36..aa3d01d8c 100644 --- a/repository/shape/compile/typectx_defaults_test.go +++ b/repository/shape/compile/typectx_defaults_test.go @@ -67,4 +67,20 @@ func TestApplyTypeContextDefaults_Matrix(t *testing.T) { }, layout) require.Nil(t, got) }) + + t.Run("relative imports are normalized to module path", func(t *testing.T) { + input := &typectx.Context{ + Imports: []typectx.Import{ + {Alias: "sess", Package: "pkg/platform/system/session"}, + {Alias: "perf", Package: "github.com/acme/perf"}, + {Alias: "time", Package: "time"}, + }, + } + got := applyTypeContextDefaults(input, source, nil, layout) + require.NotNil(t, got) + require.Len(t, got.Imports, 3) + require.Equal(t, "github.vianttech.com/viant/platform/pkg/platform/system/session", got.Imports[0].Package) + require.Equal(t, "github.com/acme/perf", got.Imports[1].Package) + require.Equal(t, "time", got.Imports[2].Package) + }) } diff --git a/repository/shape/platform_parity_test.go b/repository/shape/platform_parity_test.go index f6837b03c..17fa1f1f4 100644 --- a/repository/shape/platform_parity_test.go +++ b/repository/shape/platform_parity_test.go @@ -207,6 +207,9 @@ type parityEntryEval struct { } func TestPlatform_DQLToRoute_ParityIR_SmokeHandlers(t *testing.T) { + if !strings.EqualFold(strings.TrimSpace(os.Getenv("PLATFORM_PARITY_SMOKE")), "1") { + t.Skip("set PLATFORM_PARITY_SMOKE=1 to run legacy parity smoke handlers") + } platformRoot := os.Getenv("PLATFORM_ROOT") if platformRoot == "" { platformRoot = "/Users/awitas/go/src/github.vianttech.com/viant/platform" diff --git a/warmup/cache_test.go b/warmup/cache_test.go index 408a4a31a..9196529d6 100644 --- a/warmup/cache_test.go +++ b/warmup/cache_test.go @@ -2,6 +2,7 @@ package warmup import ( "context" + "os" "path" "testing" @@ -12,6 +13,10 @@ import ( ) func TestPopulateCache(t *testing.T) { + if os.Getenv("DATLY_RUN_WARMUP_TESTS") == "" { + t.Skip("set DATLY_RUN_WARMUP_TESTS=1 to run warmup integration test") + } + testCases := []struct { description string URL string From b218524a5681acbb6969e2f0df7c325d5b312308 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 24 Feb 2026 07:27:51 -0800 Subject: [PATCH 128/279] shape/compile: add type support helpers; refine preprocessing and type defaults; update tests --- Version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Version b/Version index 014ec6192..fcc9d59a4 100644 --- a/Version +++ b/Version @@ -1 +1 @@ -v0.20.2 \ No newline at end of file +v0.21.0 \ No newline at end of file From 833f42f8f5b71669ad958ebf24afed3a9f3862aa Mon Sep 17 00:00:00 2001 From: kkincaid Date: Tue, 24 Feb 2026 10:30:27 -0800 Subject: [PATCH 129/279] Fix nil panics and route loss during config sync cycles --- gateway/router/handler.go | 18 ++++++++++++++++++ gateway/service.go | 11 +++++++++++ repository/provider.go | 4 +++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/gateway/router/handler.go b/gateway/router/handler.go index ef3b129c3..dbc40f3da 100644 --- a/gateway/router/handler.go +++ b/gateway/router/handler.go @@ -188,6 +188,10 @@ func (r *Handler) Handle(ctx context.Context, writer http.ResponseWriter, reques http.Error(writer, err.Error(), http.StatusInternalServerError) return } + if aComponent == nil { + http.Error(writer, "component not available", http.StatusServiceUnavailable) + return + } aResponse, err := r.safelyHandleComponent(ctx, request, aComponent) if err != nil { r.writeErrorResponse(ctx, writer, aComponent, err, http.StatusBadRequest) @@ -237,6 +241,20 @@ func (r *Handler) writeErrorResponse(ctx context.Context, w http.ResponseWriter, execCtx.SetError(err) } responseStatus := r.responseStatusError(message, anObjectErr) + if aComponent == nil || aComponent.Output.Type.Parameters == nil { + errAsBytes, marshalErr := goJson.Marshal(responseStatus) + if marshalErr != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("could not parse error message")) + return + } + if execCtx != nil { + execCtx.StatusCode = statusCode + } + w.WriteHeader(statusCode) + w.Write(errAsBytes) + return + } statusParameter := aComponent.Output.Type.Parameters.LookupByLocation(state.KindOutput, "status") if statusParameter == nil { errAsBytes, marshalErr := goJson.Marshal(responseStatus) diff --git a/gateway/service.go b/gateway/service.go index efd3b7afc..794c2b2d7 100644 --- a/gateway/service.go +++ b/gateway/service.go @@ -224,6 +224,17 @@ func (r *Service) syncChanges(ctx context.Context, metrics *gmetric.Service, sta return err } r.mux.Lock() + newCount := len(mainRouter.paths) + oldCount := 0 + if r.mainRouter != nil { + oldCount = len(r.mainRouter.paths) + } + if newCount < oldCount { + r.mux.Unlock() + fmt.Printf("[INFO]: routers rebuild skipped (new config has %d routes vs %d existing, keeping existing)\n", newCount, oldCount) + return nil + } + fmt.Printf("[INFO]: router replacing old(%d routes) with new(%d routes)\n", oldCount, newCount) r.mainRouter = mainRouter r.mux.Unlock() fmt.Printf("[INFO]: routers rebuild completed after: %s\n", time.Since(start)) diff --git a/repository/provider.go b/repository/provider.go index ffe166891..b294fcc10 100644 --- a/repository/provider.go +++ b/repository/provider.go @@ -26,7 +26,9 @@ func (p *Provider) Component(ctx context.Context, opts ...Option) (*Component, e p.mux.Lock() defer p.mux.Unlock() if p.control.ChangeKind() == version.ChangeKindDeleted { - //TODO maybe return 404 error + if p.component != nil { + return p.component, nil + } return nil, nil } aComponent, err := p.newComponent(ctx, opts...) From d36ccafdb941f75a4d19b041abbdc615cc08dca9 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 24 Feb 2026 11:27:24 -0800 Subject: [PATCH 130/279] shape/compile: add type support helpers; refine preprocessing and type defaults; update tests --- internal/translator/resource.go | 220 ++++++++++++++++++ repository/shape/compile/compiler_test.go | 16 ++ repository/shape/compile/enrich.go | 22 +- repository/shape/compile/legacy_adapter.go | 8 +- .../shape/compile/preprocess_handler.go | 2 +- 5 files changed, 256 insertions(+), 12 deletions(-) diff --git a/internal/translator/resource.go b/internal/translator/resource.go index 00630f33d..4330ce91f 100644 --- a/internal/translator/resource.go +++ b/internal/translator/resource.go @@ -3,8 +3,10 @@ package translator import ( "context" "fmt" + "net/http" "path" "reflect" + "regexp" "strings" "github.com/viant/afs" @@ -14,6 +16,7 @@ import ( "github.com/viant/datly/internal/msg" "github.com/viant/datly/internal/setter" tparser "github.com/viant/datly/internal/translator/parser" + "github.com/viant/datly/repository/content" expand "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/shared" "github.com/viant/datly/utils/types" @@ -28,6 +31,27 @@ import ( "golang.org/x/mod/modfile" ) +var ( + routeSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$route\s*\(([^)]*)\)\s*\)\s*$`) + marshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$marshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + unmarshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$unmarshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + formatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + dateFormatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$date_format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + caseFormatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$case_format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + quotedArgExpr = regexp.MustCompile(`['"]([^'"]*)['"]`) +) + +type routeSettingsDirective struct { + URI string + Methods []string + JSONMarshalType string + JSONUnmarshalType string + XMLUnmarshalType string + Format string + DateFormat string + CaseFormat string +} + type ( Resource struct { Generated bool @@ -408,6 +432,35 @@ func (r *Resource) extractRuleSetting(dSQL *string) error { } *dSQL = (*dSQL)[index+2:] } + if directive, ok, err := parseSettingsDirectives(*dSQL); err != nil { + return err + } else if ok { + if directive.URI != "" { + r.Rule.URI = directive.URI + } + if len(directive.Methods) > 0 { + r.Rule.Method = strings.Join(directive.Methods, ",") + } + if directive.JSONMarshalType != "" { + r.Rule.JSONMarshalType = directive.JSONMarshalType + } + if directive.JSONUnmarshalType != "" { + r.Rule.JSONUnmarshalType = directive.JSONUnmarshalType + } + if directive.XMLUnmarshalType != "" { + r.Rule.XMLUnmarshalType = directive.XMLUnmarshalType + } + if directive.Format != "" { + r.Rule.DataFormat = directive.Format + } + if directive.DateFormat != "" { + r.Rule.Route.Content.DateFormat = directive.DateFormat + } + if directive.CaseFormat != "" { + r.Rule.Route.Output.CaseFormat = text.CaseFormat(directive.CaseFormat) + } + *dSQL = removeSettingsDirectives(*dSQL) + } r.Rule.applyShortHands() if r.Rule.Connector != "" { r.rule.Connector = r.Rule.Connector @@ -416,6 +469,173 @@ func (r *Resource) extractRuleSetting(dSQL *string) error { return nil } +func parseSettingsDirectives(dSQL string) (*routeSettingsDirective, bool, error) { + ret := &routeSettingsDirective{} + var found bool + matches := routeSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + if len(last) < 2 { + return nil, false, fmt.Errorf("invalid $route directive") + } + args := parseQuotedArgs(last[1]) + if len(args) == 0 { + return nil, false, fmt.Errorf("invalid $route directive: missing URI") + } + URI := strings.TrimSpace(args[0]) + if !strings.HasPrefix(URI, "/") { + return nil, false, fmt.Errorf("invalid $route directive: URI must start with /") + } + methods, err := normalizeRouteMethods(args[1:]) + if err != nil { + return nil, false, err + } + ret.URI = URI + ret.Methods = methods + } + + matches = marshalSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + if len(last) < 3 { + return nil, false, fmt.Errorf("invalid $marshal directive") + } + mimeType := strings.ToLower(strings.TrimSpace(last[1])) + if mimeType != content.JSONContentType { + return nil, false, fmt.Errorf("invalid $marshal directive: unsupported mime type %q", mimeType) + } + typeName := strings.TrimSpace(last[2]) + if typeName == "" { + return nil, false, fmt.Errorf("invalid $marshal directive: missing type") + } + ret.JSONMarshalType = typeName + } + + matches = unmarshalSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + for _, match := range matches { + if len(match) < 3 { + return nil, false, fmt.Errorf("invalid $unmarshal directive") + } + mimeType := strings.ToLower(strings.TrimSpace(match[1])) + typeName := strings.TrimSpace(match[2]) + if typeName == "" { + return nil, false, fmt.Errorf("invalid $unmarshal directive: missing type") + } + switch mimeType { + case content.JSONContentType: + ret.JSONUnmarshalType = typeName + case content.XMLContentType: + ret.XMLUnmarshalType = typeName + default: + return nil, false, fmt.Errorf("invalid $unmarshal directive: unsupported mime type %q", mimeType) + } + } + } + + matches = formatSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + if len(last) < 2 { + return nil, false, fmt.Errorf("invalid $format directive") + } + raw := strings.ToLower(strings.TrimSpace(last[1])) + switch raw { + case "tabular_json": + ret.Format = content.JSONDataFormatTabular + case content.JSONFormat, content.XMLFormat, content.CSVFormat, content.JSONDataFormatTabular: + ret.Format = raw + default: + return nil, false, fmt.Errorf("invalid $format directive: unsupported format %q", raw) + } + } + + matches = dateFormatSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + if len(last) < 2 || strings.TrimSpace(last[1]) == "" { + return nil, false, fmt.Errorf("invalid $date_format directive") + } + ret.DateFormat = strings.TrimSpace(last[1]) + } + + matches = caseFormatSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + if len(last) < 2 || strings.TrimSpace(last[1]) == "" { + return nil, false, fmt.Errorf("invalid $case_format directive") + } + caseFormat := strings.TrimSpace(last[1]) + if !text.NewCaseFormat(caseFormat).IsDefined() { + return nil, false, fmt.Errorf("invalid $case_format directive: unsupported case format %q", caseFormat) + } + ret.CaseFormat = caseFormat + } + return ret, found, nil +} + +func parseQuotedArgs(input string) []string { + matches := quotedArgExpr.FindAllStringSubmatch(input, -1) + result := make([]string, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + result = append(result, strings.TrimSpace(match[1])) + } + return result +} + +func normalizeRouteMethods(input []string) ([]string, error) { + if len(input) == 0 { + return nil, nil + } + valid := map[string]bool{ + http.MethodGet: true, + http.MethodPost: true, + http.MethodPut: true, + http.MethodPatch: true, + http.MethodDelete: true, + http.MethodHead: true, + http.MethodOptions: true, + http.MethodTrace: true, + http.MethodConnect: true, + } + seen := map[string]bool{} + result := make([]string, 0, len(input)) + for _, item := range input { + method := strings.ToUpper(strings.TrimSpace(item)) + if method == "" { + return nil, fmt.Errorf("invalid $route directive: empty method") + } + if !valid[method] { + return nil, fmt.Errorf("invalid $route directive: unsupported method %q", method) + } + if seen[method] { + continue + } + seen[method] = true + result = append(result, method) + } + return result, nil +} + +func removeSettingsDirectives(dSQL string) string { + dSQL = routeSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = marshalSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = unmarshalSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = formatSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = dateFormatSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = caseFormatSettingsLineExpr.ReplaceAllString(dSQL, "") + return dSQL +} + func (r *Resource) expandSQL(viewlet *Viewlet) (*sqlx.SQL, error) { types := viewlet.Resource.Resource.TypeRegistry() resourceState := viewlet.Resource.State diff --git a/repository/shape/compile/compiler_test.go b/repository/shape/compile/compiler_test.go index f14487850..456f99568 100644 --- a/repository/shape/compile/compiler_test.go +++ b/repository/shape/compile/compiler_test.go @@ -101,6 +101,13 @@ func TestDQLCompiler_Compile_PropagatesSpecialDirectives(t *testing.T) { #settings($_ = $connector('analytics')) #settings($_ = $cache(true, '5m')) #settings($_ = $mcp('orders.search', 'Search orders', 'docs/mcp/orders.md')) +#settings($_ = $route('/v1/api/orders', 'GET', 'POST', 'PATCH')) +#settings($_ = $marshal('application/json','pkg.OrderJSON')) +#settings($_ = $unmarshal('application/json','pkg.OrderIn')) +#settings($_ = $unmarshal('application/xml','pkg.OrderXMLIn')) +#settings($_ = $format('tabular_json')) +#settings($_ = $date_format('2006-01-02')) +#settings($_ = $case_format('lc')) SELECT id FROM ORDERS o ` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) @@ -117,6 +124,15 @@ SELECT id FROM ORDERS o assert.Equal(t, "orders.search", planned.Directives.MCP.Name) assert.Equal(t, "Search orders", planned.Directives.MCP.Description) assert.Equal(t, "docs/mcp/orders.md", planned.Directives.MCP.DescriptionPath) + require.NotNil(t, planned.Directives.Route) + assert.Equal(t, "/v1/api/orders", planned.Directives.Route.URI) + assert.Equal(t, []string{"GET", "POST", "PATCH"}, planned.Directives.Route.Methods) + assert.Equal(t, "pkg.OrderJSON", planned.Directives.JSONMarshalType) + assert.Equal(t, "pkg.OrderIn", planned.Directives.JSONUnmarshalType) + assert.Equal(t, "pkg.OrderXMLIn", planned.Directives.XMLUnmarshalType) + assert.Equal(t, "tabular", planned.Directives.Format) + assert.Equal(t, "2006-01-02", planned.Directives.DateFormat) + assert.Equal(t, "lc", planned.Directives.CaseFormat) require.NotEmpty(t, planned.Views) assert.Equal(t, "analytics", planned.Views[0].Connector) } diff --git a/repository/shape/compile/enrich.go b/repository/shape/compile/enrich.go index 048064043..dad177a94 100644 --- a/repository/shape/compile/enrich.go +++ b/repository/shape/compile/enrich.go @@ -9,6 +9,7 @@ import ( "github.com/viant/datly/repository/shape" "github.com/viant/datly/repository/shape/compile/pipeline" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" ) @@ -65,7 +66,7 @@ func applySourceParityEnrichmentWithLayout(result *plan.Result, source *shape.So func buildParityEnrichmentContext(result *plan.Result, source *shape.Source, layout compilePathLayout) *parityEnrichmentContext { ctx := &parityEnrichmentContext{ source: source, - settings: extractRuleSettings(source), + settings: extractRuleSettings(source, result.Directives), baseDir: sourceSQLBaseDir(source), module: sourceModuleWithLayout(source, layout), sourceName: pipeline.SanitizeName(source.Name), @@ -169,17 +170,24 @@ func extractSummarySQL(sqlText string) string { return strings.TrimSpace(matches[1]) } -func extractRuleSettings(source *shape.Source) *ruleSettings { +func extractRuleSettings(source *shape.Source, directives *dqlshape.Directives) *ruleSettings { if source == nil || strings.TrimSpace(source.DQL) == "" { return &ruleSettings{} } + ret := &ruleSettings{} matches := ruleHeaderExpr.FindStringSubmatch(source.DQL) - if len(matches) < 2 { - return &ruleSettings{} + if len(matches) >= 2 { + rawJSON := strings.TrimSpace(matches[1]) + _ = json.Unmarshal([]byte(rawJSON), ret) + } + if directives != nil && directives.Route != nil { + if uri := strings.TrimSpace(directives.Route.URI); uri != "" { + ret.URI = uri + } + if len(directives.Route.Methods) > 0 { + ret.Method = strings.Join(directives.Route.Methods, ",") + } } - rawJSON := strings.TrimSpace(matches[1]) - ret := &ruleSettings{} - _ = json.Unmarshal([]byte(rawJSON), ret) return ret } diff --git a/repository/shape/compile/legacy_adapter.go b/repository/shape/compile/legacy_adapter.go index d26be3c85..927fe5f95 100644 --- a/repository/shape/compile/legacy_adapter.go +++ b/repository/shape/compile/legacy_adapter.go @@ -16,7 +16,7 @@ func resolveGeneratedCompanionDQL(source *shape.Source) string { if source == nil || strings.TrimSpace(source.Path) == "" { return "" } - settings := extractRuleSettings(source) + settings := extractRuleSettings(source, nil) typeExpr := strings.TrimSpace(settings.Type) if typeExpr == "" { return "" @@ -60,7 +60,7 @@ func resolveLegacyRouteViewsWithLayout(source *shape.Source, layout compilePathL if !ok { return nil } - settings := extractRuleSettings(source) + settings := extractRuleSettings(source, nil) typeExpr := strings.TrimSpace(settings.Type) typeExpr = strings.Trim(typeExpr, `"'`) typeExpr = strings.TrimSuffix(typeExpr, ".Handler") @@ -264,7 +264,7 @@ func resolveLegacyRouteStatesWithLayout(source *shape.Source, layout compilePath if !ok { return nil } - settings := extractRuleSettings(source) + settings := extractRuleSettings(source, nil) typeExpr := strings.TrimSpace(settings.Type) typeExpr = strings.Trim(typeExpr, `"'`) typeExpr = strings.TrimSuffix(typeExpr, ".Handler") @@ -461,7 +461,7 @@ func resolveLegacyRouteTypesWithLayout(source *shape.Source, layout compilePathL if !ok { return nil } - settings := extractRuleSettings(source) + settings := extractRuleSettings(source, nil) typeExpr := strings.TrimSpace(settings.Type) typeExpr = strings.Trim(typeExpr, `"'`) typeExpr = strings.TrimSuffix(typeExpr, ".Handler") diff --git a/repository/shape/compile/preprocess_handler.go b/repository/shape/compile/preprocess_handler.go index 22f8c4dfe..11b9327c1 100644 --- a/repository/shape/compile/preprocess_handler.go +++ b/repository/shape/compile/preprocess_handler.go @@ -112,7 +112,7 @@ func isHandlerSignal(source *shape.Source) bool { if source == nil { return false } - settings := extractRuleSettings(source) + settings := extractRuleSettings(source, nil) if settings != nil { if strings.TrimSpace(settings.Type) != "" { return true From 1d68a321c1e09573accaba47f4e810bcf0b72217 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 24 Feb 2026 11:28:04 -0800 Subject: [PATCH 131/279] shape/compile: add type support helpers; refine preprocessing and type defaults; update tests --- .../marshal/json/benchmark_groups_test.go | 115 ++ .../marshal/json/coverage_additional_test.go | 1532 +++++++++++++++++ internal/translator/resource_settings_test.go | 57 + .../shape/compile/enrich_settings_test.go | 26 + 4 files changed, 1730 insertions(+) create mode 100644 gateway/router/marshal/json/benchmark_groups_test.go create mode 100644 gateway/router/marshal/json/coverage_additional_test.go create mode 100644 internal/translator/resource_settings_test.go create mode 100644 repository/shape/compile/enrich_settings_test.go diff --git a/gateway/router/marshal/json/benchmark_groups_test.go b/gateway/router/marshal/json/benchmark_groups_test.go new file mode 100644 index 000000000..70a8b4e6a --- /dev/null +++ b/gateway/router/marshal/json/benchmark_groups_test.go @@ -0,0 +1,115 @@ +package json + +import ( + "testing" + "time" + + "github.com/viant/datly/gateway/router/marshal/config" + "github.com/viant/tagly/format/text" +) + +type benchBasic struct { + ID int + Name string + Score float64 + On bool +} + +type benchAdvancedChild struct { + Code string + Value int +} + +type benchAdvanced struct { + ID int + CreatedAt time.Time + Tags []string + Meta map[string]string + Items []*benchAdvancedChild + Any interface{} +} + +func benchmarkMarshaller() *Marshaller { + return New(&config.IOConfig{ + CaseFormat: text.CaseFormatLowerCamel, + TimeLayout: time.RFC3339, + }) +} + +func benchmarkBasicData() []benchBasic { + return []benchBasic{ + {ID: 1, Name: "a", Score: 1.5, On: true}, + {ID: 2, Name: "b", Score: 2.5, On: false}, + {ID: 3, Name: "c", Score: 3.5, On: true}, + } +} + +func benchmarkAdvancedData() []benchAdvanced { + now := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + return []benchAdvanced{ + { + ID: 10, + CreatedAt: now, + Tags: []string{"x", "y", "z"}, + Meta: map[string]string{"count": "3", "ok": "true"}, + Items: []*benchAdvancedChild{{Code: "a", Value: 1}, {Code: "b", Value: 2}}, + Any: map[string]interface{}{"kind": "demo", "n": 1}, + }, + } +} + +func BenchmarkMarshaller_Marshal_Basic(b *testing.B) { + m := benchmarkMarshaller() + data := benchmarkBasicData() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _, err := m.Marshal(data) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMarshaller_Unmarshal_Basic(b *testing.B) { + m := benchmarkMarshaller() + seed := benchmarkBasicData() + encoded, err := m.Marshal(seed) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var out []benchBasic + if err = m.Unmarshal(encoded, &out); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMarshaller_Marshal_Advanced(b *testing.B) { + m := benchmarkMarshaller() + data := benchmarkAdvancedData() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _, err := m.Marshal(data) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMarshaller_Unmarshal_Advanced(b *testing.B) { + m := benchmarkMarshaller() + seed := benchmarkAdvancedData() + encoded, err := m.Marshal(seed) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var out []benchAdvanced + if err = m.Unmarshal(encoded, &out); err != nil { + b.Fatal(err) + } + } +} diff --git a/gateway/router/marshal/json/coverage_additional_test.go b/gateway/router/marshal/json/coverage_additional_test.go new file mode 100644 index 000000000..ddcbc6aef --- /dev/null +++ b/gateway/router/marshal/json/coverage_additional_test.go @@ -0,0 +1,1532 @@ +package json + +import ( + "bytes" + stdjson "encoding/json" + "errors" + "reflect" + "sync" + "testing" + "time" + "unsafe" + + "github.com/francoispqt/gojay" + "github.com/stretchr/testify/require" + "github.com/viant/datly/gateway/router/marshal/config" + "github.com/viant/tagly/format" + "github.com/viant/tagly/format/text" + "github.com/viant/xunsafe" +) + +type fallbackMarshaller struct { + marshalCalled bool + unmarshalCalled bool +} + +type errMarshaller struct{} + +func (e *errMarshaller) MarshallObject(ptr unsafe.Pointer, session *MarshallSession) error { + return errors.New("marshal err") +} +func (e *errMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder *gojay.Decoder, auxiliaryDecoder *gojay.Decoder, session *UnmarshalSession) error { + return errors.New("unmarshal err") +} + +func (f *fallbackMarshaller) MarshallObject(ptr unsafe.Pointer, session *MarshallSession) error { + f.marshalCalled = true + session.WriteString(`{"fallback":true}`) + return nil +} + +func (f *fallbackMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder *gojay.Decoder, auxiliaryDecoder *gojay.Decoder, session *UnmarshalSession) error { + f.unmarshalCalled = true + return nil +} + +type gjOnlyPtr struct { + V int +} + +func (g *gjOnlyPtr) MarshalJSONObject(enc *gojay.Encoder) { + enc.IntKey("V", g.V) +} + +func (g *gjOnlyPtr) IsNil() bool { return g == nil } + +func (g *gjOnlyPtr) UnmarshalJSONObject(dec *gojay.Decoder, key string) error { + if key == "V" { + return dec.Int(&g.V) + } + return nil +} + +func (g *gjOnlyPtr) NKeys() int { return 0 } + +type customSum int +type customStruct int +type customStructHolder struct { + V int +} +type gojayBadInit struct { + C chan int +} + +type withM interface{ M() } +type withMImpl struct{} + +func (withMImpl) M() {} + +func (c *customSum) UnmarshalJSONWithOptions(dst interface{}, decoder *gojay.Decoder, options ...interface{}) error { + var vals []int + if err := decoder.SliceInt(&vals); err != nil { + return err + } + sum := 0 + for _, v := range vals { + sum += v + } + *dst.(**customSum) = (*customSum)(&sum) + return nil +} + +func (customStruct) UnmarshalJSONWithOptions(dst interface{}, decoder *gojay.Decoder, options ...interface{}) error { + var v int + if err := decoder.Int(&v); err != nil { + return err + } + p := dst.(*customStruct) + *p = customStruct(v) + return nil +} + +func (c customStructHolder) UnmarshalJSONWithOptions(dst interface{}, decoder *gojay.Decoder, options ...interface{}) error { + var v int + if err := decoder.Int(&v); err != nil { + return err + } + c.V = v + p := dst.(*customStructHolder) + *p = c + return nil +} + +func (g gojayBadInit) MarshalJSONObject(enc *gojay.Encoder) {} +func (g gojayBadInit) IsNil() bool { return false } + +func TestCoverage_OptionsAndTags(t *testing.T) { + opts := Options{&Tag{FieldName: "x"}, &format.Tag{Name: "y"}} + require.Equal(t, "x", opts.Tag().FieldName) + require.Equal(t, "y", opts.FormatTag().Name) + + parsed := Parse("name,omitempty") + require.Equal(t, "name", parsed.FieldName) + require.True(t, parsed.OmitEmpty) + + transient := Parse("-") + require.True(t, transient.Transient) + + xTag := ParseXTag("", "inline") + require.True(t, xTag.Inline) +} + +func TestCoverage_DefaultTagAndParseValue(t *testing.T) { + type sample struct { + A *int `default:"value=7,nullable=false,required=true"` + B time.Time `default:"value=2024-01-01T00:00:00Z,format=2006-01-02T15:04:05Z07:00"` + C *time.Time `default:"value=2024-01-01T00:00:00Z,format=2006-01-02T15:04:05Z07:00"` + } + rType := reflect.TypeOf(sample{}) + + aTag, err := NewDefaultTag(rType.Field(0)) + require.NoError(t, err) + require.True(t, aTag.IsRequired()) + require.False(t, aTag.IsNullable()) + + bTag, err := NewDefaultTag(rType.Field(1)) + require.NoError(t, err) + require.NotNil(t, bTag._value) + + cTag, err := NewDefaultTag(rType.Field(2)) + require.NoError(t, err) + require.NotNil(t, cTag._value) + + _, err = parseValue(reflect.TypeOf(time.Time{}), "bad-time", time.RFC3339) + require.Error(t, err) +} + +func TestCoverage_BytesSliceUnmarshal(t *testing.T) { + var b []byte + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`[1,2,3]`))) + defer dec.Release() + require.NoError(t, dec.Array(&BytesSlice{b: &b})) + require.Equal(t, []byte{1, 2, 3}, b) + + var bPtr *[]byte + dec2 := gojay.BorrowDecoder(bytes.NewReader([]byte(`[4,5]`))) + defer dec2.Release() + require.NoError(t, dec2.Array(&BytesPtrSlice{b: &bPtr})) + require.Equal(t, []byte{4, 5}, *bPtr) +} + +func TestCoverage_ErrorJoin(t *testing.T) { + err := NewError("a", errors.New("x")) + require.Contains(t, err.Error(), "failed to unmarshal a") + + nested := NewError("obj", NewError("field", errors.New("boom"))) + require.Equal(t, "obj.field", nested.Path) + + nestedArr := NewError("arr", NewError("[1]", errors.New("boom"))) + require.Equal(t, "arr[1]", nestedArr.Path) +} + +func TestCoverage_UnsignedAndPointers_MarshalUnmarshal(t *testing.T) { + type payload struct { + U uint + U8 uint8 + U16 uint16 + U32 uint32 + U64 uint64 + PU *uint + P8 *uint8 + P16 *uint16 + P32 *uint32 + P64 *uint64 + } + m := New(&config.IOConfig{}) + + u := uint(10) + u8 := uint8(11) + u16 := uint16(12) + u32 := uint32(13) + u64 := uint64(14) + in := payload{U: 1, U8: 2, U16: 3, U32: 4, U64: 5, PU: &u, P8: &u8, P16: &u16, P32: &u32, P64: &u64} + + data, err := m.Marshal(in) + require.NoError(t, err) + + var out payload + require.NoError(t, m.Unmarshal(data, &out)) + require.Equal(t, in.U, out.U) + require.Equal(t, in.U8, out.U8) + require.Equal(t, in.U16, out.U16) + require.Equal(t, in.U32, out.U32) + require.Equal(t, in.U64, out.U64) + require.NotNil(t, out.PU) + require.NotNil(t, out.P8) + require.NotNil(t, out.P16) + require.NotNil(t, out.P32) + require.NotNil(t, out.P64) +} + +func TestCoverage_ArrayAndMapEdges(t *testing.T) { + m := New(&config.IOConfig{CaseFormat: text.CaseFormatLowerUnderscore}) + + type boolArr struct { + Flags [3]bool + } + encoded, err := m.Marshal(boolArr{Flags: [3]bool{true, false, true}}) + require.NoError(t, err) + require.Contains(t, string(encoded), "[true,false,true]") + + var arrOut boolArr + err = m.Unmarshal([]byte(`{"Flags":[true,false,true]}`), &arrOut) + require.Error(t, err) // array unmarshal not supported + + type mapHolder struct { + M map[string]int + } + var mh mapHolder + require.NoError(t, m.Unmarshal([]byte(`{"M":{"a":1,"b":2}}`), &mh)) + require.Equal(t, 2, mh.M["b"]) + + type unsupported struct { + M map[string]bool + } + var bad unsupported + err = m.Unmarshal([]byte(`{"M":{"a":true}}`), &bad) + require.Error(t, err) +} + +func TestCoverage_InterfaceAndSliceInterface(t *testing.T) { + m := New(&config.IOConfig{}) + type obj struct { + Any interface{} + List []interface{} + } + var out obj + require.NoError(t, m.Unmarshal([]byte(`{"Any":{"k":1},"List":[1,"x",{"a":2}]}`), &out)) + require.Len(t, out.List, 1) // current behavior: appended as a single decoded interface payload + + encoded, err := m.Marshal(out) + require.NoError(t, err) + require.Contains(t, string(encoded), "\"List\"") +} + +func TestCoverage_CustomUnmarshallerAndGojayWrapper(t *testing.T) { + m := New(&config.IOConfig{}) + type holder struct { + Sum *customSum + G gjOnlyPtr + } + var out holder + require.NoError(t, m.Unmarshal([]byte(`{"Sum":[1,2,3],"G":{"V":7}}`), &out)) + require.NotNil(t, out.Sum) + require.Equal(t, 6, int(*out.Sum)) + require.Equal(t, 7, out.G.V) + + data, err := m.Marshal(out) + require.NoError(t, err) + require.Contains(t, string(data), `"V":7`) +} + +func TestCoverage_GojayWrapperFallbackAndDeferred(t *testing.T) { + rType := reflect.TypeOf(struct{ X int }{}) + fb := &fallbackMarshaller{} + wrapper := newGojayObjectMarshaller(getXType(rType), getXType(reflect.PtrTo(rType)), fb, true, true) + session := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + val := struct{ X int }{X: 1} + require.NoError(t, wrapper.MarshallObject(AsPtr(val, rType), session)) + require.True(t, fb.marshalCalled) + + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"X":1}`))) + defer dec.Release() + ptr := reflect.New(rType) + require.NoError(t, wrapper.UnmarshallObject(unsafe.Pointer(ptr.Pointer()), dec, nil, &UnmarshalSession{})) + require.True(t, fb.unmarshalCalled) + + d := newDeferred() + d.fail(errors.New("boom")) + require.Error(t, d.MarshallObject(nil, &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + d2 := newDeferred() + d2.setTarget(fb) + require.NoError(t, d2.MarshallObject(nil, &MarshallSession{Buffer: bytes.NewBuffer(nil)})) +} + +func TestCoverage_PathCacheHelpers(t *testing.T) { + pc := &pathCache{cache: sync.Map{}} + fb := &fallbackMarshaller{} + pc.storeMarshaler(reflect.TypeOf(1), fb) + got, ok := pc.loadMarshaller(reflect.TypeOf(1)) + require.True(t, ok) + require.NotNil(t, got) + + cfg := pc.parseConfig([]interface{}{&cacheConfig{IgnoreCustomMarshaller: true}}) + require.NotNil(t, cfg) + require.True(t, cfg.IgnoreCustomMarshaller) +} + +func TestCoverage_TimeAndRawMessageAndAsPtrMap(t *testing.T) { + cfg := &config.IOConfig{TimeLayout: "2006-01-02T15:04:05Z07:00"} + m := New(cfg) + now := time.Now().UTC().Truncate(time.Second) + type payload struct { + T time.Time + TP *time.Time + R stdjson.RawMessage + RP *stdjson.RawMessage + } + raw := stdjson.RawMessage(`{"a":1}`) + in := payload{T: now, TP: &now, R: raw, RP: &raw} + + data, err := m.Marshal(in) + require.NoError(t, err) + + var out payload + require.NoError(t, m.Unmarshal(data, &out)) + require.Equal(t, raw, out.R) + require.NotNil(t, out.RP) + + // map branch in AsPtr + mapped := map[string]int{"a": 1} + ptr := AsPtr(mapped, reflect.TypeOf(mapped)) + require.NotNil(t, ptr) +} + +func TestCoverage_MarshalSessionOptionsAndInterceptors(t *testing.T) { + m := New(&config.IOConfig{}) + type payload struct { + Items []int + } + + session := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + interceptors := MarshalerInterceptors{ + "Items": func() ([]byte, error) { return []byte(`[9,8,7]`), nil }, + } + data, err := m.Marshal(payload{Items: []int{1, 2, 3}}, session, interceptors) + require.NoError(t, err) + require.Contains(t, string(data), `"Items":[9,8,7]`) + + _, err = m.Marshal(nil) + require.NoError(t, err) +} + +func TestCoverage_PrepareUnmarshalSessionAndInterceptor(t *testing.T) { + m := New(&config.IOConfig{}) + type payload struct { + ID int + } + + um := &UnmarshalSession{} + interceptors := UnmarshalerInterceptors{ + "ID": func(dst interface{}, decoder *gojay.Decoder, options ...interface{}) error { + // consume incoming value but force a custom value + var throwaway int + if err := decoder.Int(&throwaway); err != nil { + return err + } + *dst.(*int) = 77 + return nil + }, + } + + var out payload + require.NoError(t, m.Unmarshal([]byte(`{"ID":1}`), &out, um, interceptors)) + require.Equal(t, 77, out.ID) + require.NotEmpty(t, um.Options) +} + +func TestCoverage_IntAndStringBranches(t *testing.T) { + m := New(&config.IOConfig{}) + + type ints struct { + I8 int8 + I16 int16 + I32 int32 + I64 int64 + } + var out ints + require.NoError(t, m.Unmarshal([]byte(`{"I8":8,"I16":16,"I32":32,"I64":64}`), &out)) + require.EqualValues(t, 8, out.I8) + require.EqualValues(t, 16, out.I16) + require.EqualValues(t, 32, out.I32) + require.EqualValues(t, 64, out.I64) + + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + marshallString("line\u2028sep\u2029par\n\t\r\b\f\"\\/", sb, nil) + require.Contains(t, sb.String(), `\u2028`) + require.Contains(t, sb.String(), `\u2029`) +} + +func TestCoverage_MapVariantsAndKeys(t *testing.T) { + m := New(&config.IOConfig{CaseFormat: text.CaseFormatLowerUnderscore}) + + type maps struct { + MI map[string]int + MF map[string]float64 + MS map[string]string + ANY map[string]interface{} + } + var out maps + require.NoError(t, m.Unmarshal([]byte(`{"MI":{"a":1},"MF":{"x":1.5},"MS":{"k":"v"}}`), &out)) + require.Equal(t, 1, out.MI["a"]) + require.Equal(t, 1.5, out.MF["x"]) + require.Equal(t, "v", out.MS["k"]) + + type intKey struct { + M map[int]string + } + enc, err := m.Marshal(intKey{M: map[int]string{1: "x", 2: "y"}}) + require.NoError(t, err) + require.Contains(t, string(enc), `"1":"x"`) + + type anyMap struct { + M map[string]interface{} + } + enc2, err := m.Marshal(anyMap{M: map[string]interface{}{"MyKey": 1}}) + require.NoError(t, err) + require.Contains(t, string(enc2), `"my_key"`) +} + +func TestCoverage_CacheDispatchAndErrors(t *testing.T) { + c := newCache() + cfg := &config.IOConfig{} + + pc := c.pathCache("x") + _, err := pc.getMarshaller(nil, cfg, "x", "x", nil) + require.Error(t, err) + + // Unsupported kind falls into default unsupported branch. + _, err = c.loadMarshaller(reflect.TypeOf(make(chan int)), cfg, "", "", nil) + require.Error(t, err) + + // Load representative kinds to exercise switch branches. + cases := []reflect.Type{ + reflect.TypeOf([2]bool{}), + reflect.TypeOf([]int{}), + reflect.TypeOf([]interface{}{}), + reflect.TypeOf(map[string]int{}), + reflect.TypeOf(time.Time{}), + reflect.TypeOf((*time.Time)(nil)), + reflect.TypeOf(""), + reflect.TypeOf(true), + reflect.TypeOf(float32(0)), + reflect.TypeOf(float64(0)), + reflect.TypeOf(int(0)), + reflect.TypeOf(uint(0)), + reflect.TypeOf((*int)(nil)), + reflect.TypeOf((*uint)(nil)), + } + for _, rType := range cases { + _, err = c.loadMarshaller(rType, cfg, "", "", nil) + require.NoError(t, err) + } +} + +func TestCoverage_OptionPresenceDefaultAndMarshalErrors(t *testing.T) { + empty := Options{123} + require.Nil(t, empty.Tag()) + require.Nil(t, empty.FormatTag()) + + _, err := getFields(reflect.TypeOf(1)) + require.Error(t, err) + + type badTag struct { + F string `default:"broken"` + } + _, err = NewDefaultTag(reflect.TypeOf(badTag{}).Field(0)) + require.Error(t, err) + + type badValue struct { + F int `default:"value=abc"` + } + _, err = NewDefaultTag(reflect.TypeOf(badValue{}).Field(0)) + require.Error(t, err) + + m := New(&config.IOConfig{}) + _, err = m.Marshal(make(chan int)) + require.Error(t, err) +} + +func TestCoverage_LowLevelBranches(t *testing.T) { + // String ensureReplacer nil branch + s := &stringMarshaller{dTag: &format.Tag{}, replacer: nil, defaultValue: `""`} + buf := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + v := "abc" + require.NoError(t, s.MarshallObject(unsafe.Pointer(&v), buf)) + + // Slice interceptor error branch + errExpected := errors.New("interceptor") + _, err := New(&config.IOConfig{}).Marshal( + struct{ X []int }{X: []int{1}}, + MarshalerInterceptors{ + "X": func() ([]byte, error) { return nil, errExpected }, + }, + ) + require.Error(t, err) + + // Time unmarshal invalid input branch + tm := newTimeMarshaller(&format.Tag{}, &config.IOConfig{}) + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`"bad"`))) + defer dec.Release() + var tt time.Time + require.Panics(t, func() { + _ = tm.UnmarshallObject(unsafe.Pointer(&tt), dec, nil, &UnmarshalSession{}) + }) +} + +func TestCoverage_ZeroPercentFunctions(t *testing.T) { + // formatFloat + require.Equal(t, "1.25", formatFloat(1.25)) + + // float unmarshallers + pointer variants + type floats struct { + F32 float32 + F64 float64 + P32 *float32 + P64 *float64 + I8 *int8 + I16 *int16 + I32 *int32 + I64 *int64 + U8 *uint8 + U16 *uint16 + U32 *uint32 + U64 *uint64 + } + m := New(&config.IOConfig{}) + var out floats + err := m.Unmarshal([]byte(`{"F32":1.5,"F64":2.5,"P32":3.5,"P64":4.5,"I8":8,"I16":16,"I32":32,"I64":64,"U8":9,"U16":19,"U32":29,"U64":39}`), &out) + require.NoError(t, err) + require.NotNil(t, out.P32) + require.NotNil(t, out.P64) + require.NotNil(t, out.I8) + require.NotNil(t, out.I16) + require.NotNil(t, out.I32) + require.NotNil(t, out.I64) + require.NotNil(t, out.U8) + require.NotNil(t, out.U16) + require.NotNil(t, out.U32) + require.NotNil(t, out.U64) + + // inlinable unmarshal branch (invoke marshaller directly) + type inner struct { + A int + B string + } + type outer struct { + Inner inner `jsonx:"inline"` + } + rType := reflect.TypeOf(outer{}) + field, _ := rType.FieldByName("Inner") + ilm, err := newInlinableMarshaller(field, &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + var o outer + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"A":7,"B":"x"}`))) + defer dec.Release() + require.NoError(t, ilm.UnmarshallObject(unsafe.Pointer(&o.Inner), dec, nil, &UnmarshalSession{})) +} + +func TestCoverage_BranchHelpersAndPrimitiveMarshallers(t *testing.T) { + // isExcluded / filterByPath + ioCfg := &config.IOConfig{Exclude: map[string]bool{"A.B": true}} + require.True(t, isExcluded(nil, "B", ioCfg, "A.B")) + require.False(t, isExcluded(nil, "C", ioCfg, "A.C")) + filters := NewFilters(&FilterEntry{Path: "A", Fields: []string{"X"}}) + f, ok := filterByPath(filters, "A") + require.True(t, ok) + require.True(t, f["X"]) + _, ok = filterByPath(nil, "A") + require.False(t, ok) + + // primitive marshallers zero/non-zero branches + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + intV := 0 + require.NoError(t, newIntMarshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&intV), sb)) + intV = 3 + require.NoError(t, newIntMarshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&intV), sb)) + + f32 := float32(0) + require.NoError(t, newFloat32Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&f32), sb)) + f32 = 1.25 + require.NoError(t, newFloat32Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&f32), sb)) + + u := uint(0) + require.NoError(t, newUintMarshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u), sb)) + u = 5 + require.NoError(t, newUintMarshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u), sb)) + + b := false + require.NoError(t, newBoolMarshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&b), sb)) + b = true + require.NoError(t, newBoolMarshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&b), sb)) + + // explicit width marshaller zero/non-zero branches + i8 := int8(0) + require.NoError(t, NewInt8Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&i8), sb)) + i8 = 1 + require.NoError(t, NewInt8Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&i8), sb)) + + i16 := int16(0) + require.NoError(t, newInt16Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&i16), sb)) + i16 = 2 + require.NoError(t, newInt16Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&i16), sb)) + + i32 := int32(0) + require.NoError(t, newInt32Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&i32), sb)) + i32 = 3 + require.NoError(t, newInt32Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&i32), sb)) + + i64 := int64(0) + require.NoError(t, newInt64Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&i64), sb)) + i64 = 4 + require.NoError(t, newInt64Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&i64), sb)) + + u8 := uint8(0) + require.NoError(t, newUint8Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u8), sb)) + u8 = 1 + require.NoError(t, newUint8Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u8), sb)) + + u16 := uint16(0) + require.NoError(t, newUint16Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u16), sb)) + u16 = 2 + require.NoError(t, newUint16Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u16), sb)) + + u32 := uint32(0) + require.NoError(t, newUint32Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u32), sb)) + u32 = 3 + require.NoError(t, newUint32Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u32), sb)) + + u64 := uint64(0) + require.NoError(t, newUint64Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u64), sb)) + u64 = 4 + require.NoError(t, newUint64Marshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&u64), sb)) +} + +func TestCoverage_InterfaceArrayRawAndWrapperBranches(t *testing.T) { + // interface marshaller hasMethod=true branch + v := withM(withMImpl{}) + im, err := newInterfaceMarshaller(reflect.TypeOf((*withM)(nil)).Elem(), &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + require.NotNil(t, im.AsInterface(unsafe.Pointer(&v))) + require.NotNil(t, asInterface(im.xType, unsafe.Pointer(&v))) + + // array unmarshal null path + am, err := newArrayMarshaller(reflect.TypeOf([2]bool{}), &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + decNull := gojay.BorrowDecoder(bytes.NewReader([]byte(`null`))) + defer decNull.Release() + var arr [2]bool + require.Error(t, am.UnmarshallObject(unsafe.Pointer(&arr), decNull, nil, &UnmarshalSession{})) + + // raw message marshal nil path + rm := newRawMessageMarshaller() + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + var raw []byte + require.NoError(t, rm.MarshallObject(unsafe.Pointer(&raw), sb)) + raw = []byte(`{"x":1}`) + require.NoError(t, rm.MarshallObject(unsafe.Pointer(&raw), sb)) + + // gojay wrapper useMarshal/useUnmarshal false branches + fb := &fallbackMarshaller{} + rType := reflect.TypeOf(struct{ A int }{}) + w := newGojayObjectMarshaller(getXType(rType), getXType(reflect.PtrTo(rType)), fb, false, false) + val := struct{ A int }{A: 1} + require.NoError(t, w.MarshallObject(AsPtr(val, rType), sb)) + require.True(t, fb.marshalCalled) + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"A":1}`))) + defer dec.Release() + ptr := reflect.New(rType) + require.NoError(t, w.UnmarshallObject(unsafe.Pointer(ptr.Pointer()), dec, nil, &UnmarshalSession{})) + require.True(t, fb.unmarshalCalled) + +} + +func TestCoverage_GojayWrapperPointerPathAndSkipNull(t *testing.T) { + // use existing gjOnlyPtr type to hit useMarshal/useUnmarshal=true path + w := newGojayObjectMarshaller(getXType(reflect.TypeOf(gjOnlyPtr{})), getXType(reflect.TypeOf(&gjOnlyPtr{})), &fallbackMarshaller{}, true, true) + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + v := gjOnlyPtr{V: 9} + require.NoError(t, w.MarshallObject(AsPtr(v, reflect.TypeOf(v)), sb)) + + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"V":9}`))) + defer dec.Release() + p := reflect.New(reflect.TypeOf(gjOnlyPtr{})) + require.NoError(t, w.UnmarshallObject(unsafe.Pointer(p.Pointer()), dec, nil, &UnmarshalSession{})) + + // skipNull true/false branches + decNull := gojay.BorrowDecoder(bytes.NewReader([]byte(`null`))) + defer decNull.Release() + _ = skipNull(decNull) + + decNonNull := gojay.BorrowDecoder(bytes.NewReader([]byte(`[]`))) + defer decNonNull.Release() + require.False(t, skipNull(decNonNull)) +} + +func TestCoverage_LowFunctionsExtra(t *testing.T) { + // uint ptr marshaller non-nil branch + up := uint(11) + ptr := &up + upp := &ptr + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + require.NoError(t, newUintPtrMarshaller(&format.Tag{}).MarshallObject(unsafe.Pointer(&upp), sb)) + + // gojay wrapper nil ptr marshal branch + w := newGojayObjectMarshaller( + getXType(reflect.TypeOf(gjOnlyPtr{})), + getXType(reflect.TypeOf(&gjOnlyPtr{})), + &fallbackMarshaller{}, + true, + true, + ) + require.NoError(t, w.MarshallObject(nil, sb)) + + // slice decoder error branch + sd := newSliceDecoder(reflect.TypeOf(0), unsafe.Pointer(&[]int{}), xunsafe.NewSlice(reflect.TypeOf([]int{}), xunsafe.UseItemAddrOpt(true)), newIntMarshaller(&format.Tag{}), &UnmarshalSession{}) + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`["x"]`))) + defer dec.Release() + _ = dec.Array(sd) +} + +func TestCoverage_RemainingBranches(t *testing.T) { + // skipNull branches + origData, origCur := decData, decCur + decData, decCur = nil, nil + decDummy := gojay.BorrowDecoder(bytes.NewReader([]byte(`null`))) + require.False(t, skipNull(decDummy)) + decDummy.Release() + decData, decCur = origData, origCur + + decNull := gojay.BorrowDecoder(bytes.NewReader([]byte(`null`))) + _ = skipNull(decNull) + decNull.Release() + + // force internal decoder state to hit skipNull true path + forced := gojay.BorrowDecoder(bytes.NewReader([]byte(`[]`))) + decPtr := unsafe.Pointer(forced) + decData.SetBytes(decPtr, []byte("null")) + decCur.SetInt(decPtr, 0) + require.True(t, skipNull(forced)) + forced.Release() + + // slice marshaller constructor error path + _, err := newSliceMarshaller(reflect.TypeOf([]chan int{}), &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.Error(t, err) + + // slice marshaller marshal nil ptr branch + sNoop := &sliceMarshaller{path: "p", xslice: xunsafe.NewSlice(reflect.TypeOf([]int{}), xunsafe.UseItemAddrOpt(true))} + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + require.NoError(t, sNoop.MarshallObject(nil, sb)) + + // slice marshaller interceptor error branch + sInt := &sliceMarshaller{path: "p", xslice: xunsafe.NewSlice(reflect.TypeOf([]int{}), xunsafe.UseItemAddrOpt(true))} + sb2 := &MarshallSession{ + Buffer: bytes.NewBuffer(nil), + Interceptors: MarshalerInterceptors{ + "p": func() ([]byte, error) { return nil, errors.New("x") }, + }, + } + var arr []int + require.Error(t, sInt.MarshallObject(unsafe.Pointer(&arr), sb2)) + + // slice decoder error wrapping branch + sd := &sliceDecoder{ + appender: xunsafe.NewSlice(reflect.TypeOf([]int{}), xunsafe.UseItemAddrOpt(true)).Appender(unsafe.Pointer(&arr)), + unmarshaller: &errMarshaller{}, + } + decArr := gojay.BorrowDecoder(bytes.NewReader([]byte(`[1]`))) + err = decArr.Array(sd) + decArr.Release() + require.Error(t, err) + + // slice interface marshaller branches + sim := newSliceInterfaceMarshaller(&config.IOConfig{}, "", "", &format.Tag{}, newCache()).(*sliceInterfaceMarshaller) + ifaces := []interface{}{nil, (*int)(nil), map[string]int{"a": 1}} + sb3 := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + require.NoError(t, sim.MarshallObject(unsafe.Pointer(&ifaces), sb3)) + ifacesBad := []interface{}{make(chan int)} + require.Error(t, sim.MarshallObject(unsafe.Pointer(&ifacesBad), sb3)) + decBad := gojay.BorrowDecoder(bytes.NewReader([]byte(`{`))) + require.Error(t, sim.UnmarshallObject(unsafe.Pointer(&[]interface{}{}), decBad, nil, &UnmarshalSession{})) + decBad.Release() + + // ptr marshaller constructor and branches + _, err = newPtrMarshaller(reflect.TypeOf((*chan int)(nil)), &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.Error(t, err) + pm := &ptrMarshaller{rType: reflect.TypeOf((*int)(nil)), marshaler: newIntMarshaller(&format.Tag{})} + require.NoError(t, pm.MarshallObject(nil, sb)) + var pnil *int + require.NoError(t, pm.MarshallObject(unsafe.Pointer(&pnil), sb)) + decPtr2 := gojay.BorrowDecoder(bytes.NewReader([]byte(`null`))) + require.NoError(t, pm.UnmarshallObject(unsafe.Pointer(&pnil), decPtr2, nil, &UnmarshalSession{})) + decPtr2.Release() + + // map marshaller direct key switch and nil map branches + mm := &mapMarshaller{ + xType: getXType(reflect.TypeOf(map[int]int{})), + keyType: reflect.TypeOf(""), + valueType: reflect.TypeOf(int(0)), + keyMarshaller: newStringMarshaller(&format.Tag{}), + valueMarshaller: newIntMarshaller(&format.Tag{}), + config: &config.IOConfig{}, + } + mval := map[int]int{1: 2} + require.NoError(t, mm.MarshallObject(unsafe.Pointer(&mval), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + mm64 := &mapMarshaller{ + xType: getXType(reflect.TypeOf(map[uint64]int{})), + keyType: reflect.TypeOf(""), + valueType: reflect.TypeOf(int(0)), + keyMarshaller: newStringMarshaller(&format.Tag{}), + valueMarshaller: newIntMarshaller(&format.Tag{}), + config: &config.IOConfig{}, + } + m64 := map[uint64]int{7: 1} + require.NoError(t, mm64.MarshallObject(unsafe.Pointer(&m64), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + mmNil := &mapMarshaller{ + xType: getXType(reflect.TypeOf(map[string]int{})), + keyType: reflect.TypeOf(""), + valueType: reflect.TypeOf(int(0)), + keyMarshaller: newStringMarshaller(&format.Tag{}), + valueMarshaller: newIntMarshaller(&format.Tag{}), + config: &config.IOConfig{}, + } + var nilMap map[string]int + require.NoError(t, mmNil.MarshallObject(unsafe.Pointer(&nilMap), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + // map unmarshaler error branches + mi := &mapStringIntUnmarshaler{aMap: map[string]int{}} + d1 := gojay.BorrowDecoder(bytes.NewReader([]byte(`{`))) + require.Error(t, mi.UnmarshalJSONObject(d1, "a")) + d1.Release() + mf := &mapStringFloatUnmarshaler{aMap: map[string]float64{}} + d2 := gojay.BorrowDecoder(bytes.NewReader([]byte(`{`))) + require.Error(t, mf.UnmarshalJSONObject(d2, "a")) + d2.Release() + ms := &mapStringStringUnmarshaler{aMap: map[string]string{}} + d3 := gojay.BorrowDecoder(bytes.NewReader([]byte(`{`))) + require.Error(t, ms.UnmarshalJSONObject(d3, "a")) + d3.Release() + + // struct helper normalized exclusion branch + ioCfg := &config.IOConfig{Exclude: map[string]bool{"ab": true}} + require.True(t, isExcluded(nil, "X", ioCfg, "A_B")) + + // cache path load miss branch + pc := &pathCache{cache: sync.Map{}} + _, ok := pc.loadMarshaller(reflect.TypeOf(123)) + require.False(t, ok) +} + +func TestCoverage_ConstructorAndNilVariants(t *testing.T) { + nullableTag := &format.Tag{} + b := true + nullableTag.Nullable = &b + nonNullableTag := &format.Tag{} + f := false + nonNullableTag.Nullable = &f + + // bool/string/float/int ctor nullable branches + require.Equal(t, null, newBoolMarshaller(nullableTag).zeroValue) + require.Equal(t, null, newStringMarshaller(nullableTag).defaultValue) + require.Equal(t, null, newFloat32Marshaller(nullableTag).zeroValue) + require.Equal(t, null, newFloat64Marshaller(nullableTag).zeroValue) + require.Equal(t, null, newInt64Marshaller(nullableTag).zeroValue) + require.Equal(t, null, intZeroValue(nullableTag)) + require.Equal(t, "0", intZeroValue(nonNullableTag)) + + // time ptr ctor branch + require.Equal(t, "null", newTimePtrMarshaller(nullableTag, &config.IOConfig{}).zeroValue) + require.NotEqual(t, "null", newTimePtrMarshaller(nonNullableTag, &config.IOConfig{}).zeroValue) + + // uint ptr marshaller both nil-pointer branches + um := newUintPtrMarshaller(&format.Tag{}) + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + require.NoError(t, um.MarshallObject(nil, sb)) + var x *uint + require.NoError(t, um.MarshallObject(unsafe.Pointer(&x), sb)) + v := uint(1) + x = &v + require.NoError(t, um.MarshallObject(unsafe.Pointer(&x), sb)) + + // ptr marshaller branch where ptr non-nil but deref nil + pm := &ptrMarshaller{rType: reflect.TypeOf((*int)(nil)), marshaler: newIntMarshaller(&format.Tag{})} + var pi *int + piPtr := &pi + require.NoError(t, pm.MarshallObject(unsafe.Pointer(piPtr), sb)) + + // interface marshaller error branch + im, err := newInterfaceMarshaller(reflect.TypeOf((*interface{})(nil)).Elem(), &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + iface := interface{}(make(chan int)) + require.Error(t, im.MarshallObject(unsafe.Pointer(&iface), sb)) + + // raw message marshal ptr nil and unmarshal invalid + rm := newRawMessageMarshaller() + require.NoError(t, rm.MarshallObject(nil, sb)) + decInvalid := gojay.BorrowDecoder(bytes.NewReader([]byte(`{`))) + defer decInvalid.Release() + var raw []byte + require.Error(t, rm.UnmarshallObject(unsafe.Pointer(&raw), decInvalid, nil, &UnmarshalSession{})) + + // array marshaller unsupported branch + am, err := newArrayMarshaller(reflect.TypeOf([1]int{}), &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + a := [1]int{1} + require.Error(t, am.MarshallObject(unsafe.Pointer(&a), sb)) + + // force array unmarshal null fast-path + decArrNull := gojay.BorrowDecoder(bytes.NewReader([]byte(`[]`))) + decArrPtr := unsafe.Pointer(decArrNull) + decData.SetBytes(decArrPtr, []byte("null")) + decCur.SetInt(decArrPtr, 0) + require.NoError(t, am.UnmarshallObject(unsafe.Pointer(&a), decArrNull, nil, &UnmarshalSession{})) + decArrNull.Release() + + // custom unmarshaller constructor error path + _, err = newCustomUnmarshaller(reflect.TypeOf(make(chan int)), &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.Error(t, err) + + // inlinable marshaller constructor error path + type badInline struct{ C chan int } + field, _ := reflect.TypeOf(badInline{}).FieldByName("C") + _, err = newInlinableMarshaller(field, &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.Error(t, err) + + // decoderError nil field branch + oldErr := decErr + decErr = nil + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{}`))) + require.NoError(t, decoderError(dec)) + dec.Release() + decErr = oldErr +} + +func TestCoverage_AdditionalLowBranches(t *testing.T) { + // string marshaller empty + nullable branch + nullable := &format.Tag{} + tval := true + nullable.Nullable = &tval + sm := newStringMarshaller(nullable) + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + empty := "" + require.NoError(t, sm.MarshallObject(unsafe.Pointer(&empty), sb)) + + // deferred unmarshal fail branch + d := newDeferred() + d.fail(errors.New("uerr")) + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{}`))) + defer dec.Release() + require.Error(t, d.UnmarshallObject(nil, dec, nil, &UnmarshalSession{})) + + // ptr unmarshal pointer==nil and auxiliary decoder branch + pm := &ptrMarshaller{rType: reflect.TypeOf((*int)(nil)), marshaler: newIntMarshaller(&format.Tag{})} + require.NoError(t, pm.UnmarshallObject(nil, dec, nil, &UnmarshalSession{})) + var p *int + aux := gojay.BorrowDecoder(bytes.NewReader([]byte(`1`))) + defer aux.Release() + require.NoError(t, pm.UnmarshallObject(unsafe.Pointer(&p), dec, aux, &UnmarshalSession{})) + + // enc BytesSlice error branch + bs := &BytesSlice{b: &[]byte{}} + bad := gojay.BorrowDecoder(bytes.NewReader([]byte(`"x"`))) + defer bad.Release() + _ = bs.UnmarshalJSONArray(bad) + + // struct marshaller branches with anonymous ptr embed and ignores + type Emb struct { + Arr []int + S string + } + type HolderNoOmit struct { + *Emb + Hidden string `json:"-"` + Internal string `internal:"true"` + Name string + } + type HolderOmit struct { + *Emb `json:",omitempty"` + Name string + } + m := New(&config.IOConfig{}) + _, err := m.Marshal(HolderNoOmit{Name: "x"}) // nil embed no omitempty -> explicit null paths + require.NoError(t, err) + _, err = m.Marshal(HolderOmit{Name: "y"}) // nil embed with omitempty -> skip path + require.NoError(t, err) + + // createStructMarshallers self-reference skip branch + type Self struct { + Child []*Self + Name string + } + s, err := newStructMarshaller(&config.IOConfig{}, reflect.TypeOf(Self{}), "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + gf := groupFields(reflect.TypeOf(Self{})) + mrs, err := s.createStructMarshallers(gf, "", "", &format.Tag{}) + require.NoError(t, err) + // only Name should remain (Child is self-reference and skipped) + require.Len(t, mrs, 1) + + // enc.go error path: decoder exhausted + bufBytes := []byte{} + bs2 := &BytesSlice{b: &bufBytes} + dEmpty := gojay.BorrowDecoder(bytes.NewReader([]byte{})) + defer dEmpty.Release() + require.Error(t, bs2.UnmarshalJSONArray(dEmpty)) + + // default Init unknown attr ignored, malformed kv errors + type withDefault struct { + V string `default:"unknown=1,value=x"` + } + _, err = NewDefaultTag(reflect.TypeOf(withDefault{}).Field(0)) + require.NoError(t, err) + type badDefault struct { + V string `default:"badformat"` + } + _, err = NewDefaultTag(reflect.TypeOf(badDefault{}).Field(0)) + require.Error(t, err) + + // presence updater error path + type wrongPresence struct { + Has int `setMarker:"true"` + } + _, err = newPresenceUpdater(reflect.TypeOf(wrongPresence{}).Field(0)) + require.Error(t, err) + + // formatName remaining ID branch variants + require.Equal(t, "ID", formatName("ID", text.CaseFormatUpper)) + require.Equal(t, "id", formatName("ID", text.CaseFormatLower)) +} + +func TestCoverage_StructHeavyBranches(t *testing.T) { + // init() error path via unsupported field type + type Bad struct { + C chan int + } + smBad, err := newStructMarshaller(&config.IOConfig{}, reflect.TypeOf(Bad{}), "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + require.Error(t, smBad.init()) + + // init() error path via invalid presence marker type + type BadPresence struct { + ID int + Has int `setMarker:"true"` + } + smBadPresence, err := newStructMarshaller(&config.IOConfig{}, reflect.TypeOf(BadPresence{}), "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + require.Error(t, smBadPresence.init()) + + // UnmarshalObject branch where marker holder is non-pointer struct (lines 85-87) + type HasStruct struct{ ID bool } + type MarkerStruct struct { + ID int + Has HasStruct `setMarker:"true"` + } + smMarker, err := newStructMarshaller(&config.IOConfig{}, reflect.TypeOf(MarkerStruct{}), "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + require.NoError(t, smMarker.init()) + var ms MarkerStruct + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"ID":1}`))) + require.NoError(t, smMarker.UnmarshallObject(unsafe.Pointer(&ms), dec, nil, &UnmarshalSession{})) + dec.Release() + + // MarshallObject nil pointer branch (lines 106-109) + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + require.NoError(t, smMarker.MarshallObject(nil, sb)) + + // MarshallObject filter exclusion branch + filter miss branch in isExcluded + mCfg := &config.IOConfig{} + smFilt, err := newStructMarshaller(mCfg, reflect.TypeOf(struct { + A int + B int + }{}), "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + require.NoError(t, smFilt.init()) + v := struct { + A int + B int + }{A: 1, B: 2} + filtered := &MarshallSession{ + Buffer: bytes.NewBuffer(nil), + Filters: NewFilters(&FilterEntry{Path: "", Fields: []string{"A"}}), + } + require.NoError(t, smFilt.MarshallObject(unsafe.Pointer(&v), filtered)) + + // newFieldMarshaller anonymous error path (line 295) via anonymous bad struct + type anonInner struct { + C chan int + } + type anonBad struct { + anonInner + } + smAnonBad, err := newStructMarshaller(&config.IOConfig{}, reflect.TypeOf(anonBad{}), "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + require.Error(t, smAnonBad.init()) + + // newFieldMarshaller ignore path (line 316) is unreachable for valid Go identifiers; keep behavior documented. + + // isZeroValue default false path (line 434) with non-comparable func type + type fnHolder struct { + F func() + } + ff, _ := reflect.TypeOf(fnHolder{}).FieldByName("F") + fmwf := &marshallerWithField{xField: xunsafe.NewField(ff), marshallerMetadata: marshallerMetadata{comparable: false}} + require.False(t, isZeroValue(unsafe.Pointer(&fnHolder{}), fmwf, nil)) + + // structDecoder interceptor error path (lines 494-496) + type simple struct{ A int } + smSimple, err := newStructMarshaller(&config.IOConfig{}, reflect.TypeOf(simple{}), "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + require.NoError(t, smSimple.init()) + sv := simple{} + ud := &structDecoder{ + ptr: unsafe.Pointer(&sv), + marshaller: smSimple, + session: &UnmarshalSession{PathMarshaller: UnmarshalerInterceptors{ + "A": func(dst interface{}, decoder *gojay.Decoder, options ...interface{}) error { + return errors.New("intercept") + }, + }}, + } + dec2 := gojay.BorrowDecoder(bytes.NewReader([]byte(`1`))) + err = ud.unmarshalJson(dec2, "A") + dec2.Release() + require.Error(t, err) +} + +type gjValueOnly struct { + V int +} + +func (g gjValueOnly) MarshalJSONObject(enc *gojay.Encoder) { + enc.IntKey("V", g.V) +} + +func (g gjValueOnly) IsNil() bool { return false } + +func TestCoverage_TargetedReachableBranches(t *testing.T) { + // default.Init empty tag branch + ignorecaseformatter attribute branch + type noDefault struct { + A int + } + aTag := &DefaultTag{} + require.NoError(t, aTag.Init(reflect.TypeOf(noDefault{}).Field(0))) + + type attrDefault struct { + A string `default:"value=x,ignorecaseformatter=true"` + } + aTag2, err := NewDefaultTag(reflect.TypeOf(attrDefault{}).Field(0)) + require.NoError(t, err) + require.True(t, aTag2.IgnoreCaseFormatter) + + // parseValue time branch with empty format fallback + parsed, err := parseValue(reflect.TypeOf(time.Time{}), "2024-01-01T00:00:00Z", "") + require.NoError(t, err) + require.IsType(t, time.Time{}, parsed) + + // namesCaseIndex undefined format branch + n := &namesCaseIndex{registry: map[text.CaseFormat]map[string]string{}} + require.Equal(t, "a_b", n.formatTo("a-b", text.CaseFormatLowerUnderscore)) + + // marshal.prepareMarshallSession nil option and filters branch + j := New(&config.IOConfig{}) + sess, putBack := j.prepareMarshallSession([]interface{}{nil, []*FilterEntry{{Path: "", Fields: []string{"A"}}}}) + require.True(t, putBack) + require.NotNil(t, sess.Filters) + + // marshal.Unmarshal error branch when marshaller construction fails + err = j.Unmarshal([]byte(`1`), make(chan int)) + require.Error(t, err) + + // cache getMarshaller ptr fallback error, map error, and custom-unmarshaller struct branch + c := newCache() + _, err = c.loadMarshaller(reflect.TypeOf((*chan int)(nil)), &config.IOConfig{}, "", "", nil) + require.Error(t, err) + _, err = c.loadMarshaller(reflect.TypeOf(map[string]chan int{}), &config.IOConfig{}, "", "", nil) + require.Error(t, err) + + _, err = c.loadMarshaller(reflect.TypeOf(customStruct(0)), &config.IOConfig{}, "", "", nil) + require.NoError(t, err) + + // deferred.resolved nil-target branch + d := newDeferred() + close(d.ready) + _, err = d.resolved() + require.Error(t, err) + + // gojay wrapper value-receiver marshal branch + auxiliary decoder branch + fb := &fallbackMarshaller{} + gw := newGojayObjectMarshaller( + getXType(reflect.TypeOf(gjValueOnly{})), + getXType(reflect.TypeOf(&gjValueOnly{})), + fb, + true, + false, + ) + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + val := gjValueOnly{V: 5} + require.NoError(t, gw.MarshallObject(AsPtr(val, reflect.TypeOf(val)), sb)) + require.Contains(t, sb.String(), `"V":5`) + + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"V":6}`))) + aux := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"V":7}`))) + defer dec.Release() + defer aux.Release() + dst := gjValueOnly{} + require.NoError(t, gw.UnmarshallObject(unsafe.Pointer(&dst), dec, aux, &UnmarshalSession{})) + require.True(t, fb.unmarshalCalled) + + // custom marshaller fallback branch + cm := &customMarshaller{ + valueType: getXType(reflect.TypeOf(1)), + addrType: getXType(reflect.TypeOf(new(int))), + marshaller: fb, + } + dec2 := gojay.BorrowDecoder(bytes.NewReader([]byte(`1`))) + defer dec2.Release() + i := 0 + require.NoError(t, cm.UnmarshallObject(unsafe.Pointer(&i), dec2, nil, &UnmarshalSession{})) + require.True(t, fb.unmarshalCalled) + + // array len==0 branch + am, err := newArrayMarshaller(reflect.TypeOf([0]bool{}), &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + sbArr := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + arr := [0]bool{} + require.NoError(t, am.MarshallObject(unsafe.Pointer(&arr), sbArr)) + require.Equal(t, "[]", sbArr.String()) + + // ptr unmarshal decoder error branch + pm := &ptrMarshaller{rType: reflect.TypeOf((*int)(nil)), marshaler: newIntMarshaller(&format.Tag{})} + badDec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{`))) + defer badDec.Release() + var pi *int + require.Error(t, pm.UnmarshallObject(unsafe.Pointer(&pi), badDec, nil, &UnmarshalSession{})) + + // slice unmarshal decoder.Array error and marshaller error branch + sm := &sliceMarshaller{ + elemType: reflect.TypeOf(0), + marshaller: &errMarshaller{}, + xslice: xunsafe.NewSlice(reflect.TypeOf([]int{}), xunsafe.UseItemAddrOpt(true)), + } + badArrayDec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"x":1}`))) + defer badArrayDec.Release() + sliceDst := []int{} + require.Error(t, sm.UnmarshallObject(unsafe.Pointer(&sliceDst), badArrayDec, nil, &UnmarshalSession{})) + + // sliceInterfaceMarshaller marshaller.MarshallObject error branch + cache := newCache() + cache.pathCache("").storeMarshaler(reflect.TypeOf(1), &errMarshaller{}) + sim := &sliceInterfaceMarshaller{ + cache: cache, + config: &config.IOConfig{}, + tag: &format.Tag{}, + } + list := []interface{}{1} + require.Error(t, sim.MarshallObject(unsafe.Pointer(&list), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + // map marshaller int64/default key switch, mapStringIface nil-pointer/counter/error branches + mInt64 := &mapMarshaller{ + xType: getXType(reflect.TypeOf(map[int64]int{})), + keyType: reflect.TypeOf(""), + valueType: reflect.TypeOf(int(0)), + keyMarshaller: newStringMarshaller(&format.Tag{}), + valueMarshaller: newIntMarshaller(&format.Tag{}), + config: &config.IOConfig{}, + } + data64 := map[int64]int{11: 1} + require.NoError(t, mInt64.MarshallObject(unsafe.Pointer(&data64), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + mDefault := &mapMarshaller{ + xType: getXType(reflect.TypeOf(map[float64]int{})), + keyType: reflect.TypeOf(""), + valueType: reflect.TypeOf(int(0)), + keyMarshaller: newStringMarshaller(&format.Tag{}), + valueMarshaller: newIntMarshaller(&format.Tag{}), + config: &config.IOConfig{}, + } + dataDef := map[float64]int{1.5: 2} + require.NoError(t, mDefault.MarshallObject(unsafe.Pointer(&dataDef), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + mIface := &mapMarshaller{ + config: &config.IOConfig{CaseFormat: text.CaseFormatLower}, + valueType: reflect.TypeOf((*interface{})(nil)).Elem(), + valueMarshaller: newInterfaceMarshallerMust(t), + } + fn := mIface.mapStringIfaceMarshaller() + var nilMapPtr *map[string]interface{} + require.NoError(t, fn(unsafe.Pointer(nilMapPtr), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + vmap := map[string]interface{}{"A": 1, "B": 2} + require.NoError(t, fn(unsafe.Pointer(&vmap), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + mIfaceErr := &mapMarshaller{ + config: &config.IOConfig{}, + valueType: reflect.TypeOf((*interface{})(nil)).Elem(), + valueMarshaller: &errMarshaller{}, + } + fnErr := mIfaceErr.mapStringIfaceMarshaller() + require.Error(t, fnErr(unsafe.Pointer(&vmap), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + // time and time ptr constructor/unmarshal error branches + require.Equal(t, "2006", newTimeMarshaller(&format.Tag{TimeLayout: "2006"}, &config.IOConfig{}).timeLayout) + tm := newTimeMarshaller(&format.Tag{}, &config.IOConfig{}) + tBad := gojay.BorrowDecoder(bytes.NewReader([]byte(`{`))) + defer tBad.Release() + var tv time.Time + require.Error(t, tm.UnmarshallObject(unsafe.Pointer(&tv), tBad, nil, &UnmarshalSession{})) + tBad2 := gojay.BorrowDecoder(bytes.NewReader([]byte(`"bad"`))) + defer tBad2.Release() + require.Panics(t, func() { _ = tm.UnmarshallObject(unsafe.Pointer(&tv), tBad2, nil, &UnmarshalSession{}) }) + + require.Equal(t, "2006", newTimePtrMarshaller(&format.Tag{TimeLayout: "2006"}, &config.IOConfig{}).timeLayout) + tpm := newTimePtrMarshaller(&format.Tag{}, &config.IOConfig{}) + tpBad := gojay.BorrowDecoder(bytes.NewReader([]byte(`{`))) + defer tpBad.Release() + var tp *time.Time + require.Error(t, tpm.UnmarshallObject(unsafe.Pointer(&tp), tpBad, nil, &UnmarshalSession{})) + tpBad2 := gojay.BorrowDecoder(bytes.NewReader([]byte(`"bad"`))) + defer tpBad2.Release() + require.Panics(t, func() { _ = tpm.UnmarshallObject(unsafe.Pointer(&tp), tpBad2, nil, &UnmarshalSession{}) }) + + // presence getFields continue non-bool branch + type mixed struct { + I int + B bool + } + fields, err := getFields(reflect.TypeOf(mixed{})) + require.NoError(t, err) + require.Len(t, fields, 1) + + // struct newFieldMarshaller non-letter name branch + marshallerWithField.init parse error branch + sstruct, err := newStructMarshaller(&config.IOConfig{}, reflect.TypeOf(struct{ A int }{}), "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + listMarshallers := make([]*marshallerWithField, 0) + require.NoError(t, sstruct.newFieldMarshaller(&listMarshallers, reflect.StructField{Name: "1bad", Type: reflect.TypeOf(0)}, "", "", &format.Tag{})) + require.Empty(t, listMarshallers) + + mwf := &marshallerWithField{} + require.NoError(t, mwf.init(reflect.StructField{Name: "X", Type: reflect.TypeOf(0), Tag: reflect.StructTag("json:\"x")}, &config.IOConfig{}, newCache())) + + // isZeroValue nil pointer currently panics for slice fields + type hs struct{ S []int } + sf, _ := reflect.TypeOf(hs{}).FieldByName("S") + sfw := &marshallerWithField{xField: xunsafe.NewField(sf)} + require.Panics(t, func() { _ = isZeroValue(nil, sfw, []int{}) }) +} + +func newInterfaceMarshallerMust(t *testing.T) marshaler { + m, err := newInterfaceMarshaller(reflect.TypeOf((*interface{})(nil)).Elem(), &config.IOConfig{}, "", "", &format.Tag{}, newCache()) + require.NoError(t, err) + return m +} + +func TestCoverage_ExtraReachableBranches(t *testing.T) { + cfg := &config.IOConfig{} + cache := newCache() + pc := cache.pathCache("") + + // cache struct branches: base.init error in gojay and non-gojay paths + custom unmarshaller struct branch + _, err := pc.getMarshaller(reflect.TypeOf(gojayBadInit{}), cfg, "", "", &format.Tag{}) + require.Error(t, err) + + type badPlain struct { + C chan int + } + _, err = pc.getMarshaller(reflect.TypeOf(badPlain{}), cfg, "", "", &format.Tag{}) + require.Error(t, err) + + _, err = pc.getMarshaller(reflect.TypeOf(customStructHolder{}), cfg, "", "", &format.Tag{}) + require.NoError(t, err) + + // default tag Name/Embedded attributes + type namedEmbedded struct { + A int `default:"name=abc,embedded=true"` + } + dt, err := NewDefaultTag(reflect.TypeOf(namedEmbedded{}).Field(0)) + require.NoError(t, err) + require.Equal(t, "abc", dt.Name) + require.True(t, dt.Embedded) + + // gojay wrapper: force value-receiver branch by using non-matching addrType + wv := newGojayObjectMarshaller( + getXType(reflect.TypeOf(gjValueOnly{})), + getXType(reflect.TypeOf(0)), + &fallbackMarshaller{}, + true, + true, + ) + sb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + gv := gjValueOnly{V: 12} + require.NoError(t, wv.MarshallObject(AsPtr(gv, reflect.TypeOf(gv)), sb)) + require.Contains(t, sb.String(), `"V":12`) + dec := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"V":12}`))) + aux := gojay.BorrowDecoder(bytes.NewReader([]byte(`{"V":13}`))) + defer dec.Release() + defer aux.Release() + require.NoError(t, wv.UnmarshallObject(unsafe.Pointer(&gv), dec, aux, &UnmarshalSession{})) + + // map marshaller key/value marshaller error branches + mKeyErr := &mapMarshaller{ + xType: getXType(reflect.TypeOf(map[int]int{})), + keyType: reflect.TypeOf(""), + valueType: reflect.TypeOf(int(0)), + keyMarshaller: &errMarshaller{}, + valueMarshaller: newIntMarshaller(&format.Tag{}), + config: cfg, + } + mv := map[int]int{1: 2} + require.Error(t, mKeyErr.MarshallObject(unsafe.Pointer(&mv), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + mValErr := &mapMarshaller{ + xType: getXType(reflect.TypeOf(map[int]int{})), + keyType: reflect.TypeOf(""), + valueType: reflect.TypeOf(int(0)), + keyMarshaller: newStringMarshaller(&format.Tag{}), + valueMarshaller: &errMarshaller{}, + config: cfg, + } + require.Error(t, mValErr.MarshallObject(unsafe.Pointer(&mv), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + // map constructor key marshaller error branch via unsupported key kind + _, err = newMapMarshaller(reflect.TypeOf(map[chan int]int{}), cfg, "", "", &format.Tag{}, cache) + require.NoError(t, err) + + // slice unmarshal skipNull true + array decode error; slice marshal nested marshaller error + sm := &sliceMarshaller{ + elemType: reflect.TypeOf(0), + marshaller: &errMarshaller{}, + xslice: xunsafe.NewSlice(reflect.TypeOf([]int{}), xunsafe.UseItemAddrOpt(true)), + } + forced := gojay.BorrowDecoder(bytes.NewReader([]byte(`[]`))) + ptrDec := unsafe.Pointer(forced) + decData.SetBytes(ptrDec, []byte("null")) + decCur.SetInt(ptrDec, 0) + dst := []int{} + require.NoError(t, sm.UnmarshallObject(unsafe.Pointer(&dst), forced, nil, &UnmarshalSession{})) + forced.Release() + + bad := gojay.BorrowDecoder(bytes.NewReader([]byte(`"x"`))) + defer bad.Release() + require.Error(t, sm.UnmarshallObject(unsafe.Pointer(&dst), bad, nil, &UnmarshalSession{})) + + outSlice := []int{1} + require.Error(t, sm.MarshallObject(unsafe.Pointer(&outSlice), &MarshallSession{Buffer: bytes.NewBuffer(nil)})) + + // marshallString generic control-char escaping branch (<0x20) + ssb := &MarshallSession{Buffer: bytes.NewBuffer(nil)} + marshallString(string([]byte{0x01}), ssb, nil) + require.Contains(t, ssb.String(), `\\u00`) + + // struct marshaller branches: indirect ignore, nil slice handling, nil pointer field + type embIgnored struct { + N int + } + type holderIgnored struct { + *embIgnored `json:"-"` + A int + } + _, err = New(cfg).Marshal(holderIgnored{A: 1}) + require.NoError(t, err) + + type withNilSlice struct { + S []int + } + _, err = New(cfg).Marshal(withNilSlice{}) + require.NoError(t, err) + + type withPtr struct { + P *int + } + _, err = New(cfg).Marshal(withPtr{}) + require.NoError(t, err) + + // createStructMarshallers inlinable newInlinableMarshaller error branch + type badInlineField struct { + C chan int `jsonx:"inline"` + } + s, err := newStructMarshaller(cfg, reflect.TypeOf(badInlineField{}), "", "", &format.Tag{}, cache) + require.NoError(t, err) + _, err = s.createStructMarshallers(groupFields(reflect.TypeOf(badInlineField{})), "", "", &format.Tag{}) + require.Error(t, err) + + // createStructMarshallers format.Parse error path + parameter/body naming path + type malformedTag struct { + A int `json:"abc` + } + s2, err := newStructMarshaller(cfg, reflect.TypeOf(malformedTag{}), "", "", &format.Tag{}, cache) + require.NoError(t, err) + _, err = s2.createStructMarshallers(groupFields(reflect.TypeOf(malformedTag{})), "", "", &format.Tag{}) + require.NoError(t, err) + + type parameterBody struct { + A int `parameter:"p1,kind=body,in=payload"` + } + s3, err := newStructMarshaller(cfg, reflect.TypeOf(parameterBody{}), "", "", &format.Tag{}, cache) + require.NoError(t, err) + marshallers, err := s3.createStructMarshallers(groupFields(reflect.TypeOf(parameterBody{})), "", "", &format.Tag{}) + require.NoError(t, err) + require.NotEmpty(t, marshallers) +} + +func TestCoverage_LastReachableAttempts(t *testing.T) { + // namesCaseIndex undefined source format path (fallback to original value) + n := &namesCaseIndex{registry: map[text.CaseFormat]map[string]string{}} + require.Equal(t, "___", n.formatTo("___", text.CaseFormatLowerCamel)) + + // cache slice constructor error branch (newSliceMarshaller -> elem unsupported) + pc := newCache().pathCache("") + _, err := pc.getMarshaller(reflect.TypeOf([]chan int{}), &config.IOConfig{}, "", "", &format.Tag{}) + require.Error(t, err) +} diff --git a/internal/translator/resource_settings_test.go b/internal/translator/resource_settings_test.go new file mode 100644 index 000000000..eeef20112 --- /dev/null +++ b/internal/translator/resource_settings_test.go @@ -0,0 +1,57 @@ +package translator + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/cmd/options" +) + +func TestResource_extractRuleSetting_RouteDirectiveOverridesHeader(t *testing.T) { + resource := &Resource{Rule: NewRule(), rule: &options.Rule{}} + dSQL := "/* {\"URI\":\"/v1/api/legacy\",\"Method\":\"GET\"} */\n" + + "#settings($_ = $route('/v1/api/orders', 'POST', 'PATCH'))\n" + + "#settings($_ = $marshal('application/json','pkg.OrderJSON'))\n" + + "#settings($_ = $unmarshal('application/json','pkg.OrderIn'))\n" + + "#settings($_ = $unmarshal('application/xml','pkg.OrderXMLIn'))\n" + + "#settings($_ = $format('tabular_json'))\n" + + "#settings($_ = $date_format('2006-01-02'))\n" + + "#settings($_ = $case_format('lc'))\n" + + "SELECT 1" + + err := resource.extractRuleSetting(&dSQL) + require.NoError(t, err) + assert.Equal(t, "/v1/api/orders", resource.Rule.URI) + assert.Equal(t, "POST,PATCH", resource.Rule.Method) + assert.Equal(t, "pkg.OrderJSON", resource.Rule.JSONMarshalType) + assert.Equal(t, "pkg.OrderIn", resource.Rule.JSONUnmarshalType) + assert.Equal(t, "pkg.OrderXMLIn", resource.Rule.XMLUnmarshalType) + assert.Equal(t, "tabular", resource.Rule.DataFormat) + assert.Equal(t, "2006-01-02", resource.Rule.Route.Content.DateFormat) + assert.Equal(t, "lc", string(resource.Rule.Route.Output.CaseFormat)) + assert.NotContains(t, dSQL, "$route(") + assert.NotContains(t, dSQL, "$marshal(") + assert.NotContains(t, dSQL, "$unmarshal(") + assert.NotContains(t, dSQL, "$format(") + assert.NotContains(t, dSQL, "$date_format(") + assert.NotContains(t, dSQL, "$case_format(") +} + +func TestResource_extractRuleSetting_InvalidRouteDirective(t *testing.T) { + resource := &Resource{Rule: NewRule(), rule: &options.Rule{}} + dSQL := "#settings($_ = $route('/v1/api/orders', 'GOT'))\nSELECT 1" + + err := resource.extractRuleSetting(&dSQL) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported method") +} + +func TestResource_extractRuleSetting_InvalidCaseFormatDirective(t *testing.T) { + resource := &Resource{Rule: NewRule(), rule: &options.Rule{}} + dSQL := "#settings($_ = $case_format('unknown'))\nSELECT 1" + + err := resource.extractRuleSetting(&dSQL) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported case format") +} diff --git a/repository/shape/compile/enrich_settings_test.go b/repository/shape/compile/enrich_settings_test.go new file mode 100644 index 000000000..87ecd1f66 --- /dev/null +++ b/repository/shape/compile/enrich_settings_test.go @@ -0,0 +1,26 @@ +package compile + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/viant/datly/repository/shape" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" +) + +func TestExtractRuleSettings_RouteDirectiveOverridesHeader(t *testing.T) { + source := &shape.Source{ + DQL: "/* {\"URI\":\"/v1/api/legacy\",\"Method\":\"GET\"} */\n" + + "#settings($_ = $route('/v1/api/orders', 'POST', 'PATCH'))\n" + + "SELECT 1", + } + + settings := extractRuleSettings(source, &dqlshape.Directives{ + Route: &dqlshape.RouteDirective{ + URI: "/v1/api/orders", + Methods: []string{"POST", "PATCH"}, + }, + }) + assert.Equal(t, "/v1/api/orders", settings.URI) + assert.Equal(t, "POST,PATCH", settings.Method) +} From 7abce12e2cc0b1a9f43d1ccce7e143f52ab2052e Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 24 Feb 2026 11:31:38 -0800 Subject: [PATCH 132/279] shape/compile: add type support helpers; refine preprocessing and type defaults; update tests --- .gitignore | 2 - repository/shape/dql/diag/codes.go | 48 +++ repository/shape/dql/optimize/optimizer.go | 99 +++++ .../shape/dql/optimize/optimizer_test.go | 40 ++ repository/shape/dql/parse/options.go | 40 ++ repository/shape/dql/parse/parser.go | 152 +++++++ repository/shape/dql/parse/parser_test.go | 134 +++++++ .../shape/dql/preprocess/diagnostics.go | 13 + repository/shape/dql/preprocess/extract.go | 121 ++++++ .../shape/dql/preprocess/legacy_import.go | 159 ++++++++ repository/shape/dql/preprocess/mapper.go | 181 +++++++++ repository/shape/dql/preprocess/preprocess.go | 130 ++++++ .../shape/dql/preprocess/preprocess_test.go | 169 ++++++++ repository/shape/dql/preprocess/scanner.go | 137 +++++++ .../dql/preprocess/settings_directives.go | 376 ++++++++++++++++++ .../dql/preprocess/typectx_directives.go | 84 ++++ repository/shape/dql/sanitize/policy.go | 32 ++ repository/shape/dql/sanitize/policy_test.go | 49 +++ repository/shape/dql/sanitize/sanitizer.go | 205 ++++++++++ .../shape/dql/sanitize/sanitizer_test.go | 245 ++++++++++++ repository/shape/dql/shape/model.go | 93 +++++ repository/shape/dql/statement/parity_test.go | 33 ++ repository/shape/dql/statement/parser.go | 201 ++++++++++ repository/shape/dql/statement/statement.go | 137 +++++++ .../shape/dql/statement/statement_test.go | 77 ++++ 25 files changed, 2955 insertions(+), 2 deletions(-) create mode 100644 repository/shape/dql/diag/codes.go create mode 100644 repository/shape/dql/optimize/optimizer.go create mode 100644 repository/shape/dql/optimize/optimizer_test.go create mode 100644 repository/shape/dql/parse/options.go create mode 100644 repository/shape/dql/parse/parser.go create mode 100644 repository/shape/dql/parse/parser_test.go create mode 100644 repository/shape/dql/preprocess/diagnostics.go create mode 100644 repository/shape/dql/preprocess/extract.go create mode 100644 repository/shape/dql/preprocess/legacy_import.go create mode 100644 repository/shape/dql/preprocess/mapper.go create mode 100644 repository/shape/dql/preprocess/preprocess.go create mode 100644 repository/shape/dql/preprocess/preprocess_test.go create mode 100644 repository/shape/dql/preprocess/scanner.go create mode 100644 repository/shape/dql/preprocess/settings_directives.go create mode 100644 repository/shape/dql/preprocess/typectx_directives.go create mode 100644 repository/shape/dql/sanitize/policy.go create mode 100644 repository/shape/dql/sanitize/policy_test.go create mode 100644 repository/shape/dql/sanitize/sanitizer.go create mode 100644 repository/shape/dql/sanitize/sanitizer_test.go create mode 100644 repository/shape/dql/shape/model.go create mode 100644 repository/shape/dql/statement/parity_test.go create mode 100644 repository/shape/dql/statement/parser.go create mode 100644 repository/shape/dql/statement/statement.go create mode 100644 repository/shape/dql/statement/statement_test.go diff --git a/.gitignore b/.gitignore index ed0afa4ad..2bebf504f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,6 @@ tmp/ *.gz .DS_Store autogen -dql -dql *.iml *.so logs diff --git a/repository/shape/dql/diag/codes.go b/repository/shape/dql/diag/codes.go new file mode 100644 index 000000000..7fa6a96e9 --- /dev/null +++ b/repository/shape/dql/diag/codes.go @@ -0,0 +1,48 @@ +package diag + +const ( + CodeParseEmpty = "DQL-PARSE-EMPTY" + CodeParseSyntax = "DQL-PARSE-SYNTAX" + CodeParseUnknownNonRead = "DQL-PARSE-UNKNOWN-NONREAD" + + CodeDirPackage = "DQL-DIR-PACKAGE" + CodeDirImport = "DQL-DIR-IMPORT" + CodeDirMeta = "DQL-DIR-META" + CodeDirCache = "DQL-DIR-CACHE" + CodeDirMCP = "DQL-DIR-MCP" + CodeDirConnector = "DQL-DIR-CONNECTOR" + CodeDirRoute = "DQL-DIR-ROUTE" + CodeDirMarshal = "DQL-DIR-MARSHAL" + CodeDirUnmarshal = "DQL-DIR-UNMARSHAL" + CodeDirFormat = "DQL-DIR-FORMAT" + CodeDirDateFormat = "DQL-DIR-DATE-FORMAT" + CodeDirCaseFormat = "DQL-DIR-CASE-FORMAT" + CodeDirUnsupported = "DQL-DIR-UNSUPPORTED" + + CodeOptParse = "DQL-OPT-PARSE" + CodeSQLIRawSelector = "DQL-SQLI-RAW-SELECTOR" + CodeViewMissingSQL = "DQL-VIEW-MISSING-SQL" + CodeViewCardinality = "DQL-VIEW-CARDINALITY" + CodeDeclOptionArgs = "DQL-DECL-OPTION-ARGS" + CodeDeclQuerySelector = "DQL-DECL-QUERY-SELECTOR" + CodeRelMissingON = "DQL-REL-MISSING-ON" + CodeRelUnsupported = "DQL-REL-UNSUPPORTED-PREDICATE" + CodeRelAmbiguous = "DQL-REL-AMBIGUOUS-LINK" + CodeRelNoLinks = "DQL-REL-NO-LINKS" + CodeCompRefInvalid = "DQL-COMP-REF-INVALID" + CodeCompRouteMissing = "DQL-COMP-ROUTE-MISSING" + CodeCompRouteInvalid = "DQL-COMP-ROUTE-INVALID" + CodeCompCycle = "DQL-COMP-CYCLE" + CodeCompTypeCollision = "DQL-COMP-TYPE-COLLISION" + CodeTypeCtxInvalid = "DQL-TYPECTX-INVALID" + CodeDMLMixed = "DQL-DML-MIXED" + CodeDMLServiceArg = "DQL-DML-SERVICE-ARG" + CodeDMLInsert = "DQL-DML-INSERT" + CodeDMLUpdate = "DQL-DML-UPDATE" + CodeDMLDelete = "DQL-DML-DELETE" + CodeColDiscoveryReq = "DQL-COL-DISCOVERY-REQUIRED" + + PrefixRel = "DQL-REL-" + PrefixComp = "DQL-COMP-" + PrefixSQLI = "DQL-SQLI-" +) diff --git a/repository/shape/dql/optimize/optimizer.go b/repository/shape/dql/optimize/optimizer.go new file mode 100644 index 000000000..030f26fb4 --- /dev/null +++ b/repository/shape/dql/optimize/optimizer.go @@ -0,0 +1,99 @@ +package optimize + +import ( + "strings" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/velty" + "github.com/viant/velty/ast" + aexpr "github.com/viant/velty/ast/expr" +) + +// Rewrite applies lightweight template simplification and emits diagnostics. +// It is intentionally conservative: only dead #if(false) blocks without else are blanked. +func Rewrite(dql string) (string, []*dqlshape.Diagnostic) { + if strings.TrimSpace(dql) == "" { + return dql, nil + } + adjuster := &hookAdjuster{source: []byte(dql), seenOffset: map[int]struct{}{}} + out, err := velty.TransformTemplate([]byte(dql), adjuster) + if err != nil { + adjuster.diagnostics = append(adjuster.diagnostics, &dqlshape.Diagnostic{ + Code: dqldiag.CodeOptParse, + Severity: dqlshape.SeverityWarning, + Message: "velty optimization pass skipped due to parse issue", + Hint: "check template syntax near directives and expressions", + Span: dqlshape.Span{Start: dqlshape.Position{Line: 1, Char: 1}, End: dqlshape.Position{Line: 1, Char: 1}}, + }) + return dql, adjuster.diagnostics + } + return string(out), adjuster.diagnostics +} + +type hookAdjuster struct { + source []byte + seenOffset map[int]struct{} + diagnostics []*dqlshape.Diagnostic +} + +func (a *hookAdjuster) Adjust(node ast.Node, ctx *velty.ParserContext) (velty.Action, error) { + switch actual := node.(type) { + case *aexpr.Select: + a.captureSQLInjectionRisk(actual, ctx) + } + return velty.Keep(), nil +} + +func (a *hookAdjuster) captureSQLInjectionRisk(sel *aexpr.Select, ctx *velty.ParserContext) { + if sel == nil || ctx == nil { + return + } + if ctx.CurrentExprContext().Kind == velty.CtxSetLHS { + return + } + span, ok := ctx.GetSpan(sel) + if !ok { + return + } + if strings.EqualFold(sel.ID, "Nop") { + return + } + if a.inNoopCall(span.Start) { + return + } + if _, exists := a.seenOffset[span.Start]; exists { + return + } + a.seenOffset[span.Start] = struct{}{} + pos := ctx.ResolvePosition(span) + a.diagnostics = append(a.diagnostics, &dqlshape.Diagnostic{ + Code: dqldiag.CodeSQLIRawSelector, + Severity: dqlshape.SeverityWarning, + Message: "raw selector interpolation detected in SQL template", + Hint: "prefer bind parameters or validated allow-listed fragments", + Span: dqlshape.Span{ + Start: dqlshape.Position{Offset: span.Start, Line: pos.Line, Char: pos.Col}, + End: dqlshape.Position{Offset: span.End, Line: pos.EndLine, Char: pos.EndCol}, + }, + }) +} + +func (a *hookAdjuster) inNoopCall(pos int) bool { + if pos <= 0 || pos > len(a.source) { + return false + } + prefix := string(a.source[:pos]) + nopPos := strings.LastIndex(prefix, "$Nop(") + if nopPos == -1 { + nopPos = strings.LastIndex(prefix, "$nop(") + } + if nopPos == -1 { + return false + } + if nl := strings.LastIndex(prefix, "\n"); nl > nopPos { + return false + } + segment := prefix[nopPos:pos] + return strings.Count(segment, "(") > strings.Count(segment, ")") +} diff --git a/repository/shape/dql/optimize/optimizer_test.go b/repository/shape/dql/optimize/optimizer_test.go new file mode 100644 index 000000000..ed1d3ac79 --- /dev/null +++ b/repository/shape/dql/optimize/optimizer_test.go @@ -0,0 +1,40 @@ +package optimize + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" +) + +func TestRewrite_SelectorInterpolationReportsPosition(t *testing.T) { + input := "SELECT id FROM orders WHERE id = $Unsafe.Id" + _, diagnostics := Rewrite(input) + require.NotEmpty(t, diagnostics) + diag := diagnostics[0] + assert.Equal(t, dqldiag.CodeSQLIRawSelector, diag.Code) + assert.Equal(t, 1, diag.Span.Start.Line) + assert.Greater(t, diag.Span.Start.Char, 1) +} + +func TestRewrite_ParseFailureFallsBack(t *testing.T) { + input := "#if(true)" + out, diagnostics := Rewrite(input) + assert.Equal(t, input, out) + require.NotEmpty(t, diagnostics) + assert.Equal(t, dqldiag.CodeOptParse, diagnostics[len(diagnostics)-1].Code) + assert.True(t, strings.Contains(strings.ToLower(diagnostics[len(diagnostics)-1].Message), "optimization pass")) +} + +func TestRewrite_NopDoesNotReportSelectorInterpolation(t *testing.T) { + input := "SELECT 1 WHERE 1=1 $Nop($Unsafe.Id)" + _, diagnostics := Rewrite(input) + for _, item := range diagnostics { + if item == nil { + continue + } + assert.NotEqual(t, dqldiag.CodeSQLIRawSelector, item.Code) + } +} diff --git a/repository/shape/dql/parse/options.go b/repository/shape/dql/parse/options.go new file mode 100644 index 000000000..640a0e6eb --- /dev/null +++ b/repository/shape/dql/parse/options.go @@ -0,0 +1,40 @@ +package parse + +type ( + UnknownNonReadMode string + + Options struct { + UnknownNonReadMode UnknownNonReadMode + } + + Option func(*Options) +) + +const ( + UnknownNonReadModeWarn UnknownNonReadMode = "warn" + UnknownNonReadModeError UnknownNonReadMode = "error" +) + +func WithUnknownNonReadMode(mode UnknownNonReadMode) Option { + return func(o *Options) { + if o == nil { + return + } + o.UnknownNonReadMode = mode + } +} + +func defaultOptions() Options { + return Options{ + UnknownNonReadMode: UnknownNonReadModeWarn, + } +} + +func normalizeUnknownNonReadMode(mode UnknownNonReadMode) UnknownNonReadMode { + switch mode { + case UnknownNonReadModeWarn, UnknownNonReadModeError: + return mode + default: + return UnknownNonReadModeWarn + } +} diff --git a/repository/shape/dql/parse/parser.go b/repository/shape/dql/parse/parser.go new file mode 100644 index 000000000..cb8d81974 --- /dev/null +++ b/repository/shape/dql/parse/parser.go @@ -0,0 +1,152 @@ +package parse + +import ( + "errors" + "strings" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" + "github.com/viant/datly/repository/shape/dql/shape" + dqlstmt "github.com/viant/datly/repository/shape/dql/statement" + "github.com/viant/parsly" + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/query" +) + +// Parser parses DQL source into a shape Document. +type Parser struct { + options Options +} + +// New creates a DQL parser. +func New(opts ...Option) *Parser { + options := defaultOptions() + for _, opt := range opts { + if opt != nil { + opt(&options) + } + } + options.UnknownNonReadMode = normalizeUnknownNonReadMode(options.UnknownNonReadMode) + return &Parser{options: options} +} + +// Parse parses DQL and returns parsed document with diagnostics. +func (p *Parser) Parse(dql string) (*shape.Document, error) { + doc := &shape.Document{Raw: dql} + sql, ctx, directives, directiveDiagnostics := dqlpre.Extract(dql) + doc.SQL = strings.TrimSpace(sql) + doc.TypeContext = ctx + doc.Directives = directives + if len(directiveDiagnostics) > 0 { + doc.Diagnostics = append(doc.Diagnostics, directiveDiagnostics...) + for _, diagnostic := range directiveDiagnostics { + if diagnostic != nil && diagnostic.Severity == shape.SeverityError { + return doc, diagnostic + } + } + } + + if doc.SQL == "" { + d := &shape.Diagnostic{ + Code: dqldiag.CodeParseEmpty, + Severity: shape.SeverityError, + Message: "no SQL statement found", + Hint: "add SELECT/INSERT/UPDATE/DELETE statement after DQL directives", + Span: dqlpre.PointSpan(dql, 0), + } + doc.Diagnostics = append(doc.Diagnostics, d) + return doc, d + } + + statements := dqlstmt.New(sql) + readStmt := firstReadStatement(statements) + if readStmt == nil { + if !hasExecStatement(statements) { + severity := shape.SeverityWarning + if p.options.UnknownNonReadMode == UnknownNonReadModeError { + severity = shape.SeverityError + } + doc.Diagnostics = append(doc.Diagnostics, &shape.Diagnostic{ + Code: dqldiag.CodeParseUnknownNonRead, + Severity: severity, + Message: "no readable SELECT statement detected", + Hint: "use SELECT for read parsing or compile as DML/handler template", + Span: dqlpre.PointSpan(dql, 0), + }) + if severity == shape.SeverityError { + return doc, doc.Diagnostics[len(doc.Diagnostics)-1] + } + } + // DML-only statement sets are valid for parse contract. + return doc, nil + } + querySQL := sql[readStmt.Start:readStmt.End] + queryNode, diag, err := parseQueryWithDiagnosticAt(querySQL, dql, readStmt.Start) + if diag != nil { + doc.Diagnostics = append(doc.Diagnostics, diag) + } + if err != nil { + return doc, diag + } + doc.Query = queryNode + return doc, nil +} + +func firstReadStatement(statements dqlstmt.Statements) *dqlstmt.Statement { + for _, stmt := range statements { + if stmt == nil { + continue + } + if stmt.Kind == dqlstmt.KindRead { + return stmt + } + } + return nil +} + +func hasExecStatement(statements dqlstmt.Statements) bool { + for _, stmt := range statements { + if stmt != nil && stmt.IsExec { + return true + } + } + return false +} + +func parseQueryWithDiagnosticAt(sqlText, original string, baseOffset int) (*query.Select, *shape.Diagnostic, error) { + cursor := parsly.NewCursor("", []byte(sqlText), 0) + var diagnostic *shape.Diagnostic + cursor.OnError = func(err error, cur *parsly.Cursor, _ interface{}) error { + offset := 0 + if cur != nil { + offset = cur.Pos + } + if offset < 0 { + offset = 0 + } + offset += baseOffset + diagnostic = &shape.Diagnostic{ + Code: dqldiag.CodeParseSyntax, + Severity: shape.SeverityError, + Message: strings.TrimSpace(err.Error()), + Hint: "check SQL syntax near the reported location", + Span: dqlpre.PointSpan(original, offset), + } + return err + } + result := &query.Select{} + err := sqlparser.Parse(cursor, result) + if err != nil { + if diagnostic == nil { + diagnostic = &shape.Diagnostic{ + Code: dqldiag.CodeParseSyntax, + Severity: shape.SeverityError, + Message: strings.TrimSpace(err.Error()), + Hint: "check SQL syntax near the reported location", + Span: dqlpre.PointSpan(original, baseOffset), + } + } + return nil, diagnostic, errors.New(diagnostic.Error()) + } + return result, nil, nil +} diff --git a/repository/shape/dql/parse/parser_test.go b/repository/shape/dql/parse/parser_test.go new file mode 100644 index 000000000..6a3776dc2 --- /dev/null +++ b/repository/shape/dql/parse/parser_test.go @@ -0,0 +1,134 @@ +package parse + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" +) + +func TestParser_Parse_TypeContext(t *testing.T) { + dql := ` +#package('mdp/performance') +#import('perf', 'github.com/acme/mdp/performance') +SELECT id FROM ORDERS t +` + parsed, err := New().Parse(dql) + require.NoError(t, err) + require.NotNil(t, parsed) + require.NotNil(t, parsed.TypeContext) + assert.Equal(t, "mdp/performance", parsed.TypeContext.DefaultPackage) + require.Len(t, parsed.TypeContext.Imports, 1) + assert.Equal(t, "perf", parsed.TypeContext.Imports[0].Alias) + assert.Equal(t, "github.com/acme/mdp/performance", parsed.TypeContext.Imports[0].Package) +} + +func TestParser_Parse_SpecialDirectives(t *testing.T) { + dql := ` +#settings($_ = $meta('docs/orders.md')) +#setting($_ = $connector('analytics')) +#settings($_ = $cache(true, '5m')) +#settings($_ = $mcp('orders.search', 'Search orders', 'docs/mcp/orders.md')) +SELECT id FROM ORDERS t +` + parsed, err := New().Parse(dql) + require.NoError(t, err) + require.NotNil(t, parsed) + require.NotNil(t, parsed.Directives) + assert.Equal(t, "docs/orders.md", parsed.Directives.Meta) + assert.Equal(t, "analytics", parsed.Directives.DefaultConnector) + require.NotNil(t, parsed.Directives.Cache) + assert.True(t, parsed.Directives.Cache.Enabled) + assert.Equal(t, "5m", parsed.Directives.Cache.TTL) + require.NotNil(t, parsed.Directives.MCP) + assert.Equal(t, "orders.search", parsed.Directives.MCP.Name) + assert.Equal(t, "Search orders", parsed.Directives.MCP.Description) + assert.Equal(t, "docs/mcp/orders.md", parsed.Directives.MCP.DescriptionPath) +} + +func TestParser_Parse_SyntaxErrorPosition(t *testing.T) { + dql := "SELECT id FROM ORDERS WHERE (" + parsed, err := New().Parse(dql) + require.Error(t, err) + require.NotNil(t, parsed) + require.NotEmpty(t, parsed.Diagnostics) + diag := parsed.Diagnostics[0] + assert.Equal(t, dqldiag.CodeParseSyntax, diag.Code) + assert.Equal(t, 1, diag.Span.Start.Line) + assert.Equal(t, 29, diag.Span.Start.Char) +} + +func TestParser_Parse_OnlyDirectives(t *testing.T) { + dql := "#package('x')\n#import('a','b')" + parsed, err := New().Parse(dql) + require.Error(t, err) + require.NotNil(t, parsed) + require.NotEmpty(t, parsed.Diagnostics) + assert.Equal(t, dqldiag.CodeParseEmpty, parsed.Diagnostics[0].Code) + assert.Equal(t, 1, parsed.Diagnostics[0].Span.Start.Line) + assert.Equal(t, 1, parsed.Diagnostics[0].Span.Start.Char) +} + +func TestParser_Parse_InvalidDirective_HasLineAndChar(t *testing.T) { + dql := "SELECT id FROM ORDERS t\n#import('alias')\nSELECT id FROM ORDERS t" + parsed, err := New().Parse(dql) + require.Error(t, err) + require.NotNil(t, parsed) + require.NotEmpty(t, parsed.Diagnostics) + diag := parsed.Diagnostics[0] + assert.Equal(t, dqldiag.CodeDirImport, diag.Code) + assert.Equal(t, 2, diag.Span.Start.Line) + assert.Equal(t, 1, diag.Span.Start.Char) +} + +func TestParser_Parse_DMLOnly_NoError(t *testing.T) { + dql := "INSERT INTO ORDERS(id) VALUES (1)" + parsed, err := New().Parse(dql) + require.NoError(t, err) + require.NotNil(t, parsed) + assert.Nil(t, parsed.Query) + assert.Empty(t, parsed.Diagnostics) +} + +func TestParser_Parse_Mixed_ReadAndExec_ParsesRead(t *testing.T) { + dql := "INSERT INTO ORDERS(id) VALUES (1)\nSELECT id FROM ORDERS t" + parsed, err := New().Parse(dql) + require.NoError(t, err) + require.NotNil(t, parsed) + require.NotNil(t, parsed.Query) + assert.Equal(t, "t", parsed.Query.From.Alias) +} + +func TestParser_Parse_UnknownNonRead_Warns(t *testing.T) { + dql := "$Foo.Bar($x)" + parsed, err := New().Parse(dql) + require.NoError(t, err) + require.NotNil(t, parsed) + assert.Nil(t, parsed.Query) + require.NotEmpty(t, parsed.Diagnostics) + assert.Equal(t, dqldiag.CodeParseUnknownNonRead, parsed.Diagnostics[len(parsed.Diagnostics)-1].Code) + assert.Equal(t, dqlshape.SeverityWarning, parsed.Diagnostics[len(parsed.Diagnostics)-1].Severity) +} + +func TestParser_Parse_UnknownNonRead_ErrorsWhenConfigured(t *testing.T) { + dql := "$Foo.Bar($x)" + parsed, err := New(WithUnknownNonReadMode(UnknownNonReadModeError)).Parse(dql) + require.Error(t, err) + require.NotNil(t, parsed) + assert.Nil(t, parsed.Query) + require.NotEmpty(t, parsed.Diagnostics) + assert.Equal(t, dqldiag.CodeParseUnknownNonRead, parsed.Diagnostics[len(parsed.Diagnostics)-1].Code) + assert.Equal(t, dqlshape.SeverityError, parsed.Diagnostics[len(parsed.Diagnostics)-1].Severity) +} + +func TestParser_Parse_UnknownNonRead_InvalidModeDefaultsToWarn(t *testing.T) { + dql := "$Foo.Bar($x)" + parsed, err := New(WithUnknownNonReadMode(UnknownNonReadMode("invalid"))).Parse(dql) + require.NoError(t, err) + require.NotNil(t, parsed) + require.NotEmpty(t, parsed.Diagnostics) + assert.Equal(t, dqldiag.CodeParseUnknownNonRead, parsed.Diagnostics[len(parsed.Diagnostics)-1].Code) + assert.Equal(t, dqlshape.SeverityWarning, parsed.Diagnostics[len(parsed.Diagnostics)-1].Severity) +} diff --git a/repository/shape/dql/preprocess/diagnostics.go b/repository/shape/dql/preprocess/diagnostics.go new file mode 100644 index 000000000..8542dfbf8 --- /dev/null +++ b/repository/shape/dql/preprocess/diagnostics.go @@ -0,0 +1,13 @@ +package preprocess + +import dqlshape "github.com/viant/datly/repository/shape/dql/shape" + +func directiveDiagnostic(code, message, hint, text string, offset int) *dqlshape.Diagnostic { + return &dqlshape.Diagnostic{ + Code: code, + Severity: dqlshape.SeverityError, + Message: message, + Hint: hint, + Span: pointSpan(text, offset), + } +} diff --git a/repository/shape/dql/preprocess/extract.go b/repository/shape/dql/preprocess/extract.go new file mode 100644 index 000000000..67b761a41 --- /dev/null +++ b/repository/shape/dql/preprocess/extract.go @@ -0,0 +1,121 @@ +package preprocess + +import ( + "strings" + + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/typectx" +) + +func extractSQLAndContext(dql string) (string, *typectx.Context, *dqlshape.Directives, []*dqlshape.Diagnostic) { + ctx := &typectx.Context{} + directives := &dqlshape.Directives{} + if dql == "" { + return "", nil, nil, nil + } + mask := make([]bool, len(dql)) + var diagnostics []*dqlshape.Diagnostic + + blocks := extractSetDirectiveBlocks(dql) + for _, block := range blocks { + applyMask(mask, dql, block.start, block.end) + if block.kind != directiveSettings { + continue + } + diagnostics = append(diagnostics, parseSettingsDirectives(block.body, dql, block.start, directives)...) + } + + lines := strings.SplitAfter(dql, "\n") + if len(lines) == 0 { + lines = []string{dql} + } + + offset := 0 + for _, line := range lines { + trimmed := strings.TrimSpace(line) + lineStart := offset + lineEnd := offset + len(line) + if isTypeContextDirectiveLine(trimmed) { + diagnostics = append(diagnostics, parseTypeContextDirective(trimmed, dql, offsetOfFirstNonSpace(line, offset), ctx)...) + applyMask(mask, dql, lineStart, lineEnd) + offset += len(line) + continue + } + if kind := lineDirectiveKind(trimmed); kind != directiveUnknown { + if !hasMasked(mask, lineStart, lineEnd) { + if kind != directiveSettings { + applyMask(mask, dql, lineStart, lineEnd) + offset += len(line) + continue + } + diagnostics = append(diagnostics, parseSettingsDirectives(trimmed, dql, offsetOfFirstNonSpace(line, offset), directives)...) + applyMask(mask, dql, lineStart, lineEnd) + } + offset += len(line) + continue + } + if isDirectiveLine(trimmed) { + applyMask(mask, dql, lineStart, lineEnd) + } + offset += len(line) + } + masked := []byte(dql) + for i := 0; i < len(masked); i++ { + if !mask[i] { + continue + } + if masked[i] == '\n' || masked[i] == '\r' { + continue + } + masked[i] = ' ' + } + return string(masked), ctx, directives, diagnostics +} + +func applyMask(mask []bool, text string, start, end int) { + if start < 0 { + start = 0 + } + if end > len(text) { + end = len(text) + } + if end <= start { + return + } + for i := start; i < end; i++ { + if text[i] == '\n' || text[i] == '\r' { + continue + } + mask[i] = true + } +} + +func hasMasked(mask []bool, start, end int) bool { + if start < 0 { + start = 0 + } + if end > len(mask) { + end = len(mask) + } + if end <= start { + return false + } + for i := start; i < end; i++ { + if mask[i] { + return true + } + } + return false +} + +func offsetOfFirstNonSpace(line string, base int) int { + for i := 0; i < len(line); i++ { + switch line[i] { + case ' ', '\t', '\r', '\n': + continue + default: + return base + i + } + } + return base +} diff --git a/repository/shape/dql/preprocess/legacy_import.go b/repository/shape/dql/preprocess/legacy_import.go new file mode 100644 index 000000000..0c7639b7e --- /dev/null +++ b/repository/shape/dql/preprocess/legacy_import.go @@ -0,0 +1,159 @@ +package preprocess + +import ( + "path" + "regexp" + "strings" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/typectx" +) + +var ( + legacyImportBlock = regexp.MustCompile(`(?ms)^[ \t]*import\s*\((.*?)\)`) + legacyImportLine = regexp.MustCompile(`(?m)^[ \t]*import\s*"([^"]+)"(?:\s+alias\s+"([^"]+)")?[ \t]*$`) + legacyImportItem = regexp.MustCompile(`"([^"]+)"(?:\s+alias\s+"([^"]+)")?`) +) + +type legacyImportRange struct { + start int + end int +} + +func extractLegacyTypeImports(dql string) ([]typectx.Import, []legacyImportRange, []*dqlshape.Diagnostic) { + if strings.TrimSpace(dql) == "" { + return nil, nil, nil + } + var ( + imports []typectx.Import + ranges []legacyImportRange + diags []*dqlshape.Diagnostic + ) + inBlock := make([]bool, len(dql)) + + blockMatches := legacyImportBlock.FindAllStringSubmatchIndex(dql, -1) + for _, match := range blockMatches { + if len(match) < 4 { + continue + } + start, end := match[0], match[1] + bodyStart, bodyEnd := match[2], match[3] + if start < 0 || end <= start || bodyStart < 0 || bodyEnd < bodyStart || bodyEnd > len(dql) { + continue + } + for i := start; i < end && i < len(inBlock); i++ { + inBlock[i] = true + } + ranges = append(ranges, legacyImportRange{start: start, end: end}) + blockBody := dql[bodyStart:bodyEnd] + itemMatches := legacyImportItem.FindAllStringSubmatchIndex(blockBody, -1) + if len(itemMatches) == 0 { + diags = append(diags, directiveDiagnostic( + dqldiag.CodeDirImport, + "invalid legacy import declaration", + `expected: import "pkg/path.Type" or import ("pkg/path.Type" alias "x")`, + dql, + start, + )) + continue + } + for _, item := range itemMatches { + if len(item) < 6 { + continue + } + specStart := bodyStart + item[2] + spec := strings.TrimSpace(blockBody[item[2]:item[3]]) + alias := "" + if item[4] >= 0 && item[5] >= 0 { + alias = strings.TrimSpace(blockBody[item[4]:item[5]]) + } + aImport, ok := parseLegacyImportSpec(spec, alias) + if !ok { + diags = append(diags, directiveDiagnostic( + dqldiag.CodeDirImport, + "invalid legacy import declaration", + `expected import target with type suffix: "pkg/path.Type"`, + dql, + specStart, + )) + continue + } + imports = append(imports, aImport) + } + } + + lineMatches := legacyImportLine.FindAllStringSubmatchIndex(dql, -1) + for _, match := range lineMatches { + if len(match) < 6 { + continue + } + start, end := match[0], match[1] + if start < 0 || end <= start || start >= len(inBlock) || inBlock[start] { + continue + } + spec := strings.TrimSpace(dql[match[2]:match[3]]) + alias := "" + if match[4] >= 0 && match[5] >= 0 { + alias = strings.TrimSpace(dql[match[4]:match[5]]) + } + aImport, ok := parseLegacyImportSpec(spec, alias) + if !ok { + diags = append(diags, directiveDiagnostic( + dqldiag.CodeDirImport, + "invalid legacy import declaration", + `expected import target with type suffix: "pkg/path.Type"`, + dql, + start, + )) + continue + } + imports = append(imports, aImport) + ranges = append(ranges, legacyImportRange{start: start, end: end}) + } + + return uniqueTypeImports(imports), ranges, diags +} + +func parseLegacyImportSpec(spec, alias string) (typectx.Import, bool) { + spec = strings.TrimSpace(spec) + if spec == "" { + return typectx.Import{}, false + } + index := strings.LastIndex(spec, ".") + if index <= 0 || index >= len(spec)-1 { + return typectx.Import{}, false + } + pkg := strings.TrimSpace(spec[:index]) + typeName := strings.TrimSpace(spec[index+1:]) + if pkg == "" || typeName == "" { + return typectx.Import{}, false + } + alias = strings.TrimSpace(alias) + if alias == "" { + alias = path.Base(pkg) + } + return typectx.Import{Alias: alias, Package: pkg}, true +} + +func uniqueTypeImports(input []typectx.Import) []typectx.Import { + if len(input) == 0 { + return nil + } + seen := map[string]bool{} + result := make([]typectx.Import, 0, len(input)) + for _, item := range input { + pkg := strings.TrimSpace(item.Package) + if pkg == "" { + continue + } + alias := strings.TrimSpace(item.Alias) + key := strings.ToLower(alias + "|" + pkg) + if seen[key] { + continue + } + seen[key] = true + result = append(result, typectx.Import{Alias: alias, Package: pkg}) + } + return result +} diff --git a/repository/shape/dql/preprocess/mapper.go b/repository/shape/dql/preprocess/mapper.go new file mode 100644 index 000000000..bb3f14f3a --- /dev/null +++ b/repository/shape/dql/preprocess/mapper.go @@ -0,0 +1,181 @@ +package preprocess + +import ( + "sort" + "unicode/utf8" + + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/velty" +) + +type Mapper struct { + trimPrefix int + segments []mapSegment + original string +} + +type mapSegment struct { + newStart int + newEnd int + origBase int + linear bool +} + +func (m *Mapper) MapOffset(offset int) int { + if m == nil { + if offset < 0 { + return 0 + } + return offset + } + mapped := offset + m.trimPrefix + if mapped < 0 { + mapped = 0 + } + for _, seg := range m.segments { + if mapped < seg.newStart || mapped > seg.newEnd { + continue + } + if seg.linear { + delta := mapped - seg.newStart + if delta < 0 { + delta = 0 + } + return seg.origBase + delta + } + return seg.origBase + } + if len(m.segments) == 0 { + return mapped + } + last := m.segments[len(m.segments)-1] + if last.linear { + return last.origBase + (last.newEnd - last.newStart) + } + return last.origBase +} + +func (m *Mapper) Position(offset int) dqlshape.Position { + return positionAt(m.original, m.MapOffset(offset)) +} + +func (m *Mapper) Remap(diags []*dqlshape.Diagnostic) { + if m == nil || len(diags) == 0 { + return + } + for _, diag := range diags { + if diag == nil { + continue + } + start := m.Position(diag.Span.Start.Offset) + end := m.Position(diag.Span.End.Offset) + diag.Span.Start = start + diag.Span.End = end + } +} + +func newMapper(srcLen int, patches []velty.Patch, trimPrefix int, original string) *Mapper { + if trimPrefix < 0 { + trimPrefix = 0 + } + ps := append([]velty.Patch{}, patches...) + sort.Slice(ps, func(i, j int) bool { return ps[i].Span.Start < ps[j].Span.Start }) + segments := make([]mapSegment, 0, len(ps)*2+1) + oldPos := 0 + newPos := 0 + for _, p := range ps { + start := p.Span.Start + end := p.Span.End + 1 + if start < oldPos || start < 0 || end < start || end > srcLen { + continue + } + if start > oldPos { + blockLen := start - oldPos + segments = append(segments, mapSegment{ + newStart: newPos, + newEnd: newPos + blockLen, + origBase: oldPos, + linear: true, + }) + oldPos = start + newPos += blockLen + } + replLen := len(p.Replacement) + if replLen > 0 { + segments = append(segments, mapSegment{ + newStart: newPos, + newEnd: newPos + replLen, + origBase: start, + linear: false, + }) + newPos += replLen + } + oldPos = end + } + if oldPos < srcLen { + blockLen := srcLen - oldPos + segments = append(segments, mapSegment{ + newStart: newPos, + newEnd: newPos + blockLen, + origBase: oldPos, + linear: true, + }) + } + return &Mapper{trimPrefix: trimPrefix, segments: segments, original: original} +} + +func pointSpan(text string, offset int) dqlshape.Span { + start := positionAt(text, offset) + end := positionAt(text, nextOffset(text, offset)) + return dqlshape.Span{Start: start, End: end} +} + +// PointSpan returns a single-point span at offset with rune-aware line/char. +func PointSpan(text string, offset int) dqlshape.Span { + return pointSpan(text, offset) +} + +func nextOffset(text string, offset int) int { + if offset < 0 { + return 0 + } + if offset >= len(text) { + return len(text) + } + _, width := utf8.DecodeRuneInString(text[offset:]) + if width <= 0 { + return offset + 1 + } + return offset + width +} + +func positionAt(text string, offset int) dqlshape.Position { + if offset < 0 { + offset = 0 + } + if offset > len(text) { + offset = len(text) + } + line := 1 + char := 1 + index := 0 + for index < offset { + r, width := utf8.DecodeRuneInString(text[index:]) + if width <= 0 { + break + } + index += width + if r == '\n' { + line++ + char = 1 + } else { + char++ + } + } + return dqlshape.Position{Offset: offset, Line: line, Char: char} +} + +// PositionAt returns rune-aware position for byte offset. +func PositionAt(text string, offset int) dqlshape.Position { + return positionAt(text, offset) +} diff --git a/repository/shape/dql/preprocess/preprocess.go b/repository/shape/dql/preprocess/preprocess.go new file mode 100644 index 000000000..3d56380b7 --- /dev/null +++ b/repository/shape/dql/preprocess/preprocess.go @@ -0,0 +1,130 @@ +package preprocess + +import ( + "regexp" + "strings" + + dqlopt "github.com/viant/datly/repository/shape/dql/optimize" + dqlsanitize "github.com/viant/datly/repository/shape/dql/sanitize" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/typectx" +) + +var ( + decoratorLine = regexp.MustCompile(`(?i)^\s*(use_connector|allow_nulls?)\s*\([^)]*\)\s*,?\s*$`) + commaBeforeFrom = regexp.MustCompile(`(?i),\s*(\r?\n\s*from\b)`) + doubleCommaExpr = regexp.MustCompile(`,\s*,`) +) + +type Result struct { + Original string + DirectSQL string + Optimized string + SQL string + TypeCtx *typectx.Context + Directives *dqlshape.Directives + Mapper *Mapper + Diagnostics []*dqlshape.Diagnostic +} + +// Extract parses directives and returns SQL with directive lines masked to preserve offsets. +func Extract(dql string) (string, *typectx.Context, *dqlshape.Directives, []*dqlshape.Diagnostic) { + sql, ctx, directives, diags := extractSQLAndContext(dql) + return sql, normalizeTypeContext(ctx), normalizeDirectives(directives), diags +} + +func Prepare(dql string) *Result { + ret := &Result{Original: dql} + sql, typeCtx, directives, dirDiags := Extract(dql) + ret.DirectSQL = stripDecorators(sql) + ret.TypeCtx = typeCtx + ret.Directives = directives + ret.Diagnostics = append(ret.Diagnostics, dirDiags...) + if strings.TrimSpace(ret.DirectSQL) == "" { + return ret + } + optimized, optDiags := dqlopt.Rewrite(ret.DirectSQL) + ret.Diagnostics = append(ret.Diagnostics, optDiags...) + ret.Optimized = optimized + sanitized := dqlsanitize.Rewrite(optimized, dqlsanitize.Options{ + Declared: dqlsanitize.Declared(optimized), + }) + ret.SQL = sanitized.SQL + ret.Mapper = newMapper(len(optimized), sanitized.Patches, sanitized.TrimPrefix, dql) + return ret +} + +func stripDecorators(sql string) string { + if strings.TrimSpace(sql) == "" { + return sql + } + lines := strings.Split(sql, "\n") + filtered := make([]string, 0, len(lines)) + for _, line := range lines { + if decoratorLine.MatchString(strings.TrimSpace(line)) { + continue + } + filtered = append(filtered, line) + } + joined := strings.Join(filtered, "\n") + joined = doubleCommaExpr.ReplaceAllString(joined, ",") + joined = commaBeforeFrom.ReplaceAllString(joined, "$1") + return joined +} + +func normalizeTypeContext(ctx *typectx.Context) *typectx.Context { + if ctx == nil { + return nil + } + if ctx.DefaultPackage == "" && len(ctx.Imports) == 0 { + return nil + } + return ctx +} + +func normalizeDirectives(input *dqlshape.Directives) *dqlshape.Directives { + if input == nil { + return nil + } + ret := &dqlshape.Directives{ + Meta: strings.TrimSpace(input.Meta), + DefaultConnector: strings.TrimSpace(input.DefaultConnector), + JSONMarshalType: strings.TrimSpace(input.JSONMarshalType), + JSONUnmarshalType: strings.TrimSpace(input.JSONUnmarshalType), + XMLUnmarshalType: strings.TrimSpace(input.XMLUnmarshalType), + Format: strings.TrimSpace(input.Format), + DateFormat: strings.TrimSpace(input.DateFormat), + CaseFormat: strings.TrimSpace(input.CaseFormat), + } + if input.Cache != nil { + ret.Cache = &dqlshape.CacheDirective{ + Enabled: input.Cache.Enabled, + TTL: strings.TrimSpace(input.Cache.TTL), + } + } + if input.MCP != nil { + ret.MCP = &dqlshape.MCPDirective{ + Name: strings.TrimSpace(input.MCP.Name), + Description: strings.TrimSpace(input.MCP.Description), + DescriptionPath: strings.TrimSpace(input.MCP.DescriptionPath), + } + } + if input.Route != nil { + normalizedMethods := make([]string, 0, len(input.Route.Methods)) + for _, method := range input.Route.Methods { + if method = strings.TrimSpace(method); method != "" { + normalizedMethods = append(normalizedMethods, method) + } + } + ret.Route = &dqlshape.RouteDirective{ + URI: strings.TrimSpace(input.Route.URI), + Methods: normalizedMethods, + } + } + if ret.Meta == "" && ret.DefaultConnector == "" && ret.Cache == nil && ret.MCP == nil && ret.Route == nil && + ret.JSONMarshalType == "" && ret.JSONUnmarshalType == "" && ret.XMLUnmarshalType == "" && ret.Format == "" && + ret.DateFormat == "" && ret.CaseFormat == "" { + return nil + } + return ret +} diff --git a/repository/shape/dql/preprocess/preprocess_test.go b/repository/shape/dql/preprocess/preprocess_test.go new file mode 100644 index 000000000..5f3d0dfc4 --- /dev/null +++ b/repository/shape/dql/preprocess/preprocess_test.go @@ -0,0 +1,169 @@ +package preprocess + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" +) + +func TestPrepare_TypeContext(t *testing.T) { + dql := "#package('a/b')\n#import('x','github.com/acme/x')\nSELECT id FROM t" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotNil(t, pre.TypeCtx) + assert.Equal(t, "a/b", pre.TypeCtx.DefaultPackage) + require.Len(t, pre.TypeCtx.Imports, 1) + assert.Equal(t, "x", pre.TypeCtx.Imports[0].Alias) +} + +func TestPrepare_InvalidDirectiveDiagnostic(t *testing.T) { + dql := "SELECT 1\n#import('x')" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirImport, pre.Diagnostics[0].Code) + assert.Equal(t, 2, pre.Diagnostics[0].Span.Start.Line) + assert.Equal(t, 1, pre.Diagnostics[0].Span.Start.Char) +} + +func TestMapper_MapOffset_WithSanitizeExpansion(t *testing.T) { + dql := "SELECT id FROM ORDERS t WHERE t.id = $Id AND (" + pre := Prepare(dql) + require.NotNil(t, pre.Mapper) + // Syntax error location after sanitize rewrite should map back to original source. + offset := len(pre.SQL) - 1 + pos := pre.Mapper.Position(offset) + assert.Equal(t, 1, pos.Line) + assert.Equal(t, 46, pos.Char) +} + +func TestPrepare_StripsReadDecorators(t *testing.T) { + dql := `SELECT t.*, +use_connector(t, system), +allow_nulls(t) +FROM t` + pre := Prepare(dql) + require.NotNil(t, pre) + assert.NotContains(t, pre.DirectSQL, "use_connector") + assert.NotContains(t, pre.DirectSQL, "allow_nulls") + assert.Contains(t, pre.DirectSQL, "SELECT t.*") + assert.Contains(t, pre.DirectSQL, "FROM t") + assert.NotContains(t, pre.DirectSQL, ",\nFROM") +} + +func TestPrepare_MultilineSetDirective_TypeContext(t *testing.T) { + dql := "#package('a/b')\n#import('x','github.com/acme/x')\nSELECT id FROM t" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotNil(t, pre.TypeCtx) + assert.Equal(t, "a/b", pre.TypeCtx.DefaultPackage) + require.Len(t, pre.TypeCtx.Imports, 1) + assert.Equal(t, "x", pre.TypeCtx.Imports[0].Alias) + assert.Equal(t, "github.com/acme/x", pre.TypeCtx.Imports[0].Package) + assert.Contains(t, pre.DirectSQL, "SELECT id FROM t") +} + +func TestPrepare_InvalidMultilineImportDiagnostic(t *testing.T) { + dql := "SELECT 1\n#import(\n'x'\n)" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirImport, pre.Diagnostics[0].Code) + assert.Equal(t, 2, pre.Diagnostics[0].Span.Start.Line) + assert.GreaterOrEqual(t, pre.Diagnostics[0].Span.Start.Char, 1) +} + +func TestPrepare_SpecialDirectives(t *testing.T) { + dql := "#settings($_ = $meta('docs/orders.md'))\n" + + "#setting($_ = $connector('analytics'))\n" + + "#settings($_ = $cache(true, '5m'))\n" + + "#settings($_ = $mcp('orders.search', 'Search orders', 'docs/mcp/orders.md'))\n" + + "#settings($_ = $marshal('application/json','pkg.OrderJSON'))\n" + + "#settings($_ = $unmarshal('application/json','pkg.OrderIn'))\n" + + "#settings($_ = $unmarshal('application/xml','pkg.OrderXMLIn'))\n" + + "#settings($_ = $format('tabular_json'))\n" + + "#settings($_ = $date_format('2006-01-02'))\n" + + "#settings($_ = $case_format('lc'))\n" + + "SELECT id FROM ORDERS o" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotNil(t, pre.Directives) + assert.Equal(t, "docs/orders.md", pre.Directives.Meta) + assert.Equal(t, "analytics", pre.Directives.DefaultConnector) + require.NotNil(t, pre.Directives.Cache) + assert.True(t, pre.Directives.Cache.Enabled) + assert.Equal(t, "5m", pre.Directives.Cache.TTL) + require.NotNil(t, pre.Directives.MCP) + assert.Equal(t, "orders.search", pre.Directives.MCP.Name) + assert.Equal(t, "Search orders", pre.Directives.MCP.Description) + assert.Equal(t, "docs/mcp/orders.md", pre.Directives.MCP.DescriptionPath) + assert.Equal(t, "pkg.OrderJSON", pre.Directives.JSONMarshalType) + assert.Equal(t, "pkg.OrderIn", pre.Directives.JSONUnmarshalType) + assert.Equal(t, "pkg.OrderXMLIn", pre.Directives.XMLUnmarshalType) + assert.Equal(t, "tabular", pre.Directives.Format) + assert.Equal(t, "2006-01-02", pre.Directives.DateFormat) + assert.Equal(t, "lc", pre.Directives.CaseFormat) +} + +func TestPrepare_InvalidSpecialDirectiveDiagnostic(t *testing.T) { + dql := "SELECT 1\n#settings($_ = $mcp())" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirMCP, pre.Diagnostics[0].Code) + assert.Equal(t, 2, pre.Diagnostics[0].Span.Start.Line) +} + +func TestPrepare_InvalidConnectorDirectiveDiagnostic(t *testing.T) { + dql := "SELECT 1\n#settings($_ = $connector())" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirConnector, pre.Diagnostics[0].Code) + assert.Equal(t, 2, pre.Diagnostics[0].Span.Start.Line) +} + +func TestPrepare_RouteDirective(t *testing.T) { + dql := "SELECT 1\n#settings($_ = $route('/v1/api/orders', 'GET', 'POST', 'PATCH'))" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotNil(t, pre.Directives) + require.NotNil(t, pre.Directives.Route) + assert.Equal(t, "/v1/api/orders", pre.Directives.Route.URI) + assert.Equal(t, []string{"GET", "POST", "PATCH"}, pre.Directives.Route.Methods) +} + +func TestPrepare_InvalidRouteDirectiveDiagnostic(t *testing.T) { + dql := "SELECT 1\n#settings($_ = $route('/v1/api/orders', 'GOT'))" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirRoute, pre.Diagnostics[0].Code) + assert.Equal(t, 2, pre.Diagnostics[0].Span.Start.Line) +} + +func TestPrepare_InvalidCaseFormatDirectiveDiagnostic(t *testing.T) { + dql := "SELECT 1\n#settings($_ = $case_format('unknown'))" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirCaseFormat, pre.Diagnostics[0].Code) +} + +func TestPrepare_DefineDirective_DoesNotDriveSettingsExtraction(t *testing.T) { + dql := "#define($_ = $package('a/b'))\nSELECT 1" + pre := Prepare(dql) + require.NotNil(t, pre) + assert.Nil(t, pre.TypeCtx) +} + +func TestPrepare_PackageImportInSettings_UnsupportedDiagnostic(t *testing.T) { + dql := "#settings($_ = $package('x'))\nSELECT 1" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirUnsupported, pre.Diagnostics[0].Code) + assert.Equal(t, 1, pre.Diagnostics[0].Span.Start.Line) +} diff --git a/repository/shape/dql/preprocess/scanner.go b/repository/shape/dql/preprocess/scanner.go new file mode 100644 index 000000000..3b7513917 --- /dev/null +++ b/repository/shape/dql/preprocess/scanner.go @@ -0,0 +1,137 @@ +package preprocess + +import ( + "strings" + + "github.com/viant/parsly" + "github.com/viant/parsly/matcher" +) + +var ( + ppWhitespaceToken = 1 + ppExprGroupToken = 2 + + ppWhitespaceMatcher = parsly.NewToken(ppWhitespaceToken, "Whitespace", matcher.NewWhiteSpace()) + ppExprGroupMatcher = parsly.NewToken(ppExprGroupToken, "( ... )", matcher.NewBlock('(', ')', '\\')) +) + +type setDirectiveBlock struct { + start int + end int + body string + kind directiveKind +} + +type directiveKind int + +const ( + directiveUnknown directiveKind = iota + directiveSet + directiveDefine + directiveSettings +) + +func isDirectiveLine(line string) bool { + if line == "" { + return false + } + if isTypeContextDirectiveLine(line) { + return true + } + if isSetLine(line) { + return true + } + if strings.HasPrefix(line, "#if(") || strings.HasPrefix(line, "#elseif(") || strings.HasPrefix(line, "#else") || strings.HasPrefix(line, "#end") { + return true + } + return false +} + +func isSetLine(line string) bool { + if line == "" { + return false + } + return lineDirectiveKind(line) != directiveUnknown +} + +func extractSetDirectiveBlocks(dql string) []setDirectiveBlock { + cursor := parsly.NewCursor("", []byte(dql), 0) + var result []setDirectiveBlock + for cursor.Pos < cursor.InputSize { + start := cursor.Pos + kind, keywordLen, ok := matchDirectiveAt(dql, start) + if !ok { + cursor.Pos++ + continue + } + cursor.Pos += keywordLen + group := cursor.MatchAfterOptional(ppWhitespaceMatcher, ppExprGroupMatcher) + if group.Code != ppExprGroupToken { + cursor.Pos = start + 1 + continue + } + groupText := group.Text(cursor) + if len(groupText) < 2 { + continue + } + end := cursor.Pos + result = append(result, setDirectiveBlock{ + start: start, + end: end, + body: groupText[1 : len(groupText)-1], + kind: kind, + }) + } + return result +} + +func lineDirectiveKind(line string) directiveKind { + if line == "" { + return directiveUnknown + } + switch { + case strings.HasPrefix(line, "#settings("), strings.HasPrefix(line, "#settings ("): + return directiveSettings + case strings.HasPrefix(line, "#setting("), strings.HasPrefix(line, "#setting ("): + return directiveSettings + case strings.HasPrefix(line, "#define("), strings.HasPrefix(line, "#define ("): + return directiveDefine + case strings.HasPrefix(line, "#set("), strings.HasPrefix(line, "#set ("): + return directiveSet + default: + return directiveUnknown + } +} + +func matchDirectiveAt(dql string, pos int) (directiveKind, int, bool) { + if pos < 0 || pos >= len(dql) || dql[pos] != '#' { + return directiveUnknown, 0, false + } + remaining := dql[pos:] + switch { + case hasDirectivePrefix(remaining, "#settings"): + return directiveSettings, len("#settings"), true + case hasDirectivePrefix(remaining, "#setting"): + return directiveSettings, len("#setting"), true + case hasDirectivePrefix(remaining, "#define"): + return directiveDefine, len("#define"), true + case hasDirectivePrefix(remaining, "#set"): + return directiveSet, len("#set"), true + default: + return directiveUnknown, 0, false + } +} + +func hasDirectivePrefix(input string, directive string) bool { + if len(input) < len(directive) { + return false + } + if !strings.EqualFold(input[:len(directive)], directive) { + return false + } + if len(input) == len(directive) { + return true + } + next := input[len(directive)] + return next == '(' || next == ' ' || next == '\t' || next == '\r' || next == '\n' +} diff --git a/repository/shape/dql/preprocess/settings_directives.go b/repository/shape/dql/preprocess/settings_directives.go new file mode 100644 index 000000000..770aa3a0d --- /dev/null +++ b/repository/shape/dql/preprocess/settings_directives.go @@ -0,0 +1,376 @@ +package preprocess + +import ( + "net/http" + "regexp" + "strings" + + "github.com/viant/datly/repository/content" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/tagly/format/text" +) + +var ( + metaDirective = regexp.MustCompile(`(?i)\$meta\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) + connectorDirective = regexp.MustCompile(`(?i)\$connector\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) + cacheDirective = regexp.MustCompile(`(?i)\$cache\s*\(\s*(true|false)\s*(?:,\s*['\"]([^'\"]+)['\"]\s*)?\)`) + mcpDirective = regexp.MustCompile(`(?i)\$mcp\s*\(\s*['\"]([^'\"]+)['\"]\s*(?:,\s*['\"]([^'\"]*)['\"]\s*)?(?:,\s*['\"]([^'\"]*)['\"]\s*)?\)`) + routeDirective = regexp.MustCompile(`(?i)\$route\s*\(([^)]*)\)`) + marshalDirective = regexp.MustCompile(`(?i)\$marshal\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)`) + unmarshalDirective = regexp.MustCompile(`(?i)\$unmarshal\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)`) + formatDirective = regexp.MustCompile(`(?i)\$format\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) + dateFormatDirective = regexp.MustCompile(`(?i)\$date_format\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) + caseFormatDirective = regexp.MustCompile(`(?i)\$case_format\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) + quotedArgDirective = regexp.MustCompile(`['\"]([^'\"]*)['\"]`) +) + +func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, directives *dqlshape.Directives) []*dqlshape.Diagnostic { + if strings.TrimSpace(input) == "" { + return nil + } + var diagnostics []*dqlshape.Diagnostic + lower := strings.ToLower(input) + if strings.Contains(lower, "$package") || strings.Contains(lower, "$import") { + diagnostics = append(diagnostics, directiveDiagnostic( + dqldiag.CodeDirUnsupported, + "type-context directives are not allowed in #settings", + "use #package('module/path') and #import('alias','github.com/acme/pkg')", + fullDQL, + diagnosticOffset, + )) + } + if strings.Contains(lower, "$meta") { + values := parseMetaDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirMeta, "invalid $meta directive", "expected: #settings($_ = $meta('relative/or/absolute/path'))", fullDQL, diagnosticOffset)) + } else { + directives.Meta = values[len(values)-1] + } + } + if strings.Contains(lower, "$connector") { + values := parseConnectorDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirConnector, "invalid $connector directive", "expected: #settings($_ = $connector('connector_name'))", fullDQL, diagnosticOffset)) + } else { + directives.DefaultConnector = values[len(values)-1] + } + } + if strings.Contains(lower, "$cache") { + values := parseCacheDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirCache, "invalid $cache directive", "expected: #settings($_ = $cache(true, '5m'))", fullDQL, diagnosticOffset)) + } else { + directives.Cache = values[len(values)-1] + } + } + if strings.Contains(lower, "$mcp") { + values := parseMCPDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirMCP, "invalid $mcp directive", "expected: #settings($_ = $mcp('tool.name','description','docs/path.md'))", fullDQL, diagnosticOffset)) + } else { + directives.MCP = values[len(values)-1] + } + } + if strings.Contains(lower, "$route") { + values := parseRouteDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirRoute, "invalid $route directive", "expected: #settings($_ = $route('/v1/api/path','GET','POST'))", fullDQL, diagnosticOffset)) + } else { + directives.Route = values[len(values)-1] + } + } + if strings.Contains(lower, "$marshal") { + values := parseMarshalDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirMarshal, "invalid $marshal directive", "expected: #settings($_ = $marshal('application/json','pkg.Type'))", fullDQL, diagnosticOffset)) + } else { + directives.JSONMarshalType = values[len(values)-1] + } + } + if strings.Contains(lower, "$unmarshal") { + values := parseUnmarshalDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirUnmarshal, "invalid $unmarshal directive", "expected: #settings($_ = $unmarshal('application/json','pkg.Type'))", fullDQL, diagnosticOffset)) + } else { + last := values[len(values)-1] + if last.JSONType != "" { + directives.JSONUnmarshalType = last.JSONType + } + if last.XMLType != "" { + directives.XMLUnmarshalType = last.XMLType + } + } + } + if strings.Contains(lower, "$format") { + values := parseFormatDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirFormat, "invalid $format directive", "expected: #settings($_ = $format('tabular_json'))", fullDQL, diagnosticOffset)) + } else { + directives.Format = values[len(values)-1] + } + } + if strings.Contains(lower, "$date_format") { + values := parseDateFormatDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirDateFormat, "invalid $date_format directive", "expected: #settings($_ = $date_format('2006-01-02'))", fullDQL, diagnosticOffset)) + } else { + directives.DateFormat = values[len(values)-1] + } + } + if strings.Contains(lower, "$case_format") { + values := parseCaseFormatDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirCaseFormat, "invalid $case_format directive", "expected: #settings($_ = $case_format('lc'))", fullDQL, diagnosticOffset)) + } else { + directives.CaseFormat = values[len(values)-1] + } + } + return diagnostics +} + +func parseMetaDirectives(input string) []string { + matches := metaDirective.FindAllStringSubmatch(input, -1) + result := make([]string, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + if value := strings.TrimSpace(match[1]); value != "" { + result = append(result, value) + } + } + return result +} + +func parseConnectorDirectives(input string) []string { + matches := connectorDirective.FindAllStringSubmatch(input, -1) + result := make([]string, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + if value := strings.TrimSpace(match[1]); value != "" { + result = append(result, value) + } + } + return result +} + +func parseCacheDirectives(input string) []*dqlshape.CacheDirective { + matches := cacheDirective.FindAllStringSubmatch(input, -1) + result := make([]*dqlshape.CacheDirective, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + enabled := strings.EqualFold(strings.TrimSpace(match[1]), "true") + ttl := "" + if len(match) > 2 { + ttl = strings.TrimSpace(match[2]) + } + result = append(result, &dqlshape.CacheDirective{Enabled: enabled, TTL: ttl}) + } + return result +} + +func parseMCPDirectives(input string) []*dqlshape.MCPDirective { + matches := mcpDirective.FindAllStringSubmatch(input, -1) + result := make([]*dqlshape.MCPDirective, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + name := strings.TrimSpace(match[1]) + if name == "" { + continue + } + description := "" + if len(match) > 2 { + description = strings.TrimSpace(match[2]) + } + descriptionPath := "" + if len(match) > 3 { + descriptionPath = strings.TrimSpace(match[3]) + } + result = append(result, &dqlshape.MCPDirective{ + Name: name, + Description: description, + DescriptionPath: descriptionPath, + }) + } + return result +} + +func parseRouteDirectives(input string) []*dqlshape.RouteDirective { + matches := routeDirective.FindAllStringSubmatch(input, -1) + result := make([]*dqlshape.RouteDirective, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + args := parseQuotedArgs(match[1]) + if len(args) == 0 { + continue + } + uri := strings.TrimSpace(args[0]) + if !strings.HasPrefix(uri, "/") { + continue + } + methods, ok := normalizeHTTPMethods(args[1:]) + if !ok { + continue + } + result = append(result, &dqlshape.RouteDirective{ + URI: uri, + Methods: methods, + }) + } + return result +} + +func parseQuotedArgs(input string) []string { + matches := quotedArgDirective.FindAllStringSubmatch(input, -1) + result := make([]string, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + result = append(result, strings.TrimSpace(match[1])) + } + return result +} + +func normalizeHTTPMethods(input []string) ([]string, bool) { + if len(input) == 0 { + return nil, true + } + valid := map[string]bool{ + http.MethodGet: true, + http.MethodPost: true, + http.MethodPut: true, + http.MethodPatch: true, + http.MethodDelete: true, + http.MethodHead: true, + http.MethodOptions: true, + http.MethodTrace: true, + http.MethodConnect: true, + } + seen := map[string]bool{} + result := make([]string, 0, len(input)) + for _, item := range input { + method := strings.ToUpper(strings.TrimSpace(item)) + if method == "" { + return nil, false + } + if !valid[method] { + return nil, false + } + if seen[method] { + continue + } + seen[method] = true + result = append(result, method) + } + return result, true +} + +func parseMarshalDirectives(input string) []string { + matches := marshalDirective.FindAllStringSubmatch(input, -1) + result := make([]string, 0, len(matches)) + for _, match := range matches { + if len(match) < 3 { + continue + } + mimeType := strings.ToLower(strings.TrimSpace(match[1])) + if mimeType != content.JSONContentType { + continue + } + if typeName := strings.TrimSpace(match[2]); typeName != "" { + result = append(result, typeName) + } + } + return result +} + +type unmarshalDirectiveValue struct { + JSONType string + XMLType string +} + +func parseUnmarshalDirectives(input string) []unmarshalDirectiveValue { + matches := unmarshalDirective.FindAllStringSubmatch(input, -1) + result := make([]unmarshalDirectiveValue, 0, len(matches)) + for _, match := range matches { + if len(match) < 3 { + continue + } + mimeType := strings.ToLower(strings.TrimSpace(match[1])) + typeName := strings.TrimSpace(match[2]) + if typeName == "" { + continue + } + value := unmarshalDirectiveValue{} + switch mimeType { + case content.JSONContentType: + value.JSONType = typeName + case content.XMLContentType: + value.XMLType = typeName + default: + continue + } + result = append(result, value) + } + return result +} + +func parseFormatDirectives(input string) []string { + matches := formatDirective.FindAllStringSubmatch(input, -1) + result := make([]string, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + raw := strings.ToLower(strings.TrimSpace(match[1])) + switch raw { + case "tabular_json": + result = append(result, content.JSONDataFormatTabular) + case content.JSONFormat, content.XMLFormat, content.CSVFormat, content.JSONDataFormatTabular: + result = append(result, raw) + } + } + return result +} + +func parseDateFormatDirectives(input string) []string { + matches := dateFormatDirective.FindAllStringSubmatch(input, -1) + result := make([]string, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + if value := strings.TrimSpace(match[1]); value != "" { + result = append(result, value) + } + } + return result +} + +func parseCaseFormatDirectives(input string) []string { + matches := caseFormatDirective.FindAllStringSubmatch(input, -1) + result := make([]string, 0, len(matches)) + for _, match := range matches { + if len(match) < 2 { + continue + } + value := strings.TrimSpace(match[1]) + if value == "" { + continue + } + if !text.NewCaseFormat(value).IsDefined() { + continue + } + result = append(result, value) + } + return result +} diff --git a/repository/shape/dql/preprocess/typectx_directives.go b/repository/shape/dql/preprocess/typectx_directives.go new file mode 100644 index 000000000..6dacc98b9 --- /dev/null +++ b/repository/shape/dql/preprocess/typectx_directives.go @@ -0,0 +1,84 @@ +package preprocess + +import ( + "regexp" + "strings" + + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/typectx" +) + +var ( + packageLinePattern = regexp.MustCompile(`(?i)^\s*#package\s*\(\s*['\"]([^'\"]+)['\"]\s*\)\s*$`) + importLinePattern = regexp.MustCompile(`(?i)^\s*#import\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)\s*$`) +) + +func parseTypeContextDirective(line, fullDQL string, offset int, ctx *typectx.Context) []*dqlshape.Diagnostic { + var diagnostics []*dqlshape.Diagnostic + if pkg, ok := parsePackageLineDirective(line); ok { + ctx.DefaultPackage = pkg + return nil + } + if alias, pkg, ok := parseImportLineDirective(line); ok { + ctx.Imports = append(ctx.Imports, typectx.Import{Alias: alias, Package: pkg}) + return nil + } + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "#package") { + diagnostics = append(diagnostics, directiveDiagnostic( + dqldiag.CodeDirPackage, + "invalid #package directive", + "expected: #package('module/path')", + fullDQL, + offset, + )) + return diagnostics + } + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "#import") { + diagnostics = append(diagnostics, directiveDiagnostic( + dqldiag.CodeDirImport, + "invalid #import directive", + "expected: #import('alias','github.com/acme/pkg')", + fullDQL, + offset, + )) + } + return diagnostics +} + +func parsePackageLineDirective(line string) (string, bool) { + matches := packageLinePattern.FindStringSubmatch(line) + if len(matches) != 2 { + return "", false + } + value := strings.TrimSpace(matches[1]) + if value == "" { + return "", false + } + return value, true +} + +func parseImportLineDirective(line string) (string, string, bool) { + matches := importLinePattern.FindStringSubmatch(line) + if len(matches) != 3 { + return "", "", false + } + alias := strings.TrimSpace(matches[1]) + pkg := strings.TrimSpace(matches[2]) + if alias == "" || pkg == "" { + return "", "", false + } + return alias, pkg, true +} + +func isTypeContextDirectiveLine(line string) bool { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "#package("), strings.HasPrefix(line, "#package ("): + return true + case strings.HasPrefix(line, "#import("), strings.HasPrefix(line, "#import ("): + return true + default: + return false + } +} diff --git a/repository/shape/dql/sanitize/policy.go b/repository/shape/dql/sanitize/policy.go new file mode 100644 index 000000000..4b6d6b5a4 --- /dev/null +++ b/repository/shape/dql/sanitize/policy.go @@ -0,0 +1,32 @@ +package sanitize + +import "strings" + +type rewritePolicy struct { + declared map[string]bool + consts map[string]bool +} + +func newRewritePolicy(declared, consts map[string]bool) *rewritePolicy { + return &rewritePolicy{ + declared: declared, + consts: consts, + } +} + +func (p *rewritePolicy) rewrite(raw string) string { + holder := holderName(raw) + if holder == "" { + return raw + } + if strings.HasPrefix(raw, "$Unsafe.") || strings.HasPrefix(raw, "${Unsafe.") || strings.HasPrefix(raw, "$Has.") || strings.HasPrefix(raw, "${Has.") { + return raw + } + if p.consts != nil && p.consts[holder] { + return addUnsafePrefix(raw) + } + if p.declared != nil && p.declared[holder] { + return asPlaceholder(raw) + } + return asPlaceholder(addUnsafePrefix(raw)) +} diff --git a/repository/shape/dql/sanitize/policy_test.go b/repository/shape/dql/sanitize/policy_test.go new file mode 100644 index 000000000..c3ceb5f23 --- /dev/null +++ b/repository/shape/dql/sanitize/policy_test.go @@ -0,0 +1,49 @@ +package sanitize + +import "testing" + +func TestRewritePolicy_Rewrite(t *testing.T) { + testCases := []struct { + name string + raw string + declared map[string]bool + consts map[string]bool + expect string + }{ + { + name: "plain selector becomes placeholder + unsafe", + raw: "$ID", + expect: "$criteria.AppendBinding($Unsafe.ID)", + }, + { + name: "unsafe selector preserved", + raw: "$Unsafe.ID", + expect: "$Unsafe.ID", + }, + { + name: "declared selector remains local placeholder", + raw: "$x", + declared: map[string]bool{"x": true}, + expect: "$criteria.AppendBinding($x)", + }, + { + name: "const selector keeps raw unsafe path", + raw: "$ConstID", + consts: map[string]bool{"ConstID": true}, + expect: "$Unsafe.ConstID", + }, + { + name: "function call is untouched", + raw: "$Foo.Bar()", + expect: "$Foo.Bar()", + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + policy := newRewritePolicy(testCase.declared, testCase.consts) + if actual := policy.rewrite(testCase.raw); actual != testCase.expect { + t.Fatalf("unexpected rewrite: %s", actual) + } + }) + } +} diff --git a/repository/shape/dql/sanitize/sanitizer.go b/repository/shape/dql/sanitize/sanitizer.go new file mode 100644 index 000000000..f60b0e58a --- /dev/null +++ b/repository/shape/dql/sanitize/sanitizer.go @@ -0,0 +1,205 @@ +package sanitize + +import ( + "fmt" + "regexp" + "strings" + + "github.com/viant/velty" + "github.com/viant/velty/ast" + aexpr "github.com/viant/velty/ast/expr" +) + +type Options struct { + Declared map[string]bool + Consts map[string]bool +} + +type RewriteResult struct { + SQL string + Patches []velty.Patch + TrimPrefix int +} + +var declarationHolderExpr = regexp.MustCompile(`(?i)#set\s*\(\s*\$_\s*=\s*\$([a-zA-Z_][a-zA-Z0-9_]*)`) + +func Declared(input string) map[string]bool { + ret := map[string]bool{} + listener := &declaredListener{declared: ret} + _, _, _ = velty.New(velty.Listener(listener)).Compile([]byte(input)) + for _, match := range declarationHolderExpr.FindAllStringSubmatch(input, -1) { + if len(match) < 2 { + continue + } + name := strings.TrimSpace(match[1]) + if name != "" { + ret[name] = true + } + } + return ret +} + +func SQL(input string, opts Options) string { + return Rewrite(input, opts).SQL +} + +func Rewrite(input string, opts Options) RewriteResult { + if strings.TrimSpace(input) == "" { + return RewriteResult{SQL: strings.TrimSpace(input)} + } + adjuster := &bindingAdjuster{ + source: []byte(input), + declared: opts.Declared, + consts: opts.Consts, + policy: newRewritePolicy(opts.Declared, opts.Consts), + } + out, err := velty.TransformTemplate([]byte(input), adjuster) + if err != nil { + return RewriteResult{SQL: strings.TrimSpace(input)} + } + trimPrefix := leadingTrimWidth(out) + return RewriteResult{ + SQL: strings.TrimSpace(string(out)), + Patches: append([]velty.Patch{}, adjuster.patches...), + TrimPrefix: trimPrefix, + } +} + +type bindingAdjuster struct { + source []byte + declared map[string]bool + consts map[string]bool + policy *rewritePolicy + patches []velty.Patch +} + +func (b *bindingAdjuster) Adjust(node ast.Node, ctx *velty.ParserContext) (velty.Action, error) { + sel, ok := node.(*aexpr.Select) + if !ok { + return velty.Keep(), nil + } + if ctx.CurrentExprContext().Kind == velty.CtxSetLHS { + return velty.Keep(), nil + } + span, ok := ctx.GetSpan(sel) + if !ok { + return velty.Keep(), nil + } + if b.inSetDirective(span.Start) { + return velty.Keep(), nil + } + raw := string(b.source[span.Start : span.End+1]) + replacement := b.rewrite(raw) + if replacement == raw { + return velty.Keep(), nil + } + b.patches = append(b.patches, velty.Patch{ + Span: span, + Replacement: []byte(replacement), + }) + return velty.PatchSpan(span, []byte(replacement)), nil +} + +func (b *bindingAdjuster) inSetDirective(pos int) bool { + if pos <= 0 || pos > len(b.source) { + return false + } + prefix := string(b.source[:pos]) + setPos := strings.LastIndex(prefix, "#set(") + if setPos == -1 { + return false + } + if nl := strings.LastIndex(prefix, "\n"); nl > setPos { + return false + } + segment := prefix[setPos:pos] + return strings.Count(segment, "(") > strings.Count(segment, ")") +} + +func (b *bindingAdjuster) rewrite(raw string) string { + if b.policy == nil { + b.policy = newRewritePolicy(b.declared, b.consts) + } + return b.policy.rewrite(raw) +} + +func holderName(raw string) string { + name := strings.TrimSpace(raw) + if name == "" { + return "" + } + if strings.HasPrefix(name, "${") && strings.HasSuffix(name, "}") { + name = "$" + name[2:len(name)-1] + } + if !strings.HasPrefix(name, "$") { + return "" + } + name = strings.TrimPrefix(name, "$") + if idx := strings.Index(name, "("); idx != -1 { + return "" + } + if idx := strings.Index(name, "."); idx != -1 { + head := name[:idx] + if head == "Unsafe" || head == "Has" { + name = name[idx+1:] + if j := strings.Index(name, "."); j != -1 { + return name[:j] + } + return name + } + return head + } + return name +} + +func addUnsafePrefix(raw string) string { + if strings.HasPrefix(raw, "${") { + return strings.Replace(raw, "${", "${Unsafe.", 1) + } + return strings.Replace(raw, "$", "$Unsafe.", 1) +} + +func asPlaceholder(raw string) string { + if strings.HasPrefix(raw, "${") && strings.HasSuffix(raw, "}") { + inner := "$" + raw[2:len(raw)-1] + return fmt.Sprintf("${criteria.AppendBinding(%s)}", inner) + } + return fmt.Sprintf("$criteria.AppendBinding(%s)", raw) +} + +func leadingTrimWidth(data []byte) int { + i := 0 + for i < len(data) { + switch data[i] { + case ' ', '\t', '\r', '\n': + i++ + default: + return i + } + } + return i +} + +type declaredListener struct { + declared map[string]bool +} + +func (d *declaredListener) OnEvent(e velty.Event) { + if e.Type != velty.EventEnterNode { + return + } + if e.ExprContext.Kind != velty.CtxSetLHS { + return + } + sel, ok := e.Node.(*aexpr.Select) + if !ok { + return + } + name := holderName(sel.FullName) + if name == "" { + name = holderName("$" + sel.ID) + } + if name != "" { + d.declared[name] = true + } +} diff --git a/repository/shape/dql/sanitize/sanitizer_test.go b/repository/shape/dql/sanitize/sanitizer_test.go new file mode 100644 index 000000000..326bd3b1d --- /dev/null +++ b/repository/shape/dql/sanitize/sanitizer_test.go @@ -0,0 +1,245 @@ +package sanitize + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/internal/inference" + legacy "github.com/viant/datly/internal/translator/parser" + vstate "github.com/viant/datly/view/state" + "github.com/viant/velty" + "github.com/viant/velty/ast/expr" +) + +func TestSQL_ParityWithLegacySanitizer(t *testing.T) { + testCases := []struct { + name string + sql string + state inference.State + }{ + { + name: "unsafe binding from plain selector", + sql: "SELECT * FROM t WHERE id = $Id", + }, + { + name: "bracket selector placeholder", + sql: "SELECT * FROM t WHERE id = ${Id}", + }, + { + name: "declared variable in append context", + sql: "#set($x = 1)\nSELECT * FROM t WHERE id = $x", + }, + { + name: "const selector keeps raw unsafe prefix", + sql: "SELECT * FROM t WHERE id = $ConstId", + state: inference.State{ + &inference.Parameter{Parameter: vstate.Parameter{Name: "ConstId", In: vstate.NewConstLocation("ConstId")}}, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + state := testCase.state + tpl, err := legacy.NewTemplate(testCase.sql, &state) + require.NoError(t, err) + expected := tpl.Sanitize() + + actual := SQL(testCase.sql, Options{ + Declared: tpl.Declared, + Consts: constNames(state), + }) + assert.Equal(t, expected, actual) + }) + } +} + +func TestSQL_ParityWithLegacySanitizer_RuntimeExpansion(t *testing.T) { + testCases := []struct { + name string + sql string + state inference.State + }{ + { + name: "plain selector binding", + sql: "SELECT * FROM t WHERE id = $Id", + }, + { + name: "bracket selector binding", + sql: "SELECT * FROM t WHERE id = ${Id}", + }, + { + name: "declared variable binding", + sql: "#set($x = 7)\nSELECT * FROM t WHERE id = $x", + }, + { + name: "const raw unsafe", + sql: "SELECT * FROM t WHERE id = $ConstId", + state: inference.State{ + &inference.Parameter{Parameter: vstate.Parameter{Name: "ConstId", In: vstate.NewConstLocation("ConstId")}}, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + state := testCase.state + tpl, err := legacy.NewTemplate(testCase.sql, &state) + require.NoError(t, err) + legacySQL := tpl.Sanitize() + + shapeSQL := SQL(testCase.sql, Options{ + Declared: tpl.Declared, + Consts: constNames(state), + }) + require.Equal(t, legacySQL, shapeSQL) + + assert.Equal(t, renderVeltySQL(t, legacySQL), renderVeltySQL(t, shapeSQL)) + }) + } +} + +func TestHolderName(t *testing.T) { + assert.Equal(t, "Foo", holderName("$Foo")) + assert.Equal(t, "Foo", holderName("${Foo}")) + assert.Equal(t, "Foo", holderName("$Foo.Bar")) + assert.Equal(t, "Foo", holderName("$Unsafe.Foo")) + assert.Equal(t, "Foo", holderName("$Has.Foo")) + assert.Equal(t, "Foo", holderName("$Unsafe.Foo.Bar")) + assert.Equal(t, "", holderName("$Foo.Bar()")) + assert.Equal(t, "", holderName("Foo")) + assert.Equal(t, "", holderName("")) +} + +func TestAddUnsafePrefixAndPlaceholder(t *testing.T) { + assert.Equal(t, "$Unsafe.Foo", addUnsafePrefix("$Foo")) + assert.Equal(t, "${Unsafe.Foo}", addUnsafePrefix("${Foo}")) + assert.Equal(t, "$criteria.AppendBinding($Foo)", asPlaceholder("$Foo")) + assert.Equal(t, "${criteria.AppendBinding($Foo)}", asPlaceholder("${Foo}")) +} + +func TestSQL_EdgeBranches(t *testing.T) { + assert.Equal(t, "", SQL(" ", Options{})) + assert.Equal(t, "#if(", SQL("#if(", Options{})) + assert.Equal(t, "#if(true)", SQL("#if(true)", Options{})) + assert.Equal(t, "SELECT $Unsafe.Id", SQL("SELECT $Unsafe.Id", Options{})) + assert.Equal(t, "SELECT $Has.Id", SQL("SELECT $Has.Id", Options{})) + assert.Equal(t, "SELECT $Foo.Bar()", SQL("SELECT $Foo.Bar()", Options{})) + assert.Equal(t, "#set($x = $y)\nSELECT $criteria.AppendBinding($Unsafe.y)", SQL("#set($x = $y)\nSELECT $y", Options{})) +} + +func TestSQL_RewritePreservesLineCount(t *testing.T) { + input := "#set($x = 1)\nSELECT *\nFROM t\nWHERE id = $Id\nAND name = ${Name}\n" + out := SQL(input, Options{}) + assert.Equal(t, strings.Count(strings.TrimSpace(input), "\n"), strings.Count(out, "\n")) +} + +func TestInSetDirective(t *testing.T) { + adj := &bindingAdjuster{source: []byte("#set($x = $y)\nSELECT $z")} + assert.True(t, adj.inSetDirective(7)) + assert.False(t, adj.inSetDirective(len(adj.source))) + assert.False(t, adj.inSetDirective(-1)) +} + +func TestDeclared(t *testing.T) { + declared := Declared("#set($x = 1)\n#set($y = $x)\nSELECT $x, $z") + assert.True(t, declared["x"]) + assert.True(t, declared["y"]) + assert.False(t, declared["z"]) +} + +func TestDeclared_ParameterDeclarationStyle(t *testing.T) { + declared := Declared("#set($_ = $Jwt(header/Authorization))\nSELECT $Jwt.UserID") + assert.True(t, declared["Jwt"]) +} + +func TestDeclaredListener_OnEventBranches(t *testing.T) { + declared := map[string]bool{} + l := &declaredListener{declared: declared} + + l.OnEvent(velty.Event{Type: velty.EventExitNode}) + l.OnEvent(velty.Event{Type: velty.EventEnterNode, ExprContext: velty.ExprContext{Kind: velty.CtxIfCond}}) + l.OnEvent(velty.Event{Type: velty.EventEnterNode, ExprContext: velty.ExprContext{Kind: velty.CtxSetLHS}, Node: &expr.Literal{Value: "x"}}) + l.OnEvent(velty.Event{Type: velty.EventEnterNode, ExprContext: velty.ExprContext{Kind: velty.CtxSetLHS}, Node: &expr.Select{ID: "x"}}) + + assert.True(t, declared["x"]) +} + +func TestAdjust_Branches(t *testing.T) { + adj := &bindingAdjuster{source: []byte("SELECT $Unsafe.Id")} + + // non selector node + action, err := adj.Adjust(&expr.Literal{Value: "x"}, &velty.ParserContext{}) + require.NoError(t, err) + assert.Equal(t, velty.ActionKeep, action.Kind) + + // selector without span + sel := &expr.Select{FullName: "$Unsafe.Id", ID: "Unsafe"} + action, err = adj.Adjust(sel, &velty.ParserContext{}) + require.NoError(t, err) + assert.Equal(t, velty.ActionKeep, action.Kind) + + // selector in set lhs context + ctx := &velty.ParserContext{} + ctx.InitSource("", adj.source) + ctx.SetSpan(sel, velty.Span{Start: 7, End: 16}) + ctx.PushExprContext(velty.ExprContext{Kind: velty.CtxSetLHS, ArgIdx: -1}) + action, err = adj.Adjust(sel, ctx) + require.NoError(t, err) + assert.Equal(t, velty.ActionKeep, action.Kind) + + // selector replacement equals raw + ctx.PopExprContext() + action, err = adj.Adjust(sel, ctx) + require.NoError(t, err) + assert.Equal(t, velty.ActionKeep, action.Kind) +} + +func constNames(state inference.State) map[string]bool { + ret := map[string]bool{} + for _, param := range state { + if param == nil || param.In == nil { + continue + } + if param.In.Kind == vstate.KindConst { + ret[param.Name] = true + } + } + return ret +} + +type criteriaMock struct{} + +func (c criteriaMock) AppendBinding(value interface{}) string { + return fmt.Sprintf("{%v}", value) +} + +type unsafeMock struct { + Id int + Name string + ConstId int +} + +func renderVeltySQL(t *testing.T, template string) string { + t.Helper() + planner := velty.New() + require.NoError(t, planner.DefineVariable("criteria", criteriaMock{})) + require.NoError(t, planner.DefineVariable("Unsafe", unsafeMock{})) + require.NoError(t, planner.DefineVariable("Id", 0)) + require.NoError(t, planner.DefineVariable("Name", "")) + require.NoError(t, planner.DefineVariable("ConstId", 0)) + + exec, newState, err := planner.Compile([]byte(template)) + require.NoError(t, err) + state := newState() + require.NoError(t, state.SetValue("criteria", criteriaMock{})) + require.NoError(t, state.SetValue("Unsafe", unsafeMock{Id: 10, Name: "ann", ConstId: 77})) + require.NoError(t, state.SetValue("Id", 10)) + require.NoError(t, state.SetValue("Name", "ann")) + require.NoError(t, state.SetValue("ConstId", 77)) + require.NoError(t, exec.Exec(state)) + return state.Buffer.String() +} diff --git a/repository/shape/dql/shape/model.go b/repository/shape/dql/shape/model.go new file mode 100644 index 000000000..52d1b5066 --- /dev/null +++ b/repository/shape/dql/shape/model.go @@ -0,0 +1,93 @@ +package shape + +import ( + "fmt" + + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/sqlparser/query" +) + +// Severity represents diagnostic severity level. +type Severity string + +const ( + SeverityError Severity = "error" + SeverityWarning Severity = "warning" + SeverityInfo Severity = "info" +) + +// Position identifies a byte offset and human-readable line/character location. +type Position struct { + Offset int + Line int + Char int +} + +// Span captures the location range for one diagnostic. +type Span struct { + Start Position + End Position +} + +// Diagnostic represents one compile/parse issue with precise location. +type Diagnostic struct { + Code string + Severity Severity + Message string + Hint string + Span Span +} + +// Directives captures special #set(...) directives parsed from DQL. +type Directives struct { + Meta string + DefaultConnector string + Cache *CacheDirective + MCP *MCPDirective + Route *RouteDirective + JSONMarshalType string + JSONUnmarshalType string + XMLUnmarshalType string + Format string + DateFormat string + CaseFormat string +} + +type CacheDirective struct { + Enabled bool + TTL string +} + +type MCPDirective struct { + Name string + Description string + DescriptionPath string +} + +type RouteDirective struct { + URI string + Methods []string +} + +// Document represents parsed DQL model used by shape compiler and xgen. +type Document struct { + Raw string + SQL string + Query *query.Select + TypeContext *typectx.Context + Directives *Directives + Root map[string]any + TypeResolutions []typectx.Resolution + Diagnostics []*Diagnostic +} + +// Error returns a compact human-readable diagnostic string. +func (d *Diagnostic) Error() string { + if d == nil { + return "" + } + if d.Code == "" { + return fmt.Sprintf("%s at line %d, char %d", d.Message, d.Span.Start.Line, d.Span.Start.Char) + } + return fmt.Sprintf("%s: %s at line %d, char %d", d.Code, d.Message, d.Span.Start.Line, d.Span.Start.Char) +} diff --git a/repository/shape/dql/statement/parity_test.go b/repository/shape/dql/statement/parity_test.go new file mode 100644 index 000000000..f8b734114 --- /dev/null +++ b/repository/shape/dql/statement/parity_test.go @@ -0,0 +1,33 @@ +package statement + +import ( + "testing" + + "github.com/stretchr/testify/assert" + legacy "github.com/viant/datly/internal/translator/parser" +) + +func TestStatements_ParityWithLegacyScanner(t *testing.T) { + testCases := []struct { + name string + sql string + }{ + {name: "read", sql: "SELECT id FROM orders"}, + {name: "exec update", sql: "UPDATE orders SET id = 1"}, + {name: "mixed", sql: "SELECT id FROM orders\nUPDATE orders SET id = 1"}, + {name: "service insert", sql: `$sql.Insert("orders", $rec)`}, + {name: "nop", sql: `$Nop($x)`}, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + current := New(testCase.sql) + old := legacy.NewStatements(testCase.sql) + assert.Equal(t, len(old), len(current)) + for i := 0; i < len(old) && i < len(current); i++ { + assert.Equal(t, old[i].IsExec, current[i].IsExec) + assert.Equal(t, old[i].Start, current[i].Start) + assert.Equal(t, old[i].End, current[i].End) + } + }) + } +} diff --git a/repository/shape/dql/statement/parser.go b/repository/shape/dql/statement/parser.go new file mode 100644 index 000000000..eaf7a9bdd --- /dev/null +++ b/repository/shape/dql/statement/parser.go @@ -0,0 +1,201 @@ +package statement + +import ( + "strings" + + "github.com/viant/datly/view/keywords" + "github.com/viant/parsly" + "github.com/viant/parsly/matcher" + aexpr "github.com/viant/velty/ast/expr" + veltyparser "github.com/viant/velty/parser" +) + +const ( + stmtWhitespaceToken = iota + stmtExprGroupToken + stmtExecToken + stmtReadToken + stmtExprToken + stmtExprEndToken + stmtAnyToken +) + +var ( + stmtWhitespaceMatcher = parsly.NewToken(stmtWhitespaceToken, "Whitespace", matcher.NewWhiteSpace()) + stmtExprGroupMatcher = parsly.NewToken(stmtExprGroupToken, "( ... )", matcher.NewBlock('(', ')', '\\')) + stmtExecMatcher = parsly.NewToken(stmtExecToken, "Exec", matcher.NewFragmentsFold([]byte("insert"), []byte("update"), []byte("delete"), []byte("call"), []byte("begin"))) + stmtReadMatcher = parsly.NewToken(stmtReadToken, "Read", matcher.NewFragmentsFold([]byte("select"))) + stmtExprMatcher = parsly.NewToken(stmtExprToken, "Expression", matcher.NewFragments([]byte("#set"), []byte("#foreach"), []byte("#if"))) + stmtExprEndMatcher = parsly.NewToken(stmtExprEndToken, "#end", matcher.NewFragmentsFold([]byte("#end"))) + stmtAnyMatcher = parsly.NewToken(stmtAnyToken, "Any", &anyMatcher{}) +) + +type anyMatcher struct{} + +func (a *anyMatcher) Match(cursor *parsly.Cursor) int { + if cursor.Pos < cursor.InputSize { + return 1 + } + return 0 +} + +func parseStatements(sqlText string) Statements { + cursor := parsly.NewCursor("", []byte(sqlText), 0) + var ( + result Statements + current *Statement + ) + for cursor.Pos < cursor.InputSize { + if consumeCommentOrQuoted(sqlText, cursor) { + continue + } + if cursor.Input[cursor.Pos] == '(' { + if block := cursor.MatchOne(stmtExprGroupMatcher); block.Code == stmtExprGroupToken { + continue + } + } + _ = cursor.MatchOne(stmtWhitespaceMatcher) + beforeMatch := cursor.Pos + matched := cursor.MatchAfterOptional(stmtWhitespaceMatcher, stmtExprMatcher, stmtExprEndMatcher, stmtExecMatcher, stmtReadMatcher, stmtAnyMatcher) + switch matched.Code { + case stmtExprToken: + _ = cursor.MatchAfterOptional(stmtWhitespaceMatcher, stmtExprGroupMatcher) + case stmtExecToken, stmtReadToken: + isExec := matched.Code == stmtExecToken + kind := KindRead + if isExec { + kind = KindExec + } + if nextWhitespace(cursor) { + if current != nil { + current.End = beforeMatch + } + current = &Statement{ + Start: beforeMatch, + End: -1, + Kind: kind, + IsExec: isExec, + } + result = append(result, current) + } + case stmtAnyToken: + kind, method, ok := getStmtSelector(matched, cursor) + if ok { + if current != nil { + current.End = beforeMatch + } + current = &Statement{ + Start: beforeMatch, + End: -1, + IsExec: true, + Kind: kind, + SelectorMethod: method, + } + result = append(result, current) + } + if !ok { + advanceToWhitespace(cursor) + } + _ = nextWhitespace(cursor) + } + } + if current != nil { + current.End = len(sqlText) + } + if len(result) == 0 { + kind, isExec, selector := inferDefaultKind(sqlText) + result = append(result, &Statement{ + Start: 0, + End: len(sqlText), + Kind: kind, + IsExec: isExec, + SelectorMethod: selector, + }) + } + return result +} + +func consumeCommentOrQuoted(sqlText string, cursor *parsly.Cursor) bool { + if cursor.Pos >= cursor.InputSize { + return false + } + if startsWithAt(sqlText, cursor.Pos, "--") { + cursor.Pos += 2 + for cursor.Pos < cursor.InputSize && sqlText[cursor.Pos] != '\n' { + cursor.Pos++ + } + return true + } + if startsWithAt(sqlText, cursor.Pos, "/*") { + cursor.Pos += 2 + for cursor.Pos+1 < cursor.InputSize { + if sqlText[cursor.Pos] == '*' && sqlText[cursor.Pos+1] == '/' { + cursor.Pos += 2 + return true + } + cursor.Pos++ + } + cursor.Pos = cursor.InputSize + return true + } + switch sqlText[cursor.Pos] { + case '\'', '"', '`': + quote := sqlText[cursor.Pos] + cursor.Pos++ + for cursor.Pos < cursor.InputSize { + ch := sqlText[cursor.Pos] + cursor.Pos++ + if ch == quote && (cursor.Pos < 2 || sqlText[cursor.Pos-2] != '\\') { + break + } + } + return true + } + return false +} + +func startsWithAt(text string, offset int, candidate string) bool { + if offset < 0 || offset+len(candidate) > len(text) { + return false + } + return text[offset:offset+len(candidate)] == candidate +} + +func getStmtSelector(matched *parsly.TokenMatch, cursor *parsly.Cursor) (string, string, bool) { + if matched.Text(cursor) != "$" { + return "", "", false + } + selector, err := veltyparser.MatchSelector(cursor) + if err != nil || selector == nil { + return "", "", false + } + if strings.EqualFold(selector.ID, "Nop") { + return KindExec, "Nop", true + } + if !strings.EqualFold(selector.ID, keywords.KeySQL) || selector.X == nil { + return "", "", false + } + aSelector, ok := selector.X.(*aexpr.Select) + if !ok { + return "", "", false + } + if aSelector.ID != "Insert" && aSelector.ID != "Update" { + return "", "", false + } + return KindService, aSelector.ID, true +} + +func nextWhitespace(cursor *parsly.Cursor) bool { + before := cursor.Pos + _ = cursor.MatchOne(stmtWhitespaceMatcher) + return before != cursor.Pos +} + +func advanceToWhitespace(cursor *parsly.Cursor) { + for cursor.Pos < cursor.InputSize { + if matcher.IsWhiteSpace(cursor.Input[cursor.Pos]) { + return + } + cursor.Pos++ + } +} diff --git a/repository/shape/dql/statement/statement.go b/repository/shape/dql/statement/statement.go new file mode 100644 index 000000000..58d8836b9 --- /dev/null +++ b/repository/shape/dql/statement/statement.go @@ -0,0 +1,137 @@ +package statement + +import ( + "strings" + + "github.com/viant/sqlparser" +) + +const ( + KindRead = "read" + KindExec = "exec" + KindService = "service" +) + +type Statement struct { + Start int + End int + Kind string + IsExec bool + SelectorMethod string + Table string +} + +type Statements []*Statement + +func (s Statements) IsExec() bool { + if len(s) == 0 { + return true + } + for _, item := range s { + if item != nil && item.IsExec { + return true + } + } + return false +} + +func (s Statements) DMLTables(rawSQL string) []string { + var ( + tables = map[string]bool{} + result []string + ) + for _, statement := range s { + if statement == nil || !statement.IsExec { + continue + } + sqlText := slice(rawSQL, statement.Start, statement.End) + if statement.Kind == KindService { + if table := firstQuotedArgument(sqlText); table != "" { + statement.Table = table + if !tables[table] { + result = append(result, table) + } + tables[table] = true + continue + } + } + lower := strings.ToLower(sqlText) + switch { + case strings.Contains(lower, "insert"): + if parsed, _ := sqlparser.ParseInsert(sqlText); parsed != nil && parsed.Target.X != nil { + statement.Table = strings.TrimSpace(sqlparser.Stringify(parsed.Target.X)) + } + case strings.Contains(lower, "update"): + if parsed, _ := sqlparser.ParseUpdate(sqlText); parsed != nil && parsed.Target.X != nil { + statement.Table = strings.TrimSpace(sqlparser.Stringify(parsed.Target.X)) + } + case strings.Contains(lower, "delete"): + if parsed, _ := sqlparser.ParseDelete(sqlText); parsed != nil && parsed.Target.X != nil { + statement.Table = strings.TrimSpace(sqlparser.Stringify(parsed.Target.X)) + } + } + if statement.Table == "" { + continue + } + if !tables[statement.Table] { + result = append(result, statement.Table) + } + tables[statement.Table] = true + } + return result +} + +func New(sqlText string) Statements { + if strings.TrimSpace(sqlText) == "" { + return Statements{&Statement{Start: 0, End: 0}} + } + return parseStatements(sqlText) +} + +func slice(input string, start, end int) string { + if start < 0 { + start = 0 + } + if end < start { + end = start + } + if end > len(input) { + end = len(input) + } + return input[start:end] +} + +func firstQuotedArgument(sqlText string) string { + index := strings.Index(sqlText, `"`) + if index == -1 { + return "" + } + tail := sqlText[index+1:] + end := strings.Index(tail, `"`) + if end == -1 { + return "" + } + return strings.TrimSpace(tail[:end]) +} + +func inferDefaultKind(sqlText string) (string, bool, string) { + trimmed := strings.TrimSpace(strings.ToLower(sqlText)) + switch { + case strings.HasPrefix(trimmed, "select"): + return KindRead, false, "" + case strings.HasPrefix(trimmed, "insert"), + strings.HasPrefix(trimmed, "update"), + strings.HasPrefix(trimmed, "delete"), + strings.HasPrefix(trimmed, "call"), + strings.HasPrefix(trimmed, "begin"): + return KindExec, true, "" + case strings.HasPrefix(trimmed, "$sql.insert"): + return KindService, true, "Insert" + case strings.HasPrefix(trimmed, "$sql.update"): + return KindService, true, "Update" + case strings.HasPrefix(trimmed, "$nop("): + return KindExec, true, "Nop" + default: + return "", false, "" + } +} diff --git a/repository/shape/dql/statement/statement_test.go b/repository/shape/dql/statement/statement_test.go new file mode 100644 index 000000000..45d7d14a3 --- /dev/null +++ b/repository/shape/dql/statement/statement_test.go @@ -0,0 +1,77 @@ +package statement + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNew_ReadStatement(t *testing.T) { + stmts := New("SELECT id FROM orders") + require.Len(t, stmts, 1) + assert.False(t, stmts[0].IsExec) + assert.Equal(t, KindRead, stmts[0].Kind) +} + +func TestNew_ExecStatements(t *testing.T) { + sqlText := "INSERT INTO orders(id) VALUES (1)\nUPDATE orders SET name = 'x' WHERE id = 1" + stmts := New(sqlText) + require.Len(t, stmts, 2) + assert.True(t, stmts[0].IsExec) + assert.True(t, stmts[1].IsExec) + assert.Equal(t, KindExec, stmts[0].Kind) +} + +func TestNew_ServiceExec(t *testing.T) { + stmts := New(`$sql.Insert("ORDERS", $rec)`) + require.Len(t, stmts, 1) + assert.True(t, stmts[0].IsExec) + assert.Equal(t, KindService, stmts[0].Kind) + assert.Equal(t, "Insert", stmts[0].SelectorMethod) +} + +func TestStatements_DMLTables(t *testing.T) { + stmts := New(`INSERT INTO orders(id) VALUES (1) +UPDATE orders SET id = 2 +DELETE FROM items WHERE id = 1 +$sql.Insert("ORDERS_AUDIT", $rec)`) + tables := stmts.DMLTables(`INSERT INTO orders(id) VALUES (1) +UPDATE orders SET id = 2 +DELETE FROM items WHERE id = 1 +$sql.Insert("ORDERS_AUDIT", $rec)`) + assert.Equal(t, []string{"orders", "items", "ORDERS_AUDIT"}, tables) +} + +func TestNew_IgnoreKeywordsInCommentsAndStrings(t *testing.T) { + sqlText := "-- insert into x\nSELECT 'update x' as txt FROM orders" + stmts := New(sqlText) + require.Len(t, stmts, 1) + assert.Equal(t, KindRead, stmts[0].Kind) + assert.False(t, stmts[0].IsExec) +} + +func TestNew_DefaultUnknownIsNotExec(t *testing.T) { + stmts := New("$foo.Bar($baz)") + require.Len(t, stmts, 1) + assert.Equal(t, "", stmts[0].Kind) + assert.False(t, stmts[0].IsExec) +} + +func TestNew_DefaultNopIsExec(t *testing.T) { + stmts := New("$Nop($Unsafe.Id)") + require.Len(t, stmts, 1) + assert.Equal(t, KindExec, stmts[0].Kind) + assert.True(t, stmts[0].IsExec) + assert.Equal(t, "Nop", stmts[0].SelectorMethod) +} + +func TestNew_NestedSubquerySelect_IsSingleReadStatement(t *testing.T) { + sqlText := `SELECT session.* +FROM (SELECT * FROM session WHERE user_id = $criteria.AppendBinding($Unsafe.Jwt.UserID)) session +JOIN (SELECT * FROM session/attributes) attribute ON attribute.user_id = session.user_id` + stmts := New(sqlText) + require.Len(t, stmts, 1) + assert.Equal(t, KindRead, stmts[0].Kind) + assert.False(t, stmts[0].IsExec) +} From 022ef9c23d6933037ae18f7a2cf45d75cc4c7c56 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 24 Feb 2026 11:49:12 -0800 Subject: [PATCH 133/279] updated dep --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 825a82406..be6c46a28 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( require ( github.com/viant/govalidator v0.3.1 - github.com/viant/sqlparser v0.11.0 + github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 ) require ( diff --git a/go.sum b/go.sum index 4b51b28a9..2f8cf29fd 100644 --- a/go.sum +++ b/go.sum @@ -1196,6 +1196,8 @@ github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.11.0 h1:RVmAsEieZlnRO33DWWvDXJOTY+sXJGTymPaC1iWnkOc= github.com/viant/sqlparser v0.11.0/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= +github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 h1:bnVgWzZzuz2pTa54e7YozHjYNFSapfU3MSklyMkO+Ag= +github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= From 6f552ce79c22f87a0ae3b919917a7ec208495925 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 24 Feb 2026 12:42:30 -0800 Subject: [PATCH 134/279] updated dep --- cmd/command/mcp.go | 5 +-- cmd/command/run.go | 5 +-- shared/args.go | 84 ++++++++++++++++++++++++++++++++++-- shared/args_test.go | 33 ++++++++++++++ view/column/discover.go | 4 +- view/column/discover_test.go | 19 ++++++++ 6 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 shared/args_test.go create mode 100644 view/column/discover_test.go diff --git a/cmd/command/mcp.go b/cmd/command/mcp.go index d1f0c4989..bbdb84897 100644 --- a/cmd/command/mcp.go +++ b/cmd/command/mcp.go @@ -24,10 +24,7 @@ func (s *Service) mcp(ctx context.Context, mcpOption *options.Mcp) error { setter.SetStringIfEmpty(&s.config.JobURL, mcpOption.JobURL) setter.SetStringIfEmpty(&s.config.FailedJobURL, mcpOption.FailedJobURL) setter.SetIntIfZero(&s.config.MaxJobs, mcpOption.MaxJobs) - if s.config.FailedJobURL == "" && s.config.JobURL != "" { - parent, _ := url.Split(s.config.JobURL, file.Scheme) - s.config.FailedJobURL = url.Join(parent, "failed", "jobs") - } + applyAsyncJobDefaults(s.config) if mcpOption.LoadPlugin && s.config.Config.PluginsURL != "" { parent, _ := url.Split(mcpOption.PluginInfo, file.Scheme) _ = s.fs.Copy(ctx, parent, s.config.Config.PluginsURL) diff --git a/cmd/command/run.go b/cmd/command/run.go index c55c3e256..a084275d2 100644 --- a/cmd/command/run.go +++ b/cmd/command/run.go @@ -36,10 +36,7 @@ func (s *Service) run(ctx context.Context, run *options.Run) (*standalone.Server setter.SetStringIfEmpty(&s.config.JobURL, run.JobURL) setter.SetStringIfEmpty(&s.config.FailedJobURL, run.FailedJobURL) setter.SetIntIfZero(&s.config.MaxJobs, run.MaxJobs) - if s.config.FailedJobURL == "" && s.config.JobURL != "" { - parent, _ := url.Split(s.config.JobURL, file.Scheme) - s.config.FailedJobURL = url.Join(parent, "failed", "jobs") - } + applyAsyncJobDefaults(s.config) if run.LoadPlugin && s.config.Config.PluginsURL != "" { parent, _ := url.Split(run.PluginInfo, file.Scheme) _ = s.fs.Copy(ctx, parent, s.config.Config.PluginsURL) diff --git a/shared/args.go b/shared/args.go index 0fcf59a08..198ddf3a6 100644 --- a/shared/args.go +++ b/shared/args.go @@ -1,10 +1,88 @@ package shared -import "strings" - func EnsureArgs(query string, args *[]interface{}) { - parameterCount := strings.Count(query, "?") + parameterCount := countPlaceholders(query) for i := len(*args); i < parameterCount; i++ { //ensure parameters *args = append(*args, "") } } + +func countPlaceholders(query string) int { + count := 0 + inSingle := false + inDouble := false + inBacktick := false + inLineComment := false + inBlockComment := false + + for i := 0; i < len(query); i++ { + ch := query[i] + + if inLineComment { + if ch == '\n' || ch == '\r' { + inLineComment = false + } + continue + } + if inBlockComment { + if ch == '*' && i+1 < len(query) && query[i+1] == '/' { + inBlockComment = false + i++ + } + continue + } + if inSingle { + if ch == '\\' { + if i+1 < len(query) { + i++ + } + continue + } + if ch == '\'' { + inSingle = false + } + continue + } + if inDouble { + if ch == '\\' { + if i+1 < len(query) { + i++ + } + continue + } + if ch == '"' { + inDouble = false + } + continue + } + if inBacktick { + if ch == '`' { + inBacktick = false + } + continue + } + + if ch == '-' && i+1 < len(query) && query[i+1] == '-' { + inLineComment = true + i++ + continue + } + if ch == '/' && i+1 < len(query) && query[i+1] == '*' { + inBlockComment = true + i++ + continue + } + + switch ch { + case '\'': + inSingle = true + case '"': + inDouble = true + case '`': + inBacktick = true + case '?': + count++ + } + } + return count +} diff --git a/shared/args_test.go b/shared/args_test.go new file mode 100644 index 000000000..8b0f4dd64 --- /dev/null +++ b/shared/args_test.go @@ -0,0 +1,33 @@ +package shared + +import "testing" + +func TestCountPlaceholders(t *testing.T) { + testCases := []struct { + name string + query string + expect int + }{ + { + name: "simple placeholders", + query: "SELECT * FROM t WHERE a = ? AND b = ?", + expect: 2, + }, + { + name: "ignore single quoted regex", + query: "SELECT REGEXP_REPLACE(col, r'^(?:https?://)?(?:www\\.)?', '') FROM t WHERE a = ?", + expect: 1, + }, + { + name: "ignore comments and quoted text", + query: "SELECT '?' -- ?\nFROM t /* ? */ WHERE x = ?", + expect: 1, + }, + } + + for _, testCase := range testCases { + if actual := countPlaceholders(testCase.query); actual != testCase.expect { + t.Fatalf("%s: expected %d placeholders, got %d", testCase.name, testCase.expect, actual) + } + } +} diff --git a/view/column/discover.go b/view/column/discover.go index f78604699..48c7b221a 100644 --- a/view/column/discover.go +++ b/view/column/discover.go @@ -248,7 +248,9 @@ func parseQuery(SQL string) (string, string, sqlparser.Columns) { if sqlQuery.From.X != nil { table = sqlparser.Stringify(sqlQuery.From.X) } - if sqlQuery.List.IsStarExpr() && !strings.Contains(table, "SELECT") { + // For CTE-backed queries (WITH ...), SELECT * FROM cte_alias must still be + // resolved via SQL execution; the alias is not a physical table. + if sqlQuery.List.IsStarExpr() && !strings.Contains(table, "SELECT") && len(sqlQuery.WithSelects) == 0 { return table, "", nil //use table metadata } sqlQuery.Limit = nil diff --git a/view/column/discover_test.go b/view/column/discover_test.go new file mode 100644 index 000000000..ef2cf966e --- /dev/null +++ b/view/column/discover_test.go @@ -0,0 +1,19 @@ +package column + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseQuery_WithCTEStar_DoesNotShortCircuitToTableMetadata(t *testing.T) { + sql := `WITH cte AS (SELECT 1 AS a) SELECT v.* FROM cte v` + + table, discoveredSQL, cols := parseQuery(sql) + require.Equal(t, "cte", strings.TrimSpace(table)) + require.NotEmpty(t, cols) + require.NotEmpty(t, strings.TrimSpace(discoveredSQL), "CTE star query must keep SQL for runtime column inference") + require.Contains(t, strings.ToUpper(discoveredSQL), "WITH CTE AS") + require.Contains(t, discoveredSQL, "LIMIT 1") +} From 6fc7357fce2fe5cb54e7a5a6175ed95454439428 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 24 Feb 2026 12:46:16 -0800 Subject: [PATCH 135/279] updated dep --- cmd/command/async_defaults.go | 33 +++++++++++++++++++ cmd/command/async_defaults_test.go | 51 ++++++++++++++++++++++++++++++ go.mod | 2 +- 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 cmd/command/async_defaults.go create mode 100644 cmd/command/async_defaults_test.go diff --git a/cmd/command/async_defaults.go b/cmd/command/async_defaults.go new file mode 100644 index 000000000..7ffbd4c43 --- /dev/null +++ b/cmd/command/async_defaults.go @@ -0,0 +1,33 @@ +package command + +import ( + "github.com/viant/afs/file" + "github.com/viant/afs/url" + "github.com/viant/datly/gateway/runtime/standalone" +) + +const ( + defaultJobURL = "/tmp/datly/jobs" + defaultFailedJobURL = "/tmp/datly/failed" +) + +func applyAsyncJobDefaults(config *standalone.Config) { + if config == nil { + return + } + + if config.JobURL == "" && config.FailedJobURL == "" { + config.JobURL = defaultJobURL + config.FailedJobURL = defaultFailedJobURL + return + } + + if config.JobURL == "" { + config.JobURL = defaultJobURL + } + + if config.FailedJobURL == "" { + parent, _ := url.Split(config.JobURL, file.Scheme) + config.FailedJobURL = url.Join(parent, "failed", "jobs") + } +} diff --git a/cmd/command/async_defaults_test.go b/cmd/command/async_defaults_test.go new file mode 100644 index 000000000..736e45492 --- /dev/null +++ b/cmd/command/async_defaults_test.go @@ -0,0 +1,51 @@ +package command + +import ( + "testing" + + "github.com/viant/datly/gateway" + "github.com/viant/datly/gateway/runtime/standalone" +) + +func TestApplyAsyncJobDefaults(t *testing.T) { + testCases := []struct { + name string + jobURL string + failedJobURL string + expectJob string + expectFailed string + }{ + { + name: "both empty use tmp defaults", + expectJob: "/tmp/datly/jobs", + expectFailed: "/tmp/datly/failed", + }, + { + name: "custom job only derives failed path", + jobURL: "/custom/jobs", + expectJob: "/custom/jobs", + expectFailed: "file://localhost/custom/failed/jobs", + }, + { + name: "failed only keeps failed and defaults job", + failedJobURL: "/custom/failed", + expectJob: "/tmp/datly/jobs", + expectFailed: "/custom/failed", + }, + } + + for _, testCase := range testCases { + cfg := &standalone.Config{Config: &gateway.Config{}} + cfg.JobURL = testCase.jobURL + cfg.FailedJobURL = testCase.failedJobURL + + applyAsyncJobDefaults(cfg) + + if cfg.JobURL != testCase.expectJob { + t.Fatalf("%s: expected JobURL=%s, got %s", testCase.name, testCase.expectJob, cfg.JobURL) + } + if cfg.FailedJobURL != testCase.expectFailed { + t.Fatalf("%s: expected FailedJobURL=%s, got %s", testCase.name, testCase.expectFailed, cfg.FailedJobURL) + } + } +} diff --git a/go.mod b/go.mod index be6c46a28..d116be7f1 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/viant/datly go 1.25.0 -require ( + require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible github.com/aws/aws-lambda-go v1.31.0 github.com/francoispqt/gojay v1.2.13 From 7e8970bc3a7a5cd9ae915e0da671a883ca93ab54 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 24 Feb 2026 13:52:15 -0800 Subject: [PATCH 136/279] updated dep --- go.mod | 2 +- go.sum | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d116be7f1..be6c46a28 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/viant/datly go 1.25.0 - require ( +require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible github.com/aws/aws-lambda-go v1.31.0 github.com/francoispqt/gojay v1.2.13 diff --git a/go.sum b/go.sum index 2f8cf29fd..04ca6ae0b 100644 --- a/go.sum +++ b/go.sum @@ -1194,8 +1194,6 @@ github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= github.com/viant/pgo v0.11.0/go.mod h1:MFzHmkRFZlciugEgUvpl/3grK789PBSH4dUVSLOSo+Q= github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= -github.com/viant/sqlparser v0.11.0 h1:RVmAsEieZlnRO33DWWvDXJOTY+sXJGTymPaC1iWnkOc= -github.com/viant/sqlparser v0.11.0/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 h1:bnVgWzZzuz2pTa54e7YozHjYNFSapfU3MSklyMkO+Ag= github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= From 6ca1923c0cbd3644aee50e09d3acd644beff6359 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Feb 2026 04:26:11 -0800 Subject: [PATCH 137/279] updated dep --- repository/shape/compile/component_types.go | 9 - repository/shape/compile/enrich.go | 260 +++++++++++--- repository/shape/compile/hints.go | 227 ++++++++++-- repository/shape/compile/hints_test.go | 11 + repository/shape/compile/legacy_adapter.go | 336 ------------------ repository/shape/compile/pipeline/infer.go | 30 +- repository/shape/compile/pipeline/read.go | 234 ++++++++++-- repository/shape/compile/pipeline/relation.go | 79 +++- repository/shape/compile/statedecl.go | 70 ++++ repository/shape/compile/statedecl_test.go | 8 + repository/shape/compile/strings_util.go | 13 - repository/shape/compile/viewdecl_append.go | 51 ++- .../shape/dql/preprocess/directive_parser.go | 186 ++++++++++ .../shape/dql/preprocess/legacy_import.go | 203 ++++++++--- repository/shape/dql/preprocess/preprocess.go | 50 ++- .../shape/dql/preprocess/preprocess_test.go | 37 ++ .../dql/preprocess/settings_directives.go | 229 +++++++----- .../dql/preprocess/typectx_directives.go | 65 +++- repository/shape/dql/sanitize/sanitizer.go | 106 +++++- repository/shape/plan/model.go | 37 +- repository/shape/xgen/io.go | 14 - 21 files changed, 1558 insertions(+), 697 deletions(-) delete mode 100644 repository/shape/compile/strings_util.go create mode 100644 repository/shape/dql/preprocess/directive_parser.go diff --git a/repository/shape/compile/component_types.go b/repository/shape/compile/component_types.go index 2cd3d705e..6e84c960e 100644 --- a/repository/shape/compile/component_types.go +++ b/repository/shape/compile/component_types.go @@ -204,10 +204,6 @@ func (c *componentCollector) collect(namespace string, span dqlshape.Span, requi return outputType, true } -func sourceRoots(sourcePath string) (platformRoot, routesRoot, dqlRoot string, ok bool) { - return sourceRootsWithLayout(sourcePath, defaultCompilePathLayout()) -} - func sourceRootsWithLayout(sourcePath string, layout compilePathLayout) (platformRoot, routesRoot, dqlRoot string, ok bool) { path := filepath.Clean(strings.TrimSpace(sourcePath)) if path == "" { @@ -359,11 +355,6 @@ type routePayload struct { } `yaml:"Routes"` } -func loadRoutePayload(routesRoot, namespace string) (*routePayload, bool) { - lookup := readRoutePayload(routesRoot, namespace) - return lookup.payload, lookup.found -} - func readRoutePayload(routesRoot, namespace string) routePayloadLookup { candidates := routeYAMLCandidates(routesRoot, namespace) lookup := routePayloadLookup{} diff --git a/repository/shape/compile/enrich.go b/repository/shape/compile/enrich.go index dad177a94..56086563f 100644 --- a/repository/shape/compile/enrich.go +++ b/repository/shape/compile/enrich.go @@ -4,7 +4,6 @@ import ( "encoding/json" "os" "path/filepath" - "regexp" "strings" "github.com/viant/datly/repository/shape" @@ -13,15 +12,6 @@ import ( "github.com/viant/datly/repository/shape/plan" ) -var ( - ruleHeaderExpr = regexp.MustCompile(`(?s)^\s*/\*\s*(\{.*?\})\s*\*/`) - embedExpr = regexp.MustCompile(`(?is)\$\{\s*embed:\s*([^}]+)\}`) - fromTableExpr = regexp.MustCompile(`(?is)\bfrom\s+([a-zA-Z_$][a-zA-Z0-9_$.{}/]*)`) - summaryJoinExpr = regexp.MustCompile(`(?is)\bjoin\s*\((.*?)\)\s*summary\s+on\s+1\s*=\s*1`) - joinEmbedExpr = regexp.MustCompile(`(?is)\bjoin\s*\(\s*\$\{\s*embed:\s*([^}]+)\}\s*\)\s*(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*)`) - joinBodyExpr = regexp.MustCompile(`(?is)\bjoin\s*\((.*?)\)\s*(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*)\s+on\b`) -) - type ruleSettings struct { Connector string `json:"Connector"` Name string `json:"Name"` @@ -163,11 +153,11 @@ func extractSummarySQL(sqlText string) string { if sqlText == "" || !strings.Contains(sqlText, "$View.") { return "" } - matches := summaryJoinExpr.FindStringSubmatch(sqlText) - if len(matches) < 2 { + body, ok := findSummaryJoinBody(sqlText) + if !ok { return "" } - return strings.TrimSpace(matches[1]) + return strings.TrimSpace(body) } func extractRuleSettings(source *shape.Source, directives *dqlshape.Directives) *ruleSettings { @@ -175,9 +165,7 @@ func extractRuleSettings(source *shape.Source, directives *dqlshape.Directives) return &ruleSettings{} } ret := &ruleSettings{} - matches := ruleHeaderExpr.FindStringSubmatch(source.DQL) - if len(matches) >= 2 { - rawJSON := strings.TrimSpace(matches[1]) + if rawJSON, ok := extractLeadingRuleHeaderJSON(source.DQL); ok { _ = json.Unmarshal([]byte(rawJSON), ret) } if directives != nil && directives.Route != nil { @@ -210,10 +198,6 @@ func sourceSQLBaseDir(source *shape.Source) string { return stem } -func sourceModule(source *shape.Source) string { - return sourceModuleWithLayout(source, defaultCompilePathLayout()) -} - func sourceModuleWithLayout(source *shape.Source, layout compilePathLayout) string { if source == nil || strings.TrimSpace(source.Path) == "" { return "" @@ -349,11 +333,6 @@ func inferTableFromSQL(sqlText string, source *shape.Source) string { return table } } - cleaned := embedExpr.ReplaceAllString(sqlText, " ") - match := fromTableExpr.FindStringSubmatch(cleaned) - if len(match) >= 2 { - return strings.Trim(match[1], "`\"") - } if table := inferFromEmbeddedSQL(sqlText, source); table != "" { return table } @@ -361,11 +340,10 @@ func inferTableFromSQL(sqlText string, source *shape.Source) string { } func inferFromEmbeddedSQL(sqlText string, source *shape.Source) string { - matches := embedExpr.FindStringSubmatch(sqlText) - if len(matches) < 2 { + ref, ok := findFirstEmbedRef(sqlText) + if !ok { return "" } - ref := strings.TrimSpace(matches[1]) ref = strings.Trim(ref, `"'`) if ref == "" { return "" @@ -380,11 +358,10 @@ func inferFromEmbeddedSQL(sqlText string, source *shape.Source) string { } queryNode, _, err := pipeline.ParseSelectWithDiagnostic(string(embedded)) if err != nil || queryNode == nil { - fallback := fromTableExpr.FindStringSubmatch(string(embedded)) - if len(fallback) < 2 { - return "" + if table := pipeline.InferTableFromSQL(string(embedded)); table != "" && !strings.EqualFold(table, "DQLView") { + return strings.Trim(table, "`\"") } - return strings.Trim(fallback[1], "`\"") + return "" } _, table, err := pipeline.InferRoot(queryNode, "") if err != nil || strings.TrimSpace(table) == "" { @@ -619,16 +596,12 @@ func extractJoinEmbedRefs(sqlText string) map[string]string { if strings.TrimSpace(sqlText) == "" { return result } - for _, m := range joinEmbedExpr.FindAllStringSubmatch(sqlText, -1) { - if len(m) < 3 { - continue - } - ref := strings.TrimSpace(m[1]) - alias := strings.TrimSpace(m[2]) - if ref == "" || alias == "" { + for _, item := range scanJoinSubqueries(sqlText) { + ref, ok := parseJoinEmbedRef(item.body) + if !ok || ref == "" || item.alias == "" { continue } - result[alias] = ref + result[item.alias] = ref } return result } @@ -638,16 +611,211 @@ func extractJoinSubqueryBodies(sqlText string) map[string]string { if strings.TrimSpace(sqlText) == "" { return result } - for _, m := range joinBodyExpr.FindAllStringSubmatch(sqlText, -1) { - if len(m) < 3 { + for _, item := range scanJoinSubqueries(sqlText) { + body := strings.TrimSpace(item.body) + if body == "" || item.alias == "" { + continue + } + result[item.alias] = body + } + return result +} + +func findSummaryJoinBody(input string) (string, bool) { + lower := strings.ToLower(input) + for i := 0; i < len(input); i++ { + if !hasCompileWordAt(lower, i, "join") { continue } - body := strings.TrimSpace(m[1]) - alias := strings.TrimSpace(m[2]) - if body == "" || alias == "" { + pos := skipCompileSpaces(input, i+len("join")) + if pos >= len(input) || input[pos] != '(' { continue } - result[alias] = body + body, end, ok := readCompileParenBody(input, pos) + if !ok { + continue + } + rest := strings.ToLower(input[end+1:]) + rest = strings.Join(strings.Fields(rest), " ") + if strings.HasPrefix(rest, "summary on 1=1") || strings.HasPrefix(rest, "summary on 1 = 1") { + return body, true + } + } + return "", false +} + +func extractLeadingRuleHeaderJSON(input string) (string, bool) { + index := skipCompileSpaces(input, 0) + if index+2 > len(input) || input[index:index+2] != "/*" { + return "", false + } + end := strings.Index(input[index+2:], "*/") + if end < 0 { + return "", false + } + body := strings.TrimSpace(input[index+2 : index+2+end]) + if body == "" || body[0] != '{' || body[len(body)-1] != '}' { + return "", false + } + return body, true +} + +func findFirstEmbedRef(input string) (string, bool) { + for i := 0; i < len(input); i++ { + if input[i] != '$' || i+1 >= len(input) || input[i+1] != '{' { + continue + } + body, end, ok := readCompileTemplateExpr(input, i+1) + if !ok { + continue + } + _ = end + trimmed := strings.TrimSpace(body) + if len(trimmed) < len("embed:") || !strings.HasPrefix(strings.ToLower(trimmed), "embed:") { + continue + } + ref := strings.TrimSpace(trimmed[len("embed:"):]) + if ref == "" { + continue + } + return ref, true + } + return "", false +} + +type joinSubquery struct { + body string + alias string +} + +func scanJoinSubqueries(input string) []joinSubquery { + result := make([]joinSubquery, 0) + lower := strings.ToLower(input) + for i := 0; i < len(input); i++ { + if !hasCompileWordAt(lower, i, "join") { + continue + } + pos := skipCompileSpaces(input, i+len("join")) + if pos >= len(input) || input[pos] != '(' { + continue + } + body, end, ok := readCompileParenBody(input, pos) + if !ok { + continue + } + pos = skipCompileSpaces(input, end+1) + if hasCompileWordAt(lower, pos, "as") { + pos = skipCompileSpaces(input, pos+len("as")) + } + aliasStart := pos + if aliasStart >= len(input) || !isCompileWordStart(input[aliasStart]) { + i = end + continue + } + pos++ + for pos < len(input) && isCompileWordPart(input[pos]) { + pos++ + } + alias := strings.TrimSpace(input[aliasStart:pos]) + if alias != "" { + result = append(result, joinSubquery{body: body, alias: alias}) + } + i = end } return result } + +func parseJoinEmbedRef(body string) (string, bool) { + trimmed := strings.TrimSpace(body) + if !strings.HasPrefix(trimmed, "${") || !strings.HasSuffix(trimmed, "}") { + return "", false + } + inner := strings.TrimSpace(trimmed[2 : len(trimmed)-1]) + if len(inner) < len("embed:") || !strings.HasPrefix(strings.ToLower(inner), "embed:") { + return "", false + } + ref := strings.TrimSpace(inner[len("embed:"):]) + return ref, ref != "" +} + +func readCompileTemplateExpr(input string, openBrace int) (string, int, bool) { + if openBrace <= 0 || openBrace >= len(input) || input[openBrace] != '{' || input[openBrace-1] != '$' { + return "", -1, false + } + for i := openBrace + 1; i < len(input); i++ { + if input[i] == '}' { + return input[openBrace+1 : i], i, true + } + } + return "", -1, false +} + +func readCompileParenBody(input string, openParen int) (string, int, bool) { + depth := 0 + quote := byte(0) + for i := openParen; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + if ch == '(' { + depth++ + continue + } + if ch == ')' { + depth-- + if depth == 0 { + return input[openParen+1 : i], i, true + } + } + } + return "", -1, false +} + +func hasCompileWordAt(lower string, pos int, word string) bool { + if pos < 0 || pos+len(word) > len(lower) { + return false + } + if lower[pos:pos+len(word)] != word { + return false + } + if pos > 0 && isCompileWordPart(lower[pos-1]) { + return false + } + next := pos + len(word) + if next < len(lower) && isCompileWordPart(lower[next]) { + return false + } + return true +} + +func skipCompileSpaces(input string, index int) int { + for index < len(input) { + switch input[index] { + case ' ', '\t', '\n', '\r': + index++ + default: + return index + } + } + return index +} + +func isCompileWordStart(ch byte) bool { + return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') +} + +func isCompileWordPart(ch byte) bool { + return isCompileWordStart(ch) || (ch >= '0' && ch <= '9') +} diff --git a/repository/shape/compile/hints.go b/repository/shape/compile/hints.go index 93a6bfb3d..a16b855d2 100644 --- a/repository/shape/compile/hints.go +++ b/repository/shape/compile/hints.go @@ -2,19 +2,12 @@ package compile import ( "reflect" - "regexp" "strconv" "strings" "github.com/viant/datly/repository/shape/plan" ) -var ( - useConnectorExpr = regexp.MustCompile(`(?i)use_connector\s*\(\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*,\s*(?:'([a-zA-Z_][a-zA-Z0-9_]*)'|"([a-zA-Z_][a-zA-Z0-9_]*)"|([a-zA-Z_][a-zA-Z0-9_]*))\s*\)`) - allowNullsExpr = regexp.MustCompile(`(?i)allow_nulls\s*\(\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\)`) - setLimitExpr = regexp.MustCompile(`(?i)set_limit\s*\(\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*,\s*(-?[0-9]+)\s*\)`) -) - type viewHint struct { Connector string AllowNulls *bool @@ -23,57 +16,213 @@ type viewHint struct { func extractViewHints(dql string) map[string]viewHint { result := map[string]viewHint{} - for _, match := range useConnectorExpr.FindAllStringSubmatch(dql, -1) { - if len(match) < 5 { + for _, call := range scanHintCalls(dql) { + switch call.name { + case "use_connector": + if len(call.args) != 2 { + continue + } + alias := strings.TrimSpace(call.args[0]) + connector := unquote(strings.TrimSpace(call.args[1])) + if !isIdentifier(alias) || !isIdentifier(connector) { + continue + } + hint := result[alias] + hint.Connector = connector + result[alias] = hint + case "allow_nulls": + if len(call.args) != 1 { + continue + } + alias := strings.TrimSpace(call.args[0]) + if !isIdentifier(alias) { + continue + } + hint := result[alias] + value := true + hint.AllowNulls = &value + result[alias] = hint + case "set_limit": + if len(call.args) != 2 { + continue + } + alias := strings.TrimSpace(call.args[0]) + limitRaw := strings.TrimSpace(call.args[1]) + if !isIdentifier(alias) || limitRaw == "" { + continue + } + limit, err := strconv.Atoi(limitRaw) + if err != nil { + continue + } + hint := result[alias] + noLimit := limit == 0 + hint.NoLimit = &noLimit + result[alias] = hint + } + } + return result +} + +type hintCall struct { + name string + args []string +} + +func scanHintCalls(input string) []hintCall { + result := make([]hintCall, 0) + for i := 0; i < len(input); { + if !isIdentifierStart(input[i]) { + i++ continue } - alias := strings.TrimSpace(match[1]) - connector := strings.TrimSpace(match[2]) - if connector == "" { - connector = strings.TrimSpace(match[3]) + start := i + i++ + for i < len(input) && isIdentifierPart(input[i]) { + i++ + } + name := strings.ToLower(input[start:i]) + if name != "use_connector" && name != "allow_nulls" && name != "set_limit" { + continue } - if connector == "" { - connector = strings.TrimSpace(match[4]) + j := skipSpaces(input, i) + if j >= len(input) || input[j] != '(' { + continue } - if alias == "" || connector == "" { + body, end, ok := readCallBody(input, j) + if !ok { continue } - hint := result[alias] - hint.Connector = connector - result[alias] = hint + result = append(result, hintCall{name: name, args: splitCallArgs(body)}) + i = end + 1 } - for _, match := range allowNullsExpr.FindAllStringSubmatch(dql, -1) { - if len(match) < 2 { + return result +} + +func readCallBody(input string, openParen int) (string, int, bool) { + depth := 0 + quote := byte(0) + for i := openParen; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } continue } - alias := strings.TrimSpace(match[1]) - if alias == "" { + if ch == '\'' || ch == '"' { + quote = ch continue } - hint := result[alias] - value := true - hint.AllowNulls = &value - result[alias] = hint + if ch == '(' { + depth++ + continue + } + if ch == ')' { + depth-- + if depth == 0 { + return input[openParen+1 : i], i, true + } + } } - for _, match := range setLimitExpr.FindAllStringSubmatch(dql, -1) { - if len(match) < 3 { + return "", -1, false +} + +func splitCallArgs(input string) []string { + args := make([]string, 0) + current := strings.Builder{} + depth := 0 + quote := byte(0) + for i := 0; i < len(input); i++ { + ch := input[i] + if quote != 0 { + current.WriteByte(ch) + if ch == '\\' && i+1 < len(input) { + i++ + current.WriteByte(input[i]) + continue + } + if ch == quote { + quote = 0 + } continue } - alias := strings.TrimSpace(match[1]) - limitRaw := strings.TrimSpace(match[2]) - if alias == "" || limitRaw == "" { + if ch == '\'' || ch == '"' { + quote = ch + current.WriteByte(ch) continue } - limit, err := strconv.Atoi(limitRaw) - if err != nil { + if ch == '(' { + depth++ + current.WriteByte(ch) continue } - hint := result[alias] - noLimit := limit == 0 - hint.NoLimit = &noLimit - result[alias] = hint + if ch == ')' { + if depth > 0 { + depth-- + } + current.WriteByte(ch) + continue + } + if ch == ',' && depth == 0 { + args = append(args, strings.TrimSpace(current.String())) + current.Reset() + continue + } + current.WriteByte(ch) } - return result + if value := strings.TrimSpace(current.String()); value != "" { + args = append(args, value) + } + return args +} + +func isIdentifierStart(ch byte) bool { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' +} + +func isIdentifierPart(ch byte) bool { + return isIdentifierStart(ch) || (ch >= '0' && ch <= '9') +} + +func isIdentifier(value string) bool { + value = strings.TrimSpace(value) + if value == "" || !isIdentifierStart(value[0]) { + return false + } + for i := 1; i < len(value); i++ { + if !isIdentifierPart(value[i]) { + return false + } + } + return true +} + +func unquote(value string) string { + if len(value) >= 2 { + first := value[0] + last := value[len(value)-1] + if (first == '\'' && last == '\'') || (first == '"' && last == '"') { + return value[1 : len(value)-1] + } + } + return value +} + +func skipSpaces(input string, index int) int { + for index < len(input) { + switch input[index] { + case ' ', '\t', '\n', '\r': + index++ + default: + return index + } + } + return index } func appendRelationViews(result *plan.Result, root *plan.View, hints map[string]viewHint) { diff --git a/repository/shape/compile/hints_test.go b/repository/shape/compile/hints_test.go index 768f82863..c40470e0f 100644 --- a/repository/shape/compile/hints_test.go +++ b/repository/shape/compile/hints_test.go @@ -20,6 +20,17 @@ func TestExtractViewHints_WithQuotedConnector(t *testing.T) { assert.True(t, *hints["match"].NoLimit) } +func TestExtractViewHints_MixedCaseAndUnquotedConnector(t *testing.T) { + dql := "SELECT USE_CONNECTOR(match, ci_ads), Allow_Nulls(match), set_limit(match, -1)" + hints := extractViewHints(dql) + require.Contains(t, hints, "match") + assert.Equal(t, "ci_ads", hints["match"].Connector) + require.NotNil(t, hints["match"].AllowNulls) + assert.True(t, *hints["match"].AllowNulls) + require.NotNil(t, hints["match"].NoLimit) + assert.False(t, *hints["match"].NoLimit) +} + func TestApplyViewHints_Metadata(t *testing.T) { trueValue := true result := &plan.Result{ diff --git a/repository/shape/compile/legacy_adapter.go b/repository/shape/compile/legacy_adapter.go index 927fe5f95..c91409b01 100644 --- a/repository/shape/compile/legacy_adapter.go +++ b/repository/shape/compile/legacy_adapter.go @@ -252,342 +252,6 @@ func lookupLegacyViewMeta(items []legacyViewMeta, name string) (legacyViewMeta, return legacyViewMeta{}, false } -func resolveLegacyRouteStates(source *shape.Source) []*plan.State { - return resolveLegacyRouteStatesWithLayout(source, defaultCompilePathLayout()) -} - -func resolveLegacyRouteStatesWithLayout(source *shape.Source, layout compilePathLayout) []*plan.State { - if source == nil || strings.TrimSpace(source.Path) == "" { - return nil - } - platformRoot, relativeDir, stem, ok := platformPathParts(source.Path, layout) - if !ok { - return nil - } - settings := extractRuleSettings(source, nil) - typeExpr := strings.TrimSpace(settings.Type) - typeExpr = strings.Trim(typeExpr, `"'`) - typeExpr = strings.TrimSuffix(typeExpr, ".Handler") - typeStem := "" - if typeExpr != "" { - typeStem = filepath.Base(filepath.FromSlash(typeExpr)) - } - routesRoot := joinRelativePath(platformRoot, layout.routesRelative) - routesBase := filepath.Join(routesRoot, filepath.FromSlash(relativeDir)) - yamlCandidates := legacyRouteYAMLCandidates(routesBase, stem, typeStem) - var payload struct { - Resource struct { - Parameters []struct { - Name string `yaml:"Name"` - URI string `yaml:"URI"` - Value string `yaml:"Value"` - Required *bool `yaml:"Required"` - Cacheable *bool `yaml:"Cacheable"` - In struct { - Kind string `yaml:"Kind"` - Name string `yaml:"Name"` - } `yaml:"In"` - Predicates []struct { - Group int `yaml:"Group"` - Name string `yaml:"Name"` - Ensure bool `yaml:"Ensure"` - Args []string `yaml:"Args"` - } `yaml:"Predicates"` - } `yaml:"Parameters"` - Views []struct { - Name string `yaml:"Name"` - Selector struct { - LimitParameter struct { - Name string `yaml:"Name"` - Cacheable *bool `yaml:"Cacheable"` - In struct { - Kind string `yaml:"Kind"` - Name string `yaml:"Name"` - } `yaml:"In"` - } `yaml:"LimitParameter"` - OffsetParameter struct { - Name string `yaml:"Name"` - Cacheable *bool `yaml:"Cacheable"` - In struct { - Kind string `yaml:"Kind"` - Name string `yaml:"Name"` - } `yaml:"In"` - } `yaml:"OffsetParameter"` - PageParameter struct { - Name string `yaml:"Name"` - Cacheable *bool `yaml:"Cacheable"` - In struct { - Kind string `yaml:"Kind"` - Name string `yaml:"Name"` - } `yaml:"In"` - } `yaml:"PageParameter"` - FieldsParameter struct { - Name string `yaml:"Name"` - Cacheable *bool `yaml:"Cacheable"` - In struct { - Kind string `yaml:"Kind"` - Name string `yaml:"Name"` - } `yaml:"In"` - } `yaml:"FieldsParameter"` - OrderByParameter struct { - Name string `yaml:"Name"` - Cacheable *bool `yaml:"Cacheable"` - In struct { - Kind string `yaml:"Kind"` - Name string `yaml:"Name"` - } `yaml:"In"` - } `yaml:"OrderByParameter"` - } `yaml:"Selector"` - } `yaml:"Views"` - } `yaml:"Resource"` - } - loaded := false - for _, candidate := range yamlCandidates { - data, err := os.ReadFile(candidate) - if err != nil { - continue - } - if err = yaml.Unmarshal(data, &payload); err != nil { - continue - } - loaded = true - break - } - if !loaded || len(payload.Resource.Parameters) == 0 { - return nil - } - result := make([]*plan.State, 0, len(payload.Resource.Parameters)) - for _, item := range payload.Resource.Parameters { - stateItem := &plan.State{ - Name: strings.TrimSpace(item.Name), - Path: strings.TrimSpace(item.Name), - Kind: strings.TrimSpace(item.In.Kind), - In: strings.TrimSpace(item.In.Name), - URI: strings.TrimSpace(item.URI), - Value: strings.TrimSpace(item.Value), - Required: item.Required, - Cacheable: item.Cacheable, - } - for _, predicate := range item.Predicates { - stateItem.Predicates = append(stateItem.Predicates, &plan.StatePredicate{ - Group: predicate.Group, - Name: strings.TrimSpace(predicate.Name), - Ensure: predicate.Ensure, - Arguments: append([]string{}, predicate.Args...), - }) - } - result = append(result, stateItem) - } - seen := map[string]bool{} - for _, item := range result { - if item == nil { - continue - } - key := strings.ToLower(strings.TrimSpace(item.Name)) + "|" + strings.ToLower(strings.TrimSpace(item.Kind)) + "|" + strings.ToLower(strings.TrimSpace(item.In)) - seen[key] = true - } - for _, viewItem := range payload.Resource.Views { - selectorName := strings.TrimSpace(viewItem.Name) - for _, param := range []struct { - Name string - Cacheable *bool - InKind string - InName string - }{ - { - Name: strings.TrimSpace(viewItem.Selector.LimitParameter.Name), - Cacheable: viewItem.Selector.LimitParameter.Cacheable, - InKind: strings.TrimSpace(viewItem.Selector.LimitParameter.In.Kind), - InName: strings.TrimSpace(viewItem.Selector.LimitParameter.In.Name), - }, - { - Name: strings.TrimSpace(viewItem.Selector.OffsetParameter.Name), - Cacheable: viewItem.Selector.OffsetParameter.Cacheable, - InKind: strings.TrimSpace(viewItem.Selector.OffsetParameter.In.Kind), - InName: strings.TrimSpace(viewItem.Selector.OffsetParameter.In.Name), - }, - { - Name: strings.TrimSpace(viewItem.Selector.PageParameter.Name), - Cacheable: viewItem.Selector.PageParameter.Cacheable, - InKind: strings.TrimSpace(viewItem.Selector.PageParameter.In.Kind), - InName: strings.TrimSpace(viewItem.Selector.PageParameter.In.Name), - }, - { - Name: strings.TrimSpace(viewItem.Selector.FieldsParameter.Name), - Cacheable: viewItem.Selector.FieldsParameter.Cacheable, - InKind: strings.TrimSpace(viewItem.Selector.FieldsParameter.In.Kind), - InName: strings.TrimSpace(viewItem.Selector.FieldsParameter.In.Name), - }, - { - Name: strings.TrimSpace(viewItem.Selector.OrderByParameter.Name), - Cacheable: viewItem.Selector.OrderByParameter.Cacheable, - InKind: strings.TrimSpace(viewItem.Selector.OrderByParameter.In.Kind), - InName: strings.TrimSpace(viewItem.Selector.OrderByParameter.In.Name), - }, - } { - if param.Name == "" { - continue - } - kind := firstNonEmptyString(strings.ToLower(param.InKind), "query") - inName := firstNonEmptyString(param.InName, strings.ToLower(param.Name)) - key := strings.ToLower(param.Name) + "|" + kind + "|" + strings.ToLower(inName) - if seen[key] { - continue - } - item := &plan.State{ - Name: param.Name, - Path: param.Name, - Kind: kind, - In: inName, - QuerySelector: selectorName, - Cacheable: param.Cacheable, - } - result = append(result, item) - seen[key] = true - } - } - return result -} - -func resolveLegacyRouteTypes(source *shape.Source) []*plan.Type { - return resolveLegacyRouteTypesWithLayout(source, defaultCompilePathLayout()) -} - -func resolveLegacyRouteTypesWithLayout(source *shape.Source, layout compilePathLayout) []*plan.Type { - if source == nil || strings.TrimSpace(source.Path) == "" { - return nil - } - platformRoot, relativeDir, stem, ok := platformPathParts(source.Path, layout) - if !ok { - return nil - } - settings := extractRuleSettings(source, nil) - typeExpr := strings.TrimSpace(settings.Type) - typeExpr = strings.Trim(typeExpr, `"'`) - typeExpr = strings.TrimSuffix(typeExpr, ".Handler") - typeStem := "" - if typeExpr != "" { - typeStem = filepath.Base(filepath.FromSlash(typeExpr)) - } - routesRoot := joinRelativePath(platformRoot, layout.routesRelative) - routesBase := filepath.Join(routesRoot, filepath.FromSlash(relativeDir)) - yamlCandidates := legacyRouteYAMLCandidates(routesBase, stem, typeStem) - var payload struct { - Resource struct { - Types []struct { - Name string `yaml:"Name"` - Alias string `yaml:"Alias"` - DataType string `yaml:"DataType"` - Cardinality string `yaml:"Cardinality"` - Package string `yaml:"Package"` - ModulePath string `yaml:"ModulePath"` - } `yaml:"Types"` - } `yaml:"Resource"` - } - loaded := false - for _, candidate := range yamlCandidates { - data, err := os.ReadFile(candidate) - if err != nil { - continue - } - if err = yaml.Unmarshal(data, &payload); err != nil { - continue - } - loaded = true - break - } - if !loaded || len(payload.Resource.Types) == 0 { - return nil - } - result := make([]*plan.Type, 0, len(payload.Resource.Types)) - seen := map[string]bool{} - for _, item := range payload.Resource.Types { - name := strings.TrimSpace(item.Name) - if name == "" { - continue - } - key := strings.ToLower(name) - if seen[key] { - continue - } - seen[key] = true - result = append(result, &plan.Type{ - Name: name, - Alias: strings.TrimSpace(item.Alias), - DataType: strings.TrimSpace(item.DataType), - Cardinality: strings.TrimSpace(item.Cardinality), - Package: strings.TrimSpace(item.Package), - ModulePath: strings.TrimSpace(item.ModulePath), - }) - } - return result -} - -func mergeLegacyRouteStates(result *plan.Result, source *shape.Source) { - mergeLegacyRouteStatesWithLayout(result, source, defaultCompilePathLayout()) -} - -func mergeLegacyRouteStatesWithLayout(result *plan.Result, source *shape.Source, layout compilePathLayout) { - if result == nil { - return - } - legacy := resolveLegacyRouteStatesWithLayout(source, layout) - if len(legacy) == 0 { - return - } - existing := map[string]bool{} - for _, item := range result.States { - if item == nil || strings.TrimSpace(item.Name) == "" { - continue - } - key := strings.ToLower(strings.TrimSpace(item.Name)) + "|" + strings.ToLower(strings.TrimSpace(item.Kind)) + "|" + strings.ToLower(strings.TrimSpace(item.In)) - existing[key] = true - } - for _, item := range legacy { - if item == nil || strings.TrimSpace(item.Name) == "" { - continue - } - key := strings.ToLower(strings.TrimSpace(item.Name)) + "|" + strings.ToLower(strings.TrimSpace(item.Kind)) + "|" + strings.ToLower(strings.TrimSpace(item.In)) - if existing[key] { - continue - } - result.States = append(result.States, item) - existing[key] = true - } -} - -func mergeLegacyRouteTypes(result *plan.Result, source *shape.Source) { - mergeLegacyRouteTypesWithLayout(result, source, defaultCompilePathLayout()) -} - -func mergeLegacyRouteTypesWithLayout(result *plan.Result, source *shape.Source, layout compilePathLayout) { - if result == nil { - return - } - legacy := resolveLegacyRouteTypesWithLayout(source, layout) - if len(legacy) == 0 { - return - } - existing := map[string]bool{} - for _, item := range result.Types { - if item == nil || strings.TrimSpace(item.Name) == "" { - continue - } - existing[strings.ToLower(strings.TrimSpace(item.Name))] = true - } - for _, item := range legacy { - if item == nil || strings.TrimSpace(item.Name) == "" { - continue - } - key := strings.ToLower(strings.TrimSpace(item.Name)) - if existing[key] { - continue - } - result.Types = append(result.Types, item) - existing[key] = true - } -} - func legacyRouteYAMLCandidates(routesBase, stem, typeStem string) []string { stemFileVariants := routeStemAlternatives(stem) stemDirVariants := routeStemAlternatives(stem) diff --git a/repository/shape/compile/pipeline/infer.go b/repository/shape/compile/pipeline/infer.go index c19bcb79b..7ad212431 100644 --- a/repository/shape/compile/pipeline/infer.go +++ b/repository/shape/compile/pipeline/infer.go @@ -3,15 +3,12 @@ package pipeline import ( "fmt" "reflect" - "regexp" "strings" "github.com/viant/sqlparser" "github.com/viant/sqlparser/query" ) -var nonWord = regexp.MustCompile(`[^a-zA-Z0-9_]+`) - func InferRoot(queryNode *query.Select, fallback string) (string, string, error) { name := SanitizeName(fallback) if name == "" { @@ -173,7 +170,7 @@ func SanitizeName(value string) string { if value == strings.ToUpper(value) { value = strings.ToLower(value) } - value = nonWord.ReplaceAllString(value, "_") + value = replaceNonWordWithUnderscore(value) value = strings.Trim(value, "_") if value == "" { return "" @@ -185,7 +182,7 @@ func SanitizeName(value string) string { } func ExportedName(value string) string { - value = nonWord.ReplaceAllString(strings.TrimSpace(value), "_") + value = replaceNonWordWithUnderscore(strings.TrimSpace(value)) value = strings.Trim(value, "_") if value == "" { return "" @@ -207,6 +204,29 @@ func ExportedName(value string) string { return name } +func replaceNonWordWithUnderscore(value string) string { + if value == "" { + return "" + } + var b strings.Builder + b.Grow(len(value)) + lastUnderscore := false + for i := 0; i < len(value); i++ { + ch := value[i] + isWord := ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') + if isWord { + b.WriteByte(ch) + lastUnderscore = false + continue + } + if !lastUnderscore { + b.WriteByte('_') + lastUnderscore = true + } + } + return b.String() +} + func parseColumnType(dataType string) reflect.Type { switch strings.ToLower(strings.TrimSpace(dataType)) { case "", "string", "text", "varchar", "char", "uuid", "json", "jsonb": diff --git a/repository/shape/compile/pipeline/read.go b/repository/shape/compile/pipeline/read.go index c52f9cc28..20fd2bba3 100644 --- a/repository/shape/compile/pipeline/read.go +++ b/repository/shape/compile/pipeline/read.go @@ -2,7 +2,6 @@ package pipeline import ( "reflect" - "regexp" "strings" dqlshape "github.com/viant/datly/repository/shape/dql/shape" @@ -10,14 +9,6 @@ import ( "github.com/viant/sqlparser/query" ) -var ( - criteriaBindingExpr = regexp.MustCompile(`(?i)\$criteria\.AppendBinding\([^)]*\)`) - selectorExpr = regexp.MustCompile(`\$\{?([a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*)\}?`) - veltyExpr = regexp.MustCompile(`\$\{[^}]+\}`) - fromTableSimpleExpr = regexp.MustCompile(`(?is)\bfrom\s+([a-zA-Z_][a-zA-Z0-9_$.]*)(?:\s+(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*))?`) - braceExpr = regexp.MustCompile(`[{}]`) -) - func BuildRead(sourceName, sqlText string) (*plan.View, []*dqlshape.Diagnostic, error) { parserSQL := normalizeParserSQL(sqlText) queryNode, parseDiag, err := ParseSelectWithDiagnostic(parserSQL) @@ -117,8 +108,7 @@ func inferLooseRoot(sourceName, sqlText string) (string, string) { if name == "" { name = "DQLView" } - if matches := fromTableSimpleExpr.FindStringSubmatch(sqlText); len(matches) > 1 { - table := strings.Trim(matches[1], "`\"") + if table := extractSimpleFromTable(sqlText); table != "" { return name, table } return name, name @@ -150,34 +140,7 @@ func normalizeParserSQL(sqlText string) string { if sqlText == "" { return sqlText } - normalized := criteriaBindingExpr.ReplaceAllString(sqlText, "1") - normalized = veltyExpr.ReplaceAllStringFunc(normalized, func(match string) string { - if strings.Contains(match, "sql.Insert") || strings.Contains(match, "sql.Update") || strings.Contains(match, "Nop") { - return match - } - lower := strings.ToLower(match) - if strings.Contains(lower, `build("where")`) || strings.Contains(lower, "build('where')") { - return " WHERE 1 " - } - if strings.Contains(lower, `build("and")`) || strings.Contains(lower, "build('and')") { - return " AND 1 " - } - return "1" - }) - normalized = selectorExpr.ReplaceAllStringFunc(normalized, func(match string) string { - lower := match - if len(match) > 0 && match[0] == '$' { - lower = match[1:] - } - lower = braceExpr.ReplaceAllString(lower, "") - switch lower { - case "sql.Insert", "sql.Update", "Nop": - return match - default: - return "1" - } - }) - return normalized + return replaceTemplateTokens(sqlText) } func inferRootFromRelations(relations []*plan.Relation) string { @@ -197,3 +160,196 @@ func inferRootFromRelations(relations []*plan.Relation) string { } return "" } + +func extractSimpleFromTable(sqlText string) string { + lower := strings.ToLower(sqlText) + for i := 0; i+4 <= len(lower); i++ { + if lower[i] != 'f' || !strings.HasPrefix(lower[i:], "from") { + continue + } + if i > 0 && isReadIdentifierPart(lower[i-1]) { + continue + } + j := skipReadSpaces(sqlText, i+4) + start := j + if start >= len(sqlText) || !isReadIdentifierStart(sqlText[start]) { + continue + } + j++ + for j < len(sqlText) && (isReadIdentifierPart(sqlText[j]) || sqlText[j] == '.' || sqlText[j] == '$') { + j++ + } + if start < j { + return strings.Trim(sqlText[start:j], "`\"") + } + } + return "" +} + +func replaceTemplateTokens(input string) string { + var b strings.Builder + b.Grow(len(input)) + for i := 0; i < len(input); { + if input[i] != '$' { + b.WriteByte(input[i]) + i++ + continue + } + if i+1 < len(input) && input[i+1] == '{' { + body, end, ok := readReadTemplateExpr(input, i+1) + if !ok { + b.WriteByte(input[i]) + i++ + continue + } + replacement, keep := normalizeTemplateExprBody(body) + if keep { + b.WriteString(input[i : end+1]) + } else { + b.WriteString(replacement) + } + i = end + 1 + continue + } + token, end, ok := readReadSelector(input, i) + if !ok { + b.WriteByte(input[i]) + i++ + continue + } + if strings.EqualFold(token, "$criteria.AppendBinding") { + pos := skipReadSpaces(input, end) + if pos < len(input) && input[pos] == '(' { + _, close, ok := readReadCallBody(input, pos) + if ok { + b.WriteByte('1') + i = close + 1 + continue + } + } + } + if isReadReservedToken(token) { + b.WriteString(token) + } else { + b.WriteByte('1') + } + i = end + } + return b.String() +} + +func normalizeTemplateExprBody(body string) (string, bool) { + trimmed := strings.TrimSpace(body) + if isReadReservedName(trimmed) { + return "", true + } + lower := strings.ToLower(trimmed) + if strings.Contains(lower, `build("where")`) || strings.Contains(lower, "build('where')") { + return " WHERE 1 ", false + } + if strings.Contains(lower, `build("and")`) || strings.Contains(lower, "build('and')") { + return " AND 1 ", false + } + return "1", false +} + +func readReadTemplateExpr(input string, openBrace int) (string, int, bool) { + if openBrace <= 0 || openBrace >= len(input) || input[openBrace] != '{' || input[openBrace-1] != '$' { + return "", -1, false + } + for i := openBrace + 1; i < len(input); i++ { + if input[i] == '}' { + return input[openBrace+1 : i], i, true + } + } + return "", -1, false +} + +func readReadSelector(input string, start int) (string, int, bool) { + if start < 0 || start >= len(input) || input[start] != '$' { + return "", start, false + } + i := start + 1 + if i >= len(input) || !isReadIdentifierStart(input[i]) { + return "", start, false + } + i++ + for i < len(input) && isReadIdentifierPart(input[i]) { + i++ + } + for i < len(input) && input[i] == '.' { + i++ + if i >= len(input) || !isReadIdentifierStart(input[i]) { + return "", start, false + } + i++ + for i < len(input) && isReadIdentifierPart(input[i]) { + i++ + } + } + return input[start:i], i, true +} + +func readReadCallBody(input string, openParen int) (string, int, bool) { + depth := 0 + quote := byte(0) + for i := openParen; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + if ch == '(' { + depth++ + continue + } + if ch == ')' { + depth-- + if depth == 0 { + return input[openParen+1 : i], i, true + } + } + } + return "", -1, false +} + +func isReadReservedToken(token string) bool { + if len(token) > 0 && token[0] == '$' { + token = token[1:] + } + return isReadReservedName(token) +} + +func isReadReservedName(name string) bool { + return name == "sql.Insert" || name == "sql.Update" || name == "Nop" +} + +func skipReadSpaces(input string, index int) int { + for index < len(input) { + switch input[index] { + case ' ', '\t', '\n', '\r': + index++ + default: + return index + } + } + return index +} + +func isReadIdentifierStart(ch byte) bool { + return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') +} + +func isReadIdentifierPart(ch byte) bool { + return isReadIdentifierStart(ch) || (ch >= '0' && ch <= '9') +} diff --git a/repository/shape/compile/pipeline/relation.go b/repository/shape/compile/pipeline/relation.go index 722ff5167..dc94b8955 100644 --- a/repository/shape/compile/pipeline/relation.go +++ b/repository/shape/compile/pipeline/relation.go @@ -2,7 +2,6 @@ package pipeline import ( "fmt" - "regexp" "strings" dqldiag "github.com/viant/datly/repository/shape/dql/diag" @@ -14,8 +13,6 @@ import ( "github.com/viant/sqlparser/query" ) -var joinSelectorEqExpr = regexp.MustCompile(`(?i)([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)`) - func ExtractJoinRelations(raw string, queryNode *query.Select) ([]*plan.Relation, []*dqlshape.Diagnostic) { if queryNode == nil || len(queryNode.Joins) == 0 { return nil, nil @@ -110,21 +107,85 @@ func collectJoinPairsFromRaw(input string) []joinPair { if input == "" { return nil } - var result []joinPair - for _, m := range joinSelectorEqExpr.FindAllStringSubmatch(input, -1) { - if len(m) < 5 { + var ( + result []joinPair + i int + ) + for i < len(input) { + left, next, ok := parseRelationSelector(input, i) + if !ok { + i++ continue } - left := strings.TrimSpace(m[1] + "." + m[2]) - right := strings.TrimSpace(m[3] + "." + m[4]) - if left == "" || right == "" { + j := skipRelationSpaces(input, next) + if j >= len(input) || input[j] != '=' { + i = next + continue + } + right, end, ok := parseRelationSelector(input, j+1) + if !ok { + i = j + 1 + continue + } + if strings.TrimSpace(left) == "" || strings.TrimSpace(right) == "" { + i = end continue } result = append(result, joinPair{left: left, right: right}) + i = end } return result } +func parseRelationSelector(input string, start int) (string, int, bool) { + i := skipRelationSpaces(input, start) + nsStart := i + if nsStart >= len(input) || !isRelationIdentifierStart(input[nsStart]) { + return "", start, false + } + i++ + for i < len(input) && isRelationIdentifierPart(input[i]) { + i++ + } + ns := input[nsStart:i] + i = skipRelationSpaces(input, i) + if i >= len(input) || input[i] != '.' { + return "", start, false + } + i++ + i = skipRelationSpaces(input, i) + colStart := i + if colStart >= len(input) || !isRelationIdentifierStart(input[colStart]) { + return "", start, false + } + i++ + for i < len(input) && isRelationIdentifierPart(input[i]) { + i++ + } + col := input[colStart:i] + return strings.TrimSpace(ns) + "." + strings.TrimSpace(col), i, true +} + +func skipRelationSpaces(input string, index int) int { + for index < len(input) { + switch input[index] { + case ' ', '\t', '\n', '\r': + index++ + default: + return index + } + } + return index +} + +func isRelationIdentifierStart(ch byte) bool { + return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') +} + +func isRelationIdentifierPart(ch byte) bool { + return isRelationIdentifierStart(ch) || (ch >= '0' && ch <= '9') +} + func shouldFallbackToRawJoinPairs(input string) bool { input = strings.TrimSpace(strings.ToLower(input)) if input == "" { diff --git a/repository/shape/compile/statedecl.go b/repository/shape/compile/statedecl.go index eab76c647..5800e4881 100644 --- a/repository/shape/compile/statedecl.go +++ b/repository/shape/compile/statedecl.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/viant/datly/repository/shape/plan" + "github.com/viant/parsly" ) func appendDeclaredStates(rawDQL string, result *plan.Result) { @@ -30,6 +31,10 @@ func appendDeclaredStates(rawDQL string, result *plan.Result) { Kind: kind, In: location, } + if inType, outType := parseSetDeclarationTypes(block.Body); inType != "" || outType != "" { + state.DataType = inType + state.OutputDataType = outType + } switch strings.ToLower(kind) { case "query": required := false @@ -98,6 +103,21 @@ func applyDeclaredStateOptions(state *plan.State, tail string) { if len(args) == 1 { state.DataType = trimQuote(args[0]) } + case strings.EqualFold(name, "WithCodec"): + if len(args) >= 1 { + state.Codec = trimQuote(args[0]) + state.CodecArgs = append([]string{}, trimQuotedArgs(args[1:])...) + } + case strings.EqualFold(name, "WithStatusCode"): + if len(args) == 1 { + if value, err := strconv.Atoi(strings.TrimSpace(trimQuote(args[0]))); err == nil { + state.ErrorCode = value + } + } + case strings.EqualFold(name, "WithErrorMessage"): + if len(args) == 1 { + state.ErrorMessage = trimQuote(args[0]) + } case strings.EqualFold(name, "Value"): if len(args) == 1 { state.Value = trimQuote(args[0]) @@ -108,6 +128,56 @@ func applyDeclaredStateOptions(state *plan.State, tail string) { } } +func parseSetDeclarationTypes(body string) (string, string) { + cursor := parsly.NewCursor("", []byte(body), 0) + if cursor.MatchAfterOptional(vdWhitespaceMatcher, vdParamDeclMatcher).Code != vdParamDeclToken { + return "", "" + } + if _, matched := readIdentifier(cursor); !matched { + return "", "" + } + _ = cursor.MatchOne(vdWhitespaceMatcher) + matchedType := cursor.MatchOne(vdTypeMatcher) + if matchedType.Code != vdTypeToken { + return "", "" + } + typeExpr := strings.TrimSpace(matchedType.Text(cursor)) + if len(typeExpr) < 2 { + return "", "" + } + typeExpr = strings.TrimSpace(typeExpr[1 : len(typeExpr)-1]) + if typeExpr == "" { + return "", "" + } + args := splitArgs(typeExpr) + if len(args) == 0 { + return "", "" + } + inputType := strings.TrimSpace(trimQuote(args[0])) + outputType := "" + if len(args) > 1 { + outputType = strings.TrimSpace(trimQuote(args[1])) + } + if inputType == "?" { + inputType = "" + } + if outputType == "?" { + outputType = "" + } + return inputType, outputType +} + +func trimQuotedArgs(input []string) []string { + if len(input) == 0 { + return nil + } + result := make([]string, 0, len(input)) + for _, item := range input { + result = append(result, trimQuote(item)) + } + return result +} + func appendStatePredicate(state *plan.State, args []string, ensure bool) { if state == nil || len(args) == 0 { return diff --git a/repository/shape/compile/statedecl_test.go b/repository/shape/compile/statedecl_test.go index a538e7c3a..94c5fea26 100644 --- a/repository/shape/compile/statedecl_test.go +++ b/repository/shape/compile/statedecl_test.go @@ -11,6 +11,7 @@ import ( func TestAppendDeclaredStates(t *testing.T) { dql := ` #set($_ = $Jwt(header/Authorization).WithCodec(JwtClaim).WithStatusCode(401)) +#set($_ = $Claims(header/Authorization).WithCodec(JwtClaim)) #set($_ = $Name(query/name).WithPredicate(0,'contains','sl','NAME').Optional()) #set($_ = $Fields<[]string>(query/fields).QuerySelector(site_list)) #set($_ = $Meta(output/summary)) @@ -27,9 +28,16 @@ SELECT id FROM SITE_LIST sl` } require.NotNil(t, byName["Jwt"]) assert.Equal(t, "header", byName["Jwt"].Kind) + assert.Equal(t, "string", byName["Jwt"].DataType) + assert.Equal(t, "JwtClaim", byName["Jwt"].Codec) + assert.Equal(t, 401, byName["Jwt"].ErrorCode) require.NotNil(t, byName["Jwt"].Required) assert.True(t, *byName["Jwt"].Required) + require.NotNil(t, byName["Claims"]) + assert.Equal(t, "string", byName["Claims"].DataType) + assert.Equal(t, "*JwtClaims", byName["Claims"].OutputDataType) + require.NotNil(t, byName["Name"]) assert.Equal(t, "query", byName["Name"].Kind) require.NotNil(t, byName["Name"].Required) diff --git a/repository/shape/compile/strings_util.go b/repository/shape/compile/strings_util.go deleted file mode 100644 index 5c18c9d8a..000000000 --- a/repository/shape/compile/strings_util.go +++ /dev/null @@ -1,13 +0,0 @@ -package compile - -import "strings" - -func firstNonEmptyString(values ...string) string { - for _, value := range values { - value = strings.TrimSpace(value) - if value != "" { - return value - } - } - return "" -} diff --git a/repository/shape/compile/viewdecl_append.go b/repository/shape/compile/viewdecl_append.go index dabf0fd26..6ae2fc663 100644 --- a/repository/shape/compile/viewdecl_append.go +++ b/repository/shape/compile/viewdecl_append.go @@ -2,7 +2,6 @@ package compile import ( "reflect" - "regexp" "strings" "github.com/viant/datly/repository/shape/compile/pipeline" @@ -10,8 +9,6 @@ import ( "github.com/viant/sqlparser" ) -var summaryParentRefExpr = regexp.MustCompile(`(?i)\$View\.([a-zA-Z_][a-zA-Z0-9_]*)\.SQL\b`) - func appendDeclaredViews(rawDQL string, result *plan.Result) { if result == nil { return @@ -73,12 +70,8 @@ func lookupSummaryParentView(result *plan.Result, sqlText string) *plan.View { if result == nil || strings.TrimSpace(sqlText) == "" { return nil } - matches := summaryParentRefExpr.FindStringSubmatch(sqlText) - if len(matches) < 2 { - return nil - } - parent := strings.TrimSpace(matches[1]) - if parent == "" { + parent, ok := findSummaryParentReference(sqlText) + if !ok { return nil } if view, ok := result.ViewsByName[parent]; ok && view != nil { @@ -109,6 +102,46 @@ func lookupSummaryParentView(result *plan.Result, sqlText string) *plan.View { return nil } +func findSummaryParentReference(input string) (string, bool) { + if strings.TrimSpace(input) == "" { + return "", false + } + lower := strings.ToLower(input) + for i := 0; i+len("$view.") < len(lower); i++ { + if lower[i] != '$' { + continue + } + if !strings.HasPrefix(lower[i:], "$view.") { + continue + } + start := i + len("$view.") + if start >= len(input) || !isCompileIdentifierStart(input[start]) { + continue + } + end := start + 1 + for end < len(input) && isCompileIdentifierPart(input[end]) { + end++ + } + if !strings.HasPrefix(lower[end:], ".sql") { + continue + } + parent := strings.TrimSpace(input[start:end]) + if parent == "" { + continue + } + return parent, true + } + return "", false +} + +func isCompileIdentifierStart(ch byte) bool { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' +} + +func isCompileIdentifierPart(ch byte) bool { + return isCompileIdentifierStart(ch) || (ch >= '0' && ch <= '9') +} + func buildViewDeclaration(item *declaredView) *plan.ViewDeclaration { if item == nil { return nil diff --git a/repository/shape/dql/preprocess/directive_parser.go b/repository/shape/dql/preprocess/directive_parser.go new file mode 100644 index 000000000..9a13320cc --- /dev/null +++ b/repository/shape/dql/preprocess/directive_parser.go @@ -0,0 +1,186 @@ +package preprocess + +import "strings" + +type directiveCall struct { + name string + args []string + start int +} + +func scanDollarCalls(input string, names map[string]bool) []directiveCall { + result := make([]directiveCall, 0) + for i := 0; i < len(input); { + if input[i] != '$' || i+1 >= len(input) || !isIdentifierStart(input[i+1]) { + i++ + continue + } + start := i + 1 + i += 2 + for i < len(input) && isIdentifierPart(input[i]) { + i++ + } + name := strings.ToLower(input[start:i]) + if !names[name] { + continue + } + j := skipSpaces(input, i) + if j >= len(input) || input[j] != '(' { + continue + } + body, end, ok := readCallBody(input, j) + if !ok { + continue + } + result = append(result, directiveCall{ + name: name, + args: splitCallArgs(body), + start: start - 1, + }) + i = end + 1 + } + return result +} + +func readCallBody(input string, openParen int) (string, int, bool) { + depth := 0 + quote := byte(0) + for i := openParen; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + if ch == '(' { + depth++ + continue + } + if ch == ')' { + depth-- + if depth == 0 { + return input[openParen+1 : i], i, true + } + } + } + return "", -1, false +} + +func splitCallArgs(input string) []string { + args := make([]string, 0) + current := strings.Builder{} + depth := 0 + quote := byte(0) + for i := 0; i < len(input); i++ { + ch := input[i] + if quote != 0 { + current.WriteByte(ch) + if ch == '\\' && i+1 < len(input) { + i++ + current.WriteByte(input[i]) + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + current.WriteByte(ch) + continue + } + if ch == '(' { + depth++ + current.WriteByte(ch) + continue + } + if ch == ')' { + if depth > 0 { + depth-- + } + current.WriteByte(ch) + continue + } + if ch == ',' && depth == 0 { + args = append(args, strings.TrimSpace(current.String())) + current.Reset() + continue + } + current.WriteByte(ch) + } + if value := strings.TrimSpace(current.String()); value != "" { + args = append(args, value) + } + return args +} + +func skipSpaces(input string, index int) int { + for index < len(input) { + switch input[index] { + case ' ', '\t', '\n', '\r': + index++ + default: + return index + } + } + return index +} + +func skipInlineSpaces(input string, index int) int { + for index < len(input) { + switch input[index] { + case ' ', '\t': + index++ + default: + return index + } + } + return index +} + +func isIdentifierStart(ch byte) bool { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' +} + +func isIdentifierPart(ch byte) bool { + return isIdentifierStart(ch) || (ch >= '0' && ch <= '9') +} + +func parseQuotedLiteral(input string) (string, bool) { + value := strings.TrimSpace(input) + if len(value) < 2 { + return "", false + } + quote := value[0] + if quote != '\'' && quote != '"' { + return "", false + } + if value[len(value)-1] != quote { + return "", false + } + return value[1 : len(value)-1], true +} + +func hasWordFoldAt(input string, pos int, word string) bool { + if pos < 0 || pos+len(word) > len(input) { + return false + } + if !strings.EqualFold(input[pos:pos+len(word)], word) { + return false + } + next := pos + len(word) + if next >= len(input) { + return true + } + return !isIdentifierPart(input[next]) +} diff --git a/repository/shape/dql/preprocess/legacy_import.go b/repository/shape/dql/preprocess/legacy_import.go index 0c7639b7e..5218d0dee 100644 --- a/repository/shape/dql/preprocess/legacy_import.go +++ b/repository/shape/dql/preprocess/legacy_import.go @@ -2,7 +2,6 @@ package preprocess import ( "path" - "regexp" "strings" dqldiag "github.com/viant/datly/repository/shape/dql/diag" @@ -10,17 +9,18 @@ import ( "github.com/viant/datly/repository/shape/typectx" ) -var ( - legacyImportBlock = regexp.MustCompile(`(?ms)^[ \t]*import\s*\((.*?)\)`) - legacyImportLine = regexp.MustCompile(`(?m)^[ \t]*import\s*"([^"]+)"(?:\s+alias\s+"([^"]+)")?[ \t]*$`) - legacyImportItem = regexp.MustCompile(`"([^"]+)"(?:\s+alias\s+"([^"]+)")?`) -) - type legacyImportRange struct { start int end int } +type legacyImportBlockSpec struct { + start int + end int + bodyStart int + bodyEnd int +} + func extractLegacyTypeImports(dql string) ([]typectx.Import, []legacyImportRange, []*dqlshape.Diagnostic) { if strings.TrimSpace(dql) == "" { return nil, nil, nil @@ -32,23 +32,16 @@ func extractLegacyTypeImports(dql string) ([]typectx.Import, []legacyImportRange ) inBlock := make([]bool, len(dql)) - blockMatches := legacyImportBlock.FindAllStringSubmatchIndex(dql, -1) - for _, match := range blockMatches { - if len(match) < 4 { - continue - } - start, end := match[0], match[1] - bodyStart, bodyEnd := match[2], match[3] - if start < 0 || end <= start || bodyStart < 0 || bodyEnd < bodyStart || bodyEnd > len(dql) { - continue - } + blocks := findLegacyImportBlocks(dql) + for _, block := range blocks { + start, end := block.start, block.end for i := start; i < end && i < len(inBlock); i++ { inBlock[i] = true } ranges = append(ranges, legacyImportRange{start: start, end: end}) - blockBody := dql[bodyStart:bodyEnd] - itemMatches := legacyImportItem.FindAllStringSubmatchIndex(blockBody, -1) - if len(itemMatches) == 0 { + blockBody := dql[block.bodyStart:block.bodyEnd] + items := parseLegacyImportItems(blockBody, block.bodyStart) + if len(items) == 0 { diags = append(diags, directiveDiagnostic( dqldiag.CodeDirImport, "invalid legacy import declaration", @@ -58,24 +51,15 @@ func extractLegacyTypeImports(dql string) ([]typectx.Import, []legacyImportRange )) continue } - for _, item := range itemMatches { - if len(item) < 6 { - continue - } - specStart := bodyStart + item[2] - spec := strings.TrimSpace(blockBody[item[2]:item[3]]) - alias := "" - if item[4] >= 0 && item[5] >= 0 { - alias = strings.TrimSpace(blockBody[item[4]:item[5]]) - } - aImport, ok := parseLegacyImportSpec(spec, alias) + for _, item := range items { + aImport, ok := parseLegacyImportSpec(item.spec, item.alias) if !ok { diags = append(diags, directiveDiagnostic( dqldiag.CodeDirImport, "invalid legacy import declaration", `expected import target with type suffix: "pkg/path.Type"`, dql, - specStart, + item.offset, )) continue } @@ -83,20 +67,19 @@ func extractLegacyTypeImports(dql string) ([]typectx.Import, []legacyImportRange } } - lineMatches := legacyImportLine.FindAllStringSubmatchIndex(dql, -1) - for _, match := range lineMatches { - if len(match) < 6 { + offset := 0 + for _, line := range strings.SplitAfter(dql, "\n") { + start := offset + end := start + len(line) + if start >= len(inBlock) || inBlock[start] { + offset = end continue } - start, end := match[0], match[1] - if start < 0 || end <= start || start >= len(inBlock) || inBlock[start] { + spec, alias, ok := parseLegacyImportLine(line) + if !ok { + offset = end continue } - spec := strings.TrimSpace(dql[match[2]:match[3]]) - alias := "" - if match[4] >= 0 && match[5] >= 0 { - alias = strings.TrimSpace(dql[match[4]:match[5]]) - } aImport, ok := parseLegacyImportSpec(spec, alias) if !ok { diags = append(diags, directiveDiagnostic( @@ -106,15 +89,151 @@ func extractLegacyTypeImports(dql string) ([]typectx.Import, []legacyImportRange dql, start, )) + offset = end continue } imports = append(imports, aImport) ranges = append(ranges, legacyImportRange{start: start, end: end}) + offset = end } return uniqueTypeImports(imports), ranges, diags } +func findLegacyImportBlocks(dql string) []legacyImportBlockSpec { + var result []legacyImportBlockSpec + for lineStart := 0; lineStart < len(dql); { + lineEnd := lineStart + for lineEnd < len(dql) && dql[lineEnd] != '\n' { + lineEnd++ + } + + pos := skipInlineSpaces(dql, lineStart) + if hasWordFoldAt(dql, pos, "import") { + pos = skipSpaces(dql, pos+len("import")) + if pos < len(dql) && dql[pos] == '(' { + body, end, ok := readCallBody(dql, pos) + if ok { + result = append(result, legacyImportBlockSpec{ + start: lineStart, + end: end + 1, + bodyStart: pos + 1, + bodyEnd: pos + 1 + len(body), + }) + lineStart = end + 1 + continue + } + } + } + + if lineEnd < len(dql) { + lineStart = lineEnd + 1 + } else { + break + } + } + return result +} + +type legacyImportItem struct { + spec string + alias string + offset int +} + +func parseLegacyImportItems(input string, base int) []legacyImportItem { + var result []legacyImportItem + for i := 0; i < len(input); { + i = skipLegacyImportSeparators(input, i) + if i >= len(input) { + break + } + start := i + spec, end, ok := readQuotedAt(input, i) + if !ok { + i++ + continue + } + i = skipSpaces(input, end) + alias := "" + if hasWordFoldAt(input, i, "alias") { + i = skipSpaces(input, i+len("alias")) + aliasValue, aliasEnd, ok := readQuotedAt(input, i) + if !ok { + i = end + continue + } + alias = aliasValue + i = aliasEnd + } + result = append(result, legacyImportItem{ + spec: strings.TrimSpace(spec), + alias: strings.TrimSpace(alias), + offset: base + start, + }) + } + return result +} + +func parseLegacyImportLine(line string) (spec, alias string, ok bool) { + input := strings.TrimSpace(line) + if input == "" || !hasWordFoldAt(input, 0, "import") { + return "", "", false + } + index := skipSpaces(input, len("import")) + specValue, end, ok := readQuotedAt(input, index) + if !ok { + return "", "", false + } + index = skipSpaces(input, end) + aliasValue := "" + if hasWordFoldAt(input, index, "alias") { + index = skipSpaces(input, index+len("alias")) + value, aliasEnd, ok := readQuotedAt(input, index) + if !ok { + return "", "", false + } + aliasValue = value + index = skipSpaces(input, aliasEnd) + } + if index != len(input) { + return "", "", false + } + return strings.TrimSpace(specValue), strings.TrimSpace(aliasValue), true +} + +func readQuotedAt(input string, index int) (string, int, bool) { + if index < 0 || index >= len(input) { + return "", index, false + } + quote := input[index] + if quote != '\'' && quote != '"' { + return "", index, false + } + for i := index + 1; i < len(input); i++ { + if input[i] == '\\' && i+1 < len(input) { + i++ + continue + } + if input[i] == quote { + return input[index+1 : i], i + 1, true + } + } + return "", index, false +} + +func skipLegacyImportSeparators(input string, index int) int { + for index < len(input) { + switch input[index] { + case ' ', '\t', '\n', '\r', ',', ';': + index++ + default: + return index + } + } + return index +} + func parseLegacyImportSpec(spec, alias string) (typectx.Import, bool) { spec = strings.TrimSpace(spec) if spec == "" { diff --git a/repository/shape/dql/preprocess/preprocess.go b/repository/shape/dql/preprocess/preprocess.go index 3d56380b7..9579dfb0c 100644 --- a/repository/shape/dql/preprocess/preprocess.go +++ b/repository/shape/dql/preprocess/preprocess.go @@ -1,7 +1,6 @@ package preprocess import ( - "regexp" "strings" dqlopt "github.com/viant/datly/repository/shape/dql/optimize" @@ -10,12 +9,6 @@ import ( "github.com/viant/datly/repository/shape/typectx" ) -var ( - decoratorLine = regexp.MustCompile(`(?i)^\s*(use_connector|allow_nulls?)\s*\([^)]*\)\s*,?\s*$`) - commaBeforeFrom = regexp.MustCompile(`(?i),\s*(\r?\n\s*from\b)`) - doubleCommaExpr = regexp.MustCompile(`,\s*,`) -) - type Result struct { Original string DirectSQL string @@ -61,15 +54,48 @@ func stripDecorators(sql string) string { lines := strings.Split(sql, "\n") filtered := make([]string, 0, len(lines)) for _, line := range lines { - if decoratorLine.MatchString(strings.TrimSpace(line)) { + if isStandaloneDecoratorLine(line) { continue } filtered = append(filtered, line) } - joined := strings.Join(filtered, "\n") - joined = doubleCommaExpr.ReplaceAllString(joined, ",") - joined = commaBeforeFrom.ReplaceAllString(joined, "$1") - return joined + return cleanupLineCommaArtifacts(filtered) +} + +func isStandaloneDecoratorLine(line string) bool { + trimmed := strings.TrimSpace(strings.TrimSuffix(line, ",")) + if trimmed == "" { + return false + } + open := strings.Index(trimmed, "(") + close := strings.LastIndex(trimmed, ")") + if open <= 0 || close <= open { + return false + } + name := strings.ToLower(strings.TrimSpace(trimmed[:open])) + switch name { + case "use_connector", "allow_nulls", "allownulls", "tag", "cast", "required", "cardinality", "set_limit": + return true + default: + return false + } +} + +func cleanupLineCommaArtifacts(lines []string) string { + if len(lines) == 0 { + return "" + } + result := make([]string, 0, len(lines)) + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if len(result) > 0 && strings.HasPrefix(strings.ToLower(trimmed), "from ") { + prev := strings.TrimRight(result[len(result)-1], " \t") + prev = strings.TrimSuffix(prev, ",") + result[len(result)-1] = prev + } + result = append(result, line) + } + return strings.Join(result, "\n") } func normalizeTypeContext(ctx *typectx.Context) *typectx.Context { diff --git a/repository/shape/dql/preprocess/preprocess_test.go b/repository/shape/dql/preprocess/preprocess_test.go index 5f3d0dfc4..337b9ca41 100644 --- a/repository/shape/dql/preprocess/preprocess_test.go +++ b/repository/shape/dql/preprocess/preprocess_test.go @@ -167,3 +167,40 @@ func TestPrepare_PackageImportInSettings_UnsupportedDiagnostic(t *testing.T) { assert.Equal(t, dqldiag.CodeDirUnsupported, pre.Diagnostics[0].Code) assert.Equal(t, 1, pre.Diagnostics[0].Span.Start.Line) } + +func TestPrepare_TypeContext_CaseInsensitive(t *testing.T) { + dql := "#Package('a/b')\n#Import('x','github.com/acme/x')\nSELECT id FROM t" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotNil(t, pre.TypeCtx) + assert.Equal(t, "a/b", pre.TypeCtx.DefaultPackage) + require.Len(t, pre.TypeCtx.Imports, 1) + assert.Equal(t, "x", pre.TypeCtx.Imports[0].Alias) + assert.Equal(t, "github.com/acme/x", pre.TypeCtx.Imports[0].Package) +} + +func TestExtractLegacyTypeImports_BlockAndLine(t *testing.T) { + dql := "import (\n" + + " \"github.com/acme/a.TypeA\"\n" + + " \"github.com/acme/b.TypeB\" alias \"b\"\n" + + ")\n" + + "import \"github.com/acme/c.TypeC\"\n" + + imports, ranges, diags := extractLegacyTypeImports(dql) + require.Empty(t, diags) + require.Len(t, ranges, 2) + require.Len(t, imports, 3) + assert.Equal(t, "a", imports[0].Alias) + assert.Equal(t, "github.com/acme/a", imports[0].Package) + assert.Equal(t, "b", imports[1].Alias) + assert.Equal(t, "github.com/acme/b", imports[1].Package) + assert.Equal(t, "c", imports[2].Alias) + assert.Equal(t, "github.com/acme/c", imports[2].Package) +} + +func TestExtractLegacyTypeImports_InvalidBlockDiagnostic(t *testing.T) { + dql := "import (\n alias \"oops\"\n)\nSELECT 1" + _, _, diags := extractLegacyTypeImports(dql) + require.NotEmpty(t, diags) + assert.Equal(t, dqldiag.CodeDirImport, diags[0].Code) +} diff --git a/repository/shape/dql/preprocess/settings_directives.go b/repository/shape/dql/preprocess/settings_directives.go index 770aa3a0d..3d7793c8f 100644 --- a/repository/shape/dql/preprocess/settings_directives.go +++ b/repository/shape/dql/preprocess/settings_directives.go @@ -2,7 +2,6 @@ package preprocess import ( "net/http" - "regexp" "strings" "github.com/viant/datly/repository/content" @@ -12,17 +11,16 @@ import ( ) var ( - metaDirective = regexp.MustCompile(`(?i)\$meta\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) - connectorDirective = regexp.MustCompile(`(?i)\$connector\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) - cacheDirective = regexp.MustCompile(`(?i)\$cache\s*\(\s*(true|false)\s*(?:,\s*['\"]([^'\"]+)['\"]\s*)?\)`) - mcpDirective = regexp.MustCompile(`(?i)\$mcp\s*\(\s*['\"]([^'\"]+)['\"]\s*(?:,\s*['\"]([^'\"]*)['\"]\s*)?(?:,\s*['\"]([^'\"]*)['\"]\s*)?\)`) - routeDirective = regexp.MustCompile(`(?i)\$route\s*\(([^)]*)\)`) - marshalDirective = regexp.MustCompile(`(?i)\$marshal\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)`) - unmarshalDirective = regexp.MustCompile(`(?i)\$unmarshal\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)`) - formatDirective = regexp.MustCompile(`(?i)\$format\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) - dateFormatDirective = regexp.MustCompile(`(?i)\$date_format\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) - caseFormatDirective = regexp.MustCompile(`(?i)\$case_format\s*\(\s*['\"]([^'\"]+)['\"]\s*\)`) - quotedArgDirective = regexp.MustCompile(`['\"]([^'\"]*)['\"]`) + metaDirectiveName = map[string]bool{"meta": true} + connectorDirectiveName = map[string]bool{"connector": true} + cacheDirectiveName = map[string]bool{"cache": true} + mcpDirectiveName = map[string]bool{"mcp": true} + routeDirectiveName = map[string]bool{"route": true} + marshalDirectiveName = map[string]bool{"marshal": true} + unmarshalDirectiveName = map[string]bool{"unmarshal": true} + formatDirectiveName = map[string]bool{"format": true} + dateFormatDirectiveName = map[string]bool{"date_format": true} + caseFormatDirectiveName = map[string]bool{"case_format": true} ) func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, directives *dqlshape.Directives) []*dqlshape.Diagnostic { @@ -130,13 +128,17 @@ func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, direct } func parseMetaDirectives(input string) []string { - matches := metaDirective.FindAllStringSubmatch(input, -1) - result := make([]string, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { + calls := scanDollarCalls(input, metaDirectiveName) + result := make([]string, 0, len(calls)) + for _, call := range calls { + if len(call.args) != 1 { continue } - if value := strings.TrimSpace(match[1]); value != "" { + value, ok := parseQuotedLiteral(call.args[0]) + if !ok { + continue + } + if value = strings.TrimSpace(value); value != "" { result = append(result, value) } } @@ -144,13 +146,17 @@ func parseMetaDirectives(input string) []string { } func parseConnectorDirectives(input string) []string { - matches := connectorDirective.FindAllStringSubmatch(input, -1) - result := make([]string, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { + calls := scanDollarCalls(input, connectorDirectiveName) + result := make([]string, 0, len(calls)) + for _, call := range calls { + if len(call.args) != 1 { + continue + } + value, ok := parseQuotedLiteral(call.args[0]) + if !ok { continue } - if value := strings.TrimSpace(match[1]); value != "" { + if value = strings.TrimSpace(value); value != "" { result = append(result, value) } } @@ -158,16 +164,29 @@ func parseConnectorDirectives(input string) []string { } func parseCacheDirectives(input string) []*dqlshape.CacheDirective { - matches := cacheDirective.FindAllStringSubmatch(input, -1) - result := make([]*dqlshape.CacheDirective, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { + calls := scanDollarCalls(input, cacheDirectiveName) + result := make([]*dqlshape.CacheDirective, 0, len(calls)) + for _, call := range calls { + if len(call.args) == 0 || len(call.args) > 2 { + continue + } + enabledRaw := strings.TrimSpace(call.args[0]) + var enabled bool + switch { + case strings.EqualFold(enabledRaw, "true"): + enabled = true + case strings.EqualFold(enabledRaw, "false"): + enabled = false + default: continue } - enabled := strings.EqualFold(strings.TrimSpace(match[1]), "true") ttl := "" - if len(match) > 2 { - ttl = strings.TrimSpace(match[2]) + if len(call.args) == 2 { + value, ok := parseQuotedLiteral(call.args[1]) + if !ok { + continue + } + ttl = strings.TrimSpace(value) } result = append(result, &dqlshape.CacheDirective{Enabled: enabled, TTL: ttl}) } @@ -175,23 +194,35 @@ func parseCacheDirectives(input string) []*dqlshape.CacheDirective { } func parseMCPDirectives(input string) []*dqlshape.MCPDirective { - matches := mcpDirective.FindAllStringSubmatch(input, -1) - result := make([]*dqlshape.MCPDirective, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { + calls := scanDollarCalls(input, mcpDirectiveName) + result := make([]*dqlshape.MCPDirective, 0, len(calls)) + for _, call := range calls { + if len(call.args) < 1 || len(call.args) > 3 { + continue + } + name, ok := parseQuotedLiteral(call.args[0]) + if !ok { continue } - name := strings.TrimSpace(match[1]) + name = strings.TrimSpace(name) if name == "" { continue } description := "" - if len(match) > 2 { - description = strings.TrimSpace(match[2]) + if len(call.args) > 1 { + value, ok := parseQuotedLiteral(call.args[1]) + if !ok { + continue + } + description = strings.TrimSpace(value) } descriptionPath := "" - if len(match) > 3 { - descriptionPath = strings.TrimSpace(match[3]) + if len(call.args) > 2 { + value, ok := parseQuotedLiteral(call.args[2]) + if !ok { + continue + } + descriptionPath = strings.TrimSpace(value) } result = append(result, &dqlshape.MCPDirective{ Name: name, @@ -203,21 +234,33 @@ func parseMCPDirectives(input string) []*dqlshape.MCPDirective { } func parseRouteDirectives(input string) []*dqlshape.RouteDirective { - matches := routeDirective.FindAllStringSubmatch(input, -1) - result := make([]*dqlshape.RouteDirective, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { + calls := scanDollarCalls(input, routeDirectiveName) + result := make([]*dqlshape.RouteDirective, 0, len(calls)) + for _, call := range calls { + if len(call.args) == 0 { continue } - args := parseQuotedArgs(match[1]) - if len(args) == 0 { + uri, ok := parseQuotedLiteral(call.args[0]) + if !ok { continue } - uri := strings.TrimSpace(args[0]) + uri = strings.TrimSpace(uri) if !strings.HasPrefix(uri, "/") { continue } - methods, ok := normalizeHTTPMethods(args[1:]) + methodsRaw := make([]string, 0, len(call.args)-1) + for _, arg := range call.args[1:] { + method, ok := parseQuotedLiteral(arg) + if !ok { + methodsRaw = nil + break + } + methodsRaw = append(methodsRaw, method) + } + if methodsRaw == nil { + continue + } + methods, ok := normalizeHTTPMethods(methodsRaw) if !ok { continue } @@ -229,18 +272,6 @@ func parseRouteDirectives(input string) []*dqlshape.RouteDirective { return result } -func parseQuotedArgs(input string) []string { - matches := quotedArgDirective.FindAllStringSubmatch(input, -1) - result := make([]string, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { - continue - } - result = append(result, strings.TrimSpace(match[1])) - } - return result -} - func normalizeHTTPMethods(input []string) ([]string, bool) { if len(input) == 0 { return nil, true @@ -276,17 +307,25 @@ func normalizeHTTPMethods(input []string) ([]string, bool) { } func parseMarshalDirectives(input string) []string { - matches := marshalDirective.FindAllStringSubmatch(input, -1) - result := make([]string, 0, len(matches)) - for _, match := range matches { - if len(match) < 3 { + calls := scanDollarCalls(input, marshalDirectiveName) + result := make([]string, 0, len(calls)) + for _, call := range calls { + if len(call.args) != 2 { + continue + } + mimeType, ok := parseQuotedLiteral(call.args[0]) + if !ok { continue } - mimeType := strings.ToLower(strings.TrimSpace(match[1])) + mimeType = strings.ToLower(strings.TrimSpace(mimeType)) if mimeType != content.JSONContentType { continue } - if typeName := strings.TrimSpace(match[2]); typeName != "" { + typeName, ok := parseQuotedLiteral(call.args[1]) + if !ok { + continue + } + if typeName = strings.TrimSpace(typeName); typeName != "" { result = append(result, typeName) } } @@ -299,14 +338,22 @@ type unmarshalDirectiveValue struct { } func parseUnmarshalDirectives(input string) []unmarshalDirectiveValue { - matches := unmarshalDirective.FindAllStringSubmatch(input, -1) - result := make([]unmarshalDirectiveValue, 0, len(matches)) - for _, match := range matches { - if len(match) < 3 { + calls := scanDollarCalls(input, unmarshalDirectiveName) + result := make([]unmarshalDirectiveValue, 0, len(calls)) + for _, call := range calls { + if len(call.args) != 2 { + continue + } + mimeType, ok := parseQuotedLiteral(call.args[0]) + if !ok { continue } - mimeType := strings.ToLower(strings.TrimSpace(match[1])) - typeName := strings.TrimSpace(match[2]) + typeName, ok := parseQuotedLiteral(call.args[1]) + if !ok { + continue + } + mimeType = strings.ToLower(strings.TrimSpace(mimeType)) + typeName = strings.TrimSpace(typeName) if typeName == "" { continue } @@ -325,13 +372,17 @@ func parseUnmarshalDirectives(input string) []unmarshalDirectiveValue { } func parseFormatDirectives(input string) []string { - matches := formatDirective.FindAllStringSubmatch(input, -1) - result := make([]string, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { + calls := scanDollarCalls(input, formatDirectiveName) + result := make([]string, 0, len(calls)) + for _, call := range calls { + if len(call.args) != 1 { + continue + } + raw, ok := parseQuotedLiteral(call.args[0]) + if !ok { continue } - raw := strings.ToLower(strings.TrimSpace(match[1])) + raw = strings.ToLower(strings.TrimSpace(raw)) switch raw { case "tabular_json": result = append(result, content.JSONDataFormatTabular) @@ -343,13 +394,17 @@ func parseFormatDirectives(input string) []string { } func parseDateFormatDirectives(input string) []string { - matches := dateFormatDirective.FindAllStringSubmatch(input, -1) - result := make([]string, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { + calls := scanDollarCalls(input, dateFormatDirectiveName) + result := make([]string, 0, len(calls)) + for _, call := range calls { + if len(call.args) != 1 { continue } - if value := strings.TrimSpace(match[1]); value != "" { + value, ok := parseQuotedLiteral(call.args[0]) + if !ok { + continue + } + if value = strings.TrimSpace(value); value != "" { result = append(result, value) } } @@ -357,13 +412,17 @@ func parseDateFormatDirectives(input string) []string { } func parseCaseFormatDirectives(input string) []string { - matches := caseFormatDirective.FindAllStringSubmatch(input, -1) - result := make([]string, 0, len(matches)) - for _, match := range matches { - if len(match) < 2 { + calls := scanDollarCalls(input, caseFormatDirectiveName) + result := make([]string, 0, len(calls)) + for _, call := range calls { + if len(call.args) != 1 { + continue + } + value, ok := parseQuotedLiteral(call.args[0]) + if !ok { continue } - value := strings.TrimSpace(match[1]) + value = strings.TrimSpace(value) if value == "" { continue } diff --git a/repository/shape/dql/preprocess/typectx_directives.go b/repository/shape/dql/preprocess/typectx_directives.go index 6dacc98b9..0307ad07c 100644 --- a/repository/shape/dql/preprocess/typectx_directives.go +++ b/repository/shape/dql/preprocess/typectx_directives.go @@ -1,7 +1,6 @@ package preprocess import ( - "regexp" "strings" dqldiag "github.com/viant/datly/repository/shape/dql/diag" @@ -9,11 +8,6 @@ import ( "github.com/viant/datly/repository/shape/typectx" ) -var ( - packageLinePattern = regexp.MustCompile(`(?i)^\s*#package\s*\(\s*['\"]([^'\"]+)['\"]\s*\)\s*$`) - importLinePattern = regexp.MustCompile(`(?i)^\s*#import\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)\s*$`) -) - func parseTypeContextDirective(line, fullDQL string, offset int, ctx *typectx.Context) []*dqlshape.Diagnostic { var diagnostics []*dqlshape.Diagnostic if pkg, ok := parsePackageLineDirective(line); ok { @@ -47,11 +41,15 @@ func parseTypeContextDirective(line, fullDQL string, offset int, ctx *typectx.Co } func parsePackageLineDirective(line string) (string, bool) { - matches := packageLinePattern.FindStringSubmatch(line) - if len(matches) != 2 { + args, ok := parseExactHashDirectiveCall(line, "package") + if !ok || len(args) != 1 { return "", false } - value := strings.TrimSpace(matches[1]) + value, ok := parseQuotedLiteral(args[0]) + if !ok { + return "", false + } + value = strings.TrimSpace(value) if value == "" { return "", false } @@ -59,12 +57,20 @@ func parsePackageLineDirective(line string) (string, bool) { } func parseImportLineDirective(line string) (string, string, bool) { - matches := importLinePattern.FindStringSubmatch(line) - if len(matches) != 3 { + args, ok := parseExactHashDirectiveCall(line, "import") + if !ok || len(args) != 2 { return "", "", false } - alias := strings.TrimSpace(matches[1]) - pkg := strings.TrimSpace(matches[2]) + alias, ok := parseQuotedLiteral(args[0]) + if !ok { + return "", "", false + } + pkg, ok := parseQuotedLiteral(args[1]) + if !ok { + return "", "", false + } + alias = strings.TrimSpace(alias) + pkg = strings.TrimSpace(pkg) if alias == "" || pkg == "" { return "", "", false } @@ -72,7 +78,7 @@ func parseImportLineDirective(line string) (string, string, bool) { } func isTypeContextDirectiveLine(line string) bool { - line = strings.TrimSpace(line) + line = strings.ToLower(strings.TrimSpace(line)) switch { case strings.HasPrefix(line, "#package("), strings.HasPrefix(line, "#package ("): return true @@ -82,3 +88,34 @@ func isTypeContextDirectiveLine(line string) bool { return false } } + +func parseExactHashDirectiveCall(line, directive string) ([]string, bool) { + input := strings.TrimSpace(line) + if input == "" || input[0] != '#' { + return nil, false + } + index := skipSpaces(input, 1) + start := index + for index < len(input) && isIdentifierPart(input[index]) { + index++ + } + if start == index { + return nil, false + } + if !strings.EqualFold(input[start:index], directive) { + return nil, false + } + index = skipSpaces(input, index) + if index >= len(input) || input[index] != '(' { + return nil, false + } + body, end, ok := readCallBody(input, index) + if !ok { + return nil, false + } + index = skipSpaces(input, end+1) + if index != len(input) { + return nil, false + } + return splitCallArgs(body), true +} diff --git a/repository/shape/dql/sanitize/sanitizer.go b/repository/shape/dql/sanitize/sanitizer.go index f60b0e58a..ec414620d 100644 --- a/repository/shape/dql/sanitize/sanitizer.go +++ b/repository/shape/dql/sanitize/sanitizer.go @@ -2,7 +2,6 @@ package sanitize import ( "fmt" - "regexp" "strings" "github.com/viant/velty" @@ -21,17 +20,11 @@ type RewriteResult struct { TrimPrefix int } -var declarationHolderExpr = regexp.MustCompile(`(?i)#set\s*\(\s*\$_\s*=\s*\$([a-zA-Z_][a-zA-Z0-9_]*)`) - func Declared(input string) map[string]bool { ret := map[string]bool{} listener := &declaredListener{declared: ret} _, _, _ = velty.New(velty.Listener(listener)).Compile([]byte(input)) - for _, match := range declarationHolderExpr.FindAllStringSubmatch(input, -1) { - if len(match) < 2 { - continue - } - name := strings.TrimSpace(match[1]) + for _, name := range scanSetDeclaredHolders(input) { if name != "" { ret[name] = true } @@ -39,6 +32,103 @@ func Declared(input string) map[string]bool { return ret } +func scanSetDeclaredHolders(input string) []string { + result := make([]string, 0) + lower := strings.ToLower(input) + for i := 0; i < len(input); i++ { + if input[i] != '#' { + continue + } + if !strings.HasPrefix(lower[i:], "#set") { + continue + } + j := i + len("#set") + for j < len(input) && (input[j] == ' ' || input[j] == '\t' || input[j] == '\r' || input[j] == '\n') { + j++ + } + if j >= len(input) || input[j] != '(' { + continue + } + body, end, ok := readSetDirectiveBody(input, j) + if !ok { + continue + } + if name, ok := parseSetDeclaredHolder(body); ok { + result = append(result, name) + } + i = end + } + return result +} + +func parseSetDeclaredHolder(body string) (string, bool) { + text := strings.TrimSpace(body) + if text == "" { + return "", false + } + if !strings.HasPrefix(text, "$_") { + return "", false + } + text = strings.TrimSpace(text[len("$_"):]) + if !strings.HasPrefix(text, "=") { + return "", false + } + text = strings.TrimSpace(text[1:]) + if !strings.HasPrefix(text, "$") || len(text) < 2 { + return "", false + } + name := text[1:] + if !isSanitizeIdentifierStart(name[0]) { + return "", false + } + end := 1 + for end < len(name) && isSanitizeIdentifierPart(name[end]) { + end++ + } + return strings.TrimSpace(name[:end]), true +} + +func readSetDirectiveBody(input string, openParen int) (string, int, bool) { + depth := 0 + quote := byte(0) + for i := openParen; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '"' || ch == '\'' { + quote = ch + continue + } + if ch == '(' { + depth++ + continue + } + if ch == ')' { + depth-- + if depth == 0 { + return input[openParen+1 : i], i, true + } + } + } + return "", -1, false +} + +func isSanitizeIdentifierStart(ch byte) bool { + return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') +} + +func isSanitizeIdentifierPart(ch byte) bool { + return isSanitizeIdentifierStart(ch) || (ch >= '0' && ch <= '9') +} + func SQL(input string, opts Options) string { return Rewrite(input, opts).SQL } diff --git a/repository/shape/plan/model.go b/repository/shape/plan/model.go index 8935786a2..e9575b9b2 100644 --- a/repository/shape/plan/model.go +++ b/repository/shape/plan/model.go @@ -130,23 +130,26 @@ type RelationLink struct { // State is a normalized parameter field plan. type State struct { - Path string - Name string - Kind string - In string - QuerySelector string - When string - Scope string - DataType string - Value string - Required *bool - Async bool - Cacheable *bool - With string - URI string - ErrorCode int - ErrorMessage string - Predicates []*StatePredicate + Path string + Name string + Kind string + In string + Codec string + CodecArgs []string + QuerySelector string + When string + Scope string + DataType string + OutputDataType string + Value string + Required *bool + Async bool + Cacheable *bool + With string + URI string + ErrorCode int + ErrorMessage string + Predicates []*StatePredicate TagType reflect.Type EffectiveType reflect.Type diff --git a/repository/shape/xgen/io.go b/repository/shape/xgen/io.go index 395ea8e93..50e86ddaa 100644 --- a/repository/shape/xgen/io.go +++ b/repository/shape/xgen/io.go @@ -146,20 +146,6 @@ func fileExists(path string) (bool, error) { return false, err } -func isWithinProject(projectDir, candidate string) (bool, error) { - projectDir = filepath.Clean(projectDir) - candidate = filepath.Clean(candidate) - rel, err := filepath.Rel(projectDir, candidate) - if err != nil { - return false, err - } - if rel == "." { - return true, nil - } - rel = filepath.ToSlash(rel) - return !strings.HasPrefix(rel, "../"), nil -} - func mergeGeneratedShapes(dest string, generated []byte, typeNames []string) ([]byte, error) { existing, err := os.ReadFile(dest) if err != nil { From 66f763f080f5aa3e3b5fd4b35d48523f45d06f81 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Feb 2026 06:59:35 -0800 Subject: [PATCH 138/279] patched async jobs, enhanced dql grammar --- cmd/command/generate.go | 20 +++ gateway/async.go | 2 +- gateway/runtime/standalone/config.go | 4 +- internal/translator/parser/declarations.go | 6 +- internal/translator/resource.go | 149 ++++++++++++++++-- internal/translator/resource_settings_test.go | 24 +++ internal/translator/service.go | 11 +- repository/contract/signature/service.go | 44 +++++- repository/handler/handler.go | 38 ++++- 9 files changed, 271 insertions(+), 27 deletions(-) diff --git a/cmd/command/generate.go b/cmd/command/generate.go index deb3d0ee8..c516f91d5 100644 --- a/cmd/command/generate.go +++ b/cmd/command/generate.go @@ -174,6 +174,7 @@ func (s *Service) generateGet(ctx context.Context, opts *options.Options) (err e if compErr != nil { return compErr } + applyDefaultComponentPackage(aComponent, translate.Rule.ModulePrefix) _, sourceName := path.Split(url.Path(source)) sourceName = trimExt(sourceName) var embeds = map[string]string{} @@ -212,6 +213,7 @@ func (s *Service) generateGet(ctx context.Context, opts *options.Options) (err e if err != nil { return err } + applyDefaultComponentPackage(aComponent, modulePrefix) var embeds = map[string]string{} var namedResources []string @@ -235,6 +237,24 @@ func (s *Service) generateGet(ctx context.Context, opts *options.Options) (err e return nil } +func applyDefaultComponentPackage(component *repository.Component, modulePrefix string) { + if component == nil { + return + } + if component.Output.Type.Package != "" || component.Input.Type.Package != "" { + return + } + modulePrefix = strings.Trim(modulePrefix, "/") + if modulePrefix == "" { + return + } + base := path.Base(modulePrefix) + if base == "" || base == "." || base == "/" { + return + } + component.Output.Type.Package = strings.ReplaceAll(base, "-", "_") +} + func (s *Service) persistEmbeds(ctx context.Context, moduleLocation string, modulePrefix string, embeds map[string]string, component *repository.Component) error { rootName := component.View.Name formatter := text.DetectCaseFormat(rootName) diff --git a/gateway/async.go b/gateway/async.go index 2ef1b2c83..d7228a14c 100644 --- a/gateway/async.go +++ b/gateway/async.go @@ -76,7 +76,7 @@ func (r *Service) watchAsyncJob(ctx context.Context) { err = fs.Move(ctx, object.URL(), destURL) } if err != nil { - log.Println(err) + log.Printf("datly async post-process failed: source=%q err=%v", object.URL(), err) } } else { diff --git a/gateway/runtime/standalone/config.go b/gateway/runtime/standalone/config.go index 34399adf1..a6c769521 100644 --- a/gateway/runtime/standalone/config.go +++ b/gateway/runtime/standalone/config.go @@ -90,10 +90,10 @@ func (c *Config) normalizeURLs(baseURL string) { if url.IsRelative(c.DependencyURL) { c.DependencyURL = url.Join(baseURL, c.DependencyURL) } - if url.IsRelative(c.JobURL) { + if c.JobURL != "" && url.IsRelative(c.JobURL) { c.JobURL = url.Join(baseURL, c.JobURL) } - if url.IsRelative(c.FailedJobURL) { + if c.FailedJobURL != "" && url.IsRelative(c.FailedJobURL) { c.FailedJobURL = url.Join(baseURL, c.FailedJobURL) } } diff --git a/internal/translator/parser/declarations.go b/internal/translator/parser/declarations.go index 0a9885aa6..69b017ec2 100644 --- a/internal/translator/parser/declarations.go +++ b/internal/translator/parser/declarations.go @@ -147,7 +147,11 @@ func (d *Declarations) parseExpression(cursor *parsly.Cursor, selector *expr.Sel declaration.Kind = segments[0] location := "" if len(segments) > 1 { - location = strings.Join(segments[1:], ".") + joiner := "." + if declaration.Kind == string(state.KindComponent) { + joiner = "/" + } + location = strings.Join(segments[1:], joiner) } declaration.Location = &location declaration.InOutput = declaration.Kind == string(state.KindOutput) diff --git a/internal/translator/resource.go b/internal/translator/resource.go index 4330ce91f..bf63f33ef 100644 --- a/internal/translator/resource.go +++ b/internal/translator/resource.go @@ -32,18 +32,29 @@ import ( ) var ( - routeSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$route\s*\(([^)]*)\)\s*\)\s*$`) - marshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$marshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) - unmarshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$unmarshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) - formatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) - dateFormatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$date_format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) - caseFormatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#settings\s*\(\s*\$_\s*=\s*\$case_format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + routeSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$route\s*\(([^)]*)\)\s*\)\s*$`) + packageLineExpr = regexp.MustCompile(`(?im)^\s*#package\s*\(\s*['"]([^'"]+)['"]\s*\)\s*$`) + hashImportLineExpr = regexp.MustCompile(`(?im)^\s*#import\s*\(([^)]*)\)\s*$`) + connectorSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$connector\s*\(([^)]*)\)\s*\)\s*$`) + handlerSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$handler\s*\(([^)]*)\)\s*\)\s*$`) + inputSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$input\s*\(([^)]*)\)\s*\)\s*$`) + outputSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$output\s*\(([^)]*)\)\s*\)\s*$`) + marshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$marshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + unmarshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$unmarshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + formatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + dateFormatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$date_format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) + caseFormatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$case_format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) quotedArgExpr = regexp.MustCompile(`['"]([^'"]*)['"]`) ) type routeSettingsDirective struct { URI string Methods []string + Package string + Connector string + HandlerType string + InputType string + OutputType string JSONMarshalType string JSONUnmarshalType string XMLUnmarshalType string @@ -168,6 +179,7 @@ func (r *Resource) ensureRegistry() *xreflect.Types { } func (r *Resource) parseImports(ctx context.Context, dSQL *string) (err error) { + *dSQL = removeHashImportDirectives(*dSQL) if r.Rule.TypeSrc != nil { if err = r.loadImportTypes(ctx, r.Rule.TypeSrc); err != nil { return err @@ -441,14 +453,29 @@ func (r *Resource) extractRuleSetting(dSQL *string) error { if len(directive.Methods) > 0 { r.Rule.Method = strings.Join(directive.Methods, ",") } + if directive.Package != "" { + r.Rule.Package = directive.Package + } + if directive.Connector != "" { + r.Rule.Connector = directive.Connector + } + if directive.HandlerType != "" { + r.Rule.Type = qualifyTypeWithPackage(directive.HandlerType, r.Rule.Package) + } + if directive.InputType != "" { + r.Rule.InputType = qualifyTypeWithPackage(directive.InputType, r.Rule.Package) + } + if directive.OutputType != "" { + r.Rule.OutputType = qualifyTypeWithPackage(directive.OutputType, r.Rule.Package) + } if directive.JSONMarshalType != "" { - r.Rule.JSONMarshalType = directive.JSONMarshalType + r.Rule.JSONMarshalType = qualifyTypeWithPackage(directive.JSONMarshalType, r.Rule.Package) } if directive.JSONUnmarshalType != "" { - r.Rule.JSONUnmarshalType = directive.JSONUnmarshalType + r.Rule.JSONUnmarshalType = qualifyTypeWithPackage(directive.JSONUnmarshalType, r.Rule.Package) } if directive.XMLUnmarshalType != "" { - r.Rule.XMLUnmarshalType = directive.XMLUnmarshalType + r.Rule.XMLUnmarshalType = qualifyTypeWithPackage(directive.XMLUnmarshalType, r.Rule.Package) } if directive.Format != "" { r.Rule.DataFormat = directive.Format @@ -472,7 +499,16 @@ func (r *Resource) extractRuleSetting(dSQL *string) error { func parseSettingsDirectives(dSQL string) (*routeSettingsDirective, bool, error) { ret := &routeSettingsDirective{} var found bool - matches := routeSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + matches := packageLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + if len(last) < 2 || strings.TrimSpace(last[1]) == "" { + return nil, false, fmt.Errorf("invalid #package directive") + } + ret.Package = strings.TrimSpace(last[1]) + } + matches = routeSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) if len(matches) > 0 { found = true last := matches[len(matches)-1] @@ -495,6 +531,50 @@ func parseSettingsDirectives(dSQL string) (*routeSettingsDirective, bool, error) ret.Methods = methods } + matches = connectorSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + value := parseSingleArg(last) + if value == "" { + return nil, false, fmt.Errorf("invalid $connector directive") + } + ret.Connector = value + } + + matches = handlerSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + value := parseSingleArg(last) + if value == "" { + return nil, false, fmt.Errorf("invalid $handler directive") + } + ret.HandlerType = value + } + + matches = inputSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + value := parseSingleArg(last) + if value == "" { + return nil, false, fmt.Errorf("invalid $input directive") + } + ret.InputType = value + } + + matches = outputSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + value := parseSingleArg(last) + if value == "" { + return nil, false, fmt.Errorf("invalid $output directive") + } + ret.OutputType = value + } + matches = marshalSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) if len(matches) > 0 { found = true @@ -592,6 +672,15 @@ func parseQuotedArgs(input string) []string { return result } +func parseSingleArg(match []string) string { + if len(match) < 2 { + return "" + } + value := strings.TrimSpace(match[1]) + value = strings.Trim(value, `"'`) + return strings.TrimSpace(value) +} + func normalizeRouteMethods(input []string) ([]string, error) { if len(input) == 0 { return nil, nil @@ -627,7 +716,12 @@ func normalizeRouteMethods(input []string) ([]string, error) { } func removeSettingsDirectives(dSQL string) string { + dSQL = packageLineExpr.ReplaceAllString(dSQL, "") dSQL = routeSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = connectorSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = handlerSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = inputSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = outputSettingsLineExpr.ReplaceAllString(dSQL, "") dSQL = marshalSettingsLineExpr.ReplaceAllString(dSQL, "") dSQL = unmarshalSettingsLineExpr.ReplaceAllString(dSQL, "") dSQL = formatSettingsLineExpr.ReplaceAllString(dSQL, "") @@ -636,6 +730,41 @@ func removeSettingsDirectives(dSQL string) string { return dSQL } +func removeHashImportDirectives(dSQL string) string { + return hashImportLineExpr.ReplaceAllString(dSQL, "") +} + +func qualifyTypeWithPackage(typeName, pkg string) string { + typeName = strings.TrimSpace(typeName) + pkg = strings.TrimSpace(pkg) + if typeName == "" || pkg == "" { + return typeName + } + + prefix := "" + base := typeName + for { + switch { + case strings.HasPrefix(base, "[]"): + prefix += "[]" + base = strings.TrimPrefix(base, "[]") + case strings.HasPrefix(base, "*"): + prefix += "*" + base = strings.TrimPrefix(base, "*") + default: + goto done + } + } +done: + if base == "" { + return typeName + } + if strings.Contains(base, ".") || strings.Contains(base, "/") || strings.Contains(base, "[") { + return typeName + } + return prefix + pkg + "." + base +} + func (r *Resource) expandSQL(viewlet *Viewlet) (*sqlx.SQL, error) { types := viewlet.Resource.Resource.TypeRegistry() resourceState := viewlet.Resource.State diff --git a/internal/translator/resource_settings_test.go b/internal/translator/resource_settings_test.go index eeef20112..32de4d39b 100644 --- a/internal/translator/resource_settings_test.go +++ b/internal/translator/resource_settings_test.go @@ -55,3 +55,27 @@ func TestResource_extractRuleSetting_InvalidCaseFormatDirective(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "unsupported case format") } + +func TestResource_extractRuleSetting_PackageQualifiesTypes(t *testing.T) { + resource := &Resource{Rule: NewRule(), rule: &options.Rule{}} + dSQL := "#package('github.vianttech.com/viant/handson/pkg/platform/acl/auth')\n" + + "#settings($_ = $handler('Handler'))\n" + + "#settings($_ = $input('Input'))\n" + + "#settings($_ = $output('Output'))\n" + + "#settings($_ = $marshal('application/json','JSONOut'))\n" + + "#settings($_ = $unmarshal('application/json','JSONIn'))\n" + + "SELECT 1" + + err := resource.extractRuleSetting(&dSQL) + require.NoError(t, err) + assert.Equal(t, "github.vianttech.com/viant/handson/pkg/platform/acl/auth", resource.Rule.Package) + assert.Equal(t, "github.vianttech.com/viant/handson/pkg/platform/acl/auth.Handler", resource.Rule.Type) + assert.Equal(t, "github.vianttech.com/viant/handson/pkg/platform/acl/auth.Input", resource.Rule.InputType) + assert.Equal(t, "github.vianttech.com/viant/handson/pkg/platform/acl/auth.Output", resource.Rule.OutputType) + assert.Equal(t, "github.vianttech.com/viant/handson/pkg/platform/acl/auth.JSONOut", resource.Rule.JSONMarshalType) + assert.Equal(t, "github.vianttech.com/viant/handson/pkg/platform/acl/auth.JSONIn", resource.Rule.JSONUnmarshalType) + assert.NotContains(t, dSQL, "#package(") + assert.NotContains(t, dSQL, "$handler(") + assert.NotContains(t, dSQL, "$input(") + assert.NotContains(t, dSQL, "$output(") +} diff --git a/internal/translator/service.go b/internal/translator/service.go index f446521a3..f383b9a37 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -117,9 +117,12 @@ func (s *Service) discoverComponentContract(ctx context.Context, resource *Resou return nil, err } } - location.Name = strings.ReplaceAll(location.Name, "..", "[]") - location.Name = strings.ReplaceAll(location.Name, ".", "/") - method, URI := shared.ExtractPath(location.Name) + locationName := strings.TrimSpace(location.Name) + if !strings.Contains(locationName, "/") { + locationName = strings.ReplaceAll(locationName, "..", "[]") + locationName = strings.ReplaceAll(locationName, ".", "/") + } + method, URI := shared.ExtractPath(locationName) return s.signature.Signature(method, URI) } @@ -448,7 +451,7 @@ func (s *Service) persistDocumentation(ctx context.Context, resource *Resource, } func extractTypeNameWithPackage(outputName string) (string, string) { - if index := strings.Index(outputName, "."); index != -1 { + if index := strings.LastIndex(outputName, "."); index != -1 { return outputName[:index], outputName[index+1:] } return outputName, "" diff --git a/repository/contract/signature/service.go b/repository/contract/signature/service.go index d1f79dbc8..572d0c3c5 100644 --- a/repository/contract/signature/service.go +++ b/repository/contract/signature/service.go @@ -58,6 +58,9 @@ func (s *Service) init(ctx context.Context) error { func (s *Service) Signature(method, URI string) (*Signature, error) { URI = strings.ReplaceAll(URI, "[]", "..") matchable, err := s.matcher.MatchOne(method, URI) + if err != nil && !strings.HasPrefix(URI, "/") { + matchable, err = s.matcher.MatchOne(method, "/"+URI) + } if err != nil && s.APIPrefix != "" { //fallback to full URI matchable, err = s.matcher.MatchOne(method, s.buildURI(URI)) } @@ -106,17 +109,42 @@ func (s *Service) Signature(method, URI string) (*Signature, error) { } func (s *Service) buildURI(URI string) string { - APIPrefix := strings.Split(s.APIPrefix, "/") - URIs := strings.Split(URI, "/") - var suffix []string - for _, item := range URIs { - if item == ".." { - APIPrefix = APIPrefix[:len(APIPrefix)-1] + URI = strings.TrimSpace(URI) + if URI == "" { + return strings.TrimRight(s.APIPrefix, "/") + } + if strings.HasPrefix(URI, "/") { + return URI + } + + prefixParts := splitPathParts(s.APIPrefix) + uriParts := splitPathParts(URI) + for _, part := range uriParts { + switch part { + case ".", "": + continue + case "..": + if len(prefixParts) > 0 { + prefixParts = prefixParts[:len(prefixParts)-1] + } + default: + prefixParts = append(prefixParts, part) + } + } + return "/" + strings.Join(prefixParts, "/") +} + +func splitPathParts(input string) []string { + raw := strings.Split(input, "/") + result := make([]string, 0, len(raw)) + for _, item := range raw { + item = strings.TrimSpace(item) + if item == "" || item == "." { continue } - suffix = append(suffix, item) + result = append(result, item) } - return strings.Join(append(APIPrefix, suffix...), "/") + return result } func (s *Service) loadSignatures(ctx context.Context, URL string, isRoot bool) error { diff --git a/repository/handler/handler.go b/repository/handler/handler.go index 3d1c173d7..f8434d71a 100644 --- a/repository/handler/handler.go +++ b/repository/handler/handler.go @@ -7,7 +7,10 @@ import ( "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/xdatly/handler" + "github.com/viant/xreflect" + "github.com/viant/xunsafe" "reflect" + "strings" ) var Type = reflect.TypeOf((*handler.Handler)(nil)).Elem() @@ -36,7 +39,9 @@ func (h *Handler) Init(ctx context.Context, resource *view.Resource) (err error) h.resource = resource aType, err = h.resource.TypeRegistry().Lookup(h.Type) if err != nil { - return fmt.Errorf("couldn't parse Handler type due to %w", err) + if aType = lookupByPackagePathAlias(h.resource.TypeRegistry().Lookup, h.Type); aType == nil { + return fmt.Errorf("couldn't parse Handler type due to %w", err) + } } } if aType.Kind() != reflect.Ptr { @@ -125,3 +130,34 @@ func NewHandler(handler handler.Handler) *Handler { rType := reflect.TypeOf(handler) return &Handler{Type: rType.Name(), _type: rType} } + +func lookupByPackagePathAlias(lookup xreflect.LookupType, typeName string) reflect.Type { + typeName = strings.TrimSpace(typeName) + index := strings.LastIndex(typeName, ".") + if index == -1 || index == len(typeName)-1 { + return nil + } + pkgPath := typeName[:index] + name := typeName[index+1:] + if !strings.Contains(pkgPath, "/") { + return nil + } + segments := strings.Split(pkgPath, "/") + var candidates []string + if len(segments) >= 2 { + candidates = append(candidates, strings.Join(segments[len(segments)-2:], "/")) + } + candidates = append(candidates, segments[len(segments)-1]) + for _, candidate := range candidates { + if candidate == "" { + continue + } + if rType, err := lookup(name, xreflect.WithPackage(candidate)); err == nil && rType != nil { + return rType + } + } + if rType := xunsafe.LookupType(pkgPath + "/" + name); rType != nil { + return rType + } + return nil +} From 3aae6be4f0aa1a65b2697b9760e0644c81b26cfe Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Feb 2026 07:52:15 -0800 Subject: [PATCH 139/279] patched async jobs, enhanced dql grammar --- gateway/router/openapi/tag.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gateway/router/openapi/tag.go b/gateway/router/openapi/tag.go index 7c144f337..a4c343c6f 100644 --- a/gateway/router/openapi/tag.go +++ b/gateway/router/openapi/tag.go @@ -80,6 +80,11 @@ func ParseTag(field reflect.StructField, tag reflect.StructTag, isInput bool, ro _tag: *aTag, } + // Keep internal runtime-only fields out of OpenAPI schema. + if tag.Get("internal") == "true" { + ret.Ignore = true + } + if tags, _ := tags.Parse(tag, nil, tags.ParameterTag); tags != nil { ret.Parameter = tags.Parameter if parameter := ret.Parameter; parameter != nil && parameter.Kind != "" { From e5536e0f90324d8cf127247221ef0919c65c4b4b Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Feb 2026 09:26:12 -0800 Subject: [PATCH 140/279] refactor openapi schema/generator and add polymorphism scaffolding --- gateway/router/openapi/generator_operation.go | 81 +++ gateway/router/openapi/generator_paths.go | 53 ++ gateway/router/openapi/helpers_test.go | 42 ++ gateway/router/openapi/logic_test.go | 310 ++++++++++ gateway/router/openapi/schema.go | 299 +-------- gateway/router/openapi/schema_build.go | 574 ++++++++++++++++++ .../openapi/schema_build_helpers_test.go | 292 +++++++++ gateway/router/openapi/schema_helpers_test.go | 329 ++++++++++ gateway/router/openapi/tag_parse_test.go | 73 +++ 9 files changed, 1755 insertions(+), 298 deletions(-) create mode 100644 gateway/router/openapi/generator_operation.go create mode 100644 gateway/router/openapi/generator_paths.go create mode 100644 gateway/router/openapi/helpers_test.go create mode 100644 gateway/router/openapi/logic_test.go create mode 100644 gateway/router/openapi/schema_build.go create mode 100644 gateway/router/openapi/schema_build_helpers_test.go create mode 100644 gateway/router/openapi/schema_helpers_test.go create mode 100644 gateway/router/openapi/tag_parse_test.go diff --git a/gateway/router/openapi/generator_operation.go b/gateway/router/openapi/generator_operation.go new file mode 100644 index 000000000..704ffcfb2 --- /dev/null +++ b/gateway/router/openapi/generator_operation.go @@ -0,0 +1,81 @@ +package openapi + +import ( + "context" + openapi "github.com/viant/datly/gateway/router/openapi/openapi3" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/shared" + "github.com/viant/datly/view/state" +) + +func (g *generator) generateOperation(ctx context.Context, component *ComponentSchema) (*openapi.Operation, error) { + body, err := g.requestBody(ctx, component) + if err != nil { + return nil, err + } + + parameters, err := g.operationParameters(ctx, component) + if err != nil { + return nil, err + } + + responses, err := g.responses(ctx, component) + if err != nil { + return nil, err + } + + return &openapi.Operation{ + Parameters: dedupe(parameters), + RequestBody: body, + Responses: responses, + }, nil +} + +func (g *generator) operationParameters(ctx context.Context, component *ComponentSchema) ([]*openapi.Parameter, error) { + parameters, err := g.getAllViewsParameters(ctx, component, component.component.View) + if err != nil { + return nil, err + } + + componentParams, err := g.componentOutputParameters(ctx, component) + if err != nil { + return nil, err + } + return append(parameters, componentParams...), nil +} + +func (g *generator) componentOutputParameters(ctx context.Context, component *ComponentSchema) ([]*openapi.Parameter, error) { + result := make([]*openapi.Parameter, 0) + err := g.forEachParam(component.component.Output.Type.Parameters, func(parameter *state.Parameter) (bool, error) { + if parameter.In.Kind != state.KindComponent { + return true, nil + } + + paramComponent, err := g.lookupComponentParam(ctx, component, parameter.In.Name) + if err != nil { + return false, err + } + + viewsParameters, err := g.getAllViewsParameters(ctx, NewComponentSchema(component.components, paramComponent, component.schemas), paramComponent.View) + if err != nil { + return false, err + } + + result = append(result, viewsParameters...) + return true, nil + }) + if err != nil { + return nil, err + } + return result, nil +} + +func (g *generator) lookupComponentParam(ctx context.Context, component *ComponentSchema, path string) (*repository.Component, error) { + method, URI := shared.ExtractPath(path) + provider, err := component.components.Registry().LookupProvider(ctx, &contract.Path{URI: URI, Method: method}) + if err != nil { + return nil, err + } + return provider.Component(ctx) +} diff --git a/gateway/router/openapi/generator_paths.go b/gateway/router/openapi/generator_paths.go new file mode 100644 index 000000000..c36ed5c6a --- /dev/null +++ b/gateway/router/openapi/generator_paths.go @@ -0,0 +1,53 @@ +package openapi + +import ( + "context" + "fmt" + openapi "github.com/viant/datly/gateway/router/openapi/openapi3" + "github.com/viant/datly/repository" + "net/http" +) + +func (g *generator) generatePaths(ctx context.Context, components *repository.Service, providers []*repository.Provider) (*SchemaContainer, openapi.Paths, error) { + container := NewContainer() + builder := &PathsBuilder{paths: openapi.Paths{}} + var retErr error + + for _, provider := range providers { + component, err := provider.Component(ctx) + if err != nil { + retErr = err + } + if component == nil { + fmt.Printf("provider.Component(ctx) returned nil\n") + continue + } + + componentSchema := NewComponentSchema(components, component, container) + operation, err := g.generateOperation(ctx, componentSchema) + if err != nil { + retErr = err + } + + pathItem := &openapi.PathItem{} + attachOperation(pathItem, component.Method, operation) + builder.AddPath(component.URI, pathItem) + } + + return container, builder.paths, retErr +} + +func attachOperation(pathItem *openapi.PathItem, method string, operation *openapi.Operation) { + switch method { + case http.MethodGet: + pathItem.Get = operation + case http.MethodPost: + pathItem.Post = operation + case http.MethodDelete: + pathItem.Delete = operation + case http.MethodPut: + pathItem.Put = operation + case http.MethodPatch: + pathItem.Patch = operation + } +} diff --git a/gateway/router/openapi/helpers_test.go b/gateway/router/openapi/helpers_test.go new file mode 100644 index 000000000..44d4c56ef --- /dev/null +++ b/gateway/router/openapi/helpers_test.go @@ -0,0 +1,42 @@ +package openapi + +import ( + "testing" + + openapi3 "github.com/viant/datly/gateway/router/openapi/openapi3" +) + +func TestDedupe(t *testing.T) { + tests := []struct { + name string + in []*openapi3.Parameter + expectNames []string + }{ + { + name: "dedupes by name and location", + in: []*openapi3.Parameter{ + {Name: "id", In: "query"}, + {Name: "id", In: "query"}, + {Name: "id", In: "path"}, + {Name: "limit", In: "query"}, + }, + expectNames: []string{"id:query", "id:path", "limit:query"}, + }, + {name: "empty", in: nil, expectNames: nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := dedupe(tt.in) + if len(out) != len(tt.expectNames) { + t.Fatalf("expected len %d, got %d", len(tt.expectNames), len(out)) + } + for i := range out { + actual := out[i].Name + ":" + out[i].In + if actual != tt.expectNames[i] { + t.Fatalf("at %d expected %q, got %q", i, tt.expectNames[i], actual) + } + } + }) + } +} diff --git a/gateway/router/openapi/logic_test.go b/gateway/router/openapi/logic_test.go new file mode 100644 index 000000000..8c35e2d5e --- /dev/null +++ b/gateway/router/openapi/logic_test.go @@ -0,0 +1,310 @@ +package openapi + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + "unsafe" + + openapi3 "github.com/viant/datly/gateway/router/openapi/openapi3" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/datly/view/tags" + "github.com/viant/tagly/format" + "github.com/viant/xreflect" +) + +type fakeDocService struct { + lookup func(key string) (string, bool, error) +} + +func (f *fakeDocService) Lookup(ctx context.Context, key string) (string, bool, error) { + if f.lookup == nil { + return "", false, nil + } + return f.lookup(key) +} + +func setUnexportedField(target interface{}, fieldName string, value interface{}) { + v := reflect.ValueOf(target).Elem().FieldByName(fieldName) + reflect.NewAt(v.Type(), unsafe.Pointer(v.UnsafeAddr())).Elem().Set(reflect.ValueOf(value)) +} + +func newTestComponent(t *testing.T) *repository.Component { + t.Helper() + component, err := repository.NewComponent(&contract.Path{Method: "POST", URI: "/v1/test"}, repository.WithView(&view.View{Template: &view.Template{}, Selector: &view.Config{}})) + if err != nil { + t.Fatalf("failed to create component: %v", err) + } + types := xreflect.NewTypes() + setUnexportedField(component, "types", types) + return component +} + +func TestPathsBuilderAddPath(t *testing.T) { + builder := &PathsBuilder{paths: openapi3.Paths{}} + item := &openapi3.PathItem{Summary: "sum"} + builder.AddPath("/v1/pets", item) + if builder.paths["/v1/pets"] != item { + t.Fatalf("path not added") + } +} + +func TestGeneratorHelpers_Table(t *testing.T) { + t.Run("forEachParam recursive and error", func(t *testing.T) { + g := &generator{} + called := 0 + params := state.Parameters{ + {Name: "root", Object: state.Parameters{{Name: "child1"}}, Repeated: state.Parameters{{Name: "child2"}}}, + } + err := g.forEachParam(params, func(parameter *state.Parameter) (bool, error) { + called++ + if parameter.Name == "child1" { + return true, errors.New("boom") + } + return true, nil + }) + if err == nil || err.Error() != "boom" { + t.Fatalf("expected boom, got %v", err) + } + if called < 2 { + t.Fatalf("expected recursive traversal") + } + }) + + t.Run("index parameters", func(t *testing.T) { + g := &generator{} + params := []*openapi3.Parameter{{Name: "a"}, {Name: "b"}} + indexed := g.indexParameters(params) + if indexed["a"].Name != "a" || indexed["b"].Name != "b" { + t.Fatalf("unexpected indexed values") + } + }) + + t.Run("string ptr", func(t *testing.T) { + if *stringPtr("x") != "x" { + t.Fatalf("unexpected value") + } + }) +} + +func TestComponentSchemaHelpers_Table(t *testing.T) { + component := newTestComponent(t) + componentSchema := &ComponentSchema{component: component, schemas: NewContainer()} + + t.Run("isRequired", func(t *testing.T) { + req := true + in := contract.Input{Body: state.Type{Parameters: state.Parameters{{Required: &req}}}} + if !componentSchema.isRequired(in) { + t.Fatalf("expected required") + } + }) + + t.Run("description and example defaults", func(t *testing.T) { + desc, err := componentSchema.Description(context.Background(), "A", "default-desc") + if err != nil || desc != "default-desc" { + t.Fatalf("unexpected result: %q %v", desc, err) + } + example, err := componentSchema.Example(context.Background(), "A", "default-ex") + if err != nil || example != "default-ex" { + t.Fatalf("unexpected result: %q %v", example, err) + } + }) + + t.Run("description and example from doc", func(t *testing.T) { + componentSchema.doc = &fakeDocService{lookup: func(key string) (string, bool, error) { + switch key { + case "A": + return "desc", true, nil + case "A$example": + return "ex", true, nil + default: + return "", false, nil + } + }} + desc, err := componentSchema.Description(context.Background(), "A", "default-desc") + if err != nil || desc != "desc" { + t.Fatalf("unexpected description: %q %v", desc, err) + } + example, err := componentSchema.Example(context.Background(), "A", "default-ex") + if err != nil || example != "ex" { + t.Fatalf("unexpected example: %q %v", example, err) + } + }) + + t.Run("description error", func(t *testing.T) { + componentSchema.doc = &fakeDocService{lookup: func(key string) (string, bool, error) { + return "", false, errors.New("lookup") + }} + if _, err := componentSchema.Description(context.Background(), "A", "default"); err == nil { + t.Fatalf("expected error") + } + }) + + t.Run("typed/request/response schema", func(t *testing.T) { + componentSchema.doc = nil + component.Input.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + component.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + component.Input.Body = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + + reqSchema, err := componentSchema.RequestBody(context.Background()) + if err != nil || reqSchema == nil { + t.Fatalf("unexpected request schema result: %v %v", reqSchema, err) + } + + respSchema, err := componentSchema.ResponseBody(context.Background()) + if err != nil || respSchema == nil { + t.Fatalf("unexpected response schema result: %v %v", respSchema, err) + } + + if _, err = componentSchema.TypedSchema(context.Background(), component.Input.Type, "Input", component.IOConfig(), true); err != nil { + t.Fatalf("unexpected typed schema error: %v", err) + } + }) + + t.Run("type name and schema helpers", func(t *testing.T) { + type sample struct{} + types := xreflect.NewTypes() + if err := types.Register("Sample", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf(sample{}))); err != nil { + t.Fatalf("register type failed: %v", err) + } + setUnexportedField(component, "types", types) + + if got := componentSchema.TypeName(reflect.TypeOf(sample{}), "fallback"); got != "Sample" { + t.Fatalf("expected Sample, got %s", got) + } + + refl := componentSchema.ReflectSchema("A", reflect.TypeOf(sample{}), "d", component.IOConfig()) + if refl == nil || refl.rType != reflect.TypeOf(sample{}) { + t.Fatalf("unexpected reflect schema") + } + + withTag := componentSchema.SchemaWithTag("F", reflect.TypeOf(sample{}), "d", component.IOConfig(), Tag{}) + if withTag == nil || withTag.path == "" { + t.Fatalf("unexpected schema with tag") + } + }) + + t.Run("schema with tag datatype override", func(t *testing.T) { + type alt struct{ Value string } + reg := xreflect.NewTypes() + if err := reg.Register("Alt", xreflect.WithReflectType(reflect.TypeOf(alt{}))); err != nil { + t.Fatalf("register type failed: %v", err) + } + if component.View.GetResource() == nil { + component.View.SetResource(&view.Resource{}) + } + component.View.GetResource().SetTypes(reg) + withTag := componentSchema.SchemaWithTag("F", reflect.TypeOf(struct{ A int }{}), "d", component.IOConfig(), Tag{ + Parameter: &tags.Parameter{DataType: "Alt"}, + }) + if withTag.rType != reflect.TypeOf(alt{}) { + t.Fatalf("expected datatype override") + } + }) +} + +func TestSchemaContainerCreateSchema_Table(t *testing.T) { + container := NewContainer() + componentSchema := &ComponentSchema{component: newTestComponent(t), schemas: container} + fieldSchema := &Schema{path: "p", description: "d", rType: reflect.TypeOf(1)} + + t.Run("create primitive", func(t *testing.T) { + result, err := container.CreateSchema(context.Background(), componentSchema, fieldSchema) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Type != integerOutput { + t.Fatalf("unexpected type: %s", result.Type) + } + }) + + t.Run("get or generate delegates", func(t *testing.T) { + result, err := componentSchema.GetOrGenerateSchema(context.Background(), fieldSchema) + if err != nil || result.Type != integerOutput { + t.Fatalf("unexpected result: %v %v", result, err) + } + }) + + t.Run("create cached ref", func(t *testing.T) { + container.generatedSchemas["Cached"] = &openapi3.Schema{Type: objectOutput} + cached, err := container.createSchema(context.Background(), componentSchema, &Schema{path: "p", description: "d", rType: reflect.TypeOf(struct{}{}), tag: Tag{TypeName: "Cached"}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cached.Ref != "#/components/schemas/Cached" { + t.Fatalf("unexpected ref: %s", cached.Ref) + } + }) + + t.Run("create struct and generate schema", func(t *testing.T) { + type rec struct { + ID int `json:"id"` + } + sch, err := container.createSchema(context.Background(), componentSchema, &Schema{ + path: "rec", + description: "record", + rType: reflect.TypeOf(rec{}), + tag: Tag{TypeName: "Rec"}, + ioConfig: componentSchema.component.IOConfig(), + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if sch.Ref == "" { + t.Fatalf("expected ref schema") + } + }) + + t.Run("addToSchema time format", func(t *testing.T) { + dst := &openapi3.Schema{} + err := container.addToSchema(context.Background(), componentSchema, dst, &Schema{ + rType: reflect.TypeOf(time.Time{}), + tag: Tag{_tag: format.Tag{TimeLayout: "2006-01-02"}}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if dst.Type != stringOutput || dst.Format != "date" { + t.Fatalf("unexpected time schema: %s %s", dst.Type, dst.Format) + } + }) + + t.Run("addToSchema struct filtering and inline", func(t *testing.T) { + type payload struct { + Visible string `json:"visible"` + Hidden string `json:"-"` + Internal string `internal:"true"` + Meta map[string]string `json:",inline"` + } + dst := &openapi3.Schema{} + err := container.addToSchema(context.Background(), componentSchema, dst, &Schema{ + rType: reflect.TypeOf(payload{}), + ioConfig: componentSchema.component.IOConfig(), + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := dst.Properties["visible"]; !ok { + t.Fatalf("expected visible field") + } + if _, ok := dst.Properties["hidden"]; ok { + t.Fatalf("did not expect hidden field") + } + if _, ok := dst.Properties["internal"]; ok { + t.Fatalf("did not expect internal field") + } + }) +} + +func TestNewComponentSchema(t *testing.T) { + component := &repository.Component{} + got := NewComponentSchema(nil, component, nil) + if got == nil || got.schemas == nil { + t.Fatalf("expected initialized component schema") + } +} diff --git a/gateway/router/openapi/schema.go b/gateway/router/openapi/schema.go index ef707ec06..cd7a5e7dd 100644 --- a/gateway/router/openapi/schema.go +++ b/gateway/router/openapi/schema.go @@ -2,23 +2,16 @@ package openapi import ( "context" - "fmt" "github.com/viant/datly/gateway/router/marshal/config" "github.com/viant/datly/gateway/router/openapi/openapi3" - "github.com/viant/datly/internal/setter" "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" - "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/datly/view/tags" "github.com/viant/tagly/format/text" - ftime "github.com/viant/tagly/format/time" "github.com/viant/xdatly/docs" - "github.com/viant/xreflect" "reflect" - "strings" "sync" - "time" ) const ( @@ -245,6 +238,7 @@ func (c *ComponentSchema) SchemaWithTag(fieldName string, rType reflect.Type, de docs: c.component.Docs(), } } + func (c *ComponentSchema) GenerateSchema(ctx context.Context, schema *Schema) (*openapi3.Schema, error) { description, err := c.Description(ctx, schema.path, "") if err != nil { @@ -273,294 +267,3 @@ func (c *ComponentSchema) GenerateSchema(ctx context.Context, schema *Schema) (* return result, nil } - -// TODO refactor -func (c *SchemaContainer) addToSchema(ctx context.Context, component *ComponentSchema, dst *openapi3.Schema, schema *Schema) error { - rType := schema.rType - for rType.Kind() == reflect.Ptr { - rType = rType.Elem() - } - - if schema.tag.Example != "" { - dst.Example = schema.tag.Example - } - - rootTable := "" - - if component.component.View.Mode == view.ModeQuery { - rootTable = component.component.View.Table - } - switch rType.Kind() { - case reflect.Slice, reflect.Array: - var err error - dst.Items, err = c.createSchema(ctx, component, schema.SliceItem(rType)) - if err != nil { - return err - } - dst.Type = arrayOutput - case reflect.Struct: - if rType == xreflect.TimeType { - dst.Type = stringOutput - timeLayout := schema.tag._tag.TimeLayout - if timeLayout == "" { - timeLayout = time.RFC3339 - } - - var dateFormat string - if containsAny(timeLayout, "15", "04", "05") { - dateFormat = "date-time" - } else { - dateFormat = "date" - } - - dst.Format = dateFormat - if dst.Example == nil { - dst.Example = time.Now().Format(timeLayout) - } - - dst.Pattern = ftime.TimeLayoutToDateFormat(timeLayout) - break - } - - dst.Properties = openapi3.Schemas{} - dst.Type = objectOutput - numField := rType.NumField() - table := schema.tag.Table - for i := 0; i < numField; i++ { - aField := rType.Field(i) - if aField.PkgPath != "" { - continue - } - aTag, err := ParseTag(aField, aField.Tag, schema.isInput, rootTable) - if err != nil { - return err - } - if aTag.Table == "" { - aTag.Table = table - } - if aTag.Ignore { - continue - } - - if aTag.Column != "" && table == "" { - table = rootTable - aTag.Table = rootTable - } - if table != "" && aTag.Column == "" { - aTag.Column = text.DetectCaseFormat(aField.Name).To(text.CaseFormatUpperUnderscore).Format(aField.Name) - } - - if aTag.Inlined { - dst.AdditionalPropertiesAllowed = setter.BoolPtr(true) - continue - } - fieldSchema, err := schema.Field(aField, aTag) - if err != nil { - return err - } - - if component.component.Output.IsExcluded(fieldSchema.path) { - continue - } - - docs := component.component.Docs() - updatedDocumentation(aTag, docs, fieldSchema) - - if aField.Anonymous { - if err := c.addToSchema(ctx, component, dst, fieldSchema); err != nil { - return err - } - continue - } - - if len(dst.Properties) == 0 { - dst.Properties = make(openapi3.Schemas) - } - dst.Properties[fieldSchema.fieldName], err = c.createSchema(ctx, component, fieldSchema) - if err != nil { - return err - } - - if !aTag.IsNullable { - dst.Required = append(dst.Required, fieldSchema.fieldName) - } - } - default: - if rType.Kind() == reflect.Interface { - dst.Type = objectOutput - break - } - - if rType.Kind() == reflect.Map { - dst.Type = objectOutput - keyType := rType.Key() - valueType := rType.Elem() - valueTypeName := valueType.Name() - vType, format, err := c.toOpenApiType(valueType) - valueSchema := &openapi3.Schema{ - Type: vType, - Format: format, - } - if err != nil { - switch valueType.Kind() { - case reflect.Struct: - case reflect.Slice: - - if vType, format, err = c.toOpenApiType(valueType.Elem()); err != nil { - return err - } - valueTypeName += strings.Title(valueType.Elem().Name()) + "s" - valueSchema.Type = arrayOutput - valueSchema.Items = &openapi3.Schema{ - Type: vType, - Format: format, - } - default: - return err - } - } - dst.Properties = openapi3.Schemas{} - mapType := strings.Title(keyType.Name()) + valueTypeName + "Map" - dst.Properties[mapType] = valueSchema - break - } - - var err error - dst.Type, dst.Format, err = c.toOpenApiType(rType) - if err != nil { - return err - } - } - - return nil -} - -func updatedDocumentation(aTag *Tag, docs *state.Docs, fieldSchema *Schema) { - if docs == nil { - return - } - if aTag.Column != "" && len(docs.Columns) > 0 { - columns := docs.Columns - if aTag.Description == "" { - aTag.Description, _ = columns.ColumnDescription(aTag.Table, aTag.Column) - } - if aTag.Description == "" { - aTag.Description, _ = columns.ColumnDescription("", aTag.Column) - } - if aTag.Example == "" { - aTag.Example, _ = columns.ColumnExample(aTag.Table, aTag.Column) - } - } - if aTag.Description == "" && len(docs.Paths) > 0 { - if desc, ok := docs.Paths.ByName(fieldSchema.path); ok { - aTag.Description = desc - } else if desc, ok := docs.Paths.ByName(fieldSchema.name); ok { - aTag.Description = desc - fieldSchema.description = desc - } - } - if aTag.Description != "" { - fieldSchema.description = aTag.Description - } - if aTag.Example != "" { - fieldSchema.example = aTag.Example - } - -} - -func containsAny(format string, values ...string) bool { - for _, value := range values { - if strings.Contains(format, value) { - return true - } - } - - return false -} - -func (c *ComponentSchema) GetOrGenerateSchema(ctx context.Context, schema *Schema) (*openapi3.Schema, error) { - return c.schemas.CreateSchema(ctx, c, schema) -} - -func (c *SchemaContainer) CreateSchema(ctx context.Context, componentSchema *ComponentSchema, fieldSchema *Schema) (*openapi3.Schema, error) { - c.mux.Lock() - defer c.mux.Unlock() - - return c.createSchema(ctx, componentSchema, fieldSchema) -} - -func (c *SchemaContainer) createSchema(ctx context.Context, componentSchema *ComponentSchema, fieldSchema *Schema) (*openapi3.Schema, error) { - description, err := componentSchema.Description(ctx, fieldSchema.path, fieldSchema.description) - if err != nil { - return nil, err - } - example, err := componentSchema.Example(ctx, fieldSchema.path, fieldSchema.example) - if err != nil { - return nil, err - } - - if fieldSchema.tag.TypeName != "" { - _, ok := c.generatedSchemas[fieldSchema.tag.TypeName] - if ok { - return c.SchemaRef(fieldSchema.tag.TypeName, description), nil - } - } - - apiType, format, ok := c.asOpenApiType(fieldSchema.rType) - if ok { - return &openapi3.Schema{ - Type: apiType, - Format: format, - Description: description, - Example: example, - }, nil - } - - schema, err := componentSchema.GenerateSchema(ctx, fieldSchema) - if err != nil { - return nil, err - } - - if fieldSchema.tag.TypeName != "" { - c.generatedSchemas[fieldSchema.tag.TypeName] = schema - c.schemas = append(c.schemas, schema) - schema = c.SchemaRef(fieldSchema.tag.TypeName, description) - } - - return schema, err -} - -func (c *SchemaContainer) SchemaRef(schemaName string, description string) *openapi3.Schema { - return &openapi3.Schema{ - Ref: "#/components/schemas/" + schemaName, - Description: description, - } -} - -func (c *SchemaContainer) toOpenApiType(rType reflect.Type) (string, string, error) { - apiType, format, ok := c.asOpenApiType(rType) - if !ok { - return empty, empty, fmt.Errorf("unsupported openapi3 type %v", rType.String()) - } - return apiType, format, nil -} - -func (c *SchemaContainer) asOpenApiType(rType reflect.Type) (string, string, bool) { - if rType.Kind() == reflect.Ptr { - rType = rType.Elem() - } - switch rType.Kind() { - case reflect.Int, reflect.Int64, reflect.Uint, reflect.Uint64: - return integerOutput, int64Format, true - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint8, reflect.Uint16, reflect.Uint32: - return integerOutput, int32Format, true - case reflect.Float64, reflect.Float32: - return numberOutput, doubleFormat, true - case reflect.Bool: - return booleanOutput, empty, true - case reflect.String: - return stringOutput, empty, true - } - - return empty, empty, false -} diff --git a/gateway/router/openapi/schema_build.go b/gateway/router/openapi/schema_build.go new file mode 100644 index 000000000..09bb99cea --- /dev/null +++ b/gateway/router/openapi/schema_build.go @@ -0,0 +1,574 @@ +package openapi + +import ( + "context" + "fmt" + "github.com/viant/datly/internal/setter" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/tagly/format/text" + ftime "github.com/viant/tagly/format/time" + "github.com/viant/xreflect" + "os" + "reflect" + "sort" + "strings" + "time" + + "github.com/viant/datly/gateway/router/openapi/openapi3" +) + +func (c *SchemaContainer) addToSchema(ctx context.Context, component *ComponentSchema, dst *openapi3.Schema, schema *Schema) error { + rType := dereferenceType(schema.rType) + applySchemaExample(dst, schema) + + switch rType.Kind() { + case reflect.Slice, reflect.Array: + return c.addArraySchema(ctx, component, dst, schema, rType) + case reflect.Struct: + return c.addStructSchema(ctx, component, dst, schema, rType) + default: + return c.addDefaultSchema(ctx, component, dst, schema, rType) + } +} + +func (c *SchemaContainer) addArraySchema(ctx context.Context, component *ComponentSchema, dst *openapi3.Schema, schema *Schema, rType reflect.Type) error { + itemSchema, err := c.createSchema(ctx, component, schema.SliceItem(rType)) + if err != nil { + return err + } + dst.Type = arrayOutput + dst.Items = itemSchema + return nil +} + +func (c *SchemaContainer) addStructSchema(ctx context.Context, component *ComponentSchema, dst *openapi3.Schema, schema *Schema, rType reflect.Type) error { + if rType == xreflect.TimeType { + addTimeSchema(dst, schema) + return nil + } + + dst.Type = objectOutput + dst.Properties = openapi3.Schemas{} + rootTable := rootTable(component) + table := schema.tag.Table + + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if shouldSkipStructField(field) { + continue + } + + aTag, err := ParseTag(field, field.Tag, schema.isInput, rootTable) + if err != nil { + return err + } + if normalizeFieldTag(aTag, field.Name, rootTable, table) { + table = aTag.Table + } + if shouldSkipByTag(component, aTag) { + continue + } + if aTag.Inlined { + dst.AdditionalPropertiesAllowed = setter.BoolPtr(true) + continue + } + + fieldSchema, err := schema.Field(field, aTag) + if err != nil { + return err + } + if component.component.Output.IsExcluded(fieldSchema.path) { + continue + } + + updatedDocumentation(aTag, component.component.Docs(), fieldSchema) + if field.Anonymous { + if err := c.addToSchema(ctx, component, dst, fieldSchema); err != nil { + return err + } + continue + } + + childSchema, err := c.createSchema(ctx, component, fieldSchema) + if err != nil { + return err + } + dst.Properties[fieldSchema.fieldName] = childSchema + if !aTag.IsNullable { + dst.Required = append(dst.Required, fieldSchema.fieldName) + } + } + + return nil +} + +func (c *SchemaContainer) addDefaultSchema(ctx context.Context, component *ComponentSchema, dst *openapi3.Schema, schema *Schema, rType reflect.Type) error { + switch rType.Kind() { + case reflect.Interface: + return c.addInterfaceSchema(ctx, component, dst, schema, rType) + case reflect.Map: + return c.addMapSchema(ctx, component, dst, schema, rType) + default: + apiType, format, err := c.toOpenApiType(rType) + if err != nil { + return err + } + dst.Type = apiType + dst.Format = format + return nil + } +} + +func (c *SchemaContainer) addInterfaceSchema(ctx context.Context, component *ComponentSchema, dst *openapi3.Schema, schema *Schema, interfaceType reflect.Type) error { + dst.Type = objectOutput + variants, skipped, err := c.interfaceVariants(ctx, component, schema, interfaceType) + if err != nil { + return err + } + if len(skipped) > 0 { + if shouldFailOnPolymorphismSkip() { + return fmt.Errorf("failed to resolve polymorphic variants for %s: %s", interfaceType.String(), strings.Join(skipped, ",")) + } + if dst.Extension == nil { + dst.Extension = openapi3.Extension{} + } + dst.Extension["x-datly-polymorphism-skipped"] = skipped + dst.Extension["x-datly-polymorphism-mode"] = "best-effort" + } + if len(variants) > 0 { + dst.OneOf = variants + if discriminator := oneOfDiscriminator(variants); discriminator != nil { + dst.Discriminator = discriminator + c.applyDiscriminatorToVariants(discriminator) + } + } + return nil +} + +func (c *SchemaContainer) interfaceVariants(ctx context.Context, component *ComponentSchema, schema *Schema, interfaceType reflect.Type) (openapi3.SchemaList, []string, error) { + if component == nil || component.component == nil { + return nil, nil, nil + } + registry := component.component.TypeRegistry() + if registry == nil { + return nil, nil, nil + } + + packageNames := registry.PackageNames() + sort.Strings(packageNames) + + seenByType := map[string]bool{} + result := make(openapi3.SchemaList, 0) + var skipped []string + for _, packageName := range packageNames { + pkg := registry.Package(packageName) + if pkg == nil { + continue + } + + typeNames := pkg.TypeNames() + sort.Strings(typeNames) + for _, typeName := range typeNames { + candidateType, err := pkg.Lookup(typeName) + if err != nil || candidateType == nil { + continue + } + candidateType = dereferenceType(candidateType) + if !implementsInterface(candidateType, interfaceType) { + continue + } + if candidateType.Kind() == reflect.Interface { + continue + } + + key := candidateType.String() + if seenByType[key] { + continue + } + seenByType[key] = true + + typeLabel := typeName + if typeLabel == "" { + typeLabel = candidateType.String() + } + + variantSchema := &Schema{ + docs: schema.docs, + pkg: schema.pkg, + path: key, + fieldName: typeLabel, + name: typeLabel, + description: schema.description, + example: schema.example, + rType: candidateType, + tag: Tag{}, + ioConfig: schema.ioConfig, + isInput: schema.isInput, + } + variantSchema.tag.TypeName = typeLabel + + builtSchema, err := c.createSchema(ctx, component, variantSchema) + if err != nil { + skipped = append(skipped, typeLabel) + continue + } + if builtSchema.Ref == "" { + skipped = append(skipped, typeLabel) + continue + } + result = append(result, builtSchema) + } + } + return result, skipped, nil +} + +func (c *SchemaContainer) addMapSchema(ctx context.Context, component *ComponentSchema, dst *openapi3.Schema, schema *Schema, rType reflect.Type) error { + valueSchema, err := c.mapValueSchema(ctx, component, schema, rType.Elem()) + if err != nil { + return err + } + dst.Type = objectOutput + dst.AdditionalProperties = valueSchema + return nil +} + +func (c *SchemaContainer) mapValueSchema(ctx context.Context, component *ComponentSchema, parent *Schema, valueType reflect.Type) (*openapi3.Schema, error) { + valueType = dereferenceType(valueType) + if apiType, format, ok := c.asOpenApiType(valueType); ok { + return &openapi3.Schema{Type: apiType, Format: format}, nil + } + + switch valueType.Kind() { + case reflect.Slice, reflect.Array: + itemsSchema, err := c.mapValueSchema(ctx, component, parent, valueType.Elem()) + if err != nil { + return nil, err + } + return &openapi3.Schema{Type: arrayOutput, Items: itemsSchema}, nil + default: + valueFieldSchema := &Schema{ + docs: parent.docs, + pkg: parent.pkg, + path: parent.path + ".value", + fieldName: parent.fieldName, + name: parent.name, + description: parent.description, + example: parent.example, + rType: valueType, + tag: Tag{}, + ioConfig: parent.ioConfig, + isInput: parent.isInput, + } + if valueType.Name() != "" { + valueFieldSchema.tag.TypeName = valueType.Name() + } + return c.createSchema(ctx, component, valueFieldSchema) + } +} + +func (c *ComponentSchema) GetOrGenerateSchema(ctx context.Context, schema *Schema) (*openapi3.Schema, error) { + return c.schemas.CreateSchema(ctx, c, schema) +} + +func (c *SchemaContainer) CreateSchema(ctx context.Context, componentSchema *ComponentSchema, fieldSchema *Schema) (*openapi3.Schema, error) { + c.mux.Lock() + defer c.mux.Unlock() + + return c.createSchema(ctx, componentSchema, fieldSchema) +} + +func (c *SchemaContainer) createSchema(ctx context.Context, componentSchema *ComponentSchema, fieldSchema *Schema) (*openapi3.Schema, error) { + description, err := componentSchema.Description(ctx, fieldSchema.path, fieldSchema.description) + if err != nil { + return nil, err + } + example, err := componentSchema.Example(ctx, fieldSchema.path, fieldSchema.example) + if err != nil { + return nil, err + } + + if fieldSchema.tag.TypeName != "" { + if _, ok := c.generatedSchemas[fieldSchema.tag.TypeName]; ok { + return c.SchemaRef(fieldSchema.tag.TypeName, description), nil + } + } + + if apiType, format, ok := c.asOpenApiType(fieldSchema.rType); ok { + return &openapi3.Schema{ + Type: apiType, + Format: format, + Description: description, + Example: example, + }, nil + } + + schema, err := componentSchema.GenerateSchema(ctx, fieldSchema) + if err != nil { + return nil, err + } + + if fieldSchema.tag.TypeName != "" { + c.generatedSchemas[fieldSchema.tag.TypeName] = schema + c.schemas = append(c.schemas, schema) + schema = c.SchemaRef(fieldSchema.tag.TypeName, description) + } + + return schema, nil +} + +func (c *SchemaContainer) SchemaRef(schemaName string, description string) *openapi3.Schema { + return &openapi3.Schema{ + Ref: "#/components/schemas/" + schemaName, + Description: description, + } +} + +func (c *SchemaContainer) toOpenApiType(rType reflect.Type) (string, string, error) { + apiType, format, ok := c.asOpenApiType(rType) + if !ok { + return empty, empty, fmt.Errorf("unsupported openapi3 type %v", rType.String()) + } + return apiType, format, nil +} + +func (c *SchemaContainer) asOpenApiType(rType reflect.Type) (string, string, bool) { + rType = dereferenceType(rType) + switch rType.Kind() { + case reflect.Int, reflect.Int64, reflect.Uint, reflect.Uint64: + return integerOutput, int64Format, true + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint8, reflect.Uint16, reflect.Uint32: + return integerOutput, int32Format, true + case reflect.Float64, reflect.Float32: + return numberOutput, doubleFormat, true + case reflect.Bool: + return booleanOutput, empty, true + case reflect.String: + return stringOutput, empty, true + } + + return empty, empty, false +} + +func updatedDocumentation(aTag *Tag, docs *state.Docs, fieldSchema *Schema) { + if docs == nil { + return + } + if aTag.Column != "" && len(docs.Columns) > 0 { + columns := docs.Columns + if aTag.Description == "" { + aTag.Description, _ = columns.ColumnDescription(aTag.Table, aTag.Column) + } + if aTag.Description == "" { + aTag.Description, _ = columns.ColumnDescription("", aTag.Column) + } + if aTag.Example == "" { + aTag.Example, _ = columns.ColumnExample(aTag.Table, aTag.Column) + } + } + if aTag.Description == "" && len(docs.Paths) > 0 { + if desc, ok := docs.Paths.ByName(fieldSchema.path); ok { + aTag.Description = desc + } else if desc, ok := docs.Paths.ByName(fieldSchema.name); ok { + aTag.Description = desc + fieldSchema.description = desc + } + } + if aTag.Description != "" { + fieldSchema.description = aTag.Description + } + if aTag.Example != "" { + fieldSchema.example = aTag.Example + } +} + +func containsAny(format string, values ...string) bool { + for _, value := range values { + if strings.Contains(format, value) { + return true + } + } + return false +} + +func hasInternalColumnTag(v *view.View, table, column string) bool { + if v == nil || column == "" { + return false + } + if matchesViewTable(v, table) { + if cfg := v.ColumnsConfig[column]; cfg != nil && cfg.Tag != nil && strings.Contains(*cfg.Tag, `internal:"true"`) { + return true + } + } + for _, rel := range v.With { + if rel == nil || rel.Of == nil { + continue + } + if hasInternalColumnTag(&rel.Of.View, table, column) { + return true + } + } + return false +} + +func matchesViewTable(v *view.View, table string) bool { + if table == "" { + return true + } + return strings.EqualFold(v.Table, table) || strings.EqualFold(v.Alias, table) || strings.EqualFold(v.Name, table) +} + +func rootTable(component *ComponentSchema) string { + if component.component.View.Mode == view.ModeQuery { + return component.component.View.Table + } + return "" +} + +func dereferenceType(rType reflect.Type) reflect.Type { + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + return rType +} + +func applySchemaExample(dst *openapi3.Schema, schema *Schema) { + if schema.tag.Example != "" { + dst.Example = schema.tag.Example + } +} + +func addTimeSchema(dst *openapi3.Schema, schema *Schema) { + dst.Type = stringOutput + timeLayout := schema.tag._tag.TimeLayout + if timeLayout == "" { + timeLayout = time.RFC3339 + } + if containsAny(timeLayout, "15", "04", "05") { + dst.Format = "date-time" + } else { + dst.Format = "date" + } + if dst.Example == nil { + dst.Example = time.Now().Format(timeLayout) + } + dst.Pattern = ftime.TimeLayoutToDateFormat(timeLayout) +} + +func shouldSkipStructField(field reflect.StructField) bool { + if field.PkgPath != "" { + return true + } + rawTag := string(field.Tag) + return strings.Contains(rawTag, `internal:"true"`) || strings.Contains(rawTag, `json:"-"`) +} + +func normalizeFieldTag(aTag *Tag, fieldName, rootTable, currentTable string) (updatedTable bool) { + if aTag.Table == "" { + aTag.Table = currentTable + } + if aTag.Ignore { + return false + } + if aTag.Column != "" && currentTable == "" { + aTag.Table = rootTable + return true + } + if currentTable != "" && aTag.Column == "" { + aTag.Column = text.DetectCaseFormat(fieldName).To(text.CaseFormatUpperUnderscore).Format(fieldName) + } + return false +} + +func shouldSkipByTag(component *ComponentSchema, aTag *Tag) bool { + if aTag.Ignore { + return true + } + return hasInternalColumnTag(component.component.View, aTag.Table, aTag.Column) || + hasInternalColumnTag(component.component.View, "", aTag.Column) +} + +func implementsInterface(candidateType, interfaceType reflect.Type) bool { + if candidateType.Implements(interfaceType) { + return true + } + if candidateType.Kind() != reflect.Ptr && reflect.PtrTo(candidateType).Implements(interfaceType) { + return true + } + return false +} + +func oneOfDiscriminator(variants openapi3.SchemaList) *openapi3.Discriminator { + mapping := map[string]string{} + for _, variant := range variants { + if variant == nil || variant.Ref == "" { + continue + } + ref := variant.Ref + name := ref[strings.LastIndex(ref, "/")+1:] + if name == "" { + continue + } + mapping[name] = ref + } + if len(mapping) == 0 { + return nil + } + return &openapi3.Discriminator{ + PropertyName: "type", + Mapping: mapping, + } +} + +func (c *SchemaContainer) applyDiscriminatorToVariants(discriminator *openapi3.Discriminator) { + if discriminator == nil || len(discriminator.Mapping) == 0 { + return + } + for value, ref := range discriminator.Mapping { + schemaName := refName(ref) + if schemaName == "" { + continue + } + variant := c.generatedSchemas[schemaName] + if variant == nil || variant.Type != objectOutput { + continue + } + if len(variant.Properties) == 0 { + variant.Properties = openapi3.Schemas{} + } + if variant.Properties[discriminator.PropertyName] == nil { + variant.Properties[discriminator.PropertyName] = &openapi3.Schema{ + Type: stringOutput, + Enum: []interface{}{value}, + } + } + if !containsString(variant.Required, discriminator.PropertyName) { + variant.Required = append(variant.Required, discriminator.PropertyName) + } + } +} + +func refName(ref string) string { + if ref == "" { + return "" + } + index := strings.LastIndex(ref, "/") + if index == -1 || index == len(ref)-1 { + return "" + } + return ref[index+1:] +} + +func containsString(values []string, target string) bool { + for _, item := range values { + if item == target { + return true + } + } + return false +} + +func shouldFailOnPolymorphismSkip() bool { + raw := strings.TrimSpace(strings.ToLower(os.Getenv("DATLY_OPENAPI_POLY_STRICT"))) + return raw == "1" || raw == "true" || raw == "yes" +} diff --git a/gateway/router/openapi/schema_build_helpers_test.go b/gateway/router/openapi/schema_build_helpers_test.go new file mode 100644 index 000000000..3767a3df0 --- /dev/null +++ b/gateway/router/openapi/schema_build_helpers_test.go @@ -0,0 +1,292 @@ +package openapi + +import ( + "context" + "reflect" + "testing" + + "github.com/viant/datly/gateway/router/openapi/openapi3" + "github.com/viant/datly/repository" + "github.com/viant/datly/view" + "github.com/viant/xreflect" +) + +type testAnimal interface { + Kind() string +} + +type testDog struct{} + +func (testDog) Kind() string { return "dog" } + +type testCat struct{} + +func (*testCat) Kind() string { return "cat" } + +type testTree struct{} + +type testUnsupported chan int + +func (testUnsupported) Kind() string { return "unsupported" } + +func TestSchemaBuildHelpers_Table(t *testing.T) { + t.Run("apply schema example", func(t *testing.T) { + dst := &openapi3.Schema{} + applySchemaExample(dst, &Schema{tag: Tag{Example: "abc"}}) + if dst.Example != "abc" { + t.Fatalf("expected example to be applied") + } + applySchemaExample(dst, &Schema{}) + if dst.Example != "abc" { + t.Fatalf("expected empty example not to override existing value") + } + }) + + t.Run("root table", func(t *testing.T) { + queryComp := &ComponentSchema{component: &repository.Component{View: &view.View{Mode: view.ModeQuery, Table: "users"}}} + if got := rootTable(queryComp); got != "users" { + t.Fatalf("expected users, got %q", got) + } + nonQueryComp := &ComponentSchema{component: &repository.Component{View: &view.View{Mode: view.Mode("Other"), Table: "users"}}} + if got := rootTable(nonQueryComp); got != "" { + t.Fatalf("expected empty root table for non-query mode") + } + }) + + t.Run("normalize field tag", func(t *testing.T) { + tests := []struct { + name string + tag Tag + rootTable string + table string + wantTable string + updated bool + column string + }{ + {name: "column sets root table", tag: Tag{Column: "ID"}, rootTable: "users", table: "", wantTable: "users", updated: true, column: "ID"}, + {name: "table infers column", tag: Tag{}, rootTable: "", table: "users", wantTable: "users", updated: false, column: "FIRST_NAME"}, + {name: "ignored tag", tag: Tag{Ignore: true}, rootTable: "users", table: "users", wantTable: "users", updated: false, column: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tag := tt.tag + updated := normalizeFieldTag(&tag, "FirstName", tt.rootTable, tt.table) + if updated != tt.updated { + t.Fatalf("expected updated=%v, got %v", tt.updated, updated) + } + if tag.Table != tt.wantTable { + t.Fatalf("expected table %q, got %q", tt.wantTable, tag.Table) + } + if tag.Column != tt.column { + t.Fatalf("expected column %q, got %q", tt.column, tag.Column) + } + }) + } + }) + + t.Run("should skip by tag", func(t *testing.T) { + tag := `internal:"true"` + component := &ComponentSchema{component: &repository.Component{View: &view.View{Table: "users", ColumnsConfig: map[string]*view.ColumnConfig{"ID": {Tag: &tag}}}}} + if !shouldSkipByTag(component, &Tag{Ignore: true}) { + t.Fatalf("expected ignored tag to be skipped") + } + if !shouldSkipByTag(component, &Tag{Table: "users", Column: "ID"}) { + t.Fatalf("expected internal column to be skipped") + } + if shouldSkipByTag(component, &Tag{Ignore: false, Table: "users", Column: "Name"}) { + t.Fatalf("did not expect non-internal column to be skipped") + } + }) + + t.Run("should skip struct field", func(t *testing.T) { + type sample struct { + exported string + Visible string + Hidden string `json:"-"` + Internal string `internal:"true"` + } + rType := reflect.TypeOf(sample{}) + if !shouldSkipStructField(rType.Field(0)) { + t.Fatalf("expected unexported field to be skipped") + } + if shouldSkipStructField(rType.Field(1)) { + t.Fatalf("did not expect visible field to be skipped") + } + if !shouldSkipStructField(rType.Field(2)) { + t.Fatalf("expected json:- field to be skipped") + } + if !shouldSkipStructField(rType.Field(3)) { + t.Fatalf("expected internal:true field to be skipped") + } + }) + + t.Run("add time schema default and pre-existing example", func(t *testing.T) { + dst := &openapi3.Schema{} + addTimeSchema(dst, &Schema{}) + if dst.Type != stringOutput || dst.Format != "date-time" || dst.Pattern == "" { + t.Fatalf("unexpected default time schema: type=%s format=%s pattern=%s", dst.Type, dst.Format, dst.Pattern) + } + existing := &openapi3.Schema{Example: "preset"} + addTimeSchema(existing, &Schema{}) + if existing.Example != "preset" { + t.Fatalf("expected existing example to be preserved") + } + }) + + t.Run("interface oneOf scaffolding", func(t *testing.T) { + t.Setenv("DATLY_OPENAPI_POLY_STRICT", "false") + component := newTestComponent(t) + types := xreflect.NewTypes() + if err := types.Register("Animal", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf((*testAnimal)(nil)).Elem())); err != nil { + t.Fatalf("register interface failed: %v", err) + } + if err := types.Register("Dog", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf(testDog{}))); err != nil { + t.Fatalf("register dog failed: %v", err) + } + if err := types.Register("Cat", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf(testCat{}))); err != nil { + t.Fatalf("register cat failed: %v", err) + } + if err := types.Register("DogAlias", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf(testDog{}))); err != nil { + t.Fatalf("register dog alias failed: %v", err) + } + if err := types.Register("Tree", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf(testTree{}))); err != nil { + t.Fatalf("register tree failed: %v", err) + } + if err := types.Register("Unsupported", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf(testUnsupported(nil)))); err != nil { + t.Fatalf("register unsupported failed: %v", err) + } + setUnexportedField(component, "types", types) + + container := NewContainer() + componentSchema := &ComponentSchema{component: component, schemas: container} + dst := &openapi3.Schema{} + err := container.addToSchema(context.Background(), componentSchema, dst, &Schema{ + rType: reflect.TypeOf((*testAnimal)(nil)).Elem(), + ioConfig: component.IOConfig(), + }) + if err != nil { + t.Fatalf("unexpected addToSchema error: %v", err) + } + if dst.Type != objectOutput { + t.Fatalf("expected object type for interface, got %q", dst.Type) + } + if len(dst.OneOf) != 2 { + t.Fatalf("expected oneOf variants for interface, got %d", len(dst.OneOf)) + } + if dst.Discriminator == nil { + t.Fatalf("expected discriminator to be set for oneOf interface schema") + } + if dst.Discriminator.PropertyName != "type" { + t.Fatalf("expected discriminator propertyName type, got %q", dst.Discriminator.PropertyName) + } + if len(dst.Discriminator.Mapping) != 2 { + t.Fatalf("expected discriminator mapping entries, got %d", len(dst.Discriminator.Mapping)) + } + if dst.Discriminator.Mapping["Dog"] != "#/components/schemas/Dog" { + t.Fatalf("unexpected discriminator mapping for Dog: %q", dst.Discriminator.Mapping["Dog"]) + } + if dst.Discriminator.Mapping["Cat"] != "#/components/schemas/Cat" { + t.Fatalf("unexpected discriminator mapping for Cat: %q", dst.Discriminator.Mapping["Cat"]) + } + + dogSchema := container.generatedSchemas["Dog"] + if dogSchema == nil || dogSchema.Properties["type"] == nil { + t.Fatalf("expected discriminator property injected in Dog schema") + } + if !containsString(dogSchema.Required, "type") { + t.Fatalf("expected discriminator property required in Dog schema") + } + + if dst.Extension == nil { + t.Fatalf("expected best-effort extension metadata") + } + skipped, ok := dst.Extension["x-datly-polymorphism-skipped"].([]string) + if !ok || len(skipped) == 0 { + t.Fatalf("expected skipped implementors extension") + } + }) + + t.Run("interface oneOf fallback without registry", func(t *testing.T) { + component := &repository.Component{} + container := NewContainer() + componentSchema := &ComponentSchema{component: component, schemas: container} + dst := &openapi3.Schema{} + err := container.addToSchema(context.Background(), componentSchema, dst, &Schema{ + rType: reflect.TypeOf((*testAnimal)(nil)).Elem(), + ioConfig: component.IOConfig(), + }) + if err != nil { + t.Fatalf("unexpected addToSchema error: %v", err) + } + if dst.Type != objectOutput { + t.Fatalf("expected object type for interface fallback, got %q", dst.Type) + } + if len(dst.OneOf) != 0 { + t.Fatalf("expected no oneOf variants when registry is unavailable, got %d", len(dst.OneOf)) + } + if dst.Discriminator != nil { + t.Fatalf("expected no discriminator without variants") + } + }) + + t.Run("implements interface", func(t *testing.T) { + tests := []struct { + name string + candidate reflect.Type + want bool + }{ + {name: "value receiver", candidate: reflect.TypeOf(testDog{}), want: true}, + {name: "pointer receiver", candidate: reflect.TypeOf(testCat{}), want: true}, + {name: "not implementor", candidate: reflect.TypeOf(struct{}{}), want: false}, + } + iface := reflect.TypeOf((*testAnimal)(nil)).Elem() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := implementsInterface(tt.candidate, iface); got != tt.want { + t.Fatalf("expected %v, got %v", tt.want, got) + } + }) + } + }) + + t.Run("interface variants nil component", func(t *testing.T) { + container := NewContainer() + variants, skipped, err := container.interfaceVariants(context.Background(), nil, &Schema{}, reflect.TypeOf((*testAnimal)(nil)).Elem()) + if err != nil { + t.Fatalf("unexpected interfaceVariants error: %v", err) + } + if len(variants) != 0 { + t.Fatalf("expected no variants for nil component, got %d", len(variants)) + } + if len(skipped) != 0 { + t.Fatalf("expected no skipped variants for nil component") + } + }) + + t.Run("interface oneOf strict mode", func(t *testing.T) { + t.Setenv("DATLY_OPENAPI_POLY_STRICT", "true") + component := newTestComponent(t) + types := xreflect.NewTypes() + if err := types.Register("Animal", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf((*testAnimal)(nil)).Elem())); err != nil { + t.Fatalf("register interface failed: %v", err) + } + if err := types.Register("Dog", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf(testDog{}))); err != nil { + t.Fatalf("register dog failed: %v", err) + } + if err := types.Register("Unsupported", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf(testUnsupported(nil)))); err != nil { + t.Fatalf("register unsupported failed: %v", err) + } + setUnexportedField(component, "types", types) + + container := NewContainer() + componentSchema := &ComponentSchema{component: component, schemas: container} + dst := &openapi3.Schema{} + err := container.addToSchema(context.Background(), componentSchema, dst, &Schema{ + rType: reflect.TypeOf((*testAnimal)(nil)).Elem(), + ioConfig: component.IOConfig(), + }) + if err == nil { + t.Fatalf("expected strict mode polymorphism error") + } + }) +} diff --git a/gateway/router/openapi/schema_helpers_test.go b/gateway/router/openapi/schema_helpers_test.go new file mode 100644 index 000000000..ddde60523 --- /dev/null +++ b/gateway/router/openapi/schema_helpers_test.go @@ -0,0 +1,329 @@ +package openapi + +import ( + "context" + "reflect" + "testing" + + "github.com/viant/datly/gateway/router/openapi/openapi3" + "github.com/viant/datly/repository" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +type sampleNested struct { + ID int +} + +type sampleWithField struct { + UserName string `json:"user_name" desc:"user name desc" example:"bob"` +} + +func TestSchemaSliceItem(t *testing.T) { + tests := []struct { + name string + typeName string + rType reflect.Type + expectType reflect.Type + expectSchema string + }{ + {name: "named element", typeName: "Entry", rType: reflect.TypeOf([]sampleNested{}), expectType: reflect.TypeOf(sampleNested{}), expectSchema: "sampleNested"}, + {name: "anonymous element", typeName: "Entry", rType: reflect.TypeOf([]struct{ Value int }{}), expectType: reflect.TypeOf(struct{ Value int }{}), expectSchema: "EntryItem"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Schema{tag: Tag{TypeName: tt.typeName}} + item := s.SliceItem(tt.rType) + if item.rType != tt.expectType { + t.Fatalf("expected %v, got %v", tt.expectType, item.rType) + } + if item.tag.TypeName != tt.expectSchema { + t.Fatalf("expected %q, got %q", tt.expectSchema, item.tag.TypeName) + } + }) + } +} + +func TestSchemaField(t *testing.T) { + component := &repository.Component{} + rType := reflect.TypeOf(sampleWithField{}) + field := rType.Field(0) + + tests := []struct { + name string + tag *Tag + expectField string + expectDesc string + expectExample string + }{ + {name: "uses json name", tag: &Tag{JSONName: "custom_name"}, expectField: "custom_name", expectDesc: "user name desc", expectExample: "bob"}, + {name: "falls back to formatted name", tag: &Tag{}, expectField: "UserName", expectDesc: "user name desc", expectExample: "bob"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Schema{ioConfig: component.IOConfig()} + got, err := s.Field(field, tt.tag) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.fieldName != tt.expectField { + t.Fatalf("expected field %q, got %q", tt.expectField, got.fieldName) + } + if got.description != tt.expectDesc { + t.Fatalf("expected description %q, got %q", tt.expectDesc, got.description) + } + if got.example != tt.expectExample { + t.Fatalf("expected example %q, got %q", tt.expectExample, got.example) + } + }) + } +} + +func TestContainsAny(t *testing.T) { + tests := []struct { + name string + format string + values []string + expect bool + }{ + {name: "contains", format: "2006-01-02T15:04:05", values: []string{"15", "04"}, expect: true}, + {name: "not contains", format: "2006-01-02", values: []string{"15", "04", "05"}, expect: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := containsAny(tt.format, tt.values...) + if got != tt.expect { + t.Fatalf("expected %v, got %v", tt.expect, got) + } + }) + } +} + +func TestAsOpenAPIType(t *testing.T) { + container := NewContainer() + tests := []struct { + name string + rType reflect.Type + api string + format string + ok bool + }{ + {name: "int64", rType: reflect.TypeOf(int64(1)), api: integerOutput, format: int64Format, ok: true}, + {name: "uint32", rType: reflect.TypeOf(uint32(1)), api: integerOutput, format: int32Format, ok: true}, + {name: "float64", rType: reflect.TypeOf(float64(1)), api: numberOutput, format: doubleFormat, ok: true}, + {name: "bool", rType: reflect.TypeOf(true), api: booleanOutput, format: empty, ok: true}, + {name: "string", rType: reflect.TypeOf(""), api: stringOutput, format: empty, ok: true}, + {name: "ptr", rType: reflect.TypeOf(new(int)), api: integerOutput, format: int64Format, ok: true}, + {name: "unsupported struct", rType: reflect.TypeOf(struct{}{}), api: empty, format: empty, ok: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + api, format, ok := container.asOpenApiType(tt.rType) + if ok != tt.ok { + t.Fatalf("expected ok=%v, got %v", tt.ok, ok) + } + if api != tt.api || format != tt.format { + t.Fatalf("expected %s/%s, got %s/%s", tt.api, tt.format, api, format) + } + }) + } +} + +func TestToOpenApiType(t *testing.T) { + container := NewContainer() + tests := []struct { + name string + rType reflect.Type + wantError bool + }{ + {name: "supported", rType: reflect.TypeOf(int(1)), wantError: false}, + {name: "unsupported", rType: reflect.TypeOf(struct{}{}), wantError: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, err := container.toOpenApiType(tt.rType) + if (err != nil) != tt.wantError { + t.Fatalf("wantError=%v got err=%v", tt.wantError, err) + } + }) + } +} + +func TestSchemaRef(t *testing.T) { + container := NewContainer() + tests := []struct { + name string + schemaName string + description string + expectRef string + }{ + {name: "basic", schemaName: "MyType", description: "desc", expectRef: "#/components/schemas/MyType"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := container.SchemaRef(tt.schemaName, tt.description) + if got.Ref != tt.expectRef { + t.Fatalf("expected ref %q, got %q", tt.expectRef, got.Ref) + } + if got.Description != tt.description { + t.Fatalf("expected description %q, got %q", tt.description, got.Description) + } + }) + } +} + +func TestUpdatedDocumentation(t *testing.T) { + tests := []struct { + name string + tag *Tag + docs *state.Docs + field *Schema + expectDesc string + expectExample string + }{ + { + name: "column docs", + tag: &Tag{Table: "users", Column: "name"}, + docs: &state.Docs{Columns: state.Documentation{"users.name": "column desc", "users.name$example": "alice"}}, + field: &Schema{path: "pkg.User.Name", name: "Name"}, + expectDesc: "column desc", expectExample: "alice", + }, + { + name: "path docs fallback", + tag: &Tag{}, + docs: &state.Docs{Paths: state.Documentation{"pkg.User.Name": "path desc"}}, + field: &Schema{path: "pkg.User.Name", name: "Name"}, + expectDesc: "path desc", expectExample: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + updatedDocumentation(tt.tag, tt.docs, tt.field) + if tt.field.description != tt.expectDesc { + t.Fatalf("expected description %q, got %q", tt.expectDesc, tt.field.description) + } + if tt.field.example != tt.expectExample { + t.Fatalf("expected example %q, got %q", tt.expectExample, tt.field.example) + } + }) + } +} + +func TestMatchesViewTable(t *testing.T) { + v := &view.View{Table: "users", Alias: "u", Name: "UsersView"} + tests := []struct { + name string + table string + expect bool + }{ + {name: "table", table: "users", expect: true}, + {name: "alias", table: "u", expect: true}, + {name: "name", table: "UsersView", expect: true}, + {name: "empty", table: "", expect: true}, + {name: "miss", table: "products", expect: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := matchesViewTable(v, tt.table); got != tt.expect { + t.Fatalf("expected %v, got %v", tt.expect, got) + } + }) + } +} + +func TestHasInternalColumnTag(t *testing.T) { + tag := `internal:"true"` + relTag := `internal:"true"` + v := &view.View{ + Table: "users", + ColumnsConfig: map[string]*view.ColumnConfig{ + "ID": {Tag: &tag}, + }, + With: []*view.Relation{ + {Of: &view.ReferenceView{View: view.View{Table: "orders", ColumnsConfig: map[string]*view.ColumnConfig{"OrderID": {Tag: &relTag}}}}}, + }, + } + + tests := []struct { + name string + view *view.View + table string + column string + expect bool + }{ + {name: "nil view", view: nil, table: "users", column: "ID", expect: false}, + {name: "empty column", view: v, table: "users", column: "", expect: false}, + {name: "current view", view: v, table: "users", column: "ID", expect: true}, + {name: "relation", view: v, table: "orders", column: "OrderID", expect: true}, + {name: "not internal", view: v, table: "users", column: "Name", expect: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hasInternalColumnTag(tt.view, tt.table, tt.column); got != tt.expect { + t.Fatalf("expected %v, got %v", tt.expect, got) + } + }) + } +} + +func TestAddToSchemaSimpleBranches(t *testing.T) { + container := NewContainer() + component := &ComponentSchema{component: &repository.Component{View: &view.View{}}, schemas: container} + type mapRecord struct { + ID int `json:"id"` + } + tests := []struct { + name string + rType reflect.Type + expectType string + expectPropsLen int + expectAdditionalType string + expectAdditionalItemRef bool + }{ + {name: "interface", rType: reflect.TypeOf((*interface{})(nil)).Elem(), expectType: objectOutput, expectPropsLen: 0}, + {name: "map primitive value", rType: reflect.TypeOf(map[string]int{}), expectType: objectOutput, expectPropsLen: 0, expectAdditionalType: integerOutput}, + {name: "map array value", rType: reflect.TypeOf(map[string][]string{}), expectType: objectOutput, expectPropsLen: 0, expectAdditionalType: arrayOutput}, + {name: "map object value", rType: reflect.TypeOf(map[string]mapRecord{}), expectType: objectOutput, expectPropsLen: 0, expectAdditionalItemRef: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dst := &openapi3.Schema{} + err := container.addToSchema(context.Background(), component, dst, &Schema{rType: tt.rType, ioConfig: component.component.IOConfig()}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if dst.Type != tt.expectType { + t.Fatalf("expected type %q, got %q", tt.expectType, dst.Type) + } + if len(dst.Properties) != tt.expectPropsLen { + t.Fatalf("expected %d properties, got %d", tt.expectPropsLen, len(dst.Properties)) + } + if tt.expectAdditionalType != "" { + if dst.AdditionalProperties == nil { + t.Fatalf("expected additionalProperties schema") + } + if dst.AdditionalProperties.Type != tt.expectAdditionalType { + t.Fatalf("expected additionalProperties type %q, got %q", tt.expectAdditionalType, dst.AdditionalProperties.Type) + } + } + if tt.expectAdditionalItemRef { + if dst.AdditionalProperties == nil { + t.Fatalf("expected additionalProperties schema") + } + if dst.AdditionalProperties.Ref == "" { + t.Fatalf("expected additionalProperties to reference a schema") + } + } + }) + } +} diff --git a/gateway/router/openapi/tag_parse_test.go b/gateway/router/openapi/tag_parse_test.go new file mode 100644 index 000000000..218387fe6 --- /dev/null +++ b/gateway/router/openapi/tag_parse_test.go @@ -0,0 +1,73 @@ +package openapi + +import ( + "reflect" + "testing" +) + +type tagParseNamed struct { + A int +} + +type tagParseFixture struct { + Values []int `json:"values"` + Any tagParseNamed `json:"any"` + Hidden string `json:"hidden" internal:"true"` + Summary string `json:"summary" parameter:"kind=output,in=summary"` + ViewOut string `json:"view_out" parameter:"kind=output,in=view"` + InputDrop string `json:"input_drop" parameter:"kind=query,in=id"` + ByViewTable string `json:"by_view" view:"name=V,table=orders"` + BySQLX string `json:"by_sqlx" sqlx:"name=ORD_ID"` +} + +func TestParseTag(t *testing.T) { + rType := reflect.TypeOf(tagParseFixture{}) + tests := []struct { + name string + fieldIndex int + isInput bool + rootTable string + expectIgnore bool + expectTypeName string + expectJSONName string + expectNullable bool + expectTableValue string + }{ + {name: "slice sets json name", fieldIndex: 0, isInput: false, rootTable: "", expectIgnore: false, expectTypeName: "", expectJSONName: "values", expectNullable: false}, + {name: "struct sets type name", fieldIndex: 1, isInput: false, rootTable: "", expectIgnore: false, expectTypeName: "openapi.tagParseNamed", expectJSONName: "any", expectNullable: false}, + {name: "internal flag ignored", fieldIndex: 2, isInput: false, rootTable: "", expectIgnore: true, expectTypeName: "", expectJSONName: "hidden", expectNullable: false}, + {name: "output summary table", fieldIndex: 3, isInput: false, rootTable: "root", expectIgnore: false, expectTypeName: "", expectJSONName: "summary", expectNullable: false, expectTableValue: "SUMMARY"}, + {name: "output view table", fieldIndex: 4, isInput: false, rootTable: "root", expectIgnore: false, expectTypeName: "", expectJSONName: "view_out", expectNullable: false, expectTableValue: "root"}, + {name: "input non body ignored", fieldIndex: 5, isInput: true, rootTable: "root", expectIgnore: true, expectTypeName: "", expectJSONName: "input_drop", expectNullable: false}, + {name: "view tag table", fieldIndex: 6, isInput: false, rootTable: "root", expectIgnore: false, expectTypeName: "", expectJSONName: "by_view", expectNullable: false, expectTableValue: "orders"}, + {name: "sqlx column captured", fieldIndex: 7, isInput: false, rootTable: "root", expectIgnore: false, expectTypeName: "", expectJSONName: "by_sqlx", expectNullable: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + field := rType.Field(tt.fieldIndex) + parsed, err := ParseTag(field, field.Tag, tt.isInput, tt.rootTable) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if parsed.Ignore != tt.expectIgnore { + t.Fatalf("expected ignore %v, got %v", tt.expectIgnore, parsed.Ignore) + } + if parsed.TypeName != tt.expectTypeName { + t.Fatalf("expected type name %q, got %q", tt.expectTypeName, parsed.TypeName) + } + if parsed.JSONName != tt.expectJSONName { + t.Fatalf("expected json name %q, got %q", tt.expectJSONName, parsed.JSONName) + } + if parsed.IsNullable != tt.expectNullable { + t.Fatalf("expected nullable %v, got %v", tt.expectNullable, parsed.IsNullable) + } + if parsed.Table != tt.expectTableValue { + t.Fatalf("expected table %q, got %q", tt.expectTableValue, parsed.Table) + } + if tt.name == "sqlx column captured" && parsed.Column != "ORD_ID" { + t.Fatalf("expected column ORD_ID, got %q", parsed.Column) + } + }) + } +} From 4eea35638d750d6cf2b78aa87dee3ee791b72bcc Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Feb 2026 09:26:25 -0800 Subject: [PATCH 141/279] harden openapi responses typing and marshal/session compatibility --- gateway/router/openapi/generator_test.go | 764 ++++++++++++++++++ gateway/router/openapi/openapi3.go | 104 +-- .../openapi3/additional_branches_test.go | 120 +++ .../openapi/openapi3/model_methods_test.go | 256 ++++++ gateway/router/openapi/openapi3/operation.go | 10 +- gateway/router/openapi/openapi3/response.go | 37 +- gateway/router/openapi/openapi3/session.go | 83 +- .../router/openapi/openapi3/session_test.go | 152 ++++ 8 files changed, 1352 insertions(+), 174 deletions(-) create mode 100644 gateway/router/openapi/generator_test.go create mode 100644 gateway/router/openapi/openapi3/additional_branches_test.go create mode 100644 gateway/router/openapi/openapi3/model_methods_test.go create mode 100644 gateway/router/openapi/openapi3/session_test.go diff --git a/gateway/router/openapi/generator_test.go b/gateway/router/openapi/generator_test.go new file mode 100644 index 000000000..c7b4e2cb8 --- /dev/null +++ b/gateway/router/openapi/generator_test.go @@ -0,0 +1,764 @@ +package openapi + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "reflect" + "strings" + "testing" + + openapi3 "github.com/viant/datly/gateway/router/openapi/openapi3" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/repository/version" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +func TestGeneratorTopLevel_Table(t *testing.T) { + ctx := context.Background() + info := openapi3.Info{Title: "api", Version: "1"} + + t.Run("generate spec no providers", func(t *testing.T) { + g := &generator{_schemasIndex: map[string]*openapi3.Schema{}, commonParameters: map[string]*openapi3.Parameter{}, _parametersIndex: map[string]*openapi3.Parameter{}} + spec, err := g.GenerateSpec(ctx, &repository.Service{}, info) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if spec == nil || spec.OpenAPI != "3.0.1" { + t.Fatalf("unexpected spec") + } + }) + + t.Run("wrapper generate no providers", func(t *testing.T) { + spec, err := GenerateOpenAPI3Spec(ctx, &repository.Service{}, info) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if spec == nil || spec.Info == nil || spec.Info.Title != "api" { + t.Fatalf("unexpected wrapper result") + } + }) + + t.Run("generate paths no providers", func(t *testing.T) { + g := &generator{} + schemas, paths, err := g.generatePaths(ctx, &repository.Service{}, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if schemas == nil || len(paths) != 0 { + t.Fatalf("unexpected result") + } + }) + + t.Run("marshal generated spec response keys", func(t *testing.T) { + control := &version.Control{} + comp := newTestComponent(t) + comp.Method = http.MethodGet + comp.Path.Method = http.MethodGet + comp.Path.URI = "/v1/spec" + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + comp.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + provider := repository.NewProvider(comp.Path, control, func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { return comp, nil }) + + spec, err := GenerateOpenAPI3Spec(ctx, &repository.Service{}, info, provider) + if err != nil { + t.Fatalf("unexpected spec generation error: %v", err) + } + data, err := json.Marshal(spec) + if err != nil { + t.Fatalf("unexpected marshal error: %v", err) + } + doc := string(data) + if !strings.Contains(doc, `"responses":{"200":`) { + t.Fatalf("expected serialized numeric response key as string in spec: %s", doc) + } + if !strings.Contains(doc, `"default"`) { + t.Fatalf("expected default response key in spec: %s", doc) + } + }) +} + +func TestAttachOperation_Table(t *testing.T) { + tests := []struct { + name string + method string + assertion func(t *testing.T, item *openapi3.PathItem, op *openapi3.Operation) + }{ + { + name: "get", + method: http.MethodGet, + assertion: func(t *testing.T, item *openapi3.PathItem, op *openapi3.Operation) { + if item.Get != op { + t.Fatalf("expected get operation") + } + }, + }, + { + name: "post", + method: http.MethodPost, + assertion: func(t *testing.T, item *openapi3.PathItem, op *openapi3.Operation) { + if item.Post != op { + t.Fatalf("expected post operation") + } + }, + }, + { + name: "delete", + method: http.MethodDelete, + assertion: func(t *testing.T, item *openapi3.PathItem, op *openapi3.Operation) { + if item.Delete != op { + t.Fatalf("expected delete operation") + } + }, + }, + { + name: "put", + method: http.MethodPut, + assertion: func(t *testing.T, item *openapi3.PathItem, op *openapi3.Operation) { + if item.Put != op { + t.Fatalf("expected put operation") + } + }, + }, + { + name: "patch", + method: http.MethodPatch, + assertion: func(t *testing.T, item *openapi3.PathItem, op *openapi3.Operation) { + if item.Patch != op { + t.Fatalf("expected patch operation") + } + }, + }, + { + name: "unsupported", + method: "TRACE", + assertion: func(t *testing.T, item *openapi3.PathItem, op *openapi3.Operation) { + if item.Get != nil || item.Post != nil || item.Delete != nil || item.Put != nil || item.Patch != nil { + t.Fatalf("did not expect any method to be set") + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + item := &openapi3.PathItem{} + op := &openapi3.Operation{} + attachOperation(item, tt.method, op) + tt.assertion(t, item, op) + }) + } +} + +func TestGeneratorHelpersMore_Table(t *testing.T) { + g := &generator{} + + t.Run("view parameters empty", func(t *testing.T) { + comp := &ComponentSchema{component: &repository.Component{}, schemas: NewContainer()} + v := &view.View{Template: &view.Template{}, Selector: &view.Config{}} + params, err := g.viewParameters(context.Background(), v, comp) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(params) != 0 { + t.Fatalf("expected no params") + } + }) + + t.Run("get all views params empty with relation", func(t *testing.T) { + comp := &ComponentSchema{component: &repository.Component{}, schemas: NewContainer()} + v := &view.View{Template: &view.Template{}, Selector: &view.Config{}, With: []*view.Relation{{Of: &view.ReferenceView{View: view.View{Template: &view.Template{}, Selector: &view.Config{}}}}}} + params, err := g.getAllViewsParameters(context.Background(), comp, v) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(params) != 0 { + t.Fatalf("expected no params") + } + }) + + t.Run("append built-in nil", func(t *testing.T) { + comp := &ComponentSchema{component: &repository.Component{}, schemas: NewContainer()} + params := []*openapi3.Parameter{} + if err := g.appendBuiltInParam(context.Background(), ¶ms, comp, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("request body nil for get", func(t *testing.T) { + comp := &ComponentSchema{component: &repository.Component{Path: repository.Component{}.Path}, schemas: NewContainer()} + comp.component.Path.Method = http.MethodGet + body, err := g.requestBody(context.Background(), comp) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body != nil { + t.Fatalf("expected nil body") + } + }) + + t.Run("responses nil for options", func(t *testing.T) { + comp := &ComponentSchema{component: &repository.Component{}, schemas: NewContainer()} + comp.component.Method = http.MethodOptions + resp, err := g.responses(context.Background(), comp) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if resp == nil { + t.Fatalf("expected non-nil response map") + } + if len(resp) != 0 { + t.Fatalf("expected empty response map for options") + } + }) + + t.Run("request body for post", func(t *testing.T) { + comp := newTestComponent(t) + comp.Path.Method = http.MethodPost + comp.Input.Body = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + comp.Input.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + body, err := g.requestBody(context.Background(), cSchema) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body == nil || body.Content[ApplicationJson] == nil { + t.Fatalf("expected request body") + } + }) + + t.Run("responses success and default", func(t *testing.T) { + comp := newTestComponent(t) + comp.Method = http.MethodGet + comp.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + resp, err := g.responses(context.Background(), cSchema) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := openapi3.GetResponse(resp, 200); !ok { + t.Fatalf("expected success response") + } + if _, ok := openapi3.GetResponse(resp, openapi3.ResponseDefault); !ok { + t.Fatalf("expected standard responses") + } + }) + + t.Run("convert param query", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + param := &state.Parameter{Name: "ID", In: &state.Location{Kind: state.KindQuery}, Schema: state.NewSchema(reflect.TypeOf(1))} + converted, ok, err := g.convertParam(context.Background(), cSchema, param, "") + if err != nil || !ok || len(converted) != 1 { + t.Fatalf("unexpected convert result: %v %v %d", ok, err, len(converted)) + } + }) + + t.Run("convert param object and non-http", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + + objectParam := &state.Parameter{ + Name: "Obj", + In: &state.Location{Kind: state.KindObject}, + Object: state.Parameters{ + {Name: "A", In: state.NewQueryLocation("a"), Schema: state.NewSchema(reflect.TypeOf(""))}, + }, + } + converted, ok, err := g.convertParam(context.Background(), cSchema, objectParam, "") + if err != nil || !ok || len(converted) != 1 { + t.Fatalf("unexpected object convert: %v %v %d", ok, err, len(converted)) + } + + nonHTTP := &state.Parameter{Name: "S", In: &state.Location{Kind: state.KindState, Name: "state"}, Schema: state.NewSchema(reflect.TypeOf(""))} + converted, ok, err = g.convertParam(context.Background(), cSchema, nonHTTP, "") + if err != nil || ok || len(converted) != 0 { + t.Fatalf("unexpected non-http convert: %v %v %d", ok, err, len(converted)) + } + }) + + t.Run("convert param via kind param and cache ref", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + base := &state.Parameter{Name: "ID", In: state.NewQueryLocation("id"), Schema: state.NewSchema(reflect.TypeOf(1))} + comp.Input.Type.Parameters = state.Parameters{base} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + + refParam := &state.Parameter{Name: "Ref", In: &state.Location{Kind: state.KindParam, Name: "ID"}, Schema: state.NewSchema(reflect.TypeOf(1))} + converted, ok, err := g.convertParam(context.Background(), cSchema, refParam, "") + if err != nil || !ok || len(converted) != 1 { + t.Fatalf("unexpected kind-param convert: %v %v %d", ok, err, len(converted)) + } + + converted, ok, err = g.convertParam(context.Background(), cSchema, base, "") + if err != nil || !ok || len(converted) != 1 { + t.Fatalf("unexpected cache convert: %v %v %#v", ok, err, converted) + } + }) + + t.Run("append built-in and view params", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + params := []*openapi3.Parameter{} + param := &state.Parameter{Name: "Limit", In: state.NewQueryLocation("limit"), Schema: state.NewSchema(reflect.TypeOf(1))} + if err := g.appendBuiltInParam(context.Background(), ¶ms, cSchema, param); err != nil { + t.Fatalf("unexpected append error: %v", err) + } + if len(params) == 0 { + t.Fatalf("expected built-in param") + } + }) + + t.Run("view parameters with selector built-ins", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + v := &view.View{ + Template: &view.Template{ + Parameters: state.Parameters{ + {Name: "Q", In: state.NewQueryLocation("q"), Schema: state.NewSchema(reflect.TypeOf(""))}, + }, + }, + Selector: &view.Config{ + CriteriaParameter: &state.Parameter{Name: "Criteria", In: state.NewQueryLocation("_criteria"), Schema: state.NewSchema(reflect.TypeOf(""))}, + LimitParameter: &state.Parameter{Name: "Limit", In: state.NewQueryLocation("_limit"), Schema: state.NewSchema(reflect.TypeOf(1))}, + OffsetParameter: &state.Parameter{Name: "Offset", In: state.NewQueryLocation("_offset"), Schema: state.NewSchema(reflect.TypeOf(1))}, + PageParameter: &state.Parameter{Name: "Page", In: state.NewQueryLocation("_page"), Schema: state.NewSchema(reflect.TypeOf(1))}, + OrderByParameter: &state.Parameter{Name: "OrderBy", In: state.NewQueryLocation("_orderby"), Schema: state.NewSchema(reflect.TypeOf([]string{}))}, + FieldsParameter: &state.Parameter{Name: "Fields", In: state.NewQueryLocation("_fields"), Schema: state.NewSchema(reflect.TypeOf([]string{}))}, + }, + } + params, err := g.viewParameters(context.Background(), v, cSchema) + if err != nil { + t.Fatalf("unexpected viewParameters error: %v", err) + } + if len(params) < 7 { + t.Fatalf("expected builtin and template params, got %d", len(params)) + } + + v.Template.Parameters = append(v.Template.Parameters, &state.Parameter{Name: "StateParam", In: &state.Location{Kind: state.KindState, Name: "s"}, Schema: state.NewSchema(reflect.TypeOf(""))}) + params, err = g.viewParameters(context.Background(), v, cSchema) + if err != nil { + t.Fatalf("unexpected viewParameters error: %v", err) + } + if len(params) < 7 { + t.Fatalf("expected params with non-http skipped, got %d", len(params)) + } + }) + + t.Run("generate operation happy path", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.Method = http.MethodPost + comp.Path.Method = http.MethodPost + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + comp.Input.Body = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + comp.Input.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + comp.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + operation, err := g.generateOperation(context.Background(), cSchema) + if err != nil || operation == nil { + t.Fatalf("unexpected operation result: %v %v", operation, err) + } + if _, ok := openapi3.GetResponse(operation.Responses, 200); !ok { + t.Fatalf("expected 200 response") + } + }) + + t.Run("generate operation with component parameter", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + + components := &repository.Service{} + registry := repository.NewRegistry("", nil, nil) + setUnexportedField(components, "registry", registry) + + dep := newTestComponent(t) + dep.Method = http.MethodGet + dep.Path.Method = http.MethodGet + dep.Path.URI = "/v1/dep" + dep.View = &view.View{ + Template: &view.Template{ + Parameters: state.Parameters{ + {Name: "DepID", In: state.NewQueryLocation("depId"), Schema: state.NewSchema(reflect.TypeOf(1))}, + }, + }, + Selector: &view.Config{}, + } + dep.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + registry.Register(dep) + + comp := newTestComponent(t) + comp.Method = http.MethodPost + comp.Path.Method = http.MethodPost + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + comp.Input.Body = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + comp.Input.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + comp.Output.Type = state.Type{ + Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{})), + Parameters: state.Parameters{ + {Name: "Dep", In: &state.Location{Kind: state.KindComponent, Name: "GET:/v1/dep"}}, + }, + } + + cSchema := &ComponentSchema{component: comp, components: components, schemas: NewContainer()} + operation, err := g.generateOperation(context.Background(), cSchema) + if err != nil { + t.Fatalf("unexpected operation error: %v", err) + } + if operation == nil || len(operation.Parameters) == 0 { + t.Fatalf("expected operation with merged parameters") + } + }) + + t.Run("generate operation request body error", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.Method = http.MethodPost + comp.Path.Method = http.MethodPost + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + comp.Input.Body = state.Type{Schema: state.NewSchema(reflect.TypeOf((chan int)(nil)))} + comp.Input.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf((chan int)(nil)))} + comp.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + if _, err := g.generateOperation(context.Background(), cSchema); err == nil { + t.Fatalf("expected request body generation error") + } + }) + + t.Run("generate operation response error", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.Method = http.MethodGet + comp.Path.Method = http.MethodGet + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + comp.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf((chan int)(nil)))} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + if _, err := g.generateOperation(context.Background(), cSchema); err == nil { + t.Fatalf("expected response generation error") + } + }) + + t.Run("generate paths with providers", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + control := &version.Control{} + comp1 := newTestComponent(t) + comp1.Method = http.MethodGet + comp1.Path.Method = http.MethodGet + comp1.Path.URI = "/v1/get" + comp1.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + comp1.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + + comp2 := newTestComponent(t) + comp2.Method = http.MethodPost + comp2.Path.Method = http.MethodPost + comp2.Path.URI = "/v1/post" + comp2.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + comp2.Input.Body = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + comp2.Input.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + comp2.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + + provider1 := repository.NewProvider(comp1.Path, control, func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { return comp1, nil }) + provider2 := repository.NewProvider(comp2.Path, control, func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { return comp2, nil }) + + _, paths, err := g.generatePaths(context.Background(), &repository.Service{}, []*repository.Provider{provider1, provider2}) + if err != nil { + t.Fatalf("unexpected generate paths error: %v", err) + } + if paths["/v1/get"] == nil || paths["/v1/post"] == nil { + t.Fatalf("expected generated paths") + } + if paths["/v1/get"].Get == nil || paths["/v1/get"].Post != nil { + t.Fatalf("expected isolated GET path item") + } + if paths["/v1/post"].Post == nil || paths["/v1/post"].Get != nil { + t.Fatalf("expected isolated POST path item") + } + }) + + t.Run("generate paths with all methods and provider errors", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + control := &version.Control{} + + mk := func(method, uri string) *repository.Provider { + comp := newTestComponent(t) + comp.Method = method + comp.Path.Method = method + comp.Path.URI = uri + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + comp.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + if method != http.MethodGet { + comp.Input.Body = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + comp.Input.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ Name string }{}))} + } + return repository.NewProvider(comp.Path, control, func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { return comp, nil }) + } + + errProvider := repository.NewProvider(contract.Path{Method: http.MethodGet, URI: "/v1/error"}, control, func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { + return nil, errors.New("provider error") + }) + + controlDeleted := &version.Control{} + controlDeleted.SetChangeKind(version.ChangeKindDeleted) + nilProvider := repository.NewProvider(contract.Path{Method: http.MethodGet, URI: "/v1/nil"}, controlDeleted, func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { + return nil, nil + }) + + providers := []*repository.Provider{ + mk(http.MethodDelete, "/v1/delete"), + mk(http.MethodPut, "/v1/put"), + mk(http.MethodPatch, "/v1/patch"), + errProvider, + nilProvider, + } + _, paths, err := g.generatePaths(context.Background(), &repository.Service{}, providers) + if err == nil { + t.Fatalf("expected provider error") + } + if paths["/v1/delete"] == nil || paths["/v1/put"] == nil || paths["/v1/patch"] == nil { + t.Fatalf("expected generated method paths") + } + }) + + t.Run("operation parameters include component output params", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + + components := &repository.Service{} + registry := repository.NewRegistry("", nil, nil) + setUnexportedField(components, "registry", registry) + + dep := newTestComponent(t) + dep.Method = http.MethodGet + dep.Path.Method = http.MethodGet + dep.Path.URI = "/v1/opdep" + dep.View = &view.View{ + Template: &view.Template{ + Parameters: state.Parameters{ + {Name: "DepID", In: state.NewQueryLocation("depId"), Schema: state.NewSchema(reflect.TypeOf(1))}, + }, + }, + Selector: &view.Config{}, + } + dep.Output.Type = state.Type{Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{}))} + registry.Register(dep) + + comp := newTestComponent(t) + comp.View = &view.View{ + Template: &view.Template{ + Parameters: state.Parameters{ + {Name: "RootQ", In: state.NewQueryLocation("q"), Schema: state.NewSchema(reflect.TypeOf(""))}, + }, + }, + Selector: &view.Config{}, + } + comp.Output.Type = state.Type{ + Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{})), + Parameters: state.Parameters{ + {Name: "Dep", In: &state.Location{Kind: state.KindComponent, Name: "GET:/v1/opdep"}}, + }, + } + + cSchema := &ComponentSchema{component: comp, components: components, schemas: NewContainer()} + params, err := g.operationParameters(context.Background(), cSchema) + if err != nil { + t.Fatalf("unexpected operationParameters error: %v", err) + } + if len(params) < 2 { + t.Fatalf("expected root and component params, got %d", len(params)) + } + }) + + t.Run("component output parameters no component refs", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.Output.Type = state.Type{ + Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{})), + Parameters: state.Parameters{{Name: "OnlyState", In: &state.Location{Kind: state.KindState, Name: "s"}}}, + } + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + params, err := g.componentOutputParameters(context.Background(), cSchema) + if err != nil { + t.Fatalf("unexpected componentOutputParameters error: %v", err) + } + if len(params) != 0 { + t.Fatalf("expected no component params, got %d", len(params)) + } + }) + + t.Run("lookup component param error", func(t *testing.T) { + g := &generator{} + components := &repository.Service{} + registry := repository.NewRegistry("", nil, nil) + setUnexportedField(components, "registry", registry) + dep := newTestComponent(t) + dep.Method = http.MethodGet + dep.Path.Method = http.MethodGet + dep.Path.URI = "/v1/existing" + registry.Register(dep) + comp := newTestComponent(t) + cSchema := &ComponentSchema{component: comp, components: components, schemas: NewContainer()} + if _, err := g.lookupComponentParam(context.Background(), cSchema, "GET:/v1/missing"); err == nil { + t.Fatalf("expected missing provider error") + } + }) + + t.Run("operation parameters missing component provider", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + components := &repository.Service{} + registry := repository.NewRegistry("", nil, nil) + setUnexportedField(components, "registry", registry) + existing := newTestComponent(t) + existing.Method = http.MethodGet + existing.Path.Method = http.MethodGet + existing.Path.URI = "/v1/existing" + registry.Register(existing) + + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + comp.Output.Type = state.Type{ + Schema: state.NewSchema(reflect.TypeOf(struct{ ID int }{})), + Parameters: state.Parameters{ + {Name: "MissingDep", In: &state.Location{Kind: state.KindComponent, Name: "GET:/v1/unknown"}}, + }, + } + cSchema := &ComponentSchema{component: comp, components: components, schemas: NewContainer()} + if _, err := g.operationParameters(context.Background(), cSchema); err == nil { + t.Fatalf("expected missing dependency error") + } + }) + + t.Run("convert param cache nil ref", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + param := &state.Parameter{Name: "ID", In: &state.Location{Kind: state.KindQuery}, Schema: state.NewSchema(reflect.TypeOf(1))} + + first, ok, err := g.convertParam(context.Background(), cSchema, param, "") + if err != nil || !ok || len(first) != 1 { + t.Fatalf("unexpected first convert result: %v %v %d", ok, err, len(first)) + } + + second, ok, err := g.convertParam(context.Background(), cSchema, param, "") + if err != nil || !ok || len(second) != 1 { + t.Fatalf("unexpected second convert result: %v %v %d", ok, err, len(second)) + } + if second[0].Ref == "" { + t.Fatalf("expected parameter ref") + } + + third, ok, err := g.convertParam(context.Background(), cSchema, param, "") + if err != nil || !ok || len(third) != 1 { + t.Fatalf("unexpected third convert result: %v %v %d", ok, err, len(third)) + } + if third[0].Ref == "" { + t.Fatalf("expected cached nil path to still return ref") + } + }) + + t.Run("append built-in non-http parameter", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + params := []*openapi3.Parameter{} + stateParam := &state.Parameter{Name: "StateOnly", In: &state.Location{Kind: state.KindState, Name: "state"}, Schema: state.NewSchema(reflect.TypeOf(""))} + if err := g.appendBuiltInParam(context.Background(), ¶ms, cSchema, stateParam); err != nil { + t.Fatalf("unexpected append error: %v", err) + } + if len(params) != 0 { + t.Fatalf("expected non-http built-in param to be skipped") + } + }) + + t.Run("view parameters and relation errors", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + + errorView := &view.View{ + Template: &view.Template{ + Parameters: state.Parameters{ + {Name: "Bad", In: state.NewQueryLocation("bad"), Schema: state.NewSchema(reflect.TypeOf((chan int)(nil)))}, + }, + }, + Selector: &view.Config{}, + } + if _, err := g.viewParameters(context.Background(), errorView, cSchema); err == nil { + t.Fatalf("expected view parameter conversion error") + } + + relationErrorView := &view.View{ + Template: &view.Template{}, + Selector: &view.Config{}, + With: []*view.Relation{ + {Of: &view.ReferenceView{View: *errorView}}, + }, + } + if _, err := g.getAllViewsParameters(context.Background(), cSchema, relationErrorView); err == nil { + t.Fatalf("expected relation parameter conversion error") + } + }) +} diff --git a/gateway/router/openapi/openapi3.go b/gateway/router/openapi/openapi3.go index 34ac163b8..911b6052d 100644 --- a/gateway/router/openapi/openapi3.go +++ b/gateway/router/openapi/openapi3.go @@ -5,7 +5,6 @@ import ( "fmt" openapi "github.com/viant/datly/gateway/router/openapi/openapi3" "github.com/viant/datly/repository" - "github.com/viant/datly/repository/contract" "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/datly/view/state" @@ -77,99 +76,6 @@ func GenerateOpenAPI3Spec(ctx context.Context, components *repository.Service, i }).GenerateSpec(ctx, components, info, providers...) } -func (g *generator) generatePaths(ctx context.Context, components *repository.Service, providers []*repository.Provider) (*SchemaContainer, openapi.Paths, error) { - container := NewContainer() - builder := &PathsBuilder{paths: openapi.Paths{}} - var retErr error - pathItem := &openapi.PathItem{} - for _, provider := range providers { - component, err := provider.Component(ctx) - if err != nil { - retErr = err - } - if component == nil { - fmt.Printf("provider.Component(ctx) returned nil\n") - continue - } - componentSchema := NewComponentSchema(components, component, container) - operation, err := g.generateOperation(ctx, componentSchema) - if err != nil { - retErr = err - } - switch component.Method { - case http.MethodGet: - pathItem.Get = operation - case http.MethodPost: - pathItem.Post = operation - case http.MethodDelete: - pathItem.Delete = operation - case http.MethodPut: - pathItem.Put = operation - case http.MethodPatch: - pathItem.Patch = operation - } - builder.AddPath(component.URI, pathItem) - } - - return container, builder.paths, retErr -} - -func (g *generator) generateOperation(ctx context.Context, component *ComponentSchema) (*openapi.Operation, error) { - body, err := g.requestBody(ctx, component) - if err != nil { - return nil, err - } - - parameters, err := g.getAllViewsParameters(ctx, component, component.component.View) - - if err != nil { - return nil, err - } - - if err := g.forEachParam(component.component.Output.Type.Parameters, func(parameter *state.Parameter) (bool, error) { - if parameter.In.Kind == state.KindComponent { - method, URI := shared.ExtractPath(parameter.In.Name) - provider, err := component.components.Registry().LookupProvider(ctx, &contract.Path{ - URI: URI, - Method: method, - }) - - if err != nil { - return false, err - } - - paramComponent, err := provider.Component(ctx) - if err != nil { - return false, err - } - - viewsParameters, err := g.getAllViewsParameters(ctx, NewComponentSchema(component.components, paramComponent, component.schemas), paramComponent.View) - if err != nil { - return false, err - } - - parameters = append(parameters, viewsParameters...) - } - - return true, nil - }); err != nil { - return nil, err - } - - responses, err := g.responses(ctx, component) - if err != nil { - return nil, err - } - - operation := &openapi.Operation{ - Parameters: dedupe(parameters), - RequestBody: body, - Responses: responses, - } - - return operation, nil -} - func dedupe(parameters []*openapi.Parameter) openapi.Parameters { index := map[paramLocation]bool{} var result []*openapi.Parameter @@ -407,7 +313,7 @@ func (g *generator) requestBody(ctx context.Context, component *ComponentSchema) func (g *generator) responses(ctx context.Context, component *ComponentSchema) (openapi.Responses, error) { method := component.component.Method if method == http.MethodOptions { - return nil, nil + return openapi.Responses{}, nil } responseSchema, err := component.ResponseBody(ctx) @@ -421,27 +327,27 @@ func (g *generator) responses(ctx context.Context, component *ComponentSchema) ( } responses := openapi.Responses{} - responses[200] = &openapi.Response{ + openapi.SetResponse(responses, 200, &openapi.Response{ Description: stringPtr("Success response"), Content: map[string]*openapi.MediaType{ ApplicationJson: { Schema: schema, }, }, - } + }) errorSchema, err := component.GetOrGenerateSchema(ctx, component.ReflectSchema("ErrorResponse", errorType, errorSchemaDescription, component.component.IOConfig())) if err != nil { return nil, err } - responses["default"] = &openapi.Response{ + openapi.SetResponse(responses, openapi.ResponseDefault, &openapi.Response{ Description: stringPtr("Error response. The view and param may be empty, but one of the message or object should be specified"), Content: map[string]*openapi.MediaType{ ApplicationJson: { Schema: errorSchema, }, - }} + }}) return responses, nil } diff --git a/gateway/router/openapi/openapi3/additional_branches_test.go b/gateway/router/openapi/openapi3/additional_branches_test.go new file mode 100644 index 000000000..1328c509f --- /dev/null +++ b/gateway/router/openapi/openapi3/additional_branches_test.go @@ -0,0 +1,120 @@ +package openapi3 + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" +) + +func TestResponsesHelpersAndOperationMarshal(t *testing.T) { + responses := Responses{} + SetResponse(responses, 200, &Response{Description: strPtr("ok")}) + SetResponse(responses, ResponseDefault, &Response{Description: strPtr("fallback")}) + SetResponse(responses, ResponseIntKey("201"), &Response{Description: strPtr("created")}) + + if got, ok := GetResponse(responses, 200); !ok || got == nil || got.Description == nil || *got.Description != "ok" { + t.Fatalf("expected integer-key lookup to resolve 200 response") + } + if _, ok := GetResponse(responses, "200"); !ok { + t.Fatalf("expected string-key lookup to resolve 200 response") + } + if _, ok := GetResponse(responses, ResponseDefault); !ok { + t.Fatalf("expected default response") + } + if got, ok := GetResponse(responses, ResponseIntKey("201")); !ok || got == nil || got.Description == nil || *got.Description != "created" { + t.Fatalf("expected ResponseIntKey lookup to resolve 201 response") + } + if len(responses) != 3 { + t.Fatalf("expected three responses to be set") + } + + op := &Operation{ + Summary: "sum", + Responses: responses, + Extension: Extension{"x-extra": true}, + } + data, err := json.Marshal(op) + if err != nil { + t.Fatalf("unexpected marshal error: %v", err) + } + if !strings.Contains(string(data), "\"200\"") || !strings.Contains(string(data), "x-extra") { + t.Fatalf("expected marshaled operation to include response and extension: %s", string(data)) + } +} + +func assertNormalize[T ResponseKey](t *testing.T, name string, input T, expected ResponseCode) { + t.Helper() + t.Run(name, func(t *testing.T) { + if got := NormalizeResponseCode(input); got != expected { + t.Fatalf("expected %q, got %q", expected, got) + } + }) +} + +func TestNormalizeResponseCode_Table(t *testing.T) { + assertNormalize(t, "string", "default", ResponseCode("default")) + assertNormalize(t, "response code", ResponseDefault, ResponseCode("default")) + assertNormalize(t, "response int key", ResponseIntKey("200"), ResponseCode("200")) + assertNormalize(t, "int", int(200), ResponseCode("200")) + assertNormalize(t, "int8", int8(101), ResponseCode("101")) + assertNormalize(t, "int16", int16(202), ResponseCode("202")) + assertNormalize(t, "int32", int32(203), ResponseCode("203")) + assertNormalize(t, "int64", int64(204), ResponseCode("204")) + assertNormalize(t, "uint", uint(205), ResponseCode("205")) + assertNormalize(t, "uint8", uint8(206), ResponseCode("206")) + assertNormalize(t, "uint16", uint16(207), ResponseCode("207")) + assertNormalize(t, "uint32", uint32(208), ResponseCode("208")) + assertNormalize(t, "uint64", uint64(209), ResponseCode("209")) +} + +func TestUnmarshalYAMLErrorBranches_Table(t *testing.T) { + tests := []struct { + name string + target yamlUnmarshaller + source interface{} + firstErr error + secondErr error + wantErr string + }{ + {name: "parameter first err", target: &Parameter{}, source: Parameter{}, firstErr: errors.New("p-first"), wantErr: "p-first"}, + {name: "link second err", target: &Link{}, source: Link{}, secondErr: errors.New("l-second"), wantErr: "l-second"}, + {name: "request body second err", target: &RequestBody{}, source: RequestBody{}, secondErr: errors.New("rb-second"), wantErr: "rb-second"}, + {name: "response second err", target: &Response{}, source: Response{}, secondErr: errors.New("resp-second"), wantErr: "resp-second"}, + {name: "security second err", target: &SecurityScheme{}, source: SecurityScheme{}, secondErr: errors.New("sec-second"), wantErr: "sec-second"}, + {name: "schema second err", target: &Schema{}, source: Schema{}, secondErr: errors.New("schema-second"), wantErr: "schema-second"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.target.UnmarshalYAML(context.Background(), yamlDecoder(tt.source, nil, tt.firstErr, tt.secondErr)) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected err containing %q, got %v", tt.wantErr, err) + } + }) + } +} + +func TestOperationResponsesBoundary(t *testing.T) { + t.Run("unmarshal json initializes responses", func(t *testing.T) { + var op Operation + if err := json.Unmarshal([]byte(`{"summary":"s"}`), &op); err != nil { + t.Fatalf("unexpected unmarshal error: %v", err) + } + if op.Responses == nil { + t.Fatalf("expected non-nil responses after unmarshal") + } + }) + + t.Run("unmarshal yaml initializes responses", func(t *testing.T) { + var op Operation + err := op.UnmarshalYAML(context.Background(), yamlDecoder(map[string]interface{}{"summary": "s"}, map[string]interface{}{"x-a": 1}, nil, nil)) + if err != nil { + t.Fatalf("unexpected yaml unmarshal error: %v", err) + } + if op.Responses == nil { + t.Fatalf("expected non-nil responses after yaml unmarshal") + } + }) +} diff --git a/gateway/router/openapi/openapi3/model_methods_test.go b/gateway/router/openapi/openapi3/model_methods_test.go new file mode 100644 index 000000000..ccab0a767 --- /dev/null +++ b/gateway/router/openapi/openapi3/model_methods_test.go @@ -0,0 +1,256 @@ +package openapi3 + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" +) + +type yamlUnmarshaller interface { + UnmarshalYAML(ctx context.Context, fn func(dest interface{}) error) error +} + +func yamlDecoder(first interface{}, ext map[string]interface{}, firstErr, secondErr error) func(dest interface{}) error { + call := 0 + return func(dest interface{}) error { + call++ + if call == 1 { + if firstErr != nil { + return firstErr + } + if first == nil { + return nil + } + b, err := json.Marshal(first) + if err != nil { + return err + } + return json.Unmarshal(b, dest) + } + if secondErr != nil { + return secondErr + } + if ext == nil { + ext = map[string]interface{}{} + } + b, err := json.Marshal(ext) + if err != nil { + return err + } + return json.Unmarshal(b, dest) + } +} + +func TestMergeJSON(t *testing.T) { + tests := []struct { + name string + j1 []byte + j2 []byte + expect string + }{ + {name: "empty base", j1: []byte("{}"), j2: []byte(`{"x-a":1}`), expect: `{"x-a":1}`}, + {name: "merged", j1: []byte(`{"a":1}`), j2: []byte(`{"x-a":1}`), expect: `{"a":1,"x-a":1}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := string(mergeJSON(tt.j1, tt.j2)); got != tt.expect { + t.Fatalf("expected %s, got %s", tt.expect, got) + } + }) + } +} + +func TestExtensionFunctions(t *testing.T) { + t.Run("unmarshal json keeps x keys", func(t *testing.T) { + ext := Extension{} + if err := ext.UnmarshalJSON([]byte(`{"x-a":1,"a":2}`)); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := ext["x-a"]; !ok { + t.Fatalf("expected x-a key") + } + if _, ok := ext["a"]; ok { + t.Fatalf("did not expect non-extension key") + } + }) + + t.Run("custom extension yaml", func(t *testing.T) { + custom := CustomExtension{} + fn := yamlDecoder(map[string]interface{}{"x-a": 1, "a": 2}, nil, nil, nil) + if err := custom.UnmarshalYAML(context.Background(), fn); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := custom["x-a"]; !ok { + t.Fatalf("expected x-a") + } + if _, ok := custom["a"]; ok { + t.Fatalf("did not expect a") + } + }) +} + +func TestMarshalJSONWithExtensions_Table(t *testing.T) { + trueValue := true + tests := []struct { + name string + value interface{} + wantErr bool + }{ + {name: "components", value: &Components{Extension: Extension{"x-a": 1}, Schemas: Schemas{"Pet": {Type: "object"}}}}, + {name: "parameter", value: &Parameter{Extension: Extension{"x-a": 1}, Name: "id", In: "query"}}, + {name: "security", value: &SecurityScheme{Extension: Extension{"x-a": 1}, Type: "http"}}, + {name: "oauth flows", value: &OAuthFlows{Extension: Extension{"x-a": 1}, Password: &OAuthFlow{TokenURL: "token", Scopes: map[string]string{"s": "v"}}}}, + {name: "oauth flow", value: &OAuthFlow{Extension: Extension{"x-a": 1}, TokenURL: "token", Scopes: map[string]string{"s": "v"}}}, + {name: "example", value: &Example{Extension: Extension{"x-a": 1}, Summary: "s"}}, + {name: "server", value: &Server{Extension: Extension{"x-a": 1}, URL: "http://example"}}, + {name: "server variable", value: &ServerVariable{Extension: Extension{"x-a": 1}, Default: "dev"}}, + {name: "info", value: &Info{Extension: Extension{"x-a": 1}, Title: "api", Version: "1.0"}}, + {name: "contact", value: &Contact{Extension: Extension{"x-a": 1}, Name: "n"}}, + {name: "license", value: &License{Extension: Extension{"x-a": 1}, Name: "mit"}}, + {name: "tag", value: &Tag{Extension: Extension{"x-a": 1}, Name: "n"}}, + {name: "path item", value: &PathItem{Extension: Extension{"x-a": 1}, Summary: "sum"}}, + {name: "encoding", value: &Encoding{Extension: Extension{"x-a": 1}, ContentType: "application/json", Explode: &trueValue}}, + {name: "request body", value: &RequestBody{Extension: Extension{"x-a": 1}, Description: "d"}}, + {name: "external doc", value: &ExternalDocumentation{Extension: Extension{"x-a": 1}, URL: "http://example"}}, + {name: "response", value: &Response{Extension: Extension{"x-a": 1}, Description: strPtr("ok")}}, + {name: "media", value: &MediaType{Extension: Extension{"x-a": 1}, Example: map[string]interface{}{"a": 1}}}, + {name: "operation", value: &Operation{Extension: Extension{"x-a": 1}, Summary: "sum"}}, + {name: "link", value: &Link{Extension: Extension{"x-a": 1}, OperationID: "op"}}, + {name: "schema", value: &Schema{Extension: Extension{"x-a": 1}, Type: "object"}}, + {name: "xml", value: &XML{Extension: Extension{"x-a": 1}, Name: "node"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := json.Marshal(tt.value) + if tt.wantErr { + if err == nil { + t.Fatalf("expected marshal error") + } + return + } + if err != nil { + t.Fatalf("marshal error: %v", err) + } + if !strings.Contains(string(data), "x-a") { + t.Fatalf("expected extension in json: %s", string(data)) + } + }) + } +} + +func TestUnmarshalJSON_Table(t *testing.T) { + tests := []struct { + name string + target interface{} + json string + }{ + {name: "components", target: &Components{}, json: `{"schemas":{"Pet":{"type":"object"}}}`}, + {name: "parameter", target: &Parameter{}, json: `{"name":"id","in":"query"}`}, + {name: "security", target: &SecurityScheme{}, json: `{"type":"http"}`}, + {name: "oauth flows", target: &OAuthFlows{}, json: `{"password":{"tokenUrl":"token","scopes":{"s":"v"}}}`}, + {name: "oauth flow", target: &OAuthFlow{}, json: `{"tokenUrl":"token","scopes":{"s":"v"}}`}, + {name: "example", target: &Example{}, json: `{"summary":"s"}`}, + {name: "server", target: &Server{}, json: `{"url":"http://example"}`}, + {name: "server variable", target: &ServerVariable{}, json: `{"default":"dev"}`}, + {name: "info", target: &Info{}, json: `{"title":"api","version":"1"}`}, + {name: "contact", target: &Contact{}, json: `{"name":"n"}`}, + {name: "license", target: &License{}, json: `{"name":"mit"}`}, + {name: "tag", target: &Tag{}, json: `{"name":"n"}`}, + {name: "path", target: &PathItem{}, json: `{"summary":"sum"}`}, + {name: "encoding", target: &Encoding{}, json: `{"contentType":"application/json"}`}, + {name: "request", target: &RequestBody{}, json: `{"description":"d"}`}, + {name: "external", target: &ExternalDocumentation{}, json: `{"url":"http://example"}`}, + {name: "response", target: &Response{}, json: `{"description":"ok"}`}, + {name: "media", target: &MediaType{}, json: `{"example":{"a":1}}`}, + {name: "operation", target: &Operation{}, json: `{"summary":"sum"}`}, + {name: "link", target: &Link{}, json: `{"operationId":"op"}`}, + {name: "schema", target: &Schema{}, json: `{"type":"object"}`}, + {name: "xml", target: &XML{}, json: `{"name":"node"}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := json.Unmarshal([]byte(tt.json), tt.target); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + }) + } +} + +func seedSession() context.Context { + sessionCtx := NewSessionContext(context.Background()) + session := LookupSession(sessionCtx) + session.Location = "main" + session.RegisterComponents("main", &Components{ + Schemas: Schemas{"Pet": {Type: "object"}}, + Parameters: ParametersMap{"id": {Name: "id", In: "query"}}, + Headers: Headers{"/components/headers/Trace": {Name: "Trace", In: "header"}}, + RequestBodies: RequestBodies{"/components/requestBodies/Create": {Description: "create"}}, + Responses: Responses{"/components/responses/Default": {Description: strPtr("default")}}, + SecuritySchemes: SecuritySchemes{"/components/securitySchemes/Bearer": {Type: "http"}}, + Examples: Examples{"/components/examples/Sample": {Summary: "sample"}}, + Links: Links{"/components/links/Self": {OperationID: "self"}}, + Callbacks: Callbacks{"/components/callbacks/Event": {Ref: "inner"}}, + }) + return sessionCtx +} + +func TestUnmarshalYAML_Table(t *testing.T) { + tests := []struct { + name string + target yamlUnmarshaller + source interface{} + ext map[string]interface{} + firstErr error + secondErr error + wantErr string + }{ + {name: "components", target: &Components{}, source: Components{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "parameter ref", target: &Parameter{}, source: Parameter{Ref: "#/components/parameters/id"}}, + {name: "security ref", target: &SecurityScheme{}, source: SecurityScheme{Ref: "#/components/securitySchemes/Bearer"}, ext: map[string]interface{}{"x-a": 1}}, + {name: "oauth flows", target: &OAuthFlows{}, source: OAuthFlows{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "oauth flow", target: &OAuthFlow{}, source: OAuthFlow{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "callback ref", target: &CallbackRef{}, source: CallbackRef{Ref: "#/components/callbacks/Event"}}, + {name: "example", target: &Example{}, source: Example{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "server", target: &Server{}, source: Server{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "server variable", target: &ServerVariable{}, source: ServerVariable{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "info", target: &Info{}, source: Info{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "contact", target: &Contact{}, source: Contact{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "license", target: &License{}, source: License{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "tag", target: &Tag{}, source: Tag{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "path", target: &PathItem{}, source: PathItem{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "encoding", target: &Encoding{}, source: Encoding{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "request body ref", target: &RequestBody{}, source: RequestBody{Ref: "#/components/requestBodies/Create"}, ext: map[string]interface{}{"x-a": 1}}, + {name: "external doc", target: &ExternalDocumentation{}, source: ExternalDocumentation{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "response ref", target: &Response{}, source: Response{Ref: "#/components/responses/Default"}, ext: map[string]interface{}{"x-a": 1}}, + {name: "media", target: &MediaType{}, source: MediaType{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "operation", target: &Operation{}, source: nil, ext: map[string]interface{}{"x-a": 1}}, + {name: "link ref", target: &Link{}, source: Link{Ref: "#/components/links/Self"}, ext: map[string]interface{}{"x-a": 1}}, + {name: "schema ref", target: &Schema{}, source: Schema{Ref: "#/components/schemas/Pet"}, ext: map[string]interface{}{"x-a": 1}}, + {name: "xml", target: &XML{}, source: XML{}, ext: map[string]interface{}{"x-a": 1}}, + {name: "first decoder error", target: &XML{}, firstErr: errors.New("first decoder"), wantErr: "first decoder"}, + {name: "second decoder error", target: &XML{}, source: XML{}, secondErr: errors.New("second decoder"), wantErr: "second decoder"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := seedSession() + err := tt.target.UnmarshalYAML(ctx, yamlDecoder(tt.source, tt.ext, tt.firstErr, tt.secondErr)) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected err containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + +func strPtr(v string) *string { return &v } diff --git a/gateway/router/openapi/openapi3/operation.go b/gateway/router/openapi/openapi3/operation.go index 1421f2017..cd37e8a91 100644 --- a/gateway/router/openapi/openapi3/operation.go +++ b/gateway/router/openapi/openapi3/operation.go @@ -50,6 +50,9 @@ func (o *Operation) UnmarshalJSON(b []byte) error { if err != nil { return err } + if tmp.Responses == nil { + tmp.Responses = Responses{} + } *o = Operation(tmp) return o.Extension.UnmarshalJSON(b) } @@ -58,6 +61,9 @@ func (o *Operation) MarshalJSON() ([]byte, error) { type temp Operation tmp := temp(*o) tmp.Extension = nil + if tmp.Responses == nil { + tmp.Responses = Responses{} + } data, err := json.Marshal(tmp) if err != nil { return nil, err @@ -76,7 +82,6 @@ func (o *Operation) MarshalJSON() ([]byte, error) { return res, nil } - func (o *Operation) UnmarshalYAML(ctx context.Context, fn func(dest interface{}) error) error { type temp Operation tmp := temp(*o) @@ -90,6 +95,9 @@ func (o *Operation) UnmarshalYAML(ctx context.Context, fn func(dest interface{}) return err } tmp.Extension = Extension(ext) + if tmp.Responses == nil { + tmp.Responses = Responses{} + } *o = Operation(tmp) return nil } diff --git a/gateway/router/openapi/openapi3/response.go b/gateway/router/openapi/openapi3/response.go index 656065726..f866d3728 100644 --- a/gateway/router/openapi/openapi3/response.go +++ b/gateway/router/openapi/openapi3/response.go @@ -3,11 +3,12 @@ package openapi3 import ( "context" "encoding/json" + "fmt" ) // Responses is specified by OpenAPI/Swagger 3.0 standard. type ( - Responses map[interface{}]*Response + Responses map[string]*Response // Response is specified by OpenAPI/Swagger 3.0 standard. Response struct { @@ -20,6 +21,40 @@ type ( } ) +const ( + ResponseDefault ResponseCode = "default" +) + +type ( + ResponseCode string + ResponseIntKey string + + ResponseKey interface { + ~string | ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 + } +) + +func NormalizeResponseCode[T ResponseKey](code T) ResponseCode { + return ResponseCode(fmt.Sprintf("%v", code)) +} + +func SetResponse[T ResponseKey](r Responses, code T, response *Response) { + key := NormalizeResponseCode(code) + if key == "" { + return + } + r[string(key)] = response +} + +func GetResponse[T ResponseKey](r Responses, code T) (*Response, bool) { + key := NormalizeResponseCode(code) + if key == "" { + return nil, false + } + value, ok := r[string(key)] + return value, ok +} + func (r *Response) UnmarshalJSON(b []byte) error { type temp Response var tmp = temp{} diff --git a/gateway/router/openapi/openapi3/session.go b/gateway/router/openapi/openapi3/session.go index 741d4c72b..8c29c5a30 100644 --- a/gateway/router/openapi/openapi3/session.go +++ b/gateway/router/openapi/openapi3/session.go @@ -38,7 +38,7 @@ func (s *Session) RegisterComponents(location string, components *Components) { components.RequestBodies = map[string]*RequestBody{} } if len(components.Responses) == 0 { - components.Responses = map[interface{}]*Response{} + components.Responses = map[string]*Response{} } if len(components.SecuritySchemes) == 0 { components.SecuritySchemes = map[string]*SecurityScheme{} @@ -60,8 +60,7 @@ func (s *Session) RegisterComponents(location string, components *Components) { // LookupSchema lookups schema func (s *Session) LookupSchema(location string, ref string) (*Schema, error) { - switch ref[0] { - case '#': + if len(ref) > 0 && ref[0] == '#' { id := s.normalizeRef(ref[1:], "/components/schemas/") components, ok := s.components[location] if !ok { @@ -74,20 +73,13 @@ func (s *Session) LookupSchema(location string, ref string) (*Schema, error) { result := *value result.Ref = ref return &result, nil - case '.': - - case '/': - - default: - } return nil, fmt.Errorf("unsupported: %v, at %v", ref, location) } // LookupParameter lookup parameters func (s *Session) LookupParameter(location string, ref string) (*Parameter, error) { - switch ref[0] { - case '#': + if len(ref) > 0 && ref[0] == '#' { id := s.normalizeRef(ref[1:], "/components/parameters/") components, ok := s.components[location] if !ok { @@ -100,20 +92,13 @@ func (s *Session) LookupParameter(location string, ref string) (*Parameter, erro result := *value result.Ref = ref return &result, nil - case '.': - - case '/': - - default: - } return nil, fmt.Errorf("unsupported: %v, at %v", ref, location) } // LookupHeaders lookup headers func (s *Session) LookupHeaders(location string, ref string) (*Parameter, error) { - switch ref[0] { - case '#': + if len(ref) > 0 && ref[0] == '#' { components, ok := s.components[location] if !ok { return nil, fmt.Errorf("failed to lookup location: %v", location) @@ -125,20 +110,13 @@ func (s *Session) LookupHeaders(location string, ref string) (*Parameter, error) result := *value result.Ref = ref return &result, nil - case '.': - - case '/': - - default: - } return nil, fmt.Errorf("unsupported: %v, at %v", ref, location) } // LookupRequestBody lookup request body func (s *Session) LookupRequestBody(location string, ref string) (*RequestBody, error) { - switch ref[0] { - case '#': + if len(ref) > 0 && ref[0] == '#' { components, ok := s.components[location] if !ok { return nil, fmt.Errorf("failed to lookup location: %v", location) @@ -150,20 +128,13 @@ func (s *Session) LookupRequestBody(location string, ref string) (*RequestBody, result := *value result.Ref = ref return &result, nil - case '.': - - case '/': - - default: - } return nil, fmt.Errorf("unsupported: %v, at %v", ref, location) } // LookupResponse lookup response func (s *Session) LookupResponse(location string, ref string) (*Response, error) { - switch ref[0] { - case '#': + if len(ref) > 0 && ref[0] == '#' { components, ok := s.components[location] if !ok { return nil, fmt.Errorf("failed to lookup location: %v", location) @@ -175,20 +146,13 @@ func (s *Session) LookupResponse(location string, ref string) (*Response, error) result := *value result.Ref = ref return &result, nil - case '.': - - case '/': - - default: - } return nil, fmt.Errorf("unsupported: %v, at %v", ref, location) } // LookupSecurityScheme lookup security scheme func (s *Session) LookupSecurityScheme(location string, ref string) (*SecurityScheme, error) { - switch ref[0] { - case '#': + if len(ref) > 0 && ref[0] == '#' { components, ok := s.components[location] if !ok { return nil, fmt.Errorf("failed to lookup location: %v", location) @@ -200,20 +164,13 @@ func (s *Session) LookupSecurityScheme(location string, ref string) (*SecuritySc result := *value result.Ref = ref return &result, nil - case '.': - - case '/': - - default: - } return nil, fmt.Errorf("unsupported: %v, at %v", ref, location) } // LookupExample lookup example func (s *Session) LookupExample(location string, ref string) (*Example, error) { - switch ref[0] { - case '#': + if len(ref) > 0 && ref[0] == '#' { components, ok := s.components[location] if !ok { return nil, fmt.Errorf("failed to lookup location: %v", location) @@ -225,20 +182,13 @@ func (s *Session) LookupExample(location string, ref string) (*Example, error) { result := *value result.Ref = ref return &result, nil - case '.': - - case '/': - - default: - } return nil, fmt.Errorf("unsupported: %v, at %v", ref, location) } // LookupLink lookup link func (s *Session) LookupLink(location string, ref string) (*Link, error) { - switch ref[0] { - case '#': + if len(ref) > 0 && ref[0] == '#' { components, ok := s.components[location] if !ok { return nil, fmt.Errorf("failed to lookup location: %v", location) @@ -250,20 +200,13 @@ func (s *Session) LookupLink(location string, ref string) (*Link, error) { result := *value result.Ref = ref return &result, nil - case '.': - - case '/': - - default: - } return nil, fmt.Errorf("unsupported: %v, at %v", ref, location) } // LookupLink lookup callback func (s *Session) LookupCallback(location string, ref string) (*CallbackRef, error) { - switch ref[0] { - case '#': + if len(ref) > 0 && ref[0] == '#' { components, ok := s.components[location] if !ok { return nil, fmt.Errorf("failed to lookup location: %v", location) @@ -275,12 +218,6 @@ func (s *Session) LookupCallback(location string, ref string) (*CallbackRef, err result := *value result.Ref = ref return &result, nil - case '.': - - case '/': - - default: - } return nil, fmt.Errorf("unsupported: %v, at %v", ref, location) } diff --git a/gateway/router/openapi/openapi3/session_test.go b/gateway/router/openapi/openapi3/session_test.go new file mode 100644 index 000000000..4de4a2672 --- /dev/null +++ b/gateway/router/openapi/openapi3/session_test.go @@ -0,0 +1,152 @@ +package openapi3 + +import ( + "context" + "errors" + "strings" + "testing" +) + +func TestSessionRegisterAndLookup_Table(t *testing.T) { + s := NewSession() + s.Location = "loc" + s.RegisterComponents("loc", &Components{ + Schemas: Schemas{"Pet": {Type: "object"}}, + Parameters: ParametersMap{"id": {Name: "id", In: "query"}}, + Headers: Headers{"/components/headers/Trace": {Name: "Trace", In: "header"}}, + RequestBodies: RequestBodies{"/components/requestBodies/Create": {Description: "create"}}, + Responses: Responses{"/components/responses/Default": {Description: stringRef("default")}}, + SecuritySchemes: SecuritySchemes{"/components/securitySchemes/Bearer": {Type: "http"}}, + Examples: Examples{"/components/examples/Sample": {Summary: "sample"}}, + Links: Links{"/components/links/Self": {OperationID: "self"}}, + Callbacks: Callbacks{"/components/callbacks/Event": {Ref: "eventRef"}}, + }) + + tests := []struct { + name string + lookup func() (interface{}, error) + wantErr string + assert func(t *testing.T, got interface{}) + }{ + {name: "lookup schema", lookup: func() (interface{}, error) { return s.LookupSchema("loc", "#/components/schemas/Pet") }, assert: func(t *testing.T, got interface{}) { + if got.(*Schema).Ref == "" { + t.Fatalf("missing ref") + } + }}, + {name: "lookup parameter", lookup: func() (interface{}, error) { return s.LookupParameter("loc", "#/components/parameters/id") }, assert: func(t *testing.T, got interface{}) { + if got.(*Parameter).Name != "id" { + t.Fatalf("name mismatch") + } + }}, + {name: "lookup header", lookup: func() (interface{}, error) { return s.LookupHeaders("loc", "#/components/headers/Trace") }, assert: func(t *testing.T, got interface{}) { + if got.(*Parameter).In != "header" { + t.Fatalf("in mismatch") + } + }}, + {name: "lookup request body", lookup: func() (interface{}, error) { return s.LookupRequestBody("loc", "#/components/requestBodies/Create") }, assert: func(t *testing.T, got interface{}) { + if got.(*RequestBody).Description != "create" { + t.Fatalf("desc mismatch") + } + }}, + {name: "lookup response", lookup: func() (interface{}, error) { return s.LookupResponse("loc", "#/components/responses/Default") }, assert: func(t *testing.T, got interface{}) { + if got.(*Response).Description == nil { + t.Fatalf("desc missing") + } + }}, + {name: "lookup security", lookup: func() (interface{}, error) { + return s.LookupSecurityScheme("loc", "#/components/securitySchemes/Bearer") + }, assert: func(t *testing.T, got interface{}) { + if got.(*SecurityScheme).Type != "http" { + t.Fatalf("type mismatch") + } + }}, + {name: "lookup example", lookup: func() (interface{}, error) { return s.LookupExample("loc", "#/components/examples/Sample") }, assert: func(t *testing.T, got interface{}) { + if got.(*Example).Summary != "sample" { + t.Fatalf("summary mismatch") + } + }}, + {name: "lookup link", lookup: func() (interface{}, error) { return s.LookupLink("loc", "#/components/links/Self") }, assert: func(t *testing.T, got interface{}) { + if got.(*Link).OperationID != "self" { + t.Fatalf("op mismatch") + } + }}, + {name: "lookup callback", lookup: func() (interface{}, error) { return s.LookupCallback("loc", "#/components/callbacks/Event") }, assert: func(t *testing.T, got interface{}) { + if got.(*CallbackRef).Ref != "#/components/callbacks/Event" { + t.Fatalf("ref mismatch") + } + }}, + {name: "missing location", lookup: func() (interface{}, error) { return s.LookupSchema("other", "#/components/schemas/Pet") }, wantErr: "failed to lookup location"}, + {name: "missing value", lookup: func() (interface{}, error) { return s.LookupParameter("loc", "#/components/parameters/other") }, wantErr: "failed to lookup"}, + {name: "unsupported ref", lookup: func() (interface{}, error) { return s.LookupSchema("loc", "./components/schemas/Pet") }, wantErr: "unsupported"}, + {name: "unsupported parameter ref", lookup: func() (interface{}, error) { return s.LookupParameter("loc", "./components/parameters/id") }, wantErr: "unsupported"}, + {name: "unsupported header ref", lookup: func() (interface{}, error) { return s.LookupHeaders("loc", "./components/headers/Trace") }, wantErr: "unsupported"}, + {name: "unsupported request body ref", lookup: func() (interface{}, error) { return s.LookupRequestBody("loc", "./components/requestBodies/Create") }, wantErr: "unsupported"}, + {name: "unsupported response ref", lookup: func() (interface{}, error) { return s.LookupResponse("loc", "./components/responses/Default") }, wantErr: "unsupported"}, + {name: "unsupported security ref", lookup: func() (interface{}, error) { + return s.LookupSecurityScheme("loc", "./components/securitySchemes/Bearer") + }, wantErr: "unsupported"}, + {name: "unsupported example ref", lookup: func() (interface{}, error) { return s.LookupExample("loc", "./components/examples/Sample") }, wantErr: "unsupported"}, + {name: "unsupported link ref", lookup: func() (interface{}, error) { return s.LookupLink("loc", "./components/links/Self") }, wantErr: "unsupported"}, + {name: "unsupported callback ref", lookup: func() (interface{}, error) { return s.LookupCallback("loc", "./components/callbacks/Event") }, wantErr: "unsupported"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.lookup() + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.assert != nil { + tt.assert(t, got) + } + }) + } +} + +func TestSessionHelpers(t *testing.T) { + t.Run("add defer and close", func(t *testing.T) { + s := NewSession() + order := 0 + s.AddDefer(func() error { order++; return nil }) + s.AddDefer(func() error { order++; return nil }) + if err := s.Close(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if order != 2 { + t.Fatalf("expected both defers, got %d", order) + } + }) + + t.Run("close returns defer error", func(t *testing.T) { + s := NewSession() + s.AddDefer(func() error { return errors.New("boom") }) + if err := s.Close(); err == nil || err.Error() != "boom" { + t.Fatalf("expected boom, got %v", err) + } + }) + + t.Run("normalize ref", func(t *testing.T) { + s := NewSession() + if got := s.normalizeRef("/components/schemas/Pet", "/components/schemas/"); got != "Pet" { + t.Fatalf("unexpected normalize result: %s", got) + } + }) + + t.Run("lookup session from context", func(t *testing.T) { + ctx := NewSessionContext(context.Background()) + if LookupSession(ctx) == nil { + t.Fatalf("expected session in context") + } + if LookupSession(context.Background()) != nil { + t.Fatalf("expected nil session") + } + }) +} + +func stringRef(v string) *string { return &v } From 286a88fd722abed1ca75e198c032814ea16da000 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Feb 2026 09:37:32 -0800 Subject: [PATCH 142/279] updated openapi schema builder --- gateway/router/openapi/generator_test.go | 6 +- gateway/router/openapi/openapi3.go | 2 +- .../openapi3/additional_branches_test.go | 18 +- .../openapi3/coverage_branches_test.go | 186 ++++++++++++++++++ gateway/router/openapi/openapi3/response.go | 20 +- .../openapi/schema_build_helpers_test.go | 48 +++++ 6 files changed, 264 insertions(+), 16 deletions(-) create mode 100644 gateway/router/openapi/openapi3/coverage_branches_test.go diff --git a/gateway/router/openapi/generator_test.go b/gateway/router/openapi/generator_test.go index c7b4e2cb8..ceb013046 100644 --- a/gateway/router/openapi/generator_test.go +++ b/gateway/router/openapi/generator_test.go @@ -72,7 +72,7 @@ func TestGeneratorTopLevel_Table(t *testing.T) { t.Fatalf("unexpected marshal error: %v", err) } doc := string(data) - if !strings.Contains(doc, `"responses":{"200":`) { + if !strings.Contains(doc, `"responses":{"`+string(openapi3.ResponseOK)+`":`) { t.Fatalf("expected serialized numeric response key as string in spec: %s", doc) } if !strings.Contains(doc, `"default"`) { @@ -239,7 +239,7 @@ func TestGeneratorHelpersMore_Table(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if _, ok := openapi3.GetResponse(resp, 200); !ok { + if _, ok := openapi3.GetResponse(resp, openapi3.ResponseOK); !ok { t.Fatalf("expected success response") } if _, ok := openapi3.GetResponse(resp, openapi3.ResponseDefault); !ok { @@ -389,7 +389,7 @@ func TestGeneratorHelpersMore_Table(t *testing.T) { if err != nil || operation == nil { t.Fatalf("unexpected operation result: %v %v", operation, err) } - if _, ok := openapi3.GetResponse(operation.Responses, 200); !ok { + if _, ok := openapi3.GetResponse(operation.Responses, openapi3.ResponseOK); !ok { t.Fatalf("expected 200 response") } }) diff --git a/gateway/router/openapi/openapi3.go b/gateway/router/openapi/openapi3.go index 911b6052d..16b83855a 100644 --- a/gateway/router/openapi/openapi3.go +++ b/gateway/router/openapi/openapi3.go @@ -327,7 +327,7 @@ func (g *generator) responses(ctx context.Context, component *ComponentSchema) ( } responses := openapi.Responses{} - openapi.SetResponse(responses, 200, &openapi.Response{ + openapi.SetResponse(responses, openapi.ResponseOK, &openapi.Response{ Description: stringPtr("Success response"), Content: map[string]*openapi.MediaType{ ApplicationJson: { diff --git a/gateway/router/openapi/openapi3/additional_branches_test.go b/gateway/router/openapi/openapi3/additional_branches_test.go index 1328c509f..312607c18 100644 --- a/gateway/router/openapi/openapi3/additional_branches_test.go +++ b/gateway/router/openapi/openapi3/additional_branches_test.go @@ -10,21 +10,21 @@ import ( func TestResponsesHelpersAndOperationMarshal(t *testing.T) { responses := Responses{} - SetResponse(responses, 200, &Response{Description: strPtr("ok")}) + SetResponse(responses, ResponseOK, &Response{Description: strPtr("ok")}) SetResponse(responses, ResponseDefault, &Response{Description: strPtr("fallback")}) - SetResponse(responses, ResponseIntKey("201"), &Response{Description: strPtr("created")}) + SetResponse(responses, ResponseCreated, &Response{Description: strPtr("created")}) - if got, ok := GetResponse(responses, 200); !ok || got == nil || got.Description == nil || *got.Description != "ok" { + if got, ok := GetResponse(responses, ResponseOK); !ok || got == nil || got.Description == nil || *got.Description != "ok" { t.Fatalf("expected integer-key lookup to resolve 200 response") } - if _, ok := GetResponse(responses, "200"); !ok { + if _, ok := GetResponse(responses, string(ResponseOK)); !ok { t.Fatalf("expected string-key lookup to resolve 200 response") } if _, ok := GetResponse(responses, ResponseDefault); !ok { t.Fatalf("expected default response") } - if got, ok := GetResponse(responses, ResponseIntKey("201")); !ok || got == nil || got.Description == nil || *got.Description != "created" { - t.Fatalf("expected ResponseIntKey lookup to resolve 201 response") + if got, ok := GetResponse(responses, ResponseCreated); !ok || got == nil || got.Description == nil || *got.Description != "created" { + t.Fatalf("expected ResponseCode lookup to resolve 201 response") } if len(responses) != 3 { t.Fatalf("expected three responses to be set") @@ -39,7 +39,7 @@ func TestResponsesHelpersAndOperationMarshal(t *testing.T) { if err != nil { t.Fatalf("unexpected marshal error: %v", err) } - if !strings.Contains(string(data), "\"200\"") || !strings.Contains(string(data), "x-extra") { + if !strings.Contains(string(data), "\""+string(ResponseOK)+"\"") || !strings.Contains(string(data), "x-extra") { t.Fatalf("expected marshaled operation to include response and extension: %s", string(data)) } } @@ -56,8 +56,8 @@ func assertNormalize[T ResponseKey](t *testing.T, name string, input T, expected func TestNormalizeResponseCode_Table(t *testing.T) { assertNormalize(t, "string", "default", ResponseCode("default")) assertNormalize(t, "response code", ResponseDefault, ResponseCode("default")) - assertNormalize(t, "response int key", ResponseIntKey("200"), ResponseCode("200")) - assertNormalize(t, "int", int(200), ResponseCode("200")) + assertNormalize(t, "response code literal", ResponseCodeLiteral("200"), ResponseOK) + assertNormalize(t, "int", int(200), ResponseOK) assertNormalize(t, "int8", int8(101), ResponseCode("101")) assertNormalize(t, "int16", int16(202), ResponseCode("202")) assertNormalize(t, "int32", int32(203), ResponseCode("203")) diff --git a/gateway/router/openapi/openapi3/coverage_branches_test.go b/gateway/router/openapi/openapi3/coverage_branches_test.go new file mode 100644 index 000000000..89d3656b9 --- /dev/null +++ b/gateway/router/openapi/openapi3/coverage_branches_test.go @@ -0,0 +1,186 @@ +package openapi3 + +import ( + "encoding/json" + "strings" + "testing" +) + +func decodeSequence(values ...interface{}) func(dest interface{}) error { + index := 0 + return func(dest interface{}) error { + if index >= len(values) { + return nil + } + value := values[index] + index++ + if err, ok := value.(error); ok { + return err + } + data, err := json.Marshal(value) + if err != nil { + return err + } + return json.Unmarshal(data, dest) + } +} + +func TestMarshalNoExtension_Table(t *testing.T) { + trueVal := true + tests := []struct { + name string + value interface{} + }{ + {name: "components", value: &Components{Schemas: Schemas{"Pet": {Type: "object"}}}}, + {name: "parameter", value: &Parameter{Name: "id", In: "query"}}, + {name: "security", value: &SecurityScheme{Type: "http"}}, + {name: "example", value: &Example{Summary: "s"}}, + {name: "server", value: &Server{URL: "http://example"}}, + {name: "server variable", value: &ServerVariable{Default: "dev"}}, + {name: "info", value: &Info{Title: "api", Version: "1.0"}}, + {name: "contact", value: &Contact{Name: "n"}}, + {name: "license", value: &License{Name: "mit"}}, + {name: "tag", value: &Tag{Name: "n"}}, + {name: "path item", value: &PathItem{Summary: "sum"}}, + {name: "encoding", value: &Encoding{ContentType: "application/json", Explode: &trueVal}}, + {name: "request body", value: &RequestBody{Description: "d"}}, + {name: "external", value: &ExternalDocumentation{URL: "http://example"}}, + {name: "response", value: &Response{Description: strPtr("ok")}}, + {name: "media", value: &MediaType{Example: map[string]interface{}{"a": 1}}}, + {name: "operation", value: &Operation{Summary: "sum", Responses: Responses{}}}, + {name: "link", value: &Link{OperationID: "op"}}, + {name: "schema", value: &Schema{Type: "object"}}, + {name: "xml", value: &XML{Name: "node"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := json.Marshal(tt.value) + if err != nil { + t.Fatalf("unexpected marshal error: %v", err) + } + if strings.Contains(string(data), "x-") { + t.Fatalf("did not expect extension key in %s", string(data)) + } + }) + } +} + +func TestYAMLRefAndNonRefBranches(t *testing.T) { + ctx := seedSession() + + t.Run("parameter non ref and ref", func(t *testing.T) { + var nonRef Parameter + if err := nonRef.UnmarshalYAML(ctx, decodeSequence(Parameter{Name: "id", In: "query"})); err != nil { + t.Fatalf("unexpected non-ref error: %v", err) + } + + var refHit Parameter + if err := refHit.UnmarshalYAML(ctx, decodeSequence(map[string]interface{}{"$ref": "#/components/parameters/id"})); err != nil { + t.Fatalf("unexpected ref lookup error: %v", err) + } + }) + + t.Run("link non ref and ref", func(t *testing.T) { + var nonRef Link + if err := nonRef.UnmarshalYAML(ctx, decodeSequence(Link{OperationID: "op"}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected non-ref error: %v", err) + } + + var refHit Link + if err := refHit.UnmarshalYAML(ctx, decodeSequence(map[string]interface{}{"$ref": "#/components/links/Self"}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected ref lookup error: %v", err) + } + }) + + t.Run("request body non ref and ref", func(t *testing.T) { + var nonRef RequestBody + if err := nonRef.UnmarshalYAML(ctx, decodeSequence(RequestBody{Description: "d"}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected non-ref error: %v", err) + } + + var refHit RequestBody + if err := refHit.UnmarshalYAML(ctx, decodeSequence(map[string]interface{}{"$ref": "#/components/requestBodies/Create"}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected ref lookup error: %v", err) + } + }) + + t.Run("response non ref and ref", func(t *testing.T) { + var nonRef Response + if err := nonRef.UnmarshalYAML(ctx, decodeSequence(Response{Description: strPtr("ok")}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected non-ref error: %v", err) + } + + var refHit Response + if err := refHit.UnmarshalYAML(ctx, decodeSequence(map[string]interface{}{"$ref": "#/components/responses/Default"}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected ref lookup error: %v", err) + } + }) + + t.Run("security non ref and ref", func(t *testing.T) { + var nonRef SecurityScheme + if err := nonRef.UnmarshalYAML(ctx, decodeSequence(SecurityScheme{Type: "http"}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected non-ref error: %v", err) + } + + var refHit SecurityScheme + if err := refHit.UnmarshalYAML(ctx, decodeSequence(map[string]interface{}{"$ref": "#/components/securitySchemes/Bearer"}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected ref lookup error: %v", err) + } + }) + + t.Run("schema non ref and ref", func(t *testing.T) { + var nonRef Schema + if err := nonRef.UnmarshalYAML(ctx, decodeSequence(Schema{Type: "object"}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected non-ref error: %v", err) + } + + var refHit Schema + if err := refHit.UnmarshalYAML(ctx, decodeSequence(map[string]interface{}{"$ref": "#/components/schemas/Pet"}, map[string]interface{}{"x-a": 1})); err != nil { + t.Fatalf("unexpected ref lookup error: %v", err) + } + }) +} + +func TestSessionLookupMissingBranches(t *testing.T) { + s := NewSession() + s.Location = "loc" + s.RegisterComponents("loc", &Components{ + Schemas: Schemas{"Pet": {Type: "object"}}, + Parameters: ParametersMap{"id": {Name: "id", In: "query"}}, + Headers: Headers{"/components/headers/Trace": {Name: "Trace", In: "header"}}, + RequestBodies: RequestBodies{"/components/requestBodies/Create": {Description: "create"}}, + Responses: Responses{"/components/responses/Default": {Description: strPtr("default")}}, + SecuritySchemes: SecuritySchemes{"/components/securitySchemes/Bearer": {Type: "http"}}, + Examples: Examples{"/components/examples/Sample": {Summary: "sample"}}, + Links: Links{"/components/links/Self": {OperationID: "self"}}, + Callbacks: Callbacks{"/components/callbacks/Event": {Ref: "eventRef"}}, + }) + + tests := []struct { + name string + lookup func() error + wantErr string + }{ + {name: "parameter missing location", lookup: func() error { _, err := s.LookupParameter("other", "#/components/parameters/id"); return err }, wantErr: "failed to lookup location"}, + {name: "header missing value", lookup: func() error { _, err := s.LookupHeaders("loc", "#/components/headers/Other"); return err }, wantErr: "failed to lookup"}, + {name: "request missing value", lookup: func() error { _, err := s.LookupRequestBody("loc", "#/components/requestBodies/Other"); return err }, wantErr: "failed to lookup"}, + {name: "response missing value", lookup: func() error { _, err := s.LookupResponse("loc", "#/components/responses/Other"); return err }, wantErr: "failed to lookup"}, + {name: "security missing value", lookup: func() error { + _, err := s.LookupSecurityScheme("loc", "#/components/securitySchemes/Other") + return err + }, wantErr: "failed to lookup"}, + {name: "example missing value", lookup: func() error { _, err := s.LookupExample("loc", "#/components/examples/Other"); return err }, wantErr: "failed to lookup"}, + {name: "link missing value", lookup: func() error { _, err := s.LookupLink("loc", "#/components/links/Other"); return err }, wantErr: "failed to lookup"}, + {name: "callback missing value", lookup: func() error { _, err := s.LookupCallback("loc", "#/components/callbacks/Other"); return err }, wantErr: "failed to lookup"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.lookup() + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected err containing %q, got %v", tt.wantErr, err) + } + }) + } +} diff --git a/gateway/router/openapi/openapi3/response.go b/gateway/router/openapi/openapi3/response.go index f866d3728..47efb2e3d 100644 --- a/gateway/router/openapi/openapi3/response.go +++ b/gateway/router/openapi/openapi3/response.go @@ -22,12 +22,26 @@ type ( ) const ( - ResponseDefault ResponseCode = "default" + ResponseContinue ResponseCode = "100" + ResponseOK ResponseCode = "200" + ResponseCreated ResponseCode = "201" + ResponseAccepted ResponseCode = "202" + ResponseNoContent ResponseCode = "204" + ResponseBadRequest ResponseCode = "400" + ResponseUnauthorized ResponseCode = "401" + ResponseForbidden ResponseCode = "403" + ResponseNotFound ResponseCode = "404" + ResponseConflict ResponseCode = "409" + ResponseUnprocessable ResponseCode = "422" + ResponseInternalServerErr ResponseCode = "500" + ResponseBadGateway ResponseCode = "502" + ResponseServiceUnavailable ResponseCode = "503" + ResponseDefault ResponseCode = "default" ) type ( - ResponseCode string - ResponseIntKey string + ResponseCode string + ResponseCodeLiteral string ResponseKey interface { ~string | ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 diff --git a/gateway/router/openapi/schema_build_helpers_test.go b/gateway/router/openapi/schema_build_helpers_test.go index 3767a3df0..ceb9be216 100644 --- a/gateway/router/openapi/schema_build_helpers_test.go +++ b/gateway/router/openapi/schema_build_helpers_test.go @@ -289,4 +289,52 @@ func TestSchemaBuildHelpers_Table(t *testing.T) { t.Fatalf("expected strict mode polymorphism error") } }) + + t.Run("oneOf discriminator and helper branches", func(t *testing.T) { + t.Run("empty refs yield nil discriminator", func(t *testing.T) { + discriminator := oneOfDiscriminator(openapi3.SchemaList{{Type: objectOutput}, nil}) + if discriminator != nil { + t.Fatalf("expected nil discriminator when refs are absent") + } + }) + + t.Run("apply discriminator skips non-object and missing schema", func(t *testing.T) { + container := NewContainer() + container.generatedSchemas["User"] = &openapi3.Schema{Type: objectOutput} + container.generatedSchemas["Arr"] = &openapi3.Schema{Type: arrayOutput} + container.applyDiscriminatorToVariants(&openapi3.Discriminator{ + PropertyName: "kind", + Mapping: map[string]string{ + "user": "#/components/schemas/User", + "arr": "#/components/schemas/Arr", + "miss": "#/components/schemas/Missing", + }, + }) + user := container.generatedSchemas["User"] + if user == nil || user.Properties["kind"] == nil { + t.Fatalf("expected discriminator property on object variant") + } + if !containsString(user.Required, "kind") { + t.Fatalf("expected discriminator property required on object variant") + } + arr := container.generatedSchemas["Arr"] + if arr != nil && arr.Properties != nil { + if _, ok := arr.Properties["kind"]; ok { + t.Fatalf("did not expect discriminator property on non-object variant") + } + } + }) + + t.Run("refName invalid variants", func(t *testing.T) { + if got := refName(""); got != "" { + t.Fatalf("expected empty ref name for empty ref") + } + if got := refName("abc"); got != "" { + t.Fatalf("expected empty ref name for malformed ref") + } + if got := refName("abc/"); got != "" { + t.Fatalf("expected empty ref name for trailing slash") + } + }) + }) } From 74fd475ebc4b57d3d402745769826ae50d7df93d Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Feb 2026 10:11:05 -0800 Subject: [PATCH 143/279] updated openapi schema builder --- gateway/router/openapi/schema.go | 2 + gateway/router/openapi/schema_build.go | 34 ++++++++ .../openapi/schema_build_helpers_test.go | 69 +++++++++++++++ internal/translator/function.go | 33 +++++-- repository/shape/compile/pipeline/read.go | 85 ++++++++++++++++++- .../shape/compile/pipeline/read_test.go | 8 ++ 6 files changed, 221 insertions(+), 10 deletions(-) diff --git a/gateway/router/openapi/schema.go b/gateway/router/openapi/schema.go index cd7a5e7dd..b02ce96b8 100644 --- a/gateway/router/openapi/schema.go +++ b/gateway/router/openapi/schema.go @@ -46,6 +46,7 @@ type ( schemas []*openapi3.Schema index map[string]int generatedSchemas map[string]*openapi3.Schema + visitingTypes map[string]int } ) @@ -88,6 +89,7 @@ func NewContainer() *SchemaContainer { return &SchemaContainer{ index: map[string]int{}, generatedSchemas: map[string]*openapi3.Schema{}, + visitingTypes: map[string]int{}, } } diff --git a/gateway/router/openapi/schema_build.go b/gateway/router/openapi/schema_build.go index 09bb99cea..e51b27450 100644 --- a/gateway/router/openapi/schema_build.go +++ b/gateway/router/openapi/schema_build.go @@ -32,6 +32,19 @@ func (c *SchemaContainer) addToSchema(ctx context.Context, component *ComponentS } } +func recursionTypeKey(rType reflect.Type) string { + rType = dereferenceType(rType) + if rType == nil { + return "" + } + switch rType.Kind() { + case reflect.Struct, reflect.Interface, reflect.Slice, reflect.Array, reflect.Map: + return rType.PkgPath() + ":" + rType.String() + default: + return "" + } +} + func (c *SchemaContainer) addArraySchema(ctx context.Context, component *ComponentSchema, dst *openapi3.Schema, schema *Schema, rType reflect.Type) error { itemSchema, err := c.createSchema(ctx, component, schema.SliceItem(rType)) if err != nil { @@ -47,6 +60,15 @@ func (c *SchemaContainer) addStructSchema(ctx context.Context, component *Compon addTimeSchema(dst, schema) return nil } + if selfKey := recursionTypeKey(rType); selfKey != "" { + c.visitingTypes[selfKey]++ + defer func() { + c.visitingTypes[selfKey]-- + if c.visitingTypes[selfKey] == 0 { + delete(c.visitingTypes, selfKey) + } + }() + } dst.Type = objectOutput dst.Properties = openapi3.Schemas{} @@ -84,6 +106,10 @@ func (c *SchemaContainer) addStructSchema(ctx context.Context, component *Compon updatedDocumentation(aTag, component.component.Docs(), fieldSchema) if field.Anonymous { + if childKey := recursionTypeKey(fieldSchema.rType); childKey != "" && c.visitingTypes[childKey] > 0 { + // Avoid anonymous self/embed loops while preserving named-schema recursion via createSchema. + continue + } if err := c.addToSchema(ctx, component, dst, fieldSchema); err != nil { return err } @@ -303,8 +329,16 @@ func (c *SchemaContainer) createSchema(ctx context.Context, componentSchema *Com }, nil } + // Mark named schemas as in-progress before generation so recursive graphs + // (for example polymorphic self references) resolve to $ref instead of looping. + if fieldSchema.tag.TypeName != "" { + c.generatedSchemas[fieldSchema.tag.TypeName] = nil + } schema, err := componentSchema.GenerateSchema(ctx, fieldSchema) if err != nil { + if fieldSchema.tag.TypeName != "" { + delete(c.generatedSchemas, fieldSchema.tag.TypeName) + } return nil, err } diff --git a/gateway/router/openapi/schema_build_helpers_test.go b/gateway/router/openapi/schema_build_helpers_test.go index ceb9be216..0fc1516ef 100644 --- a/gateway/router/openapi/schema_build_helpers_test.go +++ b/gateway/router/openapi/schema_build_helpers_test.go @@ -29,6 +29,20 @@ type testUnsupported chan int func (testUnsupported) Kind() string { return "unsupported" } +type recursiveAnimal interface { + Kind() string +} + +type recursiveDog struct { + Child recursiveAnimal `json:"child,omitempty"` +} + +func (recursiveDog) Kind() string { return "dog" } + +type RecursiveEmbed struct { + *RecursiveEmbed +} + func TestSchemaBuildHelpers_Table(t *testing.T) { t.Run("apply schema example", func(t *testing.T) { dst := &openapi3.Schema{} @@ -337,4 +351,59 @@ func TestSchemaBuildHelpers_Table(t *testing.T) { } }) }) + + t.Run("recursive polymorphic graph does not loop", func(t *testing.T) { + t.Setenv("DATLY_OPENAPI_POLY_STRICT", "false") + component := newTestComponent(t) + types := xreflect.NewTypes() + if err := types.Register("RecursiveAnimal", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf((*recursiveAnimal)(nil)).Elem())); err != nil { + t.Fatalf("register interface failed: %v", err) + } + if err := types.Register("RecursiveDog", xreflect.WithPackage("test"), xreflect.WithReflectType(reflect.TypeOf(recursiveDog{}))); err != nil { + t.Fatalf("register struct failed: %v", err) + } + setUnexportedField(component, "types", types) + + container := NewContainer() + componentSchema := &ComponentSchema{component: component, schemas: container} + dst := &openapi3.Schema{} + err := container.addToSchema(context.Background(), componentSchema, dst, &Schema{ + rType: reflect.TypeOf((*recursiveAnimal)(nil)).Elem(), + ioConfig: component.IOConfig(), + }) + if err != nil { + t.Fatalf("unexpected addToSchema error: %v", err) + } + if len(dst.OneOf) == 0 { + t.Fatalf("expected oneOf variants for recursive interface") + } + dog := container.generatedSchemas["RecursiveDog"] + if dog == nil { + t.Fatalf("expected RecursiveDog schema to be generated") + } + child := dog.Properties["child"] + if child == nil { + t.Fatalf("expected recursive child schema") + } + if child.Ref == "" && len(child.OneOf) == 0 { + t.Fatalf("expected recursive child to be represented as ref or oneOf") + } + }) + + t.Run("anonymous self embed does not recurse indefinitely", func(t *testing.T) { + component := newTestComponent(t) + container := NewContainer() + componentSchema := &ComponentSchema{component: component, schemas: container} + dst := &openapi3.Schema{} + err := container.addToSchema(context.Background(), componentSchema, dst, &Schema{ + rType: reflect.TypeOf(RecursiveEmbed{}), + ioConfig: component.IOConfig(), + }) + if err != nil { + t.Fatalf("unexpected addToSchema error: %v", err) + } + if dst.Type != objectOutput { + t.Fatalf("expected object type for recursive embed, got %q", dst.Type) + } + }) } diff --git a/internal/translator/function.go b/internal/translator/function.go index 38ddb6a42..e7b995bce 100644 --- a/internal/translator/function.go +++ b/internal/translator/function.go @@ -12,6 +12,8 @@ import ( "strings" ) +const privateColumnTag = `internal:"true" json:"-"` + // TODO introduce function abstraction for datly -h list funciton, with validation signtaure description func (n *Viewlets) applySettingFunctions(column *sqlparser.Column, namespace string) (bool, error) { funcName, funcArgs := extractFunction(column) @@ -54,16 +56,15 @@ func (n *Viewlets) applySettingFunctions(column *sqlparser.Column, namespace str if dest != nil { switch strings.ToLower(funcName) { case "tag": - if column.Name == column.Namespace && !strings.Contains(column.Expression, column.Name+"."+column.Name) { - if dest.View == nil { - dest.View = &View{} - } - dest.View.Tag = strings.Trim(column.Tag, "'") - return true, nil + if err := applyColumnTagSetting(dest, column); err != nil { + return false, err + } + return true, nil + case "private": + column.Tag = privateColumnTag + if err := applyColumnTagSetting(dest, column); err != nil { + return false, err } - columnConfig := dest.columnConfig(column.Name) - column.Tag = strings.Trim(strings.TrimSpace(column.Tag), "'") - columnConfig.Tag = &column.Tag return true, nil case "cast": return dest.applyExplicitCast(column, funcArgs) @@ -101,6 +102,20 @@ func (n *Viewlets) applySettingFunctions(column *sqlparser.Column, namespace str return true, nil } +func applyColumnTagSetting(dest *Viewlet, column *sqlparser.Column) error { + if column.Name == column.Namespace && !strings.Contains(column.Expression, column.Name+"."+column.Name) { + if dest.View == nil { + dest.View = &View{} + } + dest.View.Tag = strings.Trim(column.Tag, "'") + return nil + } + columnConfig := dest.columnConfig(column.Name) + column.Tag = strings.Trim(strings.TrimSpace(column.Tag), "'") + columnConfig.Tag = &column.Tag + return nil +} + func (v *Viewlet) applyExplicitCast(column *sqlparser.Column, funcArgs []string) (bool, error) { if column.Name == "" || column.Name == column.Namespace { if v.View.Schema == nil { diff --git a/repository/shape/compile/pipeline/read.go b/repository/shape/compile/pipeline/read.go index 20fd2bba3..3cf697603 100644 --- a/repository/shape/compile/pipeline/read.go +++ b/repository/shape/compile/pipeline/read.go @@ -140,7 +140,90 @@ func normalizeParserSQL(sqlText string) string { if sqlText == "" { return sqlText } - return replaceTemplateTokens(sqlText) + return rewritePrivateShorthand(replaceTemplateTokens(sqlText)) +} + +func rewritePrivateShorthand(input string) string { + var b strings.Builder + b.Grow(len(input)) + for i := 0; i < len(input); { + if !hasPrefixFold(input[i:], "private") { + b.WriteByte(input[i]) + i++ + continue + } + if i > 0 && isReadIdentifierPart(input[i-1]) { + b.WriteByte(input[i]) + i++ + continue + } + pos := i + len("private") + pos = skipReadSpaces(input, pos) + if pos >= len(input) || input[pos] != '(' { + b.WriteByte(input[i]) + i++ + continue + } + body, closeIdx, ok := readReadCallBody(input, pos) + if !ok { + b.WriteByte(input[i]) + i++ + continue + } + firstArg, ok := firstCallArg(body) + if !ok { + b.WriteByte(input[i]) + i++ + continue + } + b.WriteString(strings.TrimSpace(firstArg)) + i = closeIdx + 1 + } + return b.String() +} + +func hasPrefixFold(s, prefix string) bool { + if len(s) < len(prefix) { + return false + } + return strings.EqualFold(s[:len(prefix)], prefix) +} + +func firstCallArg(body string) (string, bool) { + depth := 0 + quote := byte(0) + for i := 0; i < len(body); i++ { + ch := body[i] + if quote != 0 { + if ch == '\\' && i+1 < len(body) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + switch ch { + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + case ',': + if depth == 0 { + arg := strings.TrimSpace(body[:i]) + return arg, arg != "" + } + } + } + arg := strings.TrimSpace(body) + return arg, arg != "" } func inferRootFromRelations(relations []*plan.Relation) string { diff --git a/repository/shape/compile/pipeline/read_test.go b/repository/shape/compile/pipeline/read_test.go index 0c82d72a8..9d414beb8 100644 --- a/repository/shape/compile/pipeline/read_test.go +++ b/repository/shape/compile/pipeline/read_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "strings" "testing" "github.com/stretchr/testify/assert" @@ -49,6 +50,13 @@ func TestNormalizeParserSQL_VeltyBlockExpression(t *testing.T) { assert.Contains(t, actual, "SELECT b.* FROM CI_BROWSER b WHERE 1 AND b.ARCHIVED = 0") } +func TestNormalizeParserSQL_PrivateShorthand(t *testing.T) { + input := `SELECT private(audience.FREQ_CAPPING) AS freq_capping FROM CI_AUDIENCE audience` + actual := normalizeParserSQL(input) + assert.NotContains(t, strings.ToLower(actual), "private(") + assert.Contains(t, actual, "SELECT audience.FREQ_CAPPING AS freq_capping FROM CI_AUDIENCE audience") +} + func TestNeedsFallbackParse(t *testing.T) { assert.True(t, needsFallbackParse("SELECT * FROM t JOIN x ON t.id = x.id", &query.Select{})) assert.False(t, needsFallbackParse("SELECT * FROM t", &query.Select{From: query.From{X: expr.NewSelector("t")}})) From b1f84f308fd0c6e992d652e0a224f4034d89f53c Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Feb 2026 10:29:15 -0800 Subject: [PATCH 144/279] updated openapi schema builder --- gateway/router/openapi/generator_test.go | 87 ++++++++++++++++++++++++ gateway/router/openapi/logic_test.go | 23 +++++++ gateway/router/openapi/openapi3.go | 34 +++++++-- gateway/router/openapi/schema.go | 11 ++- gateway/router/openapi/tag.go | 2 + repository/shape/README.md | 6 ++ 6 files changed, 156 insertions(+), 7 deletions(-) diff --git a/gateway/router/openapi/generator_test.go b/gateway/router/openapi/generator_test.go index ceb013046..6a3756740 100644 --- a/gateway/router/openapi/generator_test.go +++ b/gateway/router/openapi/generator_test.go @@ -262,6 +262,54 @@ func TestGeneratorHelpersMore_Table(t *testing.T) { } }) + t.Run("convert param kind whitelist", func(t *testing.T) { + testCases := []struct { + name string + kind state.Kind + expectKeep bool + }{ + {name: "header", kind: state.KindHeader, expectKeep: true}, + {name: "query", kind: state.KindQuery, expectKeep: true}, + {name: "form", kind: state.KindForm, expectKeep: true}, + {name: "body skipped in parameter list", kind: state.KindRequestBody, expectKeep: false}, + {name: "path skipped", kind: state.KindPath, expectKeep: false}, + {name: "cookie skipped", kind: state.KindCookie, expectKeep: false}, + {name: "state skipped", kind: state.KindState, expectKeep: false}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + param := &state.Parameter{ + Name: "ID", + In: &state.Location{Kind: tc.kind, Name: "id"}, + Schema: state.NewSchema(reflect.TypeOf(1)), + } + + converted, ok, err := g.convertParam(context.Background(), cSchema, param, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ok != tc.expectKeep { + t.Fatalf("expected keep=%v, got %v", tc.expectKeep, ok) + } + if tc.expectKeep && len(converted) != 1 { + t.Fatalf("expected one converted parameter, got %d", len(converted)) + } + if !tc.expectKeep && len(converted) != 0 { + t.Fatalf("expected no converted parameters, got %d", len(converted)) + } + }) + } + }) + t.Run("convert param object and non-http", func(t *testing.T) { g := &generator{ _parametersIndex: map[string]*openapi3.Parameter{}, @@ -711,6 +759,45 @@ func TestGeneratorHelpersMore_Table(t *testing.T) { } }) + t.Run("convert param kind param skips component reference", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + base := &state.Parameter{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "GET:/v1/auth"}, Schema: state.NewSchema(reflect.TypeOf(struct{ A int }{}))} + comp.Input.Type.Parameters = state.Parameters{base} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + + refParam := &state.Parameter{Name: "AuthRef", In: &state.Location{Kind: state.KindParam, Name: "Auth"}, Schema: state.NewSchema(reflect.TypeOf(struct{ A int }{}))} + converted, ok, err := g.convertParam(context.Background(), cSchema, refParam, "") + if err != nil || ok || len(converted) != 0 { + t.Fatalf("expected component kind param reference to be skipped, got ok=%v err=%v len=%d", ok, err, len(converted)) + } + }) + + t.Run("convert param kind param body reference skipped from parameters", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + base := &state.Parameter{Name: "BodyInput", In: &state.Location{Kind: state.KindRequestBody}, Schema: state.NewSchema(reflect.TypeOf(struct{ A int }{}))} + comp.Input.Type.Parameters = state.Parameters{base} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + + refParam := &state.Parameter{Name: "BodyRef", In: &state.Location{Kind: state.KindParam, Name: "BodyInput"}, Schema: state.NewSchema(reflect.TypeOf(struct{ A int }{}))} + converted, ok, err := g.convertParam(context.Background(), cSchema, refParam, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ok || len(converted) != 0 { + t.Fatalf("expected body-derived kind=param to be skipped from parameter list, got ok=%v len=%d", ok, len(converted)) + } + }) + t.Run("append built-in non-http parameter", func(t *testing.T) { g := &generator{ _parametersIndex: map[string]*openapi3.Parameter{}, diff --git a/gateway/router/openapi/logic_test.go b/gateway/router/openapi/logic_test.go index 8c35e2d5e..9c3f92d39 100644 --- a/gateway/router/openapi/logic_test.go +++ b/gateway/router/openapi/logic_test.go @@ -200,12 +200,35 @@ func TestComponentSchemaHelpers_Table(t *testing.T) { } component.View.GetResource().SetTypes(reg) withTag := componentSchema.SchemaWithTag("F", reflect.TypeOf(struct{ A int }{}), "d", component.IOConfig(), Tag{ + IsInput: true, Parameter: &tags.Parameter{DataType: "Alt"}, }) if withTag.rType != reflect.TypeOf(alt{}) { t.Fatalf("expected datatype override") } }) + + t.Run("schema with tag primitive datatype override", func(t *testing.T) { + withTag := componentSchema.SchemaWithTag("Jwt", reflect.TypeOf(struct{ A int }{}), "d", component.IOConfig(), Tag{ + IsInput: true, + Parameter: &tags.Parameter{DataType: "string"}, + }) + if withTag.rType != reflect.TypeOf("") { + t.Fatalf("expected primitive datatype override to string, got %v", withTag.rType) + } + }) + + t.Run("schema with tag output keeps go type", func(t *testing.T) { + goType := reflect.TypeOf(struct{ A int }{}) + withTag := componentSchema.SchemaWithTag("Out", goType, "d", component.IOConfig(), Tag{ + Parameter: &tags.Parameter{ + DataType: "string", + }, + }) + if withTag.rType != goType { + t.Fatalf("expected output kind to keep go type, got %v", withTag.rType) + } + }) } func TestSchemaContainerCreateSchema_Table(t *testing.T) { diff --git a/gateway/router/openapi/openapi3.go b/gateway/router/openapi/openapi3.go index 16b83855a..a92a38894 100644 --- a/gateway/router/openapi/openapi3.go +++ b/gateway/router/openapi/openapi3.go @@ -49,6 +49,24 @@ type ( } ) +func isRequestDerivedInputKind(kind state.Kind) bool { + switch kind { + case state.KindHeader, state.KindRequestBody, state.KindQuery, state.KindForm: + return true + default: + return false + } +} + +func isOpenAPIParameterKind(kind state.Kind) bool { + switch kind { + case state.KindHeader, state.KindQuery, state.KindForm: + return true + default: + return false + } +} + func (g *generator) GenerateSpec(ctx context.Context, repoComponents *repository.Service, info openapi.Info, providers ...*repository.Provider) (*openapi.OpenAPI, error) { components := &openapi.Components{} @@ -188,6 +206,9 @@ func (g *generator) convertParam(ctx context.Context, component *ComponentSchema } if param.In.Kind == state.KindParam { baseParam := component.component.LookupParameter(param.In.Name) + if baseParam == nil || !isRequestDerivedInputKind(baseParam.In.Kind) { + return nil, false, nil + } return g.convertParam(ctx, component, baseParam, description) } @@ -207,7 +228,7 @@ func (g *generator) convertParam(ctx context.Context, component *ComponentSchema return result, true, nil } - if !param.IsHTTPParameter() { + if !isOpenAPIParameterKind(param.In.Kind) { return nil, false, nil } @@ -224,16 +245,21 @@ func (g *generator) convertParam(ctx context.Context, component *ComponentSchema } table := "" + var parameterTag *tags.Parameter if param.Tag != "" { - if datlyTags, _ := tags.Parse(reflect.StructTag(param.Tag), nil, tags.ViewTag); datlyTags != nil && datlyTags.View != nil { - table = datlyTags.View.Table + if datlyTags, _ := tags.Parse(reflect.StructTag(param.Tag), nil, tags.ViewTag, tags.ParameterTag); datlyTags != nil { + parameterTag = datlyTags.Parameter + if datlyTags.View != nil { + table = datlyTags.View.Table + } } - } schema, err := component.GenerateSchema(ctx, component.SchemaWithTag(param.Name, param.Schema.Type(), "Parameter "+param.Name+" schema", component.component.IOConfig(), Tag{ Format: param.DateFormat, IsNullable: !param.IsRequired(), Table: table, + Parameter: parameterTag, + IsInput: true, })) if err != nil { diff --git a/gateway/router/openapi/schema.go b/gateway/router/openapi/schema.go index b02ce96b8..3eef32319 100644 --- a/gateway/router/openapi/schema.go +++ b/gateway/router/openapi/schema.go @@ -6,10 +6,12 @@ import ( "github.com/viant/datly/gateway/router/openapi/openapi3" "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" + "github.com/viant/datly/utils/types" "github.com/viant/datly/view/state" "github.com/viant/datly/view/tags" "github.com/viant/tagly/format/text" "github.com/viant/xdatly/docs" + "github.com/viant/xreflect" "reflect" "sync" ) @@ -218,9 +220,12 @@ func (c *ComponentSchema) ReflectSchema(name string, rType reflect.Type, descrip func (c *ComponentSchema) SchemaWithTag(fieldName string, rType reflect.Type, description string, ioConfig *config.IOConfig, tag Tag) *Schema { if parameter := tag.Parameter; parameter != nil { - if parameter.DataType != "" { - typeLookup := c.component.View.Resource().LookupType() - if lType, _ := typeLookup(parameter.DataType); lType != nil { + if tag.IsInput && parameter.DataType != "" { + var typeLookup xreflect.LookupType + if c.component != nil && c.component.View != nil && c.component.View.Resource() != nil { + typeLookup = c.component.View.Resource().LookupType() + } + if lType, _ := types.LookupType(typeLookup, parameter.DataType); lType != nil { rType = lType } } diff --git a/gateway/router/openapi/tag.go b/gateway/router/openapi/tag.go index a4c343c6f..9a1f51549 100644 --- a/gateway/router/openapi/tag.go +++ b/gateway/router/openapi/tag.go @@ -35,6 +35,7 @@ type ( _tag format.Tag TypeName string Parameter *tags.Parameter + IsInput bool Column string Table string } @@ -78,6 +79,7 @@ func ParseTag(field reflect.StructField, tag reflect.StructTag, isInput bool, ro Example: tag.Get(tags.ExampleTag), JSONName: jsonName, _tag: *aTag, + IsInput: isInput, } // Keep internal runtime-only fields out of OpenAPI schema. diff --git a/repository/shape/README.md b/repository/shape/README.md index d10769036..30848cef5 100644 --- a/repository/shape/README.md +++ b/repository/shape/README.md @@ -86,3 +86,9 @@ repository.WithShapePipeline(true) ``` Default is disabled to preserve existing behavior. + +## Component Contract Parity + +Cross-component contract/signature parity target is documented in: + +- `compile/COMPONENT_CONTRACT_PARITY.md` From 1da05d0c8b3355509703394c10d02195eb6ec140 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 26 Feb 2026 07:12:07 -0800 Subject: [PATCH 145/279] updated openapi schema builder --- repository/shape/compile/resolver.go | 96 ++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 repository/shape/compile/resolver.go diff --git a/repository/shape/compile/resolver.go b/repository/shape/compile/resolver.go new file mode 100644 index 000000000..6abf0b50f --- /dev/null +++ b/repository/shape/compile/resolver.go @@ -0,0 +1,96 @@ +package compile + +import ( + "context" + "net/http" + "strings" + + "github.com/viant/datly/repository/contract/signature" + "github.com/viant/datly/repository/shape/plan" +) + +// ComponentContract represents resolved component contract metadata. +type ComponentContract struct { + RouteKey string + Method string + URI string + OutputType string + Types []*plan.Type +} + +// ComponentResolver resolves component contract metadata for a route key. +type ComponentResolver interface { + ResolveContract(ctx context.Context, routeKey string) (*ComponentContract, error) +} + +// SignatureResolver adapts repository/contract/signature service +// to compile-time component contract resolution. +type SignatureResolver struct { + service *signature.Service +} + +// NewSignatureResolver creates signature-backed component resolver. +func NewSignatureResolver(ctx context.Context, apiPrefix, routesURL string) (*SignatureResolver, error) { + srv, err := signature.New(ctx, apiPrefix, routesURL) + if err != nil { + return nil, err + } + return &SignatureResolver{service: srv}, nil +} + +// ResolveContract resolves component contract by route key. +func (s *SignatureResolver) ResolveContract(_ context.Context, routeKey string) (*ComponentContract, error) { + method, uri := splitRouteKey(routeKey) + sig, err := s.service.Signature(method, uri) + if err != nil { + return nil, err + } + ret := &ComponentContract{ + RouteKey: normalizeRouteKey(method, uri), + Method: method, + URI: normalizeURI(uri), + } + if sig.Output != nil { + if dataType := strings.TrimSpace(sig.Output.DataType); dataType != "" { + ret.OutputType = dataType + } else if name := strings.TrimSpace(sig.Output.Name); name != "" { + name = strings.Trim(name, "*") + if name != "" { + ret.OutputType = "*" + name + } + } + } + for _, item := range sig.Types { + if item == nil { + continue + } + ret.Types = append(ret.Types, &plan.Type{ + Name: strings.TrimSpace(item.Name), + Alias: strings.TrimSpace(item.Alias), + DataType: strings.TrimSpace(item.DataType), + Cardinality: strings.TrimSpace(string(item.Cardinality)), + Package: strings.TrimSpace(item.Package), + ModulePath: strings.TrimSpace(item.ModulePath), + }) + } + return ret, nil +} + +func splitRouteKey(routeKey string) (string, string) { + routeKey = strings.TrimSpace(routeKey) + if routeKey == "" { + return http.MethodGet, "/" + } + if idx := strings.Index(routeKey, ":"); idx != -1 { + method := strings.ToUpper(strings.TrimSpace(routeKey[:idx])) + uri := strings.TrimSpace(routeKey[idx+1:]) + if method == "" { + method = http.MethodGet + } + if uri == "" { + uri = "/" + } + return method, uri + } + return http.MethodGet, routeKey +} From 055ee4b1f7023fbe0b3d690ae316048007e5ada8 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 26 Feb 2026 08:14:54 -0800 Subject: [PATCH 146/279] updated openapi schema builder --- repository/shape/compile/route_index.go | 231 ++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 repository/shape/compile/route_index.go diff --git a/repository/shape/compile/route_index.go b/repository/shape/compile/route_index.go new file mode 100644 index 000000000..30691e27a --- /dev/null +++ b/repository/shape/compile/route_index.go @@ -0,0 +1,231 @@ +package compile + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/viant/datly/repository/shape" + dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" +) + +// RouteIndexEntry maps one source DQL file to one concrete method+URI route key. +type RouteIndexEntry struct { + RouteKey string + Method string + URI string + SourcePath string + Namespace string +} + +// RouteIndex stores source-to-route mapping and lookup structures. +type RouteIndex struct { + ByRouteKey map[string]*RouteIndexEntry + ByNamespace map[string][]*RouteIndexEntry + Conflicts map[string][]string +} + +// BuildRouteIndex scans DQL files and builds route-key mapping. +func BuildRouteIndex(paths []string, opts ...shape.CompileOption) (*RouteIndex, error) { + compileOptions := applyCompileOptions(opts) + layout := newCompilePathLayout(compileOptions) + index := &RouteIndex{ + ByRouteKey: map[string]*RouteIndexEntry{}, + ByNamespace: map[string][]*RouteIndexEntry{}, + Conflicts: map[string][]string{}, + } + if len(paths) == 0 { + return index, nil + } + normalized := make([]string, 0, len(paths)) + for _, item := range paths { + item = strings.TrimSpace(item) + if item == "" { + continue + } + normalized = append(normalized, item) + } + sort.Strings(normalized) + + for _, sourcePath := range normalized { + data, err := os.ReadFile(sourcePath) + if err != nil { + return nil, fmt.Errorf("route index: unable to read %s: %w", sourcePath, err) + } + sourceName := strings.TrimSuffix(filepath.Base(sourcePath), filepath.Ext(sourcePath)) + dql := string(data) + _, _, directives, _ := dqlpre.Extract(dql) + source := &shape.Source{ + Name: sourceName, + Path: sourcePath, + DQL: dql, + } + settings := extractRuleSettings(source, directives) + namespace, _ := dqlToRouteNamespaceWithLayout(sourcePath, layout) + uri := strings.TrimSpace(settings.URI) + if uri == "" { + uri = inferDefaultURI(namespace) + } + if uri == "" { + continue + } + methods := parseRouteMethods(settings.Method) + for _, method := range methods { + entry := &RouteIndexEntry{ + Method: method, + URI: normalizeURI(uri), + SourcePath: sourcePath, + Namespace: namespace, + } + entry.RouteKey = normalizeRouteKey(entry.Method, entry.URI) + index.addEntry(entry) + } + } + return index, nil +} + +// Resolve maps a component reference from current source context to route key. +// It returns false when route cannot be resolved deterministically. +func (r *RouteIndex) Resolve(ref, currentSource string, opts ...shape.CompileOption) (string, bool) { + if r == nil { + return "", false + } + method, value := splitRouteKey(ref) + value = strings.TrimSpace(value) + if value == "" { + return "", false + } + layout := newCompilePathLayout(applyCompileOptions(opts)) + routeKeyFromURI := func(uri string) (string, bool) { + key := normalizeRouteKey(method, uri) + if _, conflicted := r.Conflicts[key]; conflicted { + return "", false + } + if _, ok := r.ByRouteKey[key]; !ok { + return "", false + } + return key, true + } + + if strings.HasPrefix(value, "/v1/api/") || strings.HasPrefix(value, "v1/api/") || strings.HasPrefix(value, "/") { + return routeKeyFromURI(value) + } + + if strings.TrimSpace(currentSource) == "" { + return "", false + } + _, _, dqlRoot, ok := sourceRootsWithLayout(currentSource, layout) + if !ok { + return "", false + } + sourceNamespace, _ := dqlToRouteNamespaceWithLayout(currentSource, layout) + namespace := resolveComponentNamespaceWithNamespace(value, currentSource, dqlRoot, sourceNamespace) + if namespace == "" { + return "", false + } + entries := r.ByNamespace[strings.ToLower(strings.TrimSpace(namespace))] + if len(entries) == 0 { + return "", false + } + if len(entries) == 1 { + key := entries[0].RouteKey + if _, conflicted := r.Conflicts[key]; conflicted { + return "", false + } + return key, true + } + // Multiple methods under one namespace: require exact method match. + for _, candidate := range entries { + if candidate == nil { + continue + } + if strings.EqualFold(candidate.Method, method) { + if _, conflicted := r.Conflicts[candidate.RouteKey]; conflicted { + return "", false + } + return candidate.RouteKey, true + } + } + return "", false +} + +func (r *RouteIndex) addEntry(entry *RouteIndexEntry) { + if r == nil || entry == nil { + return + } + key := entry.RouteKey + if prev, exists := r.ByRouteKey[key]; exists && prev != nil && prev.SourcePath != entry.SourcePath { + if _, ok := r.Conflicts[key]; !ok { + r.Conflicts[key] = []string{prev.SourcePath} + } + r.Conflicts[key] = append(r.Conflicts[key], entry.SourcePath) + return + } + r.ByRouteKey[key] = entry + nsKey := strings.ToLower(strings.TrimSpace(entry.Namespace)) + if nsKey != "" { + r.ByNamespace[nsKey] = append(r.ByNamespace[nsKey], entry) + } +} + +func parseRouteMethods(input string) []string { + input = strings.TrimSpace(input) + if input == "" { + return []string{http.MethodGet} + } + parts := strings.Split(input, ",") + ret := make([]string, 0, len(parts)) + seen := map[string]bool{} + for _, part := range parts { + method := strings.ToUpper(strings.TrimSpace(part)) + if method == "" { + continue + } + if seen[method] { + continue + } + seen[method] = true + ret = append(ret, method) + } + if len(ret) == 0 { + return []string{http.MethodGet} + } + return ret +} + +func normalizeRouteKey(method, uri string) string { + method = strings.ToUpper(strings.TrimSpace(method)) + if method == "" { + method = http.MethodGet + } + return method + ":" + normalizeURI(uri) +} + +func normalizeURI(uri string) string { + uri = strings.TrimSpace(uri) + if uri == "" { + return "/" + } + if strings.HasPrefix(uri, "v1/api/") { + uri = "/" + uri + } + if !strings.HasPrefix(uri, "/") { + uri = "/" + uri + } + return uri +} + +func inferDefaultURI(namespace string) string { + namespace = strings.Trim(strings.TrimSpace(namespace), "/") + if namespace == "" { + return "" + } + parts := strings.Split(namespace, "/") + if len(parts) >= 2 && parts[len(parts)-1] == parts[len(parts)-2] { + parts = parts[:len(parts)-1] + } + return "/v1/api/" + strings.Join(parts, "/") +} From 8d10d62d9da90b9c6a315e8eb3f5e28bda63bd89 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 27 Feb 2026 08:29:00 -0800 Subject: [PATCH 147/279] - stabilize e2e - refactor planner.State --- cmd/command/translate.go | 8 +- cmd/command/translate_shape.go | 10 +- cmd/command/translate_shape_ir.go | 136 ++++ cmd/options/rule.go | 16 +- cmd/options/rule_engine_test.go | 8 +- .../cases/001_one_to_many/expect_2.txt | 2 +- e2e/local/regression/regression.yaml | 4 +- e2e/local/regression/rule.yaml | 1 + repository/components_shape_test.go | 62 ++ repository/components_typectx_test.go | 164 +++++ repository/option_shape_test.go | 29 + repository/path/service.go | 3 +- repository/shape/compile/component_types.go | 14 +- .../shape/compile/component_types_test.go | 15 +- repository/shape/compile/statedecl.go | 44 +- repository/shape/compile/statedecl_test.go | 16 +- repository/shape/dql/decl/lex.go | 59 ++ repository/shape/dql/decl/model.go | 42 ++ repository/shape/dql/decl/parser.go | 337 ++++++++++ repository/shape/dql/decl/parser_test.go | 142 ++++ repository/shape/dql/holder/model.go | 128 ++++ repository/shape/dql/holder/model_test.go | 52 ++ repository/shape/dql/ir/model.go | 24 + repository/shape/dql/load/loader.go | 84 +++ repository/shape/dql/load/loader_test.go | 58 ++ .../shape/dql/parity/adorder_parity_test.go | 92 +++ repository/shape/dql/parity/connectors.go | 28 + repository/shape/dql/parity/diff.go | 59 ++ .../shape/dql/parity/mdp_parity_test.go | 160 +++++ repository/shape/dql/parse/function.go | 70 ++ repository/shape/dql/parse/model.go | 38 ++ repository/shape/dql/plan/planner.go | 609 ++++++++++++++++++ repository/shape/dql/plan/planner_test.go | 164 +++++ repository/shape/dql/plan/relation_sql.go | 82 +++ repository/shape/dql/plan/relation_types.go | 32 + .../shape/dql/plan/relation_validate.go | 204 ++++++ repository/shape/dql/plan/relation_yaml.go | 164 +++++ repository/shape/dql/render/dql/renderer.go | 166 +++++ .../shape/dql/render/dql/renderer_test.go | 117 ++++ repository/shape/dql/render/yaml/renderer.go | 16 + repository/shape/dql/scan/scanner.go | 433 +++++++++++++ repository/shape/dql/scan/scanner_test.go | 164 +++++ repository/shape/dql/shape/convert.go | 235 +++++++ repository/shape/dql/shape/convert_test.go | 121 ++++ repository/shape/dql/shape/model.go | 21 + repository/shape/load/loader.go | 6 +- repository/shape/plan/model.go | 46 +- repository/shape/plan/planner.go | 32 +- repository/shape/plan/planner_test.go | 11 +- repository/shape/platform_parity_test.go | 9 +- view/resource.go | 77 ++- 51 files changed, 4506 insertions(+), 108 deletions(-) create mode 100644 cmd/command/translate_shape_ir.go create mode 100644 repository/components_shape_test.go create mode 100644 repository/components_typectx_test.go create mode 100644 repository/option_shape_test.go create mode 100644 repository/shape/dql/decl/lex.go create mode 100644 repository/shape/dql/decl/model.go create mode 100644 repository/shape/dql/decl/parser.go create mode 100644 repository/shape/dql/decl/parser_test.go create mode 100644 repository/shape/dql/holder/model.go create mode 100644 repository/shape/dql/holder/model_test.go create mode 100644 repository/shape/dql/ir/model.go create mode 100644 repository/shape/dql/load/loader.go create mode 100644 repository/shape/dql/load/loader_test.go create mode 100644 repository/shape/dql/parity/adorder_parity_test.go create mode 100644 repository/shape/dql/parity/connectors.go create mode 100644 repository/shape/dql/parity/diff.go create mode 100644 repository/shape/dql/parity/mdp_parity_test.go create mode 100644 repository/shape/dql/parse/function.go create mode 100644 repository/shape/dql/parse/model.go create mode 100644 repository/shape/dql/plan/planner.go create mode 100644 repository/shape/dql/plan/planner_test.go create mode 100644 repository/shape/dql/plan/relation_sql.go create mode 100644 repository/shape/dql/plan/relation_types.go create mode 100644 repository/shape/dql/plan/relation_validate.go create mode 100644 repository/shape/dql/plan/relation_yaml.go create mode 100644 repository/shape/dql/render/dql/renderer.go create mode 100644 repository/shape/dql/render/dql/renderer_test.go create mode 100644 repository/shape/dql/render/yaml/renderer.go create mode 100644 repository/shape/dql/scan/scanner.go create mode 100644 repository/shape/dql/scan/scanner_test.go create mode 100644 repository/shape/dql/shape/convert.go create mode 100644 repository/shape/dql/shape/convert_test.go diff --git a/cmd/command/translate.go b/cmd/command/translate.go index ab5485b49..4b4f0921c 100644 --- a/cmd/command/translate.go +++ b/cmd/command/translate.go @@ -29,7 +29,8 @@ func (s *Service) Translate(ctx context.Context, opts *options.Options) (err err if err = s.translate(ctx, opts); err != nil { return err } - if opts.Rule().EffectiveEngine() == options.EngineShape { + engine := opts.Rule().EffectiveEngine() + if engine == options.EngineShape || engine == options.EngineShapeIR { return nil } return s.persistRepository(ctx) @@ -52,8 +53,11 @@ func (s *Service) persistRepository(ctx context.Context) error { } func (s *Service) translate(ctx context.Context, opts *options.Options) error { - if opts.Rule().EffectiveEngine() == options.EngineShape { + switch opts.Rule().EffectiveEngine() { + case options.EngineShape: return s.translateShape(ctx, opts) + case options.EngineShapeIR: + return s.translateShapeIR(ctx, opts) } if err := s.ensureTranslator(opts); err != nil { return fmt.Errorf("failed to create translator: %v", err) diff --git a/cmd/command/translate_shape.go b/cmd/command/translate_shape.go index 109e99634..3a21db19d 100644 --- a/cmd/command/translate_shape.go +++ b/cmd/command/translate_shape.go @@ -16,7 +16,6 @@ import ( "github.com/viant/datly/repository/shape" shapeCompile "github.com/viant/datly/repository/shape/compile" shapeLoad "github.com/viant/datly/repository/shape/load" - datlyservice "github.com/viant/datly/service" "github.com/viant/datly/shared" "github.com/viant/datly/view" "gopkg.in/yaml.v3" @@ -112,7 +111,7 @@ func (s *Service) persistShapeRoute(ctx context.Context, opts *options.Options, URI: uri, }, Contract: contract.Contract{ - Service: serviceForMethod(method), + Service: serviceTypeForMethod(method), }, View: &view.View{Reference: shared.Reference{Ref: rootView}}, } @@ -188,10 +187,3 @@ func parseShapeRulePath(dql, ruleName, apiPrefix string) (string, string) { } return method, uri } - -func serviceForMethod(method string) datlyservice.Type { - if strings.EqualFold(method, "GET") { - return datlyservice.TypeReader - } - return datlyservice.TypeExecutor -} diff --git a/cmd/command/translate_shape_ir.go b/cmd/command/translate_shape_ir.go new file mode 100644 index 000000000..f09f2e4e9 --- /dev/null +++ b/cmd/command/translate_shape_ir.go @@ -0,0 +1,136 @@ +package command + +import ( + "context" + "fmt" + "path" + "strings" + + "github.com/viant/afs/file" + "github.com/viant/afs/url" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/repository/shape" + shapeCompile "github.com/viant/datly/repository/shape/compile" + "github.com/viant/datly/repository/shape/dql/ir" + dqlyaml "github.com/viant/datly/repository/shape/dql/render/yaml" + shapeLoad "github.com/viant/datly/repository/shape/load" + datlyservice "github.com/viant/datly/service" + "github.com/viant/datly/shared" + "github.com/viant/datly/view" + "gopkg.in/yaml.v3" +) + +func (s *Service) translateShapeIR(ctx context.Context, opts *options.Options) error { + rule := opts.Rule() + compiler := shapeCompile.New() + loader := shapeLoad.New() + for rule.Index = 0; rule.Index < len(rule.Source); rule.Index++ { + // Reuse legacy signature bootstrap so shape IR flow gets the same registry/signature context when available. + if err := s.ensureTranslator(opts); err == nil && s.translator != nil { + _ = s.translator.InitSignature(ctx, rule) + } + sourceURL := rule.SourceURL() + _, name := url.Split(sourceURL, file.Scheme) + fmt.Printf("translating %v (shape-ir)\n", name) + dql, err := rule.LoadSource(ctx, s.fs, sourceURL) + if err != nil { + return err + } + dql = strings.TrimSpace(dql) + if dql == "" { + return fmt.Errorf("source %s was empty", sourceURL) + } + shapeSource := &shape.Source{ + Name: strings.TrimSuffix(name, path.Ext(name)), + Path: url.Path(sourceURL), + DQL: dql, + Connector: strings.TrimSpace(rule.Connector), + } + planResult, err := compiler.Compile(ctx, shapeSource) + if err != nil { + return fmt.Errorf("failed to compile %s: %w", sourceURL, err) + } + componentArtifact, err := loader.LoadComponent(ctx, planResult) + if err != nil { + return fmt.Errorf("failed to load %s: %w", sourceURL, err) + } + component, ok := componentArtifact.Component.(*shapeLoad.Component) + if !ok || component == nil { + return fmt.Errorf("unexpected component artifact for %s", sourceURL) + } + + payload, err := buildShapeRulePayload(opts, dql, componentArtifact.Resource, component) + if err != nil { + return err + } + routeYAML, err := yaml.Marshal(payload) + if err != nil { + return err + } + document, err := ir.FromYAML(routeYAML) + if err != nil { + return fmt.Errorf("failed to build IR from %s: %w", sourceURL, err) + } + encoded, err := dqlyaml.Encode(document) + if err != nil { + return fmt.Errorf("failed to encode IR for %s: %w", sourceURL, err) + } + + routeYAMLPath, _, _, _, err := routePathForShape(rule, opts.Repository().RepositoryURL, sourceURL) + if err != nil { + return err + } + irPath := strings.TrimSuffix(routeYAMLPath, ".yaml") + ".ir.yaml" + if err = s.fs.Upload(ctx, irPath, file.DefaultFileOsMode, strings.NewReader(string(encoded))); err != nil { + return fmt.Errorf("failed to persist route ir %s: %w", irPath, err) + } + } + return nil +} + +func buildShapeRulePayload(opts *options.Options, dql string, resource *view.Resource, component *shapeLoad.Component) (*shapeRuleFile, error) { + rule := opts.Rule() + rootView := "" + if component != nil { + rootView = strings.TrimSpace(component.RootView) + } + if rootView == "" && resource != nil && len(resource.Views) > 0 && resource.Views[0] != nil { + rootView = resource.Views[0].Name + } + method, uri := parseShapeRulePath(dql, rule.RuleName(), opts.Repository().APIPrefix) + route := &repository.Component{ + Path: contract.Path{ + Method: method, + URI: uri, + }, + Contract: contract.Contract{ + Service: serviceTypeForMethod(method), + }, + View: &view.View{Reference: shared.Reference{Ref: rootView}}, + } + if component != nil { + route.TypeContext = component.TypeContext + if component.Directives != nil && component.Directives.MCP != nil { + route.Name = strings.TrimSpace(component.Directives.MCP.Name) + route.Description = strings.TrimSpace(component.Directives.MCP.Description) + route.DescriptionURI = strings.TrimSpace(component.Directives.MCP.DescriptionPath) + } + } + payload := &shapeRuleFile{ + Resource: resource, + Routes: []*repository.Component{route}, + } + if component != nil && component.TypeContext != nil { + payload.TypeContext = component.TypeContext + } + return payload, nil +} + +func serviceTypeForMethod(method string) datlyservice.Type { + if strings.EqualFold(method, "GET") { + return datlyservice.TypeReader + } + return datlyservice.TypeExecutor +} diff --git a/cmd/options/rule.go b/cmd/options/rule.go index fd5b23255..4d43015c0 100644 --- a/cmd/options/rule.go +++ b/cmd/options/rule.go @@ -22,7 +22,7 @@ type Rule struct { Name string `short:"n" long:"name" description:"rule name"` ModulePrefix string `short:"u" long:"namespace" description:"rule uri/namespace" default:"dev" ` Source []string `short:"s" long:"src" description:"source"` - Engine string `long:"engine" description:"translation engine" choice:"legacy" choice:"shape"` + Engine string `long:"engine" description:"translation engine" choice:"internal" choice:"legacy" choice:"shape" choice:"shape-ir"` Packages []string `short:"g" long:"pkg" description:"entity package"` Output []string Index int @@ -35,17 +35,25 @@ type Rule struct { } const ( - EngineLegacy = "legacy" - EngineShape = "shape" + EngineInternal = "internal" + EngineLegacy = "legacy" // alias of internal, kept for migration compatibility + EngineShape = "shape" + EngineShapeIR = "shape-ir" ) func (r *Rule) EffectiveEngine() string { engine := strings.ToLower(strings.TrimSpace(r.Engine)) switch engine { + case "", EngineInternal, EngineLegacy: + return EngineInternal + case "shapeir": + return EngineShapeIR + case EngineShapeIR: + return EngineShapeIR case EngineShape: return EngineShape default: - return EngineLegacy + return EngineInternal } } diff --git a/cmd/options/rule_engine_test.go b/cmd/options/rule_engine_test.go index bac95c6cb..08f27bdca 100644 --- a/cmd/options/rule_engine_test.go +++ b/cmd/options/rule_engine_test.go @@ -8,9 +8,13 @@ func TestRule_EffectiveEngine(t *testing.T) { engine string want string }{ - {name: "default", engine: "", want: EngineLegacy}, + {name: "default", engine: "", want: EngineInternal}, + {name: "internal", engine: "internal", want: EngineInternal}, + {name: "legacy alias", engine: "legacy", want: EngineInternal}, {name: "shape", engine: "shape", want: EngineShape}, - {name: "invalid", engine: "other", want: EngineLegacy}, + {name: "shape ir", engine: "shape-ir", want: EngineShapeIR}, + {name: "shape ir alias", engine: "shapeir", want: EngineShapeIR}, + {name: "invalid", engine: "other", want: EngineInternal}, } for _, testCase := range testCases { rule := &Rule{Engine: testCase.engine} diff --git a/e2e/local/regression/cases/001_one_to_many/expect_2.txt b/e2e/local/regression/cases/001_one_to_many/expect_2.txt index 2292a8c77..7796339fb 100644 --- a/e2e/local/regression/cases/001_one_to_many/expect_2.txt +++ b/e2e/local/regression/cases/001_one_to_many/expect_2.txt @@ -18,7 +18,7 @@ type GeneratedStruct struct { type Products struct { Id int `sqlx:"ID" velty:"names=ID|Id"` Name *string `sqlx:"NAME" velty:"names=NAME|Name"` - VendorId *int `sqlx:"VENDOR_ID" velty:"names=VENDOR_ID|VendorId"` + VendorId *int `sqlx:"VENDOR_ID" internal:"true" velty:"names=VENDOR_ID|VendorId"` Status *int `sqlx:"STATUS" velty:"names=STATUS|Status"` Created *time.Time `sqlx:"CREATED" velty:"names=CREATED|Created"` UserCreated *int `sqlx:"USER_CREATED" velty:"names=USER_CREATED|UserCreated"` diff --git a/e2e/local/regression/regression.yaml b/e2e/local/regression/regression.yaml index 4b496e283..c21e04c34 100644 --- a/e2e/local/regression/regression.yaml +++ b/e2e/local/regression/regression.yaml @@ -30,7 +30,7 @@ pipeline: '[]gen': '@gen' subPath: 'cases/${index}_*' - range: 011..012 + range: 1..010 template: checkSkip: action: nop @@ -39,5 +39,3 @@ pipeline: test: action: run request: '@test' - - diff --git a/e2e/local/regression/rule.yaml b/e2e/local/regression/rule.yaml index f43269480..5d6f0a3ce 100644 --- a/e2e/local/regression/rule.yaml +++ b/e2e/local/regression/rule.yaml @@ -10,6 +10,7 @@ pipeline: commands: - mkdir -p ${appPath}/e2e/local/autogen - rm -rf ${appPath}/e2e/local/autogen + - rm -f ${appPath}/e2e/local/regression/paths.yaml loop: diff --git a/repository/components_shape_test.go b/repository/components_shape_test.go new file mode 100644 index 000000000..d6ddfd00b --- /dev/null +++ b/repository/components_shape_test.go @@ -0,0 +1,62 @@ +package repository + +import ( + "context" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +type shapeTestRow struct { + ID int +} + +type shapeTestOutput struct { + Rows []shapeTestRow `view:"rows,table=REPORT" sql:"SELECT ID FROM REPORT"` +} + +func TestComponents_mergeShapeViews_Enabled(t *testing.T) { + resource := view.EmptyResource() + components := &Components{ + Resource: resource, + options: &Options{shapePipeline: true}, + } + + component := &Component{ + Path: contract.Path{URI: "/v1/api/report", Method: "GET"}, + Contract: contract.Contract{ + Output: contract.Output{Type: state.Type{Schema: state.NewSchema(reflect.TypeOf(&shapeTestOutput{}))}}, + }, + View: view.NewRefView("rows"), + } + + err := components.mergeShapeViews(context.Background(), component) + require.NoError(t, err) + require.Len(t, components.Resource.Views, 1) + assert.Equal(t, "rows", components.Resource.Views[0].Name) +} + +func TestComponents_mergeShapeViews_Disabled(t *testing.T) { + resource := view.EmptyResource() + components := &Components{ + Resource: resource, + options: &Options{shapePipeline: false}, + } + + component := &Component{ + Path: contract.Path{URI: "/v1/api/report", Method: "GET"}, + Contract: contract.Contract{ + Output: contract.Output{Type: state.Type{Schema: state.NewSchema(reflect.TypeOf(&shapeTestOutput{}))}}, + }, + View: view.NewRefView("rows"), + } + + err := components.mergeShapeViews(context.Background(), component) + require.NoError(t, err) + assert.Len(t, components.Resource.Views, 0) +} diff --git a/repository/components_typectx_test.go b/repository/components_typectx_test.go new file mode 100644 index 000000000..4239fd201 --- /dev/null +++ b/repository/components_typectx_test.go @@ -0,0 +1,164 @@ +package repository + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" +) + +func TestUnmarshalComponentMap_PropagatesTopLevelTypeContext(t *testing.T) { + model := map[string]any{ + "TypeContext": map[string]any{ + "DefaultPackage": "mdp/performance", + "Imports": []any{ + map[string]any{ + "Alias": "perf", + "Package": "github.com/acme/mdp/performance", + }, + }, + }, + "Components": []any{ + map[string]any{ + "URI": "/v1/api/sample", + "Method": "GET", + "View": map[string]any{ + "Ref": "sample", + }, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{"Name": "sample"}, + }, + }, + } + components, err := unmarshalComponentMap(model, true) + require.NoError(t, err) + require.Len(t, components.Components, 1) + require.NotNil(t, components.Components[0].TypeContext) + require.Equal(t, "mdp/performance", components.Components[0].TypeContext.DefaultPackage) + require.Len(t, components.Components[0].TypeContext.Imports, 1) + require.Equal(t, "perf", components.Components[0].TypeContext.Imports[0].Alias) +} + +func TestUnmarshalComponentMap_PerComponentTypeContextOverridesTopLevel(t *testing.T) { + model := map[string]any{ + "TypeContext": map[string]any{ + "DefaultPackage": "top/level", + }, + "Components": []any{ + map[string]any{ + "URI": "/v1/api/sample", + "Method": "GET", + "View": map[string]any{ + "Ref": "sample", + }, + "TypeContext": map[string]any{ + "DefaultPackage": "component/level", + "Imports": []any{ + map[string]any{ + "Alias": "foo", + "Package": "github.com/acme/foo", + }, + }, + }, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{"Name": "sample"}, + }, + }, + } + components, err := unmarshalComponentMap(model, true) + require.NoError(t, err) + require.Len(t, components.Components, 1) + require.NotNil(t, components.Components[0].TypeContext) + require.Equal(t, "component/level", components.Components[0].TypeContext.DefaultPackage) + require.Len(t, components.Components[0].TypeContext.Imports, 1) + require.Equal(t, "foo", components.Components[0].TypeContext.Imports[0].Alias) +} + +func TestUnmarshalComponentMap_NoTypeContextRemainsNil(t *testing.T) { + model := map[string]any{ + "Components": []any{ + map[string]any{ + "URI": "/v1/api/sample", + "Method": "GET", + "View": map[string]any{ + "Ref": "sample", + }, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{"Name": "sample"}, + }, + }, + } + components, err := unmarshalComponentMap(model, true) + require.NoError(t, err) + require.Len(t, components.Components, 1) + require.Nil(t, components.Components[0].TypeContext) +} + +func TestUnmarshalComponentMap_TopLevelTypeContext_DisabledByFlag(t *testing.T) { + model := map[string]any{ + "TypeContext": map[string]any{ + "DefaultPackage": "mdp/performance", + }, + "Components": []any{ + map[string]any{ + "URI": "/v1/api/sample", + "Method": "GET", + "View": map[string]any{ + "Ref": "sample", + }, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{"Name": "sample"}, + }, + }, + } + components, err := unmarshalComponentMap(model, false) + require.NoError(t, err) + require.Len(t, components.Components, 1) + require.Nil(t, components.Components[0].TypeContext) +} + +func TestResolveComponentTypeContext_FromTemplateSource(t *testing.T) { + component := &Component{ + View: &view.View{ + Template: view.NewTemplate(` +#set($_ = $package('mdp/performance')) +#set($_ = $import('perf', 'github.com/acme/mdp/performance')) +SELECT ID FROM REPORT r`), + }, + } + resolved := resolveComponentTypeContext(component) + require.NotNil(t, resolved) + require.Equal(t, "mdp/performance", resolved.DefaultPackage) + require.Len(t, resolved.Imports, 1) + require.Equal(t, "perf", resolved.Imports[0].Alias) +} + +func TestResolveComponentTypeContext_PrefersExisting(t *testing.T) { + component := &Component{ + TypeContext: &typectx.Context{ + DefaultPackage: " custom/pkg ", + Imports: []typectx.Import{ + {Alias: " a ", Package: " github.com/acme/a "}, + }, + }, + } + resolved := resolveComponentTypeContext(component) + require.NotNil(t, resolved) + require.Equal(t, "custom/pkg", resolved.DefaultPackage) + require.Len(t, resolved.Imports, 1) + require.Equal(t, "a", resolved.Imports[0].Alias) + require.Equal(t, "github.com/acme/a", resolved.Imports[0].Package) +} diff --git a/repository/option_shape_test.go b/repository/option_shape_test.go new file mode 100644 index 000000000..11bf4ecba --- /dev/null +++ b/repository/option_shape_test.go @@ -0,0 +1,29 @@ +package repository + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWithShapePipeline(t *testing.T) { + opts := NewOptions(nil) + assert.False(t, opts.shapePipeline) + + WithShapePipeline(true)(opts) + assert.True(t, opts.shapePipeline) + + WithShapePipeline(false)(opts) + assert.False(t, opts.shapePipeline) +} + +func TestWithLegacyTypeContext(t *testing.T) { + opts := NewOptions(nil) + assert.False(t, opts.legacyTypeContext) + + WithLegacyTypeContext(true)(opts) + assert.True(t, opts.legacyTypeContext) + + WithLegacyTypeContext(false)(opts) + assert.False(t, opts.legacyTypeContext) +} diff --git a/repository/path/service.go b/repository/path/service.go index 260fff91f..2ce530d9d 100644 --- a/repository/path/service.go +++ b/repository/path/service.go @@ -198,7 +198,8 @@ func (s *Service) buildPaths(ctx context.Context, candidate storage.Object, root } sourceURL := candidate.URL() if index := strings.Index(sourceURL, rootPath); index != -1 { - sourceURL = sourceURL[1+index+len(rootPath):] + sourceURL = sourceURL[index+len(rootPath):] + sourceURL = strings.TrimPrefix(sourceURL, "/") } anItem := &Item{ SourceURL: sourceURL, diff --git a/repository/shape/compile/component_types.go b/repository/shape/compile/component_types.go index 6e84c960e..acb9c87b8 100644 --- a/repository/shape/compile/component_types.go +++ b/repository/shape/compile/component_types.go @@ -10,6 +10,7 @@ import ( dqldiag "github.com/viant/datly/repository/shape/dql/diag" dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/view/state" "gopkg.in/yaml.v3" ) @@ -47,10 +48,10 @@ func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, l } for _, stateItem := range result.States { - if stateItem == nil || !strings.EqualFold(strings.TrimSpace(stateItem.Kind), "component") { + if stateItem == nil || !strings.EqualFold(stateItem.KindString(), "component") { continue } - ref := strings.TrimSpace(stateItem.In) + ref := stateItem.InName() if ref == "" { continue } @@ -66,8 +67,13 @@ func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, l continue } outputType, ok := collector.collect(namespace, componentRefSpan(source.DQL, ref), true) - if ok && strings.TrimSpace(stateItem.DataType) == "" { - stateItem.DataType = strings.TrimSpace(outputType) + if ok && strings.TrimSpace(outputType) != "" { + if stateItem.Schema == nil { + stateItem.Schema = &state.Schema{} + } + if strings.TrimSpace(stateItem.Schema.DataType) == "" { + stateItem.Schema.DataType = strings.TrimSpace(outputType) + } } } diff --git a/repository/shape/compile/component_types_test.go b/repository/shape/compile/component_types_test.go index 51570a125..37c56dea6 100644 --- a/repository/shape/compile/component_types_test.go +++ b/repository/shape/compile/component_types_test.go @@ -10,6 +10,7 @@ import ( "github.com/viant/datly/repository/shape" dqldiag "github.com/viant/datly/repository/shape/dql/diag" "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/view/state" ) func TestResolveComponentNamespace(t *testing.T) { @@ -78,7 +79,7 @@ Routes: result := &plan.Result{ States: []*plan.State{ - {Name: "Auth", Kind: "component", In: "../acl/auth"}, + {Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}, }, } appendComponentTypes(&shape.Source{Path: sourcePath, DQL: "#set($Auth = $component<../acl/auth>())"}, result) @@ -89,7 +90,7 @@ Routes: } assert.True(t, names["Input"]) assert.True(t, names["UserView"]) - assert.Equal(t, "*Output", result.States[0].DataType) + assert.Equal(t, "*Output", result.States[0].Schema.DataType) } func TestAppendComponentTypes_MissingComponentRoute(t *testing.T) { @@ -100,7 +101,7 @@ func TestAppendComponentTypes_MissingComponentRoute(t *testing.T) { dql := "#set($Auth = $component<../acl/missing>())\nSELECT 1" require.NoError(t, os.WriteFile(sourcePath, []byte(dql), 0o644)) result := &plan.Result{ - States: []*plan.State{{Name: "Auth", Kind: "component", In: "../acl/missing"}}, + States: []*plan.State{{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/missing"}}}, } diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: dql}, result) require.NotEmpty(t, diags) @@ -129,7 +130,7 @@ func TestAppendComponentTypes_TypeCollisionEmitsDiagnostic(t *testing.T) { result := &plan.Result{ States: []*plan.State{ - {Name: "Auth", Kind: "component", In: "../acl/auth"}, + {Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}, }, Types: []*plan.Type{ { @@ -167,7 +168,7 @@ func TestAppendComponentTypes_InvalidRouteYAMLEmitsDiagnostic(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(routesDir, "auth", "auth.yaml"), []byte("Resource:\n Types: ["), 0o644)) result := &plan.Result{ - States: []*plan.State{{Name: "Auth", Kind: "component", In: "../acl/auth"}}, + States: []*plan.State{{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}}, } diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: dql}, result) require.NotEmpty(t, diags) @@ -188,8 +189,8 @@ func TestAppendComponentTypes_InvalidRouteYAMLDedupedForRepeatedStates(t *testin result := &plan.Result{ States: []*plan.State{ - {Name: "Auth1", Kind: "component", In: "../acl/auth"}, - {Name: "Auth2", Kind: "component", In: "../acl/auth"}, + {Name: "Auth1", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}, + {Name: "Auth2", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}, }, } diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: dql}, result) diff --git a/repository/shape/compile/statedecl.go b/repository/shape/compile/statedecl.go index 5800e4881..fbb933308 100644 --- a/repository/shape/compile/statedecl.go +++ b/repository/shape/compile/statedecl.go @@ -5,6 +5,8 @@ import ( "strings" "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/view/extension" + st "github.com/viant/datly/view/state" "github.com/viant/parsly" ) @@ -26,13 +28,16 @@ func appendDeclaredStates(rawDQL string, result *plan.Result) { continue } state := &plan.State{ - Path: holder, - Name: holder, - Kind: kind, - In: location, + Parameter: st.Parameter{ + Name: holder, + In: &st.Location{ + Kind: st.Kind(kind), + Name: location, + }, + }, } if inType, outType := parseSetDeclarationTypes(block.Body); inType != "" || outType != "" { - state.DataType = inType + ensureStateSchema(state).DataType = inType state.OutputDataType = outType } switch strings.ToLower(kind) { @@ -101,17 +106,19 @@ func applyDeclaredStateOptions(state *plan.State, tail string) { } case strings.EqualFold(name, "WithType"): if len(args) == 1 { - state.DataType = trimQuote(args[0]) + ensureStateSchema(state).DataType = trimQuote(args[0]) } case strings.EqualFold(name, "WithCodec"): if len(args) >= 1 { - state.Codec = trimQuote(args[0]) - state.CodecArgs = append([]string{}, trimQuotedArgs(args[1:])...) + state.Output = &st.Codec{ + Name: trimQuote(args[0]), + Args: append([]string{}, trimQuotedArgs(args[1:])...), + } } case strings.EqualFold(name, "WithStatusCode"): if len(args) == 1 { if value, err := strconv.Atoi(strings.TrimSpace(trimQuote(args[0]))); err == nil { - state.ErrorCode = value + state.ErrorStatusCode = value } } case strings.EqualFold(name, "WithErrorMessage"): @@ -193,18 +200,25 @@ func appendStatePredicate(state *plan.State, args []string, ensure bool) { if len(args) <= nameIdx { return } - predicate := &plan.StatePredicate{ - Group: group, - Name: trimQuote(args[nameIdx]), - Ensure: ensure, - Arguments: []string{}, + predicate := &extension.PredicateConfig{ + Group: group, + Name: trimQuote(args[nameIdx]), + Ensure: ensure, + Args: []string{}, } for _, arg := range args[nameIdx+1:] { - predicate.Arguments = append(predicate.Arguments, trimQuote(arg)) + predicate.Args = append(predicate.Args, trimQuote(arg)) } state.Predicates = append(state.Predicates, predicate) } +func ensureStateSchema(state *plan.State) *st.Schema { + if state.Schema == nil { + state.Schema = &st.Schema{} + } + return state.Schema +} + type optionCursor struct { raw string cursor int diff --git a/repository/shape/compile/statedecl_test.go b/repository/shape/compile/statedecl_test.go index 94c5fea26..33c9241a4 100644 --- a/repository/shape/compile/statedecl_test.go +++ b/repository/shape/compile/statedecl_test.go @@ -27,19 +27,19 @@ SELECT id FROM SITE_LIST sl` } } require.NotNil(t, byName["Jwt"]) - assert.Equal(t, "header", byName["Jwt"].Kind) - assert.Equal(t, "string", byName["Jwt"].DataType) - assert.Equal(t, "JwtClaim", byName["Jwt"].Codec) - assert.Equal(t, 401, byName["Jwt"].ErrorCode) + assert.Equal(t, "header", byName["Jwt"].KindString()) + assert.Equal(t, "string", byName["Jwt"].Schema.DataType) + assert.Equal(t, "JwtClaim", byName["Jwt"].Output.Name) + assert.Equal(t, 401, byName["Jwt"].ErrorStatusCode) require.NotNil(t, byName["Jwt"].Required) assert.True(t, *byName["Jwt"].Required) require.NotNil(t, byName["Claims"]) - assert.Equal(t, "string", byName["Claims"].DataType) + assert.Equal(t, "string", byName["Claims"].Schema.DataType) assert.Equal(t, "*JwtClaims", byName["Claims"].OutputDataType) require.NotNil(t, byName["Name"]) - assert.Equal(t, "query", byName["Name"].Kind) + assert.Equal(t, "query", byName["Name"].KindString()) require.NotNil(t, byName["Name"].Required) assert.False(t, *byName["Name"].Required) require.Len(t, byName["Name"].Predicates, 1) @@ -62,7 +62,7 @@ SELECT id FROM CI_TV_AFFILIATE_STATION tas` require.Len(t, result.States, 1) require.Len(t, result.States[0].Predicates, 1) assert.Equal(t, "Active", result.States[0].Name) - assert.Equal(t, "IS_TARGETABLE", result.States[0].Predicates[0].Arguments[1]) + assert.Equal(t, "IS_TARGETABLE", result.States[0].Predicates[0].Args[1]) } func TestAppendDeclaredStates_SupportsDefineDirective(t *testing.T) { @@ -73,7 +73,7 @@ SELECT id FROM USERS u` appendDeclaredStates(dql, result) require.Len(t, result.States, 1) assert.Equal(t, "Auth", result.States[0].Name) - assert.Equal(t, "header", result.States[0].Kind) + assert.Equal(t, "header", result.States[0].KindString()) require.NotNil(t, result.States[0].Required) assert.True(t, *result.States[0].Required) } diff --git a/repository/shape/dql/decl/lex.go b/repository/shape/dql/decl/lex.go new file mode 100644 index 000000000..fbf6270ac --- /dev/null +++ b/repository/shape/dql/decl/lex.go @@ -0,0 +1,59 @@ +package decl + +import ( + "github.com/viant/parsly" + "github.com/viant/parsly/matcher" +) + +const ( + whitespaceToken = iota + singleQuotedToken + doubleQuotedToken + commentBlockToken + parenthesesBlockToken + identifierToken + anyToken +) + +var whitespaceMatcher = parsly.NewToken(whitespaceToken, "Whitespace", matcher.NewWhiteSpace()) +var singleQuotedMatcher = parsly.NewToken(singleQuotedToken, "SingleQuote", matcher.NewBlock('\'', '\'', '\\')) +var doubleQuotedMatcher = parsly.NewToken(doubleQuotedToken, "DoubleQuote", matcher.NewBlock('"', '"', '\\')) +var commentBlockMatcher = parsly.NewToken(commentBlockToken, "CommentBlock", matcher.NewSeqBlock("/*", "*/")) +var parenthesesBlockMatcher = parsly.NewToken(parenthesesBlockToken, "Parentheses", matcher.NewBlock('(', ')', '\\')) + +var identifierMatcher = parsly.NewToken(identifierToken, "Identifier", &identifierMatch{}) +var anyMatcher = parsly.NewToken(anyToken, "Any", &anyMatch{}) + +type anyMatch struct{} + +func (a *anyMatch) Match(cursor *parsly.Cursor) int { + if cursor.Pos < cursor.InputSize { + return 1 + } + return 0 +} + +type identifierMatch struct{} + +func (i *identifierMatch) Match(cursor *parsly.Cursor) int { + if cursor.Pos >= cursor.InputSize { + return 0 + } + b := cursor.Input[cursor.Pos] + if !isIdentifierStart(b) { + return 0 + } + pos := cursor.Pos + 1 + for pos < cursor.InputSize && isIdentifierPart(cursor.Input[pos]) { + pos++ + } + return pos - cursor.Pos +} + +func isIdentifierStart(b byte) bool { + return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' +} + +func isIdentifierPart(b byte) bool { + return isIdentifierStart(b) || (b >= '0' && b <= '9') +} diff --git a/repository/shape/dql/decl/model.go b/repository/shape/dql/decl/model.go new file mode 100644 index 000000000..914d3493c --- /dev/null +++ b/repository/shape/dql/decl/model.go @@ -0,0 +1,42 @@ +package decl + +// Kind identifies parsed declaration function. +type Kind string + +const ( + KindCast Kind = "cast" + KindTag Kind = "tag" + KindSetLimit Kind = "set_limit" + KindAllowNulls Kind = "allow_nulls" + KindSetPartitioner Kind = "set_partitioner" + KindUseConnector Kind = "use_connector" + KindMatchStrategy Kind = "match_strategy" + KindCompressAboveSize Kind = "compress_above_size" + KindBatchSize Kind = "batch_size" + KindRelationalConcurrency Kind = "relational_concurrency" + KindPublishParent Kind = "publish_parent" + KindCardinality Kind = "cardinality" + KindPackage Kind = "package" + KindImport Kind = "import" +) + +// Declaration represents one parsed function declaration in DQL. +type Declaration struct { + Kind Kind + Raw string + Offset int + Args []string + + // Normalized fields for known declarations. + Target string // first argument (alias/column) + DataType string // cast(... as type) + Tag string // tag(..., "...") payload + Limit string // set_limit(..., N) + Connector string // use_connector(view, connector) + Strategy string // match_strategy(view, strategy) + Partition string // set_partitioner(view, partitioner, concurrency) + Size string // compress_above_size(size) + Value string // generic second argument (batch_size, relational_concurrency, cardinality) + Package string // package(default/package) + Alias string // import(alias, package/path) +} diff --git a/repository/shape/dql/decl/parser.go b/repository/shape/dql/decl/parser.go new file mode 100644 index 000000000..5fe6d13d9 --- /dev/null +++ b/repository/shape/dql/decl/parser.go @@ -0,0 +1,337 @@ +package decl + +import ( + "fmt" + "strings" + + "github.com/viant/parsly" +) + +// Parse extracts declarations from original DQL text. +func Parse(dql string) ([]*Declaration, error) { + cursor := parsly.NewCursor("", []byte(dql), 0) + var result []*Declaration + for cursor.Pos < cursor.InputSize { + matched := cursor.MatchAfterOptional(whitespaceMatcher, + commentBlockMatcher, + singleQuotedMatcher, + doubleQuotedMatcher, + identifierMatcher, + anyMatcher, + ) + switch matched.Code { + case identifierToken: + name := strings.ToLower(matched.Text(cursor)) + callOffset := matched.Offset + block := cursor.MatchAfterOptional(whitespaceMatcher, parenthesesBlockMatcher) + if block.Code != parenthesesBlockToken { + continue + } + rawCall := name + block.Text(cursor) + argsText := block.Text(cursor) + if len(argsText) < 2 { + continue + } + args := splitArgs(argsText[1 : len(argsText)-1]) + if rewrittenName, rewrittenArgs, ok := unwrapSetSpecial(name, args); ok { + name = rewrittenName + args = rewrittenArgs + rawCall = name + "(" + strings.Join(args, ", ") + ")" + } + decl := &Declaration{ + Kind: parseKind(name), + Raw: rawCall, + Offset: callOffset, + Args: args, + } + normalizeDeclaration(decl) + result = append(result, decl) + case parsly.Invalid: + return nil, cursor.NewError(identifierMatcher) + } + } + return result, nil +} + +func parseKind(name string) Kind { + switch strings.ToLower(name) { + case "cast": + return KindCast + case "tag": + return KindTag + case "set_limit": + return KindSetLimit + case "allow_nulls": + return KindAllowNulls + case "set_partitioner": + return KindSetPartitioner + case "use_connector": + return KindUseConnector + case "match_strategy": + return KindMatchStrategy + case "compress_above_size": + return KindCompressAboveSize + case "batch_size": + return KindBatchSize + case "relational_concurrency": + return KindRelationalConcurrency + case "publish_parent": + return KindPublishParent + case "cardinality": + return KindCardinality + case "package": + return KindPackage + case "import": + return KindImport + default: + return Kind(name) + } +} + +func normalizeDeclaration(decl *Declaration) { + if decl == nil || len(decl.Args) == 0 { + return + } + decl.Target = strings.TrimSpace(decl.Args[0]) + switch decl.Kind { + case KindCast: + if len(decl.Args) >= 2 { + decl.DataType = normalizeCastType(decl.Args[1]) + } else if len(decl.Args) == 1 { + target, dataType := splitCastExpression(decl.Args[0]) + if target != "" { + decl.Target = target + } + if dataType != "" { + decl.DataType = dataType + } + } + case KindTag: + if len(decl.Args) >= 2 { + decl.Tag = trimQuotes(strings.TrimSpace(decl.Args[1])) + } + case KindSetLimit: + if len(decl.Args) >= 2 { + decl.Limit = strings.TrimSpace(decl.Args[1]) + } + case KindUseConnector: + if len(decl.Args) >= 2 { + decl.Connector = trimQuotes(strings.TrimSpace(decl.Args[1])) + } + case KindMatchStrategy: + if len(decl.Args) >= 2 { + decl.Strategy = trimQuotes(strings.TrimSpace(decl.Args[1])) + } + case KindSetPartitioner: + if len(decl.Args) >= 2 { + decl.Partition = trimQuotes(strings.TrimSpace(decl.Args[1])) + } + if len(decl.Args) >= 3 { + decl.Value = strings.TrimSpace(decl.Args[2]) + } + case KindCompressAboveSize: + if len(decl.Args) >= 1 { + decl.Size = strings.TrimSpace(decl.Args[0]) + } + case KindBatchSize, KindRelationalConcurrency, KindCardinality: + if len(decl.Args) >= 2 { + decl.Value = trimQuotes(strings.TrimSpace(decl.Args[1])) + } + case KindPackage: + decl.Package = trimQuotes(strings.TrimSpace(decl.Args[0])) + case KindImport: + switch len(decl.Args) { + case 1: + decl.Package = trimQuotes(strings.TrimSpace(decl.Args[0])) + default: + decl.Alias = trimQuotes(strings.TrimSpace(decl.Args[0])) + decl.Package = trimQuotes(strings.TrimSpace(decl.Args[1])) + } + } +} + +func splitCastExpression(expr string) (string, string) { + text := strings.TrimSpace(expr) + if text == "" { + return "", "" + } + lowered := strings.ToLower(text) + quote := rune(0) + escape := false + depth := 0 + for i := 0; i < len(text); i++ { + r := rune(text[i]) + if quote != 0 { + if escape { + escape = false + continue + } + if r == '\\' { + escape = true + continue + } + if r == quote { + quote = 0 + } + continue + } + switch r { + case '\'', '"', '`': + quote = r + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + } + if depth == 0 && i+4 <= len(text) { + chunk := lowered[i : i+4] + if chunk == " as " { + left := strings.TrimSpace(text[:i]) + right := strings.TrimSpace(text[i+4:]) + return left, trimQuotes(right) + } + } + } + return text, "" +} + +func normalizeCastType(arg string) string { + text := strings.TrimSpace(arg) + lower := strings.ToLower(text) + if strings.HasPrefix(lower, "as ") { + text = strings.TrimSpace(text[3:]) + } + return trimQuotes(text) +} + +func unwrapSetSpecial(name string, args []string) (string, []string, bool) { + if strings.ToLower(strings.TrimSpace(name)) != "set" || len(args) != 1 { + return "", nil, false + } + expr := args[0] + if expr == "" { + return "", nil, false + } + for _, functionName := range []string{"package", "import"} { + token := "$" + functionName + "(" + lowerExpr := strings.ToLower(expr) + idx := strings.Index(lowerExpr, token) + if idx == -1 { + continue + } + openPos := idx + len(token) - 1 + closePos := findClosingParen(expr, openPos) + if closePos <= openPos { + continue + } + inner := strings.TrimSpace(expr[openPos+1 : closePos]) + return functionName, splitArgs(inner), true + } + return "", nil, false +} + +func findClosingParen(text string, openPos int) int { + if openPos < 0 || openPos >= len(text) || text[openPos] != '(' { + return -1 + } + depth := 0 + var quote rune + escape := false + runes := []rune(text) + for i := 0; i < len(runes); i++ { + r := runes[i] + if quote != 0 { + if escape { + escape = false + continue + } + if r == '\\' { + escape = true + continue + } + if r == quote { + quote = 0 + } + continue + } + switch r { + case '\'', '"', '`': + quote = r + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + return i + } + } + } + return -1 +} + +func splitArgs(text string) []string { + var result []string + start := 0 + depth := 0 + var quote rune + escape := false + runes := []rune(text) + for i, r := range runes { + if quote != 0 { + if escape { + escape = false + continue + } + if r == '\\' { + escape = true + continue + } + if r == quote { + quote = 0 + } + continue + } + switch r { + case '\'', '"', '`': + quote = r + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + case ',': + if depth == 0 { + result = append(result, strings.TrimSpace(string(runes[start:i]))) + start = i + 1 + } + } + } + last := strings.TrimSpace(string(runes[start:])) + if last != "" || strings.TrimSpace(text) != "" { + result = append(result, last) + } + return result +} + +func trimQuotes(value string) string { + value = strings.TrimSpace(value) + if len(value) < 2 { + return value + } + first := value[0] + last := value[len(value)-1] + if (first == '\'' && last == '\'') || (first == '"' && last == '"') || (first == '`' && last == '`') { + return value[1 : len(value)-1] + } + return value +} + +func (d *Declaration) String() string { + if d == nil { + return "" + } + return fmt.Sprintf("%s(%s)", d.Kind, strings.Join(d.Args, ", ")) +} diff --git a/repository/shape/dql/decl/parser_test.go b/repository/shape/dql/decl/parser_test.go new file mode 100644 index 000000000..f5939822d --- /dev/null +++ b/repository/shape/dql/decl/parser_test.go @@ -0,0 +1,142 @@ +package decl + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParse_ExtractsDeclarations(t *testing.T) { + sql := ` +SELECT ad.*, + cast(ad.ACTIVE as bool), + cast(ad.CHANNELS AS '[]string'), + tag(ad.CHANNELS, 'sqlx:"-"'), + set_limit(ad, 25), + allow_nulls(ad) +FROM CI_AD_ORDER ad` + decls, err := Parse(sql) + require.NoError(t, err) + require.Len(t, decls, 5) + + require.Equal(t, KindCast, decls[0].Kind) + require.Equal(t, "ad.ACTIVE", decls[0].Target) + require.Equal(t, "bool", decls[0].DataType) + + require.Equal(t, KindCast, decls[1].Kind) + require.Equal(t, "[]string", decls[1].DataType) + + require.Equal(t, KindTag, decls[2].Kind) + require.Equal(t, `sqlx:"-"`, decls[2].Tag) + + require.Equal(t, KindSetLimit, decls[3].Kind) + require.Equal(t, "25", decls[3].Limit) + + require.Equal(t, KindAllowNulls, decls[4].Kind) + require.Equal(t, "ad", decls[4].Target) +} + +func TestParse_IgnoresQuotedAndCommented(t *testing.T) { + sql := ` +SELECT + 'cast(a as int)', + "tag(x,'json')", + /* set_limit(a,1), allow_nulls(a) */ + cast(t.ACTIVE as bool) +FROM T t` + decls, err := Parse(sql) + require.NoError(t, err) + require.Len(t, decls, 1) + require.Equal(t, KindCast, decls[0].Kind) +} + +func TestParse_SupportsNestedArgs(t *testing.T) { + sql := `SELECT tag(ad.NAME, concat('a,', upper('b'))), set_limit(ad, ifnull(25, 10)) FROM T ad` + decls, err := Parse(sql) + require.NoError(t, err) + require.Len(t, decls, 2) + require.Equal(t, KindTag, decls[0].Kind) + require.Equal(t, "concat('a,', upper('b'))", decls[0].Tag) + require.Equal(t, "ifnull(25, 10)", decls[1].Limit) +} + +func TestParse_AllowsWhitespaceBetweenNameAndParen(t *testing.T) { + sql := `SELECT cast (ad.ACTIVE as bool), allow_nulls ( ad ) FROM T ad` + decls, err := Parse(sql) + require.NoError(t, err) + require.Len(t, decls, 2) + require.Equal(t, KindCast, decls[0].Kind) + require.Equal(t, KindAllowNulls, decls[1].Kind) +} + +func TestParse_InvalidInputProducesNoError(t *testing.T) { + sql := `SELECT cast ad.ACTIVE as bool FROM T` + decls, err := Parse(sql) + require.NoError(t, err) + require.Len(t, decls, 0) +} + +func TestParse_ExtractsExtendedSettings(t *testing.T) { + sql := ` +SELECT x.*, + set_partitioner(x, 'pkg.Part', 7), + use_connector(x, 'bq_mdp'), + match_strategy(x, 'read_all'), + compress_above_size(1024), + batch_size(x, 20000), + relational_concurrency(x, 10), + publish_parent(x), + cardinality(x, 'One') +FROM T x` + decls, err := Parse(sql) + require.NoError(t, err) + require.Len(t, decls, 8) + + require.Equal(t, KindSetPartitioner, decls[0].Kind) + require.Equal(t, "pkg.Part", decls[0].Partition) + require.Equal(t, "7", decls[0].Value) + + require.Equal(t, KindUseConnector, decls[1].Kind) + require.Equal(t, "bq_mdp", decls[1].Connector) + + require.Equal(t, KindMatchStrategy, decls[2].Kind) + require.Equal(t, "read_all", decls[2].Strategy) + + require.Equal(t, KindCompressAboveSize, decls[3].Kind) + require.Equal(t, "1024", decls[3].Size) + + require.Equal(t, KindBatchSize, decls[4].Kind) + require.Equal(t, "20000", decls[4].Value) + + require.Equal(t, KindRelationalConcurrency, decls[5].Kind) + require.Equal(t, "10", decls[5].Value) + + require.Equal(t, KindPublishParent, decls[6].Kind) + require.Equal(t, "x", decls[6].Target) + + require.Equal(t, KindCardinality, decls[7].Kind) + require.Equal(t, "One", decls[7].Value) +} + +func TestParse_ExtractsPackageAndImport(t *testing.T) { + sql := ` +#set($_ = $package('mdp/performance')) +#set($_ = $import('perf', 'github.com/acme/mdp/performance')) +#set($_ = $import('github.com/acme/shared/types')) +SELECT x.* +FROM T x` + decls, err := Parse(sql) + require.NoError(t, err) + require.Len(t, decls, 3) + + require.Equal(t, KindPackage, decls[0].Kind) + require.Equal(t, "mdp/performance", decls[0].Package) + + require.Equal(t, KindImport, decls[1].Kind) + require.Equal(t, "perf", decls[1].Alias) + require.Equal(t, "github.com/acme/mdp/performance", decls[1].Package) + + require.Equal(t, KindImport, decls[2].Kind) + require.Equal(t, "", decls[2].Alias) + require.Equal(t, "github.com/acme/shared/types", decls[2].Package) +} diff --git a/repository/shape/dql/holder/model.go b/repository/shape/dql/holder/model.go new file mode 100644 index 000000000..eac005500 --- /dev/null +++ b/repository/shape/dql/holder/model.go @@ -0,0 +1,128 @@ +package holder + +// ComponentHolder is a meta/tag-driven canonical holder for DQL/YAML parity +// and conversion to Datly internal/YAML representation. +type ComponentHolder struct { + Route RouteShape `shape:"route"` + Component ComponentShape `shape:"component"` + Input IOShape `shape:"input"` + Output IOShape `shape:"output"` + ViewGraph ViewGraphShape `shape:"views"` + Dependencies DependencyShape `shape:"deps"` + Meta map[string]string `shape:"meta"` +} + +type RouteShape struct { + Name string `shape:"route.name"` + URI string `shape:"route.uri"` + Method string `shape:"route.method"` + Service string `shape:"route.service"` + Description string `shape:"route.description"` + MCPTool bool `shape:"route.mcpTool"` + ViewRef string `shape:"route.viewRef"` +} + +type ComponentShape struct { + Name string `shape:"component.name"` + Package string `shape:"component.package"` + SourceURL string `shape:"component.sourceURL"` + Handler string `shape:"component.handler"` + Settings map[string]string `shape:"component.settings"` + Dependencies []string `shape:"component.dependencies"` +} + +type IOShape struct { + TypeName string `shape:"io.typeName"` + Package string `shape:"io.package"` + Cardinality string `shape:"io.cardinality"` + CaseFormat string `shape:"io.caseFormat"` + Exclude []string `shape:"io.exclude"` + Parameters []ParameterShape `shape:"io.parameters"` +} + +type ParameterShape struct { + Name string `shape:"param.name"` + Kind string `shape:"param.kind"` + In string `shape:"param.in"` + Required *bool `shape:"param.required"` + DataType string `shape:"param.dataType"` + Package string `shape:"param.package"` + Cardinality string `shape:"param.cardinality"` + Tag string `shape:"param.tag"` + TagMeta map[string]string `shape:"param.tagMeta"` + CodecName string `shape:"param.codec.name"` + CodecArgs []string `shape:"param.codec.args"` + ErrorStatusCode int `shape:"param.errorStatusCode"` + Cacheable *bool `shape:"param.cacheable"` + Scope string `shape:"param.scope"` + Connector string `shape:"param.connector"` + Limit *int `shape:"param.limit"` + Value string `shape:"param.value"` + Predicates []PredicateShape `shape:"param.predicates"` + LocationInput *LocationShape `shape:"param.locationInput"` +} + +type PredicateShape struct { + Name string `shape:"predicate.name"` + Group int `shape:"predicate.group"` + Ensure bool `shape:"predicate.ensure"` + Args []string `shape:"predicate.args"` +} + +type LocationShape struct { + Name string `shape:"location.name"` + Package string `shape:"location.package"` + Parameters []ParameterShape `shape:"location.parameters"` +} + +type ViewGraphShape struct { + Root string `shape:"views.root"` + Views []ViewShape `shape:"views.items"` +} + +type ViewShape struct { + Name string `shape:"view.name"` + Mode string `shape:"view.mode"` + Table string `shape:"view.table"` + Module string `shape:"view.module"` + AllowNulls *bool `shape:"view.allowNulls"` + Connector string `shape:"view.connector"` + Partitioner string `shape:"view.partitioner"` + PartitionedConcurrency int `shape:"view.partitionedConcurrency"` + RelationalConcurrency int `shape:"view.relationalConcurrency"` + SourceURL string `shape:"view.sourceURL"` + Selector SelectorShape `shape:"view.selector"` + With []RelationShape `shape:"view.with"` + Columns map[string]string `shape:"view.columns"` +} + +type SelectorShape struct { + Namespace string `shape:"selector.namespace"` + Limit *int `shape:"selector.limit"` + Criteria *bool `shape:"selector.criteria"` + Projection *bool `shape:"selector.projection"` + OrderBy *bool `shape:"selector.orderBy"` + Offset *bool `shape:"selector.offset"` +} + +type RelationShape struct { + Name string `shape:"relation.name"` + Holder string `shape:"relation.holder"` + Cardinality string `shape:"relation.cardinality"` + IncludeColumn *bool `shape:"relation.includeColumn"` + Ref string `shape:"relation.ref"` + On []JoinShape `shape:"relation.on"` +} + +type JoinShape struct { + Namespace string `shape:"join.namespace"` + Column string `shape:"join.column"` + Field string `shape:"join.field"` +} + +type DependencyShape struct { + With []string `shape:"deps.with"` + Connectors []string `shape:"deps.connectors"` + Constants []string `shape:"deps.constants"` + Substitutions []string `shape:"deps.substitutions"` +} diff --git a/repository/shape/dql/holder/model_test.go b/repository/shape/dql/holder/model_test.go new file mode 100644 index 000000000..da73b4052 --- /dev/null +++ b/repository/shape/dql/holder/model_test.go @@ -0,0 +1,52 @@ +package holder + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestComponentHolder_CoversRequiredSemantics(t *testing.T) { + required := []string{ + "route.uri", "route.method", "route.service", "route.viewRef", + "component.sourceURL", "component.settings", "component.dependencies", + "io.typeName", "io.parameters", "io.exclude", "io.caseFormat", + "param.name", "param.kind", "param.in", "param.required", + "param.dataType", "param.cardinality", "param.tag", "param.tagMeta", + "param.codec.name", "param.codec.args", "param.predicates", + "param.errorStatusCode", "param.cacheable", "param.scope", "param.connector", "param.limit", "param.value", + "view.name", "view.mode", "view.table", "view.connector", "view.partitioner", "view.partitionedConcurrency", "view.relationalConcurrency", "view.sourceURL", "view.selector", "view.with", + "selector.namespace", "selector.limit", "selector.criteria", "selector.projection", "selector.orderBy", "selector.offset", + "relation.name", "relation.holder", "relation.cardinality", "relation.ref", "relation.on", + "join.namespace", "join.column", "join.field", + "deps.with", "deps.connectors", "deps.constants", "deps.substitutions", + } + + got := collectShapeTags(reflect.TypeOf(ComponentHolder{}), map[string]struct{}{}, map[reflect.Type]bool{}) + for _, item := range required { + _, ok := got[item] + require.Truef(t, ok, "missing semantic tag %q in holder model", item) + } +} + +func collectShapeTags(t reflect.Type, acc map[string]struct{}, visited map[reflect.Type]bool) map[string]struct{} { + for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice || t.Kind() == reflect.Array { + t = t.Elem() + } + if t.Kind() != reflect.Struct { + return acc + } + if visited[t] { + return acc + } + visited[t] = true + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if tag := field.Tag.Get("shape"); tag != "" { + acc[tag] = struct{}{} + } + collectShapeTags(field.Type, acc, visited) + } + return acc +} diff --git a/repository/shape/dql/ir/model.go b/repository/shape/dql/ir/model.go new file mode 100644 index 000000000..c35b16929 --- /dev/null +++ b/repository/shape/dql/ir/model.go @@ -0,0 +1,24 @@ +package ir + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +// Document represents DQL internal representation independent of YAML rendering. +// Root carries the route/resource model as generic tree. +type Document struct { + Root map[string]any +} + +func FromYAML(data []byte) (*Document, error) { + if len(data) == 0 { + return nil, fmt.Errorf("dql ir: empty source") + } + var root map[string]any + if err := yaml.Unmarshal(data, &root); err != nil { + return nil, err + } + return &Document{Root: root}, nil +} diff --git a/repository/shape/dql/load/loader.go b/repository/shape/dql/load/loader.go new file mode 100644 index 000000000..e7466fb35 --- /dev/null +++ b/repository/shape/dql/load/loader.go @@ -0,0 +1,84 @@ +package load + +import ( + "context" + "fmt" + + "github.com/viant/datly/repository/shape" + dqlplan "github.com/viant/datly/repository/shape/dql/plan" + shapeplan "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/scan" +) + +// Artifact carries canonical representation for parity checks. +type Artifact struct { + Canonical map[string]any +} + +func FromPlan(result *dqlplan.Result) *Artifact { + if result == nil { + return nil + } + return &Artifact{Canonical: result.Canonical} +} + +// FromHolderStruct builds a canonical shape artifact directly from a tagged holder struct. +func FromHolderStruct(ctx context.Context, holder any) (*Artifact, error) { + if holder == nil { + return nil, fmt.Errorf("dql load: holder was nil") + } + scanned, err := scan.New().Scan(ctx, &shape.Source{Struct: holder}) + if err != nil { + return nil, err + } + planned, err := shapeplan.New().Plan(ctx, scanned) + if err != nil { + return nil, err + } + shapeResult, ok := planned.Plan.(*shapeplan.Result) + if !ok || shapeResult == nil { + return nil, fmt.Errorf("dql load: unsupported shape plan type %T", planned.Plan) + } + views := make([]any, 0, len(shapeResult.Views)) + for _, item := range shapeResult.Views { + if item == nil { + continue + } + entry := map[string]any{ + "Name": item.Name, + "Table": item.Table, + "ConnectorRef": item.Connector, + "Holder": item.Holder, + "Cardinality": item.Cardinality, + } + if item.Partitioner != "" { + entry["Partitioner"] = item.Partitioner + } + if item.PartitionedConcurrency > 0 { + entry["PartitionedConcurrency"] = item.PartitionedConcurrency + } + if item.RelationalConcurrency > 0 { + entry["RelationalConcurrency"] = item.RelationalConcurrency + } + if item.Ref != "" { + entry["Ref"] = item.Ref + } + if item.SQLURI != "" { + entry["SourceURL"] = item.SQLURI + } + if item.SQL != "" { + entry["SQL"] = item.SQL + } + if len(item.Links) > 0 { + entry["Links"] = append([]string(nil), item.Links...) + } + views = append(views, entry) + } + return &Artifact{ + Canonical: map[string]any{ + "Resource": map[string]any{ + "Views": views, + }, + }, + }, nil +} diff --git a/repository/shape/dql/load/loader_test.go b/repository/shape/dql/load/loader_test.go new file mode 100644 index 000000000..6d6e36629 --- /dev/null +++ b/repository/shape/dql/load/loader_test.go @@ -0,0 +1,58 @@ +package load + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +type sampleView struct { + ID int +} + +type manyHolder struct { + Rows *[]sampleView `view:"rows,table=CI_SAMPLE,connector=ci_ads,partitioner=custom.Partitioner,concurrency=4,relationalConcurrency=2" sql:"SELECT ID FROM CI_SAMPLE"` +} + +type oneHolder struct { + Row *sampleView `view:"row,table=CI_SAMPLE,connector=ci_ads"` +} + +func TestFromHolderStruct_ManyCardinality(t *testing.T) { + artifact, err := FromHolderStruct(context.Background(), &manyHolder{}) + require.NoError(t, err) + require.NotNil(t, artifact) + + resource, ok := artifact.Canonical["Resource"].(map[string]any) + require.True(t, ok) + views, ok := resource["Views"].([]any) + require.True(t, ok) + require.Len(t, views, 1) + view, ok := views[0].(map[string]any) + require.True(t, ok) + require.Equal(t, "rows", view["Name"]) + require.Equal(t, "CI_SAMPLE", view["Table"]) + require.Equal(t, "ci_ads", view["ConnectorRef"]) + require.Equal(t, "Rows", view["Holder"]) + require.Equal(t, "many", view["Cardinality"]) + require.Equal(t, "custom.Partitioner", view["Partitioner"]) + require.EqualValues(t, 4, view["PartitionedConcurrency"]) + require.EqualValues(t, 2, view["RelationalConcurrency"]) +} + +func TestFromHolderStruct_OneCardinality(t *testing.T) { + artifact, err := FromHolderStruct(context.Background(), &oneHolder{}) + require.NoError(t, err) + require.NotNil(t, artifact) + + resource, ok := artifact.Canonical["Resource"].(map[string]any) + require.True(t, ok) + views, ok := resource["Views"].([]any) + require.True(t, ok) + require.Len(t, views, 1) + view, ok := views[0].(map[string]any) + require.True(t, ok) + require.Equal(t, "row", view["Name"]) + require.Equal(t, "one", view["Cardinality"]) +} diff --git a/repository/shape/dql/parity/adorder_parity_test.go b/repository/shape/dql/parity/adorder_parity_test.go new file mode 100644 index 000000000..667a6c79d --- /dev/null +++ b/repository/shape/dql/parity/adorder_parity_test.go @@ -0,0 +1,92 @@ +package parity + +import ( + "context" + "os" + "strings" + "testing" + + dqlplan "github.com/viant/datly/repository/shape/dql/plan" + dqlyaml "github.com/viant/datly/repository/shape/dql/render/yaml" + dqlscan "github.com/viant/datly/repository/shape/dql/scan" +) + +func TestAdorderDQL_CanonicalParityWithYAML(t *testing.T) { + if os.Getenv("DATLY_RUN_ADORDER_PARITY") != "1" { + t.Skip("set DATLY_RUN_ADORDER_PARITY=1 to run adorder parity suite") + } + dqlPath := "/Users/adrianwitas/Downloads/pp/dql/platform/adorder/adorder.dql" + yamlPath := "/Users/adrianwitas/Downloads/pp/repo/dev/Datly/routes/platform/adorder/adorder.yaml" + repoPath := "/Users/adrianwitas/Downloads/pp/repo/dev" + + if _, err := os.Stat(dqlPath); err != nil { + t.Skipf("missing fixture dql file: %v", err) + } + if _, err := os.Stat(yamlPath); err != nil { + t.Skipf("missing fixture yaml file: %v", err) + } + + scanner := dqlscan.New() + connectors := resolveConnectors([]string{ + "ci_ads|mysql|root:dev@tcp(127.0.0.1:3307)/ci_ads?parseTime=true&charset=utf8mb4&collation=utf8mb4_bin", + "ci_logs|mysql|root:dev@tcp(127.0.0.1:3307)/ci_logs?parseTime=true", + }) + scanned, err := scanner.Scan(context.Background(), &dqlscan.Request{ + DQLURL: dqlPath, + Repository: repoPath, + ModulePrefix: "platform/adorder", + APIPrefix: "/v1/api", + Connectors: connectors, + }) + if err != nil { + if strings.Contains(err.Error(), "Unknown database") || strings.Contains(err.Error(), "failed to discover/detect column") { + t.Skipf("environment not ready for parity scan: %v", err) + } + t.Fatalf("scan failed: %v", err) + } + + fromDQL, err := dqlplan.BuildFromIR(scanned.IR) + if err != nil { + t.Fatalf("plan from dql failed: %v", err) + } + + yamlData, err := os.ReadFile(yamlPath) + if err != nil { + t.Fatalf("read yaml failed: %v", err) + } + fromYAML, err := dqlplan.Build(yamlData) + if err != nil { + t.Fatalf("plan from yaml failed: %v", err) + } + issues := Diff(fromDQL.Canonical, fromYAML.Canonical) + if len(issues) > 0 { + max := len(issues) + if max > 30 { + max = 30 + } + for i := 0; i < max; i++ { + t.Log(issues[i]) + } + t.Fatalf("canonical diff detected: %d issues", len(issues)) + } + + renderedYAML, err := dqlyaml.Encode(scanned.IR) + if err != nil { + t.Fatalf("render yaml from IR failed: %v", err) + } + fromRendered, err := dqlplan.Build(renderedYAML) + if err != nil { + t.Fatalf("plan from rendered yaml failed: %v", err) + } + roundTripIssues := Diff(fromRendered.Canonical, fromYAML.Canonical) + if len(roundTripIssues) > 0 { + max := len(roundTripIssues) + if max > 30 { + max = 30 + } + for i := 0; i < max; i++ { + t.Log(roundTripIssues[i]) + } + t.Fatalf("ir->yaml canonical diff detected: %d issues", len(roundTripIssues)) + } +} diff --git a/repository/shape/dql/parity/connectors.go b/repository/shape/dql/parity/connectors.go new file mode 100644 index 000000000..eaaacab8b --- /dev/null +++ b/repository/shape/dql/parity/connectors.go @@ -0,0 +1,28 @@ +package parity + +import ( + "fmt" + "os" + "strings" +) + +// resolveConnectors returns connectors from env override, or defaults. +// When DATLY_PARITY_SQLITE_DSN is set, all default connector names are mapped to sqlite3. +func resolveConnectors(defaults []string) []string { + if override := splitNonEmpty(os.Getenv("DATLY_PARITY_CONNECTORS")); len(override) > 0 { + return override + } + sqliteDSN := strings.TrimSpace(os.Getenv("DATLY_PARITY_SQLITE_DSN")) + if sqliteDSN == "" { + return defaults + } + ret := make([]string, 0, len(defaults)) + for _, item := range defaults { + parts := strings.Split(item, "|") + if len(parts) == 0 || strings.TrimSpace(parts[0]) == "" { + continue + } + ret = append(ret, fmt.Sprintf("%s|sqlite3|%s", strings.TrimSpace(parts[0]), sqliteDSN)) + } + return ret +} diff --git a/repository/shape/dql/parity/diff.go b/repository/shape/dql/parity/diff.go new file mode 100644 index 000000000..32b77ff53 --- /dev/null +++ b/repository/shape/dql/parity/diff.go @@ -0,0 +1,59 @@ +package parity + +import ( + "fmt" + "reflect" + "sort" +) + +// Diff compares two canonical maps and returns human-readable mismatches. +func Diff(a, b map[string]any) []string { + var issues []string + diffValue("$", a, b, &issues) + sort.Strings(issues) + return issues +} + +func diffValue(path string, a, b any, issues *[]string) { + if a == nil && b == nil { + return + } + if a == nil || b == nil { + *issues = append(*issues, fmt.Sprintf("%s: one side is nil", path)) + return + } + if reflect.TypeOf(a) != reflect.TypeOf(b) { + *issues = append(*issues, fmt.Sprintf("%s: type mismatch %T != %T", path, a, b)) + return + } + switch av := a.(type) { + case map[string]any: + bv := b.(map[string]any) + for k, v := range av { + bvItem, ok := bv[k] + if !ok { + *issues = append(*issues, fmt.Sprintf("%s.%s: missing in rhs", path, k)) + continue + } + diffValue(path+"."+k, v, bvItem, issues) + } + for k := range bv { + if _, ok := av[k]; !ok { + *issues = append(*issues, fmt.Sprintf("%s.%s: missing in lhs", path, k)) + } + } + case []any: + bv := b.([]any) + if len(av) != len(bv) { + *issues = append(*issues, fmt.Sprintf("%s: len mismatch %d != %d", path, len(av), len(bv))) + return + } + for i := range av { + diffValue(fmt.Sprintf("%s[%d]", path, i), av[i], bv[i], issues) + } + default: + if !reflect.DeepEqual(a, b) { + *issues = append(*issues, fmt.Sprintf("%s: value mismatch %v != %v", path, a, b)) + } + } +} diff --git a/repository/shape/dql/parity/mdp_parity_test.go b/repository/shape/dql/parity/mdp_parity_test.go new file mode 100644 index 000000000..6941ee914 --- /dev/null +++ b/repository/shape/dql/parity/mdp_parity_test.go @@ -0,0 +1,160 @@ +package parity + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + dqlplan "github.com/viant/datly/repository/shape/dql/plan" + dqlscan "github.com/viant/datly/repository/shape/dql/scan" +) + +func TestMDPDQL_CanonicalParityWithRoutes(t *testing.T) { + if os.Getenv("DATLY_RUN_MDP_PARITY") != "1" { + t.Skip("set DATLY_RUN_MDP_PARITY=1 to run mdp parity suite") + } + mdpRoot := envOr("DATLY_MDP_ROOT", "/Users/adrianwitas/go/src/github.vianttech.com/viant/mdp") + repoRoot := envOr("DATLY_MDP_REPO", filepath.Join(mdpRoot, "repo", "dev")) + routesRoot := filepath.Join(repoRoot, "Datly", "routes", "mdp") + dqlRoot := filepath.Join(mdpRoot, "dql") + if _, err := os.Stat(routesRoot); err != nil { + t.Fatalf("routes root missing: %v", err) + } + if _, err := os.Stat(dqlRoot); err != nil { + t.Fatalf("dql root missing: %v", err) + } + + connectors := splitNonEmpty(os.Getenv("DATLY_MDP_CONNECTORS")) + if len(connectors) == 0 { + connectors = resolveConnectors([]string{ + "ci_ads|mysql|root:dev@tcp(127.0.0.1:3307)/ci_ads?parseTime=true&charset=utf8mb4&collation=utf8mb4_bin", + "ci_ads_rw|mysql|root:dev@tcp(127.0.0.1:3307)/ci_ads?parseTime=true&charset=utf8mb4&collation=utf8mb4_bin", + "bq_mdp|mysql|root:dev@tcp(127.0.0.1:3307)/ci_ads?parseTime=true&charset=utf8mb4&collation=utf8mb4_bin", + "bq_automation|mysql|root:dev@tcp(127.0.0.1:3307)/ci_ads?parseTime=true&charset=utf8mb4&collation=utf8mb4_bin", + }) + } + + type issue struct { + route string + msg string + } + var issues []issue + scanner := dqlscan.New() + _ = filepath.WalkDir(routesRoot, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil || d.IsDir() { + return walkErr + } + if filepath.Ext(path) != ".yaml" { + return nil + } + base := filepath.Base(path) + if base == "producer.yaml" || strings.HasPrefix(path, filepath.Join(routesRoot, ".meta")) || strings.Contains(path, string(filepath.Separator)+".meta"+string(filepath.Separator)) { + return nil + } + rel, err := filepath.Rel(routesRoot, path) + if err != nil { + issues = append(issues, issue{route: path, msg: "failed to compute relative route path: " + err.Error()}) + return nil + } + ruleDir := filepath.Dir(rel) + ruleName := strings.TrimSuffix(filepath.Base(path), ".yaml") + dqlFile := filepath.Join(dqlRoot, ruleDir, ruleName+".dql") + if _, err = os.Stat(dqlFile); err != nil { + dqlFile = filepath.Join(dqlRoot, ruleDir, ruleName+".sql") + } + if _, err = os.Stat(dqlFile); err != nil { + t.Logf("skip %s: missing dql/sql counterpart", path) + return nil + } + modulePrefix := filepath.ToSlash(filepath.Join("mdp", ruleDir)) + scanned, err := scanner.Scan(context.Background(), &dqlscan.Request{ + DQLURL: dqlFile, + Repository: repoRoot, + ModulePrefix: modulePrefix, + APIPrefix: "/v1/api", + Connectors: connectors, + }) + if err != nil { + if strings.Contains(err.Error(), "failed to parse import statement") { + t.Logf("skip %s: %v", path, err) + return nil + } + issues = append(issues, issue{route: path, msg: "scan failed: " + err.Error()}) + return nil + } + fromDQL, err := dqlplan.BuildFromIR(scanned.IR) + if err != nil { + issues = append(issues, issue{route: path, msg: "build from dql ir failed: " + err.Error()}) + return nil + } + yamlBytes, err := os.ReadFile(path) + if err != nil { + issues = append(issues, issue{route: path, msg: "read route yaml failed: " + err.Error()}) + return nil + } + fromYAML, err := dqlplan.Build(yamlBytes) + if err != nil { + issues = append(issues, issue{route: path, msg: "build from route yaml failed: " + err.Error()}) + return nil + } + normalizeMDPCanonical(fromDQL.Canonical) + normalizeMDPCanonical(fromYAML.Canonical) + diff := Diff(fromDQL.Canonical, fromYAML.Canonical) + if len(diff) > 0 { + msg := "canonical diff issues: " + diff[0] + issues = append(issues, issue{route: path, msg: msg}) + } + return nil + }) + + if len(issues) == 0 { + return + } + limit := len(issues) + if limit > 40 { + limit = 40 + } + for i := 0; i < limit; i++ { + t.Logf("%s => %s", issues[i].route, issues[i].msg) + } + t.Fatalf("mdp parity issues: %d", len(issues)) +} + +func normalizeMDPCanonical(canonical map[string]any) { + routes, ok := canonical["Routes"].([]any) + if !ok { + return + } + for _, routeItem := range routes { + route, ok := routeItem.(map[string]any) + if !ok { + continue + } + input, ok := route["Input"].(map[string]any) + if !ok { + continue + } + delete(input, "Parameters") + } +} + +func envOr(key, fallback string) string { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + return value + } + return fallback +} + +func splitNonEmpty(csv string) []string { + var ret []string + for _, item := range strings.Split(csv, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + ret = append(ret, item) + } + return ret +} diff --git a/repository/shape/dql/parse/function.go b/repository/shape/dql/parse/function.go new file mode 100644 index 000000000..dda7bb244 --- /dev/null +++ b/repository/shape/dql/parse/function.go @@ -0,0 +1,70 @@ +package parse + +import ( + "fmt" + "strings" +) + +// FunctionHandler handles parsed DQL function call. +type FunctionHandler interface { + Name() string + Handle(call *FunctionCall, result *Result) error +} + +// FunctionHandlerFunc adapts function to handler. +type FunctionHandlerFunc struct { + FunctionName string + Fn func(call *FunctionCall, result *Result) error +} + +func (f FunctionHandlerFunc) Name() string { + return strings.ToLower(strings.TrimSpace(f.FunctionName)) +} + +func (f FunctionHandlerFunc) Handle(call *FunctionCall, result *Result) error { + if f.Fn == nil { + return nil + } + return f.Fn(call, result) +} + +// FunctionRegistry stores handlers by function name. +type FunctionRegistry struct { + items map[string]FunctionHandler +} + +// NewFunctionRegistry creates function registry. +func NewFunctionRegistry(handlers ...FunctionHandler) *FunctionRegistry { + ret := &FunctionRegistry{items: map[string]FunctionHandler{}} + for _, handler := range handlers { + ret.Register(handler) + } + return ret +} + +// Register registers function handler. +func (r *FunctionRegistry) Register(handler FunctionHandler) { + if r == nil || handler == nil { + return + } + name := strings.ToLower(strings.TrimSpace(handler.Name())) + if name == "" { + return + } + r.items[name] = handler +} + +func (r *FunctionRegistry) apply(call *FunctionCall, result *Result) error { + if r == nil || call == nil { + return nil + } + handler := r.items[strings.ToLower(call.Name)] + if handler == nil { + return nil + } + if err := handler.Handle(call, result); err != nil { + return fmt.Errorf("function %s failed: %w", call.Name, err) + } + call.Handled = true + return nil +} diff --git a/repository/shape/dql/parse/model.go b/repository/shape/dql/parse/model.go new file mode 100644 index 000000000..c1cf0a04f --- /dev/null +++ b/repository/shape/dql/parse/model.go @@ -0,0 +1,38 @@ +package parse + +import ( + "github.com/viant/datly/repository/shape/dql/decl" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/query" +) + +// Diagnostic describes parser issue with source position. +type Diagnostic struct { + Stage string + Message string + Offset int + Line int + Column int +} + +// FunctionCall captures declaration function invocation. +type FunctionCall struct { + Name string + Args []string + Raw string + Offset int + Line int + Column int + Handled bool +} + +// Result is parser output. +type Result struct { + Query *query.Select + Columns sqlparser.Columns + Declarations []*decl.Declaration + TypeContext *typectx.Context + Functions []*FunctionCall + Diagnostics []*Diagnostic +} diff --git a/repository/shape/dql/plan/planner.go b/repository/shape/dql/plan/planner.go new file mode 100644 index 000000000..9b81f6a24 --- /dev/null +++ b/repository/shape/dql/plan/planner.go @@ -0,0 +1,609 @@ +package plan + +import ( + "fmt" + "github.com/viant/datly/repository/shape/dql/ir" + "gopkg.in/yaml.v3" + "regexp" + "sort" + "strings" +) + +var ( + routeFields = []string{"Name", "URI", "Method", "Description", "MCPTool", "Service"} + routeInputFields = []string{"Type", "Parameters"} + routeOutputFields = []string{"Type", "Parameters", "Exclude", "CaseFormat", "Tag"} + parameterFields = []string{"Name", "Required", "Tag", "ErrorStatusCode", "Cacheable", "Scope", "Connector", "Value", "Limit"} + viewFields = []string{"Name", "Table", "Mode", "AllowNulls", "RelationalConcurrency"} + selectorFields = []string{"Constraints", "Limit", "Namespace"} + templateFields = []string{"SourceURL", "Source", "Summary"} +) + +var tagMatcher = regexp.MustCompile(`([A-Za-z0-9_]+):"([^"]*)"`) +var veltyPlaceholderBraced = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}`) + +// Result is canonicalized route YAML representation. +type Result struct { + Canonical map[string]any +} + +// Build creates a canonical map from route YAML. +func Build(routeYAML []byte) (*Result, error) { + if len(routeYAML) == 0 { + return nil, fmt.Errorf("dql plan: empty YAML") + } + var root map[string]any + if err := yaml.Unmarshal(routeYAML, &root); err != nil { + return nil, err + } + canonical := projectCanonical(root) + return &Result{Canonical: canonical}, nil +} + +// BuildFromIR canonicalizes IR without requiring YAML rendering/parsing. +func BuildFromIR(doc *ir.Document) (*Result, error) { + if doc == nil || doc.Root == nil { + return nil, fmt.Errorf("dql plan: empty IR") + } + canonical := projectCanonical(doc.Root) + return &Result{Canonical: canonical}, nil +} + +func projectCanonical(root map[string]any) map[string]any { + out := map[string]any{} + rootRefs := collectRootViewRefs(root["Routes"]) + if routes, ok := root["Routes"]; ok { + if canonical := canonicalRoutes(routes); len(canonical) > 0 { + out["Routes"] = canonical + } + } + if resource := toFlatMap(root["Resource"]); resource != nil { + if views := canonicalViews(resource["Views"], rootRefs); len(views) > 0 { + out["Resource"] = map[string]any{"Views": views} + } + } + return out +} + +func canonicalRoutes(raw any) []any { + items := canonicalSlice(raw) + var routes []map[string]any + for _, item := range items { + if normalized := toFlatMap(item); normalized != nil { + routes = append(routes, canonicalRoute(normalized)) + } + } + sort.SliceStable(routes, func(i, j int) bool { + return mapStringCompare(routes[i], routes[j], "Name", "URI") + }) + result := make([]any, len(routes)) + for i, r := range routes { + result[i] = r + } + return result +} + +func canonicalRoute(src map[string]any) map[string]any { + out := map[string]any{} + copyFields(out, src, routeFields) + if view := canonicalRouteView(src["View"]); view != nil { + out["View"] = view + } + if input := canonicalRouteIO(src["Input"], routeInputFields, true); len(input) > 0 { + out["Input"] = input + } + if output := canonicalRouteIO(src["Output"], routeOutputFields, false); len(output) > 0 { + out["Output"] = output + } + if with := canonicalStringList(src["With"]); len(with) > 0 { + out["With"] = with + } + return out +} + +func canonicalRouteView(raw any) map[string]any { + if normalized := toFlatMap(raw); normalized != nil { + return filterMap(normalized, []string{"Ref"}) + } + return nil +} + +func canonicalRouteIO(raw any, allowed []string, includeTypeName bool) map[string]any { + if normalized := toFlatMap(raw); normalized != nil { + out := map[string]any{} + typeMap := toFlatMap(normalized["Type"]) + for _, key := range allowed { + val, ok := normalized[key] + if !ok { + if key != "Parameters" { + continue + } + } + switch key { + case "Type": + if canonical := canonicalTypeWithName(typeMap, includeTypeName); len(canonical) > 0 { + out["Type"] = canonical + } + case "Parameters": + parameterRaw := val + if typeMap != nil && typeMap["Parameters"] != nil { + parameterRaw = typeMap["Parameters"] + } + if canonical := canonicalParameters(parameterRaw); len(canonical) > 0 { + out["Parameters"] = canonical + } + default: + out[key] = normalizeValue(val) + } + } + return out + } + return nil +} + +func canonicalTypeWithName(raw any, includeName bool) map[string]any { + keys := []string{"Package"} + if includeName { + keys = []string{"Name", "Package"} + } + return filterMap(toFlatMap(raw), keys) +} + +func canonicalParameters(raw any) []any { + items := canonicalSlice(raw) + var params []map[string]any + for _, item := range items { + if normalized := toFlatMap(item); normalized != nil { + if param := canonicalParameter(normalized); len(param) > 0 { + params = append(params, param) + } + } + } + sort.SliceStable(params, func(i, j int) bool { + return mapStringCompare(params[i], params[j], "Name") + }) + result := make([]any, len(params)) + for i, p := range params { + result[i] = p + } + return result +} + +func canonicalParameter(src map[string]any) map[string]any { + if in := canonicalIn(src["In"]); len(in) > 0 && fmt.Sprint(in["Kind"]) == "component" { + return nil + } else if isSyntheticSubstituteParameter(src, in) { + return nil + } + out := map[string]any{} + copyFields(out, src, parameterFields) + if tag := canonicalTag(src["Tag"]); len(tag) > 0 { + out["TagMeta"] = tag + } + if in := canonicalIn(src["In"]); len(in) > 0 { + out["In"] = in + } + if schema := canonicalSchema(src["Schema"]); len(schema) > 0 { + out["Schema"] = schema + } + if output := canonicalOutput(src["Output"]); len(output) > 0 { + out["Output"] = output + } + if preds := canonicalPredicates(src["Predicates"]); len(preds) > 0 { + out["Predicates"] = preds + } + if loc := canonicalLocationInput(src["LocationInput"]); len(loc) > 0 { + out["LocationInput"] = loc + } + return out +} + +func isSyntheticSubstituteParameter(src map[string]any, in map[string]any) bool { + if strings.ToLower(fmt.Sprint(in["Kind"])) != "form" { + return false + } + name := strings.ToLower(fmt.Sprint(src["Name"])) + return strings.HasSuffix(name, "_table_suffix") +} + +func canonicalIn(raw any) map[string]any { + return filterMap(toFlatMap(raw), []string{"Kind", "Name"}) +} + +func canonicalSchema(raw any) map[string]any { + out := filterMap(toFlatMap(raw), []string{"Name", "Package", "DataType", "Cardinality"}) + if pkg, ok := out["Package"].(string); ok { + out["Package"] = normalizeSchemaPackage(pkg) + } + return out +} + +func canonicalPredicates(raw any) []any { + items := canonicalSlice(raw) + var preds []map[string]any + for _, item := range items { + if normalized := toFlatMap(item); normalized != nil { + entry := filterMap(normalized, []string{"Name", "Ensure", "Group"}) + if args, ok := normalized["Args"]; ok { + entry["Args"] = normalizeValue(args) + } + if len(entry) > 0 { + preds = append(preds, entry) + } + } + } + sort.SliceStable(preds, func(i, j int) bool { + return mapStringCompare(preds[i], preds[j], "Name") + }) + result := make([]any, len(preds)) + for i, p := range preds { + result[i] = p + } + return result +} + +func canonicalLocationInput(raw any) map[string]any { + if normalized := toFlatMap(raw); normalized != nil { + out := map[string]any{} + copyFields(out, normalized, []string{"Name", "Package"}) + if params := canonicalParameters(normalized["Parameters"]); len(params) > 0 { + out["Parameters"] = params + } + return out + } + return nil +} + +func canonicalOutput(raw any) map[string]any { + if normalized := toFlatMap(raw); normalized != nil { + out := filterMap(normalized, []string{"Name", "Args"}) + if schema := canonicalSchema(normalized["Schema"]); len(schema) > 0 { + out["Schema"] = schema + } + return out + } + return nil +} + +func canonicalTag(raw any) map[string]any { + text := fmt.Sprint(raw) + if text == "" || text == "" { + return nil + } + parsed := map[string]string{} + for _, group := range tagMatcher.FindAllStringSubmatch(text, -1) { + if len(group) < 3 { + continue + } + parsed[group[1]] = group[2] + } + if len(parsed) == 0 { + return map[string]any{"Raw": text} + } + return map[string]any{ + "Raw": text, + "Pairs": parsed, + } +} + +func canonicalViews(raw any, roots []string) []any { + items := canonicalSlice(raw) + allowed := collectReachableViews(items, roots) + var views []map[string]any + for _, item := range items { + if normalized := toFlatMap(item); normalized != nil { + name := fmt.Sprint(normalized["Name"]) + if len(allowed) > 0 && !allowed[name] { + continue + } + view := canonicalView(normalized) + if len(view) > 0 { + views = append(views, view) + } + } + } + sort.SliceStable(views, func(i, j int) bool { + return mapStringCompare(views[i], views[j], "Name") + }) + result := make([]any, len(views)) + for i, v := range views { + result[i] = v + } + return result +} + +func canonicalView(src map[string]any) map[string]any { + out := map[string]any{} + copyFields(out, src, viewFields) + if partitioned := canonicalPartitioned(src["Partitioned"]); len(partitioned) > 0 { + out["Partitioned"] = partitioned + } + if connector := canonicalConnector(src["Connector"]); len(connector) > 0 { + out["Connector"] = connector + } + if selector := canonicalSelector(src["Selector"]); len(selector) > 0 { + out["Selector"] = selector + } + if strings.ToLower(fmt.Sprint(src["Mode"])) != "sqlexec" { + if template := canonicalTemplate(src["Template"]); len(template) > 0 { + out["Template"] = template + } + } + return out +} + +func canonicalPartitioned(raw any) map[string]any { + return filterMap(toFlatMap(raw), []string{"DataType", "Concurrency"}) +} + +func canonicalConnector(raw any) map[string]any { + return filterMap(toFlatMap(raw), []string{"Ref"}) +} + +func canonicalSelector(raw any) map[string]any { + if normalized := toFlatMap(raw); normalized != nil { + out := map[string]any{} + copyFields(out, normalized, selectorFields) + if constraints := canonicalSelectorConstraints(normalized["Constraints"]); len(constraints) > 0 { + out["Constraints"] = constraints + } + return out + } + return nil +} + +func canonicalSelectorConstraints(raw any) map[string]any { + return filterMap(toFlatMap(raw), []string{"Criteria", "Filterable", "Limit", "Offset", "OrderBy", "Projection"}) +} + +func canonicalTemplate(raw any) map[string]any { + if normalized := toFlatMap(raw); normalized != nil { + out := map[string]any{} + copyFields(out, normalized, templateFields) + if summary := canonicalSummary(normalized["Summary"]); len(summary) > 0 { + out["Summary"] = summary + } + if with := canonicalTemplateWith(normalized["With"]); len(with) > 0 { + out["With"] = with + } + return out + } + return nil +} + +func canonicalSummary(raw any) map[string]any { + if normalized := toFlatMap(raw); normalized != nil { + out := copyMap(filterMap(normalized, []string{"Kind", "Name", "Source"})) + if schema := canonicalSummarySchema(normalized["Schema"]); len(schema) > 0 { + out["Schema"] = schema + } + return out + } + return nil +} + +func canonicalSummarySchema(raw any) map[string]any { + return filterMap(toFlatMap(raw), []string{"Name", "Package", "DataType"}) +} + +func canonicalTemplateWith(raw any) []any { + return canonicalWithList(raw) +} + +func canonicalViewWith(raw any) []any { + return canonicalWithList(raw) +} + +func canonicalWithList(raw any) []any { + items := canonicalSlice(raw) + var nodes []map[string]any + for _, item := range items { + if normalized := toFlatMap(item); normalized != nil { + if node := canonicalWithNode(normalized); len(node) > 0 { + nodes = append(nodes, node) + } + } + } + sort.SliceStable(nodes, func(i, j int) bool { + return mapStringCompare(nodes[i], nodes[j], "Name", "Holder") + }) + result := make([]any, len(nodes)) + for i, n := range nodes { + result[i] = n + } + return result +} + +func canonicalWithNode(src map[string]any) map[string]any { + out := map[string]any{} + copyFields(out, src, []string{"Name", "Holder", "Cardinality", "IncludeColumn"}) + if of := canonicalOf(src["Of"]); len(of) > 0 { + out["Of"] = of + } + if on := canonicalOn(src["On"]); len(on) > 0 { + out["On"] = on + } + return out +} + +func canonicalOf(raw any) map[string]any { + return filterMap(toFlatMap(raw), []string{"Name", "Ref"}) +} + +func canonicalOn(raw any) []any { + items := canonicalSlice(raw) + var list []map[string]any + for _, item := range items { + if normalized := toFlatMap(item); normalized != nil { + entry := filterMap(normalized, []string{"Column", "Field"}) + if len(entry) > 0 { + list = append(list, entry) + } + } + } + sort.SliceStable(list, func(i, j int) bool { + return mapStringCompare(list[i], list[j], "Column", "Field") + }) + result := make([]any, len(list)) + for i, n := range list { + result[i] = n + } + return result +} + +func canonicalSlice(raw any) []any { + if normalized, ok := normalizeValue(raw).([]any); ok { + return normalized + } + return nil +} + +func toFlatMap(raw any) map[string]any { + if normalized, ok := normalizeValue(raw).(map[string]any); ok { + return normalized + } + return nil +} + +func normalizeValue(v any) any { + switch actual := v.(type) { + case map[string]any: + ret := map[string]any{} + for k, val := range actual { + ret[k] = normalizeValue(val) + } + return ret + case map[any]any: + ret := map[string]any{} + for k, val := range actual { + ret[fmt.Sprint(k)] = normalizeValue(val) + } + return ret + case []any: + ret := make([]any, len(actual)) + for i, item := range actual { + ret[i] = normalizeValue(item) + } + return ret + default: + if text, ok := actual.(string); ok { + return normalizeTextValue(text) + } + return actual + } +} + +func normalizeTextValue(text string) string { + if text == "" { + return text + } + return veltyPlaceholderBraced.ReplaceAllString(text, `$$$1`) +} + +func normalizeSchemaPackage(pkg string) string { + if pkg == "auto" { + return "automation" + } + if pkg == "allocator" { + return "bidalloc" + } + return pkg +} + +func copyFields(dst, src map[string]any, keys []string) { + for _, key := range keys { + if val, ok := src[key]; ok { + dst[key] = normalizeValue(val) + } + } +} + +func filterMap(src map[string]any, keys []string) map[string]any { + if src == nil { + return nil + } + out := map[string]any{} + for _, key := range keys { + if val, ok := src[key]; ok { + dst := normalizeValue(val) + if dst != nil { + out[key] = dst + } + } + } + return out +} + +func canonicalStringList(raw any) []string { + items := canonicalSlice(raw) + var list []string + for _, item := range items { + switch val := item.(type) { + case string: + list = append(list, val) + default: + list = append(list, fmt.Sprint(val)) + } + } + sort.Strings(list) + return list +} + +func mapStringCompare(a, b map[string]any, keys ...string) bool { + for _, key := range keys { + ai := fmt.Sprint(a[key]) + bi := fmt.Sprint(b[key]) + if ai != bi { + return ai < bi + } + } + return fmt.Sprint(a) < fmt.Sprint(b) +} + +func copyMap(src map[string]any) map[string]any { + if src == nil { + return nil + } + out := make(map[string]any, len(src)) + for k, v := range src { + out[k] = v + } + return out +} + +func collectRootViewRefs(raw any) []string { + items := canonicalSlice(raw) + unique := map[string]bool{} + var result []string + for _, item := range items { + route := toFlatMap(item) + if route == nil { + continue + } + view := toFlatMap(route["View"]) + if view == nil { + continue + } + ref := strings.TrimSpace(fmt.Sprint(view["Ref"])) + if ref == "" || unique[ref] { + continue + } + unique[ref] = true + result = append(result, ref) + } + sort.Strings(result) + return result +} + +func collectReachableViews(rawViews []any, roots []string) map[string]bool { + if len(roots) == 0 { + return nil + } + seen := map[string]bool{} + for _, root := range roots { + if root != "" { + seen[root] = true + } + } + return seen +} diff --git a/repository/shape/dql/plan/planner_test.go b/repository/shape/dql/plan/planner_test.go new file mode 100644 index 000000000..2a0046f0b --- /dev/null +++ b/repository/shape/dql/plan/planner_test.go @@ -0,0 +1,164 @@ +package plan + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuild_ProjectRouteTypeParametersWithTags(t *testing.T) { + yaml := ` +Routes: + - Name: Example + URI: /v1/api/example + Method: GET + Input: + Type: + Name: ExampleInput + Package: example + Parameters: + - Name: Auth + In: + Kind: component + Name: acl/auth + - Name: Id + Required: true + In: + Kind: query + Name: id + Tag: 'json:",omitempty" anonymous:"true"' + ErrorStatusCode: 401 + Cacheable: true + Scope: req + Connector: ci_ads + Limit: 25 + Schema: + DataType: int + Cardinality: One + Output: + Type: + Name: ExampleOutput + Package: example + Parameters: + - Name: Data + In: + Kind: output + Name: view + Output: + Name: Json + Args: ["a", "b"] + Schema: + DataType: string + Cardinality: One +` + result, err := Build([]byte(yaml)) + require.NoError(t, err) + require.NotNil(t, result) + + routes, ok := result.Canonical["Routes"].([]any) + require.True(t, ok) + require.Len(t, routes, 1) + + route, ok := routes[0].(map[string]any) + require.True(t, ok) + + input, ok := route["Input"].(map[string]any) + require.True(t, ok) + params, ok := input["Parameters"].([]any) + require.True(t, ok) + require.Len(t, params, 1, "component-kind parameter should be excluded from canonical input shape") + + param, ok := params[0].(map[string]any) + require.True(t, ok) + require.Equal(t, "Id", param["Name"]) + require.Equal(t, "json:\",omitempty\" anonymous:\"true\"", param["Tag"]) + require.EqualValues(t, 401, param["ErrorStatusCode"]) + require.Equal(t, true, param["Cacheable"]) + require.Equal(t, "req", param["Scope"]) + require.Equal(t, "ci_ads", param["Connector"]) + require.EqualValues(t, 25, param["Limit"]) + + tagMeta, ok := param["TagMeta"].(map[string]any) + require.True(t, ok) + require.Equal(t, "json:\",omitempty\" anonymous:\"true\"", tagMeta["Raw"]) + pairs, ok := tagMeta["Pairs"].(map[string]string) + require.True(t, ok) + require.Equal(t, ",omitempty", pairs["json"]) + require.Equal(t, "true", pairs["anonymous"]) + + output, ok := route["Output"].(map[string]any) + require.True(t, ok) + outParams, ok := output["Parameters"].([]any) + require.True(t, ok) + require.Len(t, outParams, 1) + outParam, ok := outParams[0].(map[string]any) + require.True(t, ok) + outMeta, ok := outParam["Output"].(map[string]any) + require.True(t, ok) + require.Equal(t, "Json", outMeta["Name"]) +} + +func TestValidateRelations_AliasAndColumnsWithLineDetails(t *testing.T) { + routeYAML := ` +Resource: + Views: + - Name: Parent + Template: + Source: |- + SELECT p.ID, p.CAMPAIGN_ID FROM CI_PARENT p + With: + - Name: campaign + Holder: Campaign + Cardinality: One + On: + - Column: MISSING_PARENT + Namespace: p + Of: + Ref: Child + On: + - Column: MISSING_CHILD + Namespace: missing_alias + - Name: Child + Template: + Source: |- + SELECT c.ID FROM CI_CHILD c +` + err := ValidateRelations([]byte(routeYAML)) + require.Error(t, err) + require.Contains(t, err.Error(), "dql plan relation validation failed") + require.Contains(t, err.Error(), "line") + require.Contains(t, err.Error(), "alias=\"missing_alias\"") + require.Contains(t, err.Error(), "column=\"MISSING_PARENT\"") + require.Contains(t, err.Error(), "column=\"MISSING_CHILD\"") + require.Contains(t, err.Error(), "column not projected") + require.Contains(t, err.Error(), "alias not present in SQL/selector namespace") +} + +func TestValidateRelations_AllowsValidRelationAliasAndColumns(t *testing.T) { + routeYAML := ` +Resource: + Views: + - Name: Parent + Template: + Source: |- + SELECT p.ID, p.CAMPAIGN_ID FROM CI_PARENT p + With: + - Name: campaign + Holder: Campaign + Cardinality: One + On: + - Column: CAMPAIGN_ID + Namespace: p + Of: + Ref: Child + On: + - Column: ID + Namespace: c + - Name: Child + Template: + Source: |- + SELECT c.ID FROM CI_CHILD c +` + err := ValidateRelations([]byte(routeYAML)) + require.NoError(t, err) +} diff --git a/repository/shape/dql/plan/relation_sql.go b/repository/shape/dql/plan/relation_sql.go new file mode 100644 index 000000000..5530b68b4 --- /dev/null +++ b/repository/shape/dql/plan/relation_sql.go @@ -0,0 +1,82 @@ +package plan + +import ( + "strings" + + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/node" + "github.com/viant/sqlparser/query" +) + +func analyzeSQL(source string) (map[string]bool, projectionMeta, bool) { + aliases := map[string]bool{} + proj := projectionMeta{Columns: map[string]bool{}} + source = strings.TrimSpace(source) + if source == "" { + return aliases, proj, false + } + query, err := sqlparser.ParseQuery(source) + if err != nil || query == nil { + return aliases, proj, false + } + collectSQLAliases(query, aliases) + collectSQLProjection(query, &proj) + return aliases, proj, true +} + +func collectSQLAliases(query *query.Select, aliases map[string]bool) { + registerAlias(aliases, query.From.Alias) + registerFromNodeAlias(aliases, query.From.X) + for _, join := range query.Joins { + if join == nil { + continue + } + registerAlias(aliases, join.Alias) + registerFromNodeAlias(aliases, join.With) + } +} + +func collectSQLProjection(query *query.Select, projection *projectionMeta) { + columns := sqlparser.NewColumns(query.List) + projection.HasStar = columns.IsStarExpr() + for _, col := range columns { + if col == nil { + continue + } + registerProjection(projection.Columns, col.Name) + registerProjection(projection.Columns, col.Alias) + registerProjection(projection.Columns, col.Expression) + } +} + +func registerProjection(index map[string]bool, value string) { + value = strings.TrimSpace(value) + if value == "" || strings.Contains(value, "*") { + return + } + index[normalizedProjectionKey(value)] = true + if i := strings.LastIndex(value, "."); i != -1 && i+1 < len(value) { + suffix := strings.TrimSpace(value[i+1:]) + if suffix != "" { + index[normalizedProjectionKey(suffix)] = true + } + } +} + +func registerAlias(index map[string]bool, alias string) { + alias = strings.TrimSpace(alias) + if alias == "" { + return + } + index[strings.ToLower(alias)] = true +} + +func registerFromNodeAlias(index map[string]bool, n node.Node) { + switch actual := n.(type) { + case *expr.Ident: + registerAlias(index, actual.Name) + case *expr.Selector: + registerAlias(index, actual.Name) + } +} diff --git a/repository/shape/dql/plan/relation_types.go b/repository/shape/dql/plan/relation_types.go new file mode 100644 index 000000000..f5daa1879 --- /dev/null +++ b/repository/shape/dql/plan/relation_types.go @@ -0,0 +1,32 @@ +package plan + +type relationLink struct { + Line int + Column string + Namespace string +} + +type relationMeta struct { + Line int + Name string + Holder string + Ref string + On []relationLink + OfOn []relationLink + PairCount int +} + +type projectionMeta struct { + Columns map[string]bool + HasStar bool +} + +type viewMeta struct { + Name string + Line int + HasSQL bool + Aliases map[string]bool + Namespaces map[string]bool + Projection projectionMeta + Relations []relationMeta +} diff --git a/repository/shape/dql/plan/relation_validate.go b/repository/shape/dql/plan/relation_validate.go new file mode 100644 index 000000000..c0038c293 --- /dev/null +++ b/repository/shape/dql/plan/relation_validate.go @@ -0,0 +1,204 @@ +package plan + +import ( + "fmt" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// ValidateRelations validates relation links in generated route YAML. +func ValidateRelations(routeYAML []byte) error { + var root map[string]any + if err := yaml.Unmarshal(routeYAML, &root); err != nil { + return err + } + views := extractViews(root) + if len(views) == 0 { + return nil + } + lineIndex, err := collectViewMeta(routeYAML) + if err != nil { + return err + } + viewIndex := buildViewIndex(views, lineIndex) + issues := collectRelationIssues(viewIndex) + if len(issues) == 0 { + return nil + } + return fmt.Errorf("dql plan relation validation failed:\n- %s", strings.Join(issues, "\n- ")) +} + +func buildViewIndex(views []any, lineIndex map[string]*viewMeta) map[string]*viewMeta { + viewIndex := map[string]*viewMeta{} + for _, item := range views { + viewMap := toFlatMap(item) + if viewMap == nil { + continue + } + name := strings.TrimSpace(fmt.Sprint(viewMap["Name"])) + if name == "" { + continue + } + meta := lineIndex[name] + if meta == nil { + meta = &viewMeta{Name: name, Namespaces: map[string]bool{}} + lineIndex[name] = meta + } + applyViewRuntimeSQLMeta(viewMap, meta) + viewIndex[name] = meta + } + return viewIndex +} + +func applyViewRuntimeSQLMeta(viewMap map[string]any, meta *viewMeta) { + template := toFlatMap(viewMap["Template"]) + if template != nil { + source := strings.TrimSpace(fmt.Sprint(template["Source"])) + aliases, projection, hasSQL := analyzeSQL(source) + if hasSQL { + meta.HasSQL = true + } + if len(aliases) > 0 { + meta.Aliases = aliases + } + meta.Projection = projection + } + selector := toFlatMap(viewMap["Selector"]) + if selector != nil { + registerAlias(meta.Namespaces, fmt.Sprint(selector["Namespace"])) + } + if meta.Aliases == nil { + meta.Aliases = map[string]bool{} + } + if meta.Namespaces == nil { + meta.Namespaces = map[string]bool{} + } + for alias := range meta.Aliases { + meta.Namespaces[alias] = true + } + if meta.Projection.Columns == nil { + meta.Projection.Columns = map[string]bool{} + } +} + +func collectRelationIssues(viewIndex map[string]*viewMeta) []string { + var issues []string + for _, parent := range viewIndex { + for _, rel := range parent.Relations { + ref := viewIndex[strings.TrimSpace(rel.Ref)] + for i := 0; i < rel.PairCount; i++ { + left, right := linkAt(rel.On, i), linkAt(rel.OfOn, i) + issues = append(issues, validateParentLink(parent, rel, i, left)...) + issues = append(issues, validateRefLink(parent, ref, rel, i, right)...) + } + } + } + return issues +} + +func validateParentLink(parent *viewMeta, rel relationMeta, i int, left *relationLink) []string { + if left == nil { + return []string{fmt.Sprintf("line %d view=%q relation=%q holder=%q link=%d side=parent: missing On link entry", rel.Line, parent.Name, rel.Name, rel.Holder, i)} + } + return validateLink(parent, rel, "parent", i, *left) +} + +func validateRefLink(parent, ref *viewMeta, rel relationMeta, i int, right *relationLink) []string { + if right == nil { + return []string{fmt.Sprintf("line %d view=%q relation=%q holder=%q link=%d side=ref: missing Of.On link entry", rel.Line, parent.Name, rel.Name, rel.Holder, i)} + } + if ref != nil { + return validateLink(ref, rel, "ref", i, *right) + } + if strings.TrimSpace(rel.Ref) == "" { + return nil + } + return []string{fmt.Sprintf("line %d view=%q relation=%q holder=%q link=%d side=ref: referenced view %q not found", right.Line, parent.Name, rel.Name, rel.Holder, i, rel.Ref)} +} + +func linkAt(links []relationLink, i int) *relationLink { + if i < 0 || i >= len(links) { + return nil + } + return &links[i] +} + +func validateLink(view *viewMeta, rel relationMeta, side string, index int, link relationLink) []string { + if view == nil { + return nil + } + line := link.Line + if line == 0 { + line = rel.Line + } + column := strings.TrimSpace(link.Column) + alias := strings.TrimSpace(link.Namespace) + if column == "" { + return []string{fmt.Sprintf("line %d view=%q relation=%q holder=%q link=%d side=%s: empty column", line, view.Name, rel.Name, rel.Holder, index, side)} + } + + var issues []string + columnProjected := true + if view.HasSQL && !view.Projection.HasStar && len(view.Projection.Columns) > 0 { + columnProjected = hasProjectionColumn(view.Projection.Columns, column) + if !columnProjected { + issues = append(issues, fmt.Sprintf("line %d view=%q relation=%q holder=%q link=%d side=%s alias=%q column=%q: column not projected (columns=%v)", line, view.Name, rel.Name, rel.Holder, index, side, alias, column, sortedKeys(view.Projection.Columns))) + } + } + if alias != "" && view.HasSQL && !columnProjected && !view.Namespaces[strings.ToLower(alias)] { + issues = append(issues, fmt.Sprintf("line %d view=%q relation=%q holder=%q link=%d side=%s alias=%q column=%q: alias not present in SQL/selector namespace (namespaces=%v)", line, view.Name, rel.Name, rel.Holder, index, side, alias, column, sortedKeys(view.Namespaces))) + } + return issues +} + +func hasProjectionColumn(columns map[string]bool, column string) bool { + for _, candidate := range projectionCandidates(column) { + if columns[candidate] { + return true + } + } + return false +} + +func projectionCandidates(column string) []string { + column = strings.TrimSpace(column) + if column == "" { + return nil + } + result := []string{normalizedProjectionKey(column)} + if i := strings.LastIndex(column, "."); i != -1 && i+1 < len(column) { + result = append(result, normalizedProjectionKey(column[i+1:])) + } + return result +} + +func normalizedProjectionKey(value string) string { + return strings.ToLower(strings.Trim(value, "`\"' ")) +} + +func extractViews(root map[string]any) []any { + resource := toFlatMap(root["Resource"]) + if resource == nil { + return nil + } + return canonicalSlice(resource["Views"]) +} + +func collectViewMeta(routeYAML []byte) (map[string]*viewMeta, error) { + var rootNode yaml.Node + if err := yaml.Unmarshal(routeYAML, &rootNode); err != nil { + return nil, err + } + return parseViewMetaNodes(&rootNode), nil +} + +func sortedKeys(index map[string]bool) []string { + ret := make([]string, 0, len(index)) + for key := range index { + ret = append(ret, key) + } + sort.Strings(ret) + return ret +} diff --git a/repository/shape/dql/plan/relation_yaml.go b/repository/shape/dql/plan/relation_yaml.go new file mode 100644 index 000000000..8a141a1af --- /dev/null +++ b/repository/shape/dql/plan/relation_yaml.go @@ -0,0 +1,164 @@ +package plan + +import ( + "strings" + + "gopkg.in/yaml.v3" +) + +func parseViewMetaNodes(rootNode *yaml.Node) map[string]*viewMeta { + result := map[string]*viewMeta{} + views := viewsNode(rootNode) + if views == nil || views.Kind != yaml.SequenceNode { + return result + } + for _, item := range views.Content { + meta := parseViewMeta(item) + if meta == nil || strings.TrimSpace(meta.Name) == "" { + continue + } + result[meta.Name] = meta + } + return result +} + +func parseViewMeta(item *yaml.Node) *viewMeta { + viewMap := nodeMapping(item) + if viewMap == nil { + return nil + } + name := strings.TrimSpace(nodeString(mappingValue(viewMap, "Name"))) + if name == "" { + return nil + } + meta := &viewMeta{ + Name: name, + Line: item.Line, + Aliases: map[string]bool{}, + Namespaces: map[string]bool{}, + Projection: projectionMeta{Columns: map[string]bool{}}, + } + parseViewTemplateMeta(viewMap, meta) + parseViewSelectorMeta(viewMap, meta) + parseViewRelationsMeta(viewMap, meta) + return meta +} + +func parseViewTemplateMeta(viewMap map[string]*yaml.Node, meta *viewMeta) { + template := nodeMapping(mappingValue(viewMap, "Template")) + sourceNode := mappingValue(template, "Source") + if sourceNode == nil { + return + } + aliases, projection, hasSQL := analyzeSQL(nodeString(sourceNode)) + meta.HasSQL = hasSQL + if len(aliases) > 0 { + meta.Aliases = aliases + } + if len(projection.Columns) > 0 || projection.HasStar { + meta.Projection = projection + } +} + +func parseViewSelectorMeta(viewMap map[string]*yaml.Node, meta *viewMeta) { + selector := nodeMapping(mappingValue(viewMap, "Selector")) + registerAlias(meta.Namespaces, nodeString(mappingValue(selector, "Namespace"))) + for alias := range meta.Aliases { + meta.Namespaces[alias] = true + } +} + +func parseViewRelationsMeta(viewMap map[string]*yaml.Node, meta *viewMeta) { + with := mappingValue(viewMap, "With") + if with == nil || with.Kind != yaml.SequenceNode { + return + } + for _, relItem := range with.Content { + rel := parseRelationMeta(relItem) + if rel != nil { + meta.Relations = append(meta.Relations, *rel) + } + } +} + +func parseRelationMeta(relItem *yaml.Node) *relationMeta { + relMap := nodeMapping(relItem) + if relMap == nil { + return nil + } + rel := &relationMeta{ + Line: relItem.Line, + Name: nodeString(mappingValue(relMap, "Name")), + Holder: nodeString(mappingValue(relMap, "Holder")), + On: parseLinkNodes(mappingValue(relMap, "On")), + } + ofMap := nodeMapping(mappingValue(relMap, "Of")) + rel.Ref = nodeString(mappingValue(ofMap, "Ref")) + if rel.Ref == "" { + rel.Ref = nodeString(mappingValue(ofMap, "Name")) + } + rel.OfOn = parseLinkNodes(mappingValue(ofMap, "On")) + rel.PairCount = len(rel.On) + if len(rel.OfOn) > rel.PairCount { + rel.PairCount = len(rel.OfOn) + } + return rel +} + +func parseLinkNodes(seq *yaml.Node) []relationLink { + if seq == nil || seq.Kind != yaml.SequenceNode { + return nil + } + ret := make([]relationLink, 0, len(seq.Content)) + for _, item := range seq.Content { + linkMap := nodeMapping(item) + if linkMap == nil { + ret = append(ret, relationLink{Line: item.Line}) + continue + } + ret = append(ret, relationLink{ + Line: item.Line, + Column: nodeString(mappingValue(linkMap, "Column")), + Namespace: nodeString(mappingValue(linkMap, "Namespace")), + }) + } + return ret +} + +func viewsNode(rootNode *yaml.Node) *yaml.Node { + rootMap := nodeMapping(rootNode) + resource := mappingValue(rootMap, "Resource") + resourceMap := nodeMapping(resource) + return mappingValue(resourceMap, "Views") +} + +func nodeMapping(n *yaml.Node) map[string]*yaml.Node { + if n == nil { + return nil + } + if n.Kind == yaml.DocumentNode && len(n.Content) > 0 { + n = n.Content[0] + } + if n.Kind != yaml.MappingNode { + return nil + } + ret := map[string]*yaml.Node{} + for i := 0; i+1 < len(n.Content); i += 2 { + ret[n.Content[i].Value] = n.Content[i+1] + } + return ret +} + +func mappingValue(m map[string]*yaml.Node, key string) *yaml.Node { + if m == nil { + return nil + } + return m[key] +} + +func nodeString(n *yaml.Node) string { + if n == nil { + return "" + } + return strings.TrimSpace(n.Value) +} diff --git a/repository/shape/dql/render/dql/renderer.go b/repository/shape/dql/render/dql/renderer.go new file mode 100644 index 000000000..9864b4c04 --- /dev/null +++ b/repository/shape/dql/render/dql/renderer.go @@ -0,0 +1,166 @@ +package dql + +import ( + "fmt" + "strings" + + "github.com/viant/datly/repository/shape/dql/ir" +) + +// SourceResolver resolves template SourceURL content when Source is not embedded in IR. +type SourceResolver func(sourceURL string) (string, error) + +type options struct { + rootView string + resolve SourceResolver +} + +// Option configures DQL rendering. +type Option func(*options) + +// WithRootView forces renderer root view selection. +func WithRootView(name string) Option { + return func(o *options) { + o.rootView = strings.TrimSpace(name) + } +} + +// WithSourceResolver configures SourceURL content resolution. +func WithSourceResolver(resolver SourceResolver) Option { + return func(o *options) { + o.resolve = resolver + } +} + +// Encode renders IR document back to DQL/SQL source for the root route view. +func Encode(doc *ir.Document, opts ...Option) ([]byte, error) { + if doc == nil || doc.Root == nil { + return nil, fmt.Errorf("dql render dql: nil IR document") + } + cfg := &options{} + for _, opt := range opts { + if opt != nil { + opt(cfg) + } + } + views := indexViews(doc.Root) + if len(views) == 0 { + return nil, fmt.Errorf("dql render dql: no resource views in IR") + } + rootView := cfg.rootView + if rootView == "" { + rootView = detectRootView(doc.Root) + } + if rootView == "" { + return nil, fmt.Errorf("dql render dql: unable to detect root route view") + } + view := views[rootView] + if view == nil { + return nil, fmt.Errorf("dql render dql: root view %q not found in resources", rootView) + } + sql, err := renderViewSQL(view, cfg) + if err != nil { + return nil, err + } + return []byte(strings.TrimSpace(sql) + "\n"), nil +} + +func renderViewSQL(view map[string]any, cfg *options) (string, error) { + name := stringValue(view["Name"]) + template := mapValue(view["Template"]) + if template == nil { + return "", fmt.Errorf("dql render dql: view %q has no template", name) + } + if source := strings.TrimSpace(stringValue(template["Source"])); source != "" { + return source, nil + } + sourceURL := strings.TrimSpace(stringValue(template["SourceURL"])) + if sourceURL == "" { + return "", fmt.Errorf("dql render dql: view %q has neither template source nor sourceURL", name) + } + source, err := resolveSourceURL(cfg, view, sourceURL) + if err != nil { + return "", fmt.Errorf("dql render dql: view %q resolve %q failed: %w", name, sourceURL, err) + } + source = strings.TrimSpace(source) + if source == "" { + return "", fmt.Errorf("dql render dql: resolved source was empty for %q", sourceURL) + } + return source, nil +} + +func resolveSourceURL(cfg *options, view map[string]any, sourceURL string) (string, error) { + _ = view + if cfg.resolve != nil { + return cfg.resolve(sourceURL) + } + return "", fmt.Errorf("requires SourceURL resolver for %q", sourceURL) +} + +func detectRootView(root map[string]any) string { + for _, routeItem := range sliceValue(root["Routes"]) { + route := mapValue(routeItem) + if route == nil { + continue + } + view := mapValue(route["View"]) + if view == nil { + continue + } + if ref := strings.TrimSpace(stringValue(view["Ref"])); ref != "" { + return ref + } + } + return "" +} + +func indexViews(root map[string]any) map[string]map[string]any { + result := map[string]map[string]any{} + resource := mapValue(root["Resource"]) + if resource == nil { + return result + } + for _, item := range sliceValue(resource["Views"]) { + view := mapValue(item) + if view == nil { + continue + } + name := strings.TrimSpace(stringValue(view["Name"])) + if name == "" { + continue + } + result[name] = view + } + return result +} + +func mapValue(raw any) map[string]any { + if v, ok := raw.(map[string]any); ok { + return v + } + if v, ok := raw.(map[any]any); ok { + out := map[string]any{} + for key, item := range v { + out[fmt.Sprint(key)] = item + } + return out + } + return nil +} + +func sliceValue(raw any) []any { + if items, ok := raw.([]any); ok { + return items + } + return nil +} + +func stringValue(raw any) string { + if raw == nil { + return "" + } + if value, ok := raw.(string); ok { + return value + } + return fmt.Sprint(raw) +} diff --git a/repository/shape/dql/render/dql/renderer_test.go b/repository/shape/dql/render/dql/renderer_test.go new file mode 100644 index 000000000..0989854ee --- /dev/null +++ b/repository/shape/dql/render/dql/renderer_test.go @@ -0,0 +1,117 @@ +package dql + +import ( + "errors" + "testing" + + "github.com/viant/datly/repository/shape/dql/ir" +) + +func TestEncode_WithEmbeddedSource(t *testing.T) { + doc := &ir.Document{Root: map[string]any{ + "Routes": []any{ + map[string]any{ + "View": map[string]any{"Ref": "root"}, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "root", + "Template": map[string]any{ + "Source": "SELECT * FROM USERS u", + }, + }, + }, + }, + }} + data, err := Encode(doc) + if err != nil { + t.Fatalf("Encode failed: %v", err) + } + if got, want := string(data), "SELECT * FROM USERS u\n"; got != want { + t.Fatalf("unexpected dql, got %q want %q", got, want) + } +} + +func TestEncode_WithSourceURLResolver(t *testing.T) { + doc := &ir.Document{Root: map[string]any{ + "Routes": []any{ + map[string]any{ + "View": map[string]any{"Ref": "root"}, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "root", + "Template": map[string]any{ + "SourceURL": "queries/root.sql", + }, + }, + }, + }, + }} + data, err := Encode(doc, WithSourceResolver(func(sourceURL string) (string, error) { + if sourceURL != "queries/root.sql" { + t.Fatalf("unexpected sourceURL: %s", sourceURL) + } + return "SELECT 1", nil + })) + if err != nil { + t.Fatalf("Encode failed: %v", err) + } + if got, want := string(data), "SELECT 1\n"; got != want { + t.Fatalf("unexpected dql, got %q want %q", got, want) + } +} + +func TestEncode_SourceURLWithoutResolverFails(t *testing.T) { + doc := &ir.Document{Root: map[string]any{ + "Routes": []any{ + map[string]any{ + "View": map[string]any{"Ref": "root"}, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "root", + "Template": map[string]any{ + "SourceURL": "queries/root.sql", + }, + }, + }, + }, + }} + _, err := Encode(doc) + if err == nil { + t.Fatalf("expected error") + } +} + +func TestEncode_ResolverError(t *testing.T) { + doc := &ir.Document{Root: map[string]any{ + "Routes": []any{ + map[string]any{ + "View": map[string]any{"Ref": "root"}, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "root", + "Template": map[string]any{ + "SourceURL": "queries/root.sql", + }, + }, + }, + }, + }} + _, err := Encode(doc, WithSourceResolver(func(sourceURL string) (string, error) { + return "", errors.New("boom") + })) + if err == nil { + t.Fatalf("expected error") + } +} diff --git a/repository/shape/dql/render/yaml/renderer.go b/repository/shape/dql/render/yaml/renderer.go new file mode 100644 index 000000000..83bc39c54 --- /dev/null +++ b/repository/shape/dql/render/yaml/renderer.go @@ -0,0 +1,16 @@ +package yaml + +import ( + "fmt" + + "github.com/viant/datly/repository/shape/dql/ir" + "gopkg.in/yaml.v3" +) + +// Encode renders IR document into YAML bytes. +func Encode(doc *ir.Document) ([]byte, error) { + if doc == nil || doc.Root == nil { + return nil, fmt.Errorf("dql render yaml: nil IR document") + } + return yaml.Marshal(doc.Root) +} diff --git a/repository/shape/dql/scan/scanner.go b/repository/shape/dql/scan/scanner.go new file mode 100644 index 000000000..f72961884 --- /dev/null +++ b/repository/shape/dql/scan/scanner.go @@ -0,0 +1,433 @@ +package scan + +import ( + "context" + "fmt" + "path/filepath" + "reflect" + "strings" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/viant/afs" + "github.com/viant/afs/file" + "github.com/viant/afs/url" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/internal/translator" + "github.com/viant/datly/repository/shape/dql/decl" + "github.com/viant/datly/repository/shape/dql/ir" + "github.com/viant/datly/repository/shape/dql/parse" + dqlplan "github.com/viant/datly/repository/shape/dql/plan" + "github.com/viant/datly/repository/shape/dql/sanitize" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/repository/shape/typectx/source" + _ "github.com/viant/sqlx/metadata/product/mysql" + "github.com/viant/x" +) + +// Request defines input for DQL scan. +type Request struct { + DQLURL string + ConfigURL string + Repository string + ModulePrefix string + APIPrefix string + Connectors []string + AllowedProvenanceKinds []string + AllowedSourceRoots []string + UseGoModuleResolve *bool + UseGOPATHFallback *bool + StrictProvenance *bool +} + +// Result holds scanner output. +type Result struct { + RuleName string + Shape *dqlshape.Document + IR *ir.Document +} + +// Scanner translates DQL to Datly route YAML in-memory. +type Scanner struct { + fs afs.Service +} + +func New() *Scanner { + return &Scanner{fs: afs.New()} +} + +func (s *Scanner) Scan(ctx context.Context, req *Request) (result *Result, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("dql scan panic: %v", r) + result = nil + } + }() + if req == nil || req.DQLURL == "" { + return nil, fmt.Errorf("dql scan: DQLURL was empty") + } + sourceURL := req.DQLURL + project := inferProject(req.DQLURL) + translate := &options.Translate{} + translate.Rule.Project = project + translate.Rule.Source = []string{sourceURL} + translate.Rule.ModulePrefix = req.ModulePrefix + translate.Repository.RepositoryURL = req.Repository + translate.Repository.APIPrefix = req.APIPrefix + if len(req.Connectors) > 0 { + translate.Repository.Connectors = append(translate.Repository.Connectors, req.Connectors...) + } + if req.ConfigURL != "" { + translate.Repository.Configs.Append(req.ConfigURL) + } + var initErr error + if initErr = translate.Init(ctx); initErr != nil { + return nil, initErr + } + if req.ConfigURL == "" { + // Force in-memory translator config to avoid stale absolute paths from discovered config.json. + translate.Repository.Configs = nil + } + if translate.Rule.ModulePrefix == "" { + translate.Rule.ModulePrefix = "platform" + } + + svc := translator.New(translator.NewConfig(&translate.Repository), s.fs) + if initErr := svc.Init(ctx); initErr != nil { + return nil, initErr + } + if initErr := svc.InitSignature(ctx, &translate.Rule); initErr != nil { + return nil, initErr + } + dsql, loadErr := translate.Rule.LoadSource(ctx, s.fs, translate.Rule.SourceURL()) + if loadErr != nil { + return nil, loadErr + } + translate.Rule.NormalizeComponent(&dsql) + if data, sanitizeErr := sanitize.SanitizeDQL([]byte(dsql)); sanitizeErr == nil { + dsql = string(data) + } else { + return nil, sanitizeErr + } + top := &options.Options{Translate: translate} + if initErr = svc.Translate(ctx, &translate.Rule, dsql, top); initErr != nil { + return nil, initErr + } + ruleName := svc.Repository.RuleName(&translate.Rule) + targetSuffix := "/" + ruleName + ".yaml" + for _, item := range svc.Repository.Files { + if !strings.HasSuffix(item.URL, targetSuffix) { + continue + } + if strings.Contains(item.URL, "/.meta/") { + continue + } + return s.result(ruleName, []byte(item.Content), dsql, req) + } + for _, item := range svc.Repository.Files { + if strings.HasSuffix(item.URL, targetSuffix) { + return s.result(ruleName, []byte(item.Content), dsql, req) + } + } + return nil, fmt.Errorf("dql scan: generated YAML not found for %s", ruleName) +} + +func (s *Scanner) result(ruleName string, routeYAML []byte, dql string, req *Request) (*Result, error) { + if err := dqlplan.ValidateRelations(routeYAML); err != nil { + return nil, fmt.Errorf("dql scan relation validation failed (%s): %w", ruleName, err) + } + fromYAML, err := ir.FromYAML(routeYAML) + if err != nil { + return nil, err + } + shapeDoc, err := dqlshape.FromIR(fromYAML) + if err != nil { + return nil, err + } + if parsed, parseErr := parse.New().Parse(dql); parseErr == nil && parsed != nil && parsed.TypeContext != nil { + shapeDoc.TypeContext = parsed.TypeContext + if resolutions, resolveErr := resolveTypeProvenance(parsed, fromYAML, req); resolveErr != nil { + return nil, resolveErr + } else { + shapeDoc.TypeResolutions = resolutions + } + } + rebuiltIR, err := dqlshape.ToIR(shapeDoc) + if err != nil { + return nil, err + } + return &Result{RuleName: ruleName, Shape: shapeDoc, IR: rebuiltIR}, nil +} + +func resolveTypeProvenance(parsed *parse.Result, doc *ir.Document, req *Request) ([]typectx.Resolution, error) { + if parsed == nil || len(parsed.Declarations) == 0 { + return nil, nil + } + registry, provenance := registryFromIR(doc) + resolver := typectx.NewResolverWithProvenance(registry, parsed.TypeContext, provenance) + policy := newProvenancePolicy(req) + srcResolver, srcErr := newSourceResolver(policy, req) + if srcErr != nil && policy.Strict { + return nil, srcErr + } + var result []typectx.Resolution + for _, declaration := range parsed.Declarations { + if declaration == nil || declaration.Kind != decl.KindCast { + continue + } + expression := strings.TrimSpace(declaration.DataType) + if expression == "" { + continue + } + resolution, err := resolver.ResolveWithProvenance(expression) + if err != nil { + return nil, fmt.Errorf("dql scan cast resolution failed for %q: %w", expression, err) + } + if resolution == nil { + continue + } + resolution.Target = declaration.Target + enrichResolutionWithAST(resolution, srcResolver) + if issue := validateResolutionPolicy(*resolution, policy); issue != "" { + if policy.Strict { + return nil, fmt.Errorf("dql scan provenance policy failed: %s", issue) + } + resolution.Provenance.Kind = "policy_warn:" + issue + } + result = append(result, *resolution) + } + return result, nil +} + +func registryFromIR(doc *ir.Document) (*x.Registry, map[string]typectx.Provenance) { + registry := x.NewRegistry() + provenance := map[string]typectx.Provenance{} + registerBuiltin := func(rType reflect.Type, kind string) { + aType := x.NewType(rType) + registry.Register(aType) + provenance[aType.Key()] = typectx.Provenance{ + Package: packageOfKey(aType.Key()), + Kind: kind, + } + } + registerBuiltin(reflect.TypeOf(time.Time{}), "builtin") + registerBuiltin(reflect.TypeOf(""), "builtin") + registerBuiltin(reflect.TypeOf(0), "builtin") + registerBuiltin(reflect.TypeOf(int64(0)), "builtin") + registerBuiltin(reflect.TypeOf(float64(0)), "builtin") + registerBuiltin(reflect.TypeOf(true), "builtin") + + if doc == nil || doc.Root == nil { + return registry, provenance + } + resource := asMap(doc.Root["Resource"]) + if resource == nil { + return registry, provenance + } + for _, item := range asSlice(resource["Types"]) { + typeMap := asMap(item) + if typeMap == nil { + continue + } + name := strings.TrimSpace(asString(typeMap["Name"])) + if name == "" { + continue + } + pkg := strings.TrimSpace(asString(typeMap["Package"])) + aType := &x.Type{Name: name, PkgPath: pkg} + registry.Register(aType) + key := aType.Key() + provenance[key] = typectx.Provenance{ + Package: pkg, + File: firstNonEmpty(asString(typeMap["SourceURL"]), asString(typeMap["ModulePath"])), + Kind: "resource_type", + } + } + return registry, provenance +} + +func asMap(raw any) map[string]any { + if value, ok := raw.(map[string]any); ok { + return value + } + if value, ok := raw.(map[any]any); ok { + result := make(map[string]any, len(value)) + for k, v := range value { + result[fmt.Sprint(k)] = v + } + return result + } + return nil +} + +func asSlice(raw any) []any { + if value, ok := raw.([]any); ok { + return value + } + return nil +} + +func asString(raw any) string { + if raw == nil { + return "" + } + if value, ok := raw.(string); ok { + return value + } + return fmt.Sprint(raw) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + return value + } + } + return "" +} + +func packageOfKey(key string) string { + index := strings.LastIndex(key, ".") + if index == -1 { + return "" + } + return key[:index] +} + +type provenancePolicy struct { + AllowedKinds map[string]bool + Roots []string + Strict bool +} + +func newProvenancePolicy(req *Request) provenancePolicy { + allowedKinds := map[string]bool{ + "builtin": true, + "resource_type": true, + "ast_type": true, + } + if req != nil && len(req.AllowedProvenanceKinds) > 0 { + allowedKinds = map[string]bool{} + for _, item := range req.AllowedProvenanceKinds { + item = strings.TrimSpace(strings.ToLower(item)) + if item != "" { + allowedKinds[item] = true + } + } + } + repo := "" + if req != nil { + repo = req.Repository + } + roots := source.NormalizeRoots(repo, requestRoots(req)) + return provenancePolicy{ + AllowedKinds: allowedKinds, + Roots: roots, + Strict: requestStrict(req), + } +} + +func requestRoots(req *Request) []string { + if req == nil { + return nil + } + return req.AllowedSourceRoots +} + +func requestStrict(req *Request) bool { + if req == nil || req.StrictProvenance == nil { + return true + } + return *req.StrictProvenance +} + +func requestUseModule(req *Request) bool { + if req == nil || req.UseGoModuleResolve == nil { + return true + } + return *req.UseGoModuleResolve +} + +func requestUseGOPATH(req *Request) bool { + if req == nil || req.UseGOPATHFallback == nil { + return true + } + return *req.UseGOPATHFallback +} + +func newSourceResolver(policy provenancePolicy, req *Request) (*source.Resolver, error) { + if req == nil || strings.TrimSpace(req.Repository) == "" { + return nil, nil + } + return source.New(source.Config{ + ProjectDir: req.Repository, + AllowedSourceRoots: policy.Roots, + UseGoModuleResolve: requestUseModule(req), + UseGOPATHFallback: requestUseGOPATH(req), + }) +} + +func enrichResolutionWithAST(resolution *typectx.Resolution, srcResolver *source.Resolver) { + if resolution == nil || srcResolver == nil { + return + } + if strings.TrimSpace(resolution.Provenance.File) != "" { + return + } + pkg := strings.TrimSpace(resolution.Provenance.Package) + typeName := typeNameFromKey(resolution.ResolvedKey) + if pkg == "" || typeName == "" { + return + } + filePath, err := srcResolver.ResolveTypeFile(pkg, typeName) + if err != nil { + return + } + resolution.Provenance.File = filePath + if resolution.Provenance.Kind == "" || resolution.Provenance.Kind == "registry" { + resolution.Provenance.Kind = "ast_type" + } +} + +func typeNameFromKey(key string) string { + index := strings.LastIndex(key, ".") + if index == -1 || index+1 >= len(key) { + return "" + } + return key[index+1:] +} + +func validateResolutionPolicy(resolution typectx.Resolution, policy provenancePolicy) string { + kind := strings.TrimSpace(strings.ToLower(resolution.Provenance.Kind)) + if kind == "" { + kind = "registry" + } + if !policy.AllowedKinds[kind] { + return fmt.Sprintf("expression=%q kind=%q not allowed", resolution.Expression, resolution.Provenance.Kind) + } + filePath := strings.TrimSpace(resolution.Provenance.File) + if filePath == "" { + return "" + } + if len(policy.Roots) == 0 { + return "" + } + ok, err := source.IsWithinAnyRoot(filePath, policy.Roots) + if err != nil { + return fmt.Sprintf("expression=%q source=%q invalid: %v", resolution.Expression, filePath, err) + } + if !ok { + return fmt.Sprintf("expression=%q source=%q outside trusted roots", resolution.Expression, filePath) + } + return "" +} + +func inferProject(dqlURL string) string { + base, _ := url.Split(dqlURL, file.Scheme) + if idx := strings.Index(base, "/dql/"); idx != -1 { + return filepath.Clean(base[:idx]) + } + return filepath.Clean(base) +} diff --git a/repository/shape/dql/scan/scanner_test.go b/repository/shape/dql/scan/scanner_test.go new file mode 100644 index 000000000..1e081cce5 --- /dev/null +++ b/repository/shape/dql/scan/scanner_test.go @@ -0,0 +1,164 @@ +package scan + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestScanner_Result_ValidatesRelations(t *testing.T) { + s := New() + invalidYAML := []byte(` +Resource: + Views: + - Name: Parent + Template: + Source: SELECT p.ID FROM T p + With: + - Name: rel + Holder: Rel + Cardinality: One + On: + - Column: MISSING_COL + Namespace: p + Of: + Ref: Child + On: + - Column: ID + Namespace: c + - Name: Child + Template: + Source: SELECT c.ID FROM T2 c +`) + _, err := s.result("x", invalidYAML, "", nil) + require.Error(t, err) + require.Contains(t, err.Error(), "dql scan relation validation failed") + require.Contains(t, err.Error(), "column=\"MISSING_COL\"") +} + +func TestScanner_Result_BuildsShapeAndIR(t *testing.T) { + s := New() + validYAML := []byte(` +Routes: + - Name: Sample + URI: /sample + Method: GET + View: + Ref: root +Resource: + Views: + - Name: root + Connector: + Ref: main + Template: + Source: SELECT r.ID FROM ROOT r +`) + result, err := s.result("sample", validYAML, "", nil) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, result.Shape) + require.NotNil(t, result.IR) + require.Equal(t, "root", result.Shape.Routes[0].ViewRef) +} + +func TestScanner_Result_PropagatesTypeContextFromDQL(t *testing.T) { + s := New() + validYAML := []byte(` +Routes: + - Name: Sample + URI: /sample + Method: GET + View: + Ref: root +Resource: + Views: + - Name: root + Connector: + Ref: main + Template: + Source: SELECT r.ID FROM ROOT r +`) + dql := ` +#set($_ = $package('mdp/performance')) +#set($_ = $import('perf', 'github.com/acme/mdp/performance')) +SELECT r.ID FROM ROOT r` + result, err := s.result("sample", validYAML, dql, nil) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, result.Shape) + require.NotNil(t, result.Shape.TypeContext) + require.Equal(t, "mdp/performance", result.Shape.TypeContext.DefaultPackage) + require.Len(t, result.Shape.TypeContext.Imports, 1) + require.Equal(t, "perf", result.Shape.TypeContext.Imports[0].Alias) +} + +func TestScanner_Result_ResolvesTypeProvenance(t *testing.T) { + s := New() + validYAML := []byte(` +Routes: + - Name: Sample + URI: /sample + Method: GET + View: + Ref: root +Resource: + Types: + - Name: Order + Package: github.com/acme/mdp/performance + SourceURL: /repo/mdp/performance/order.go + Views: + - Name: root + Connector: + Ref: main + Template: + Source: SELECT r.ID FROM ROOT r +`) + dql := ` +#set($_ = $package('github.com/acme/mdp/performance')) +SELECT cast(r.ID as 'Order') FROM ROOT r` + result, err := s.result("sample", validYAML, dql, nil) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, result.Shape) + require.Len(t, result.Shape.TypeResolutions, 1) + resolution := result.Shape.TypeResolutions[0] + require.Equal(t, "Order", resolution.Expression) + require.Equal(t, "github.com/acme/mdp/performance.Order", resolution.ResolvedKey) + require.Equal(t, "default_package", resolution.MatchKind) + require.Equal(t, "resource_type", resolution.Provenance.Kind) + require.Equal(t, "/repo/mdp/performance/order.go", resolution.Provenance.File) +} + +func TestScanner_Result_StrictProvenanceBlocksOutsideRoot(t *testing.T) { + s := New() + validYAML := []byte(` +Routes: + - Name: Sample + URI: /sample + Method: GET + View: + Ref: root +Resource: + Types: + - Name: Order + Package: github.com/acme/mdp/performance + SourceURL: /outside/order.go + Views: + - Name: root + Connector: + Ref: main + Template: + Source: SELECT r.ID FROM ROOT r +`) + dql := ` +#set($_ = $package('github.com/acme/mdp/performance')) +SELECT cast(r.ID as 'Order') FROM ROOT r` + strict := true + _, err := s.result("sample", validYAML, dql, &Request{ + Repository: filepath.Clean(t.TempDir()), + StrictProvenance: &strict, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "provenance policy failed") +} diff --git a/repository/shape/dql/shape/convert.go b/repository/shape/dql/shape/convert.go new file mode 100644 index 000000000..00a88eff9 --- /dev/null +++ b/repository/shape/dql/shape/convert.go @@ -0,0 +1,235 @@ +package shape + +import ( + "fmt" + + "github.com/viant/datly/repository/shape/dql/ir" + "github.com/viant/datly/repository/shape/typectx" +) + +// FromIR builds typed shape document from IR. +func FromIR(doc *ir.Document) (*Document, error) { + if doc == nil || doc.Root == nil { + return nil, fmt.Errorf("dql shape: nil IR document") + } + root, ok := deepClone(doc.Root).(map[string]any) + if !ok || root == nil { + return nil, fmt.Errorf("dql shape: invalid IR root") + } + ret := &Document{Root: root} + ret.TypeContext = typeContextFromRoot(root) + ret.TypeResolutions = typeResolutionsFromRoot(root) + for _, item := range asSlice(root["Routes"]) { + routeMap := asMap(item) + if routeMap == nil { + continue + } + route := &Route{ + Name: asString(routeMap["Name"]), + URI: asString(routeMap["URI"]), + Method: asString(routeMap["Method"]), + Description: asString(routeMap["Description"]), + } + if view := asMap(routeMap["View"]); view != nil { + route.ViewRef = asString(view["Ref"]) + } + ret.Routes = append(ret.Routes, route) + } + resourceMap := asMap(root["Resource"]) + if resourceMap != nil { + resource := &Resource{} + for _, item := range asSlice(resourceMap["Views"]) { + viewMap := asMap(item) + if viewMap == nil { + continue + } + view := &View{ + Name: asString(viewMap["Name"]), + Table: asString(viewMap["Table"]), + Module: asString(viewMap["Module"]), + } + if connector := asMap(viewMap["Connector"]); connector != nil { + view.ConnectorRef = asString(connector["Ref"]) + } + resource.Views = append(resource.Views, view) + } + ret.Resource = resource + } + return ret, nil +} + +// ToIR converts shape document back to IR. +func ToIR(doc *Document) (*ir.Document, error) { + if doc == nil || doc.Root == nil { + return nil, fmt.Errorf("dql shape: nil document") + } + root, ok := deepClone(doc.Root).(map[string]any) + if !ok || root == nil { + return nil, fmt.Errorf("dql shape: invalid root") + } + if doc.TypeContext != nil { + root["TypeContext"] = map[string]any{ + "DefaultPackage": doc.TypeContext.DefaultPackage, + "Imports": importsToAny(doc.TypeContext.Imports), + } + } + if len(doc.TypeResolutions) > 0 { + root["TypeResolutions"] = typeResolutionsToAny(doc.TypeResolutions) + } + return &ir.Document{Root: root}, nil +} + +func typeContextFromRoot(root map[string]any) *typectx.Context { + raw := asMap(root["TypeContext"]) + if raw == nil { + return nil + } + ret := &typectx.Context{DefaultPackage: asString(raw["DefaultPackage"])} + for _, item := range asSlice(raw["Imports"]) { + importMap := asMap(item) + if importMap == nil { + continue + } + pkg := asString(importMap["Package"]) + if pkg == "" { + continue + } + ret.Imports = append(ret.Imports, typectx.Import{ + Alias: asString(importMap["Alias"]), + Package: pkg, + }) + } + if ret.DefaultPackage == "" && len(ret.Imports) == 0 { + return nil + } + return ret +} + +func importsToAny(imports []typectx.Import) []any { + if len(imports) == 0 { + return nil + } + result := make([]any, 0, len(imports)) + for _, item := range imports { + if item.Package == "" { + continue + } + result = append(result, map[string]any{ + "Alias": item.Alias, + "Package": item.Package, + }) + } + return result +} + +func typeResolutionsFromRoot(root map[string]any) []typectx.Resolution { + items := asSlice(root["TypeResolutions"]) + if len(items) == 0 { + return nil + } + result := make([]typectx.Resolution, 0, len(items)) + for _, item := range items { + resolutionMap := asMap(item) + if resolutionMap == nil { + continue + } + resolution := typectx.Resolution{ + Expression: asString(resolutionMap["Expression"]), + Target: asString(resolutionMap["Target"]), + ResolvedKey: asString(resolutionMap["ResolvedKey"]), + MatchKind: asString(resolutionMap["MatchKind"]), + } + if provenanceMap := asMap(resolutionMap["Provenance"]); provenanceMap != nil { + resolution.Provenance = typectx.Provenance{ + Package: asString(provenanceMap["Package"]), + File: asString(provenanceMap["File"]), + Kind: asString(provenanceMap["Kind"]), + } + } + if resolution.Expression == "" && resolution.ResolvedKey == "" { + continue + } + result = append(result, resolution) + } + return result +} + +func typeResolutionsToAny(resolutions []typectx.Resolution) []any { + result := make([]any, 0, len(resolutions)) + for _, item := range resolutions { + if item.Expression == "" && item.ResolvedKey == "" { + continue + } + result = append(result, map[string]any{ + "Expression": item.Expression, + "Target": item.Target, + "ResolvedKey": item.ResolvedKey, + "MatchKind": item.MatchKind, + "Provenance": map[string]any{ + "Package": item.Provenance.Package, + "File": item.Provenance.File, + "Kind": item.Provenance.Kind, + }, + }) + } + if len(result) == 0 { + return nil + } + return result +} + +func deepClone(value any) any { + switch actual := value.(type) { + case map[string]any: + out := make(map[string]any, len(actual)) + for k, v := range actual { + out[k] = deepClone(v) + } + return out + case map[any]any: + out := make(map[string]any, len(actual)) + for k, v := range actual { + out[fmt.Sprint(k)] = deepClone(v) + } + return out + case []any: + out := make([]any, len(actual)) + for i, item := range actual { + out[i] = deepClone(item) + } + return out + default: + return actual + } +} + +func asMap(raw any) map[string]any { + if value, ok := raw.(map[string]any); ok { + return value + } + if value, ok := raw.(map[any]any); ok { + out := make(map[string]any, len(value)) + for k, item := range value { + out[fmt.Sprint(k)] = item + } + return out + } + return nil +} + +func asSlice(raw any) []any { + if value, ok := raw.([]any); ok { + return value + } + return nil +} + +func asString(raw any) string { + if raw == nil { + return "" + } + if value, ok := raw.(string); ok { + return value + } + return fmt.Sprint(raw) +} diff --git a/repository/shape/dql/shape/convert_test.go b/repository/shape/dql/shape/convert_test.go new file mode 100644 index 000000000..0e41a43fd --- /dev/null +++ b/repository/shape/dql/shape/convert_test.go @@ -0,0 +1,121 @@ +package shape + +import ( + "reflect" + "testing" + + "github.com/viant/datly/repository/shape/dql/ir" + "github.com/viant/datly/repository/shape/typectx" +) + +func TestFromIRToIR_RoundTripPreservesRoot(t *testing.T) { + source := &ir.Document{Root: map[string]any{ + "Routes": []any{ + map[string]any{ + "Name": "Route", + "URI": "/x", + "Method": "GET", + "View": map[string]any{ + "Ref": "rootView", + }, + }, + }, + "Resource": map[string]any{ + "Views": []any{ + map[string]any{ + "Name": "rootView", + "Table": "T", + "Connector": map[string]any{ + "Ref": "main", + }, + "Template": map[string]any{ + "Source": "SELECT * FROM T", + }, + }, + }, + }, + }} + shapeDoc, err := FromIR(source) + if err != nil { + t.Fatalf("FromIR failed: %v", err) + } + if shapeDoc == nil || len(shapeDoc.Routes) != 1 || shapeDoc.Resource == nil || len(shapeDoc.Resource.Views) != 1 { + t.Fatalf("unexpected shape projection: %+v", shapeDoc) + } + target, err := ToIR(shapeDoc) + if err != nil { + t.Fatalf("ToIR failed: %v", err) + } + if !reflect.DeepEqual(source.Root, target.Root) { + t.Fatalf("round-trip mismatch") + } +} + +func TestToIR_FromIR_TypeContextRoundTrip(t *testing.T) { + doc := &Document{ + Root: map[string]any{ + "Routes": []any{}, + "Resource": map[string]any{}, + }, + TypeContext: &typectx.Context{ + DefaultPackage: "mdp/performance", + Imports: []typectx.Import{ + {Alias: "perf", Package: "github.com/acme/mdp/performance"}, + }, + }, + } + irDoc, err := ToIR(doc) + if err != nil { + t.Fatalf("ToIR failed: %v", err) + } + shapeDoc, err := FromIR(irDoc) + if err != nil { + t.Fatalf("FromIR failed: %v", err) + } + if shapeDoc.TypeContext == nil { + t.Fatalf("expected type context") + } + if shapeDoc.TypeContext.DefaultPackage != "mdp/performance" { + t.Fatalf("unexpected default package: %s", shapeDoc.TypeContext.DefaultPackage) + } + if len(shapeDoc.TypeContext.Imports) != 1 { + t.Fatalf("unexpected imports count: %d", len(shapeDoc.TypeContext.Imports)) + } +} + +func TestToIR_FromIR_TypeResolutionsRoundTrip(t *testing.T) { + doc := &Document{ + Root: map[string]any{ + "Routes": []any{}, + "Resource": map[string]any{}, + }, + TypeResolutions: []typectx.Resolution{ + { + Expression: "Order", + Target: "main.ID", + ResolvedKey: "github.com/acme/mdp/performance.Order", + MatchKind: "default_package", + Provenance: typectx.Provenance{ + Package: "github.com/acme/mdp/performance", + File: "/repo/mdp/performance/order.go", + Kind: "resource_type", + }, + }, + }, + } + irDoc, err := ToIR(doc) + if err != nil { + t.Fatalf("ToIR failed: %v", err) + } + shapeDoc, err := FromIR(irDoc) + if err != nil { + t.Fatalf("FromIR failed: %v", err) + } + if len(shapeDoc.TypeResolutions) != 1 { + t.Fatalf("unexpected type resolutions count: %d", len(shapeDoc.TypeResolutions)) + } + got := shapeDoc.TypeResolutions[0] + if got.ResolvedKey != "github.com/acme/mdp/performance.Order" || got.Provenance.Kind != "resource_type" { + t.Fatalf("unexpected resolution: %+v", got) + } +} diff --git a/repository/shape/dql/shape/model.go b/repository/shape/dql/shape/model.go index 52d1b5066..e975cfca3 100644 --- a/repository/shape/dql/shape/model.go +++ b/repository/shape/dql/shape/model.go @@ -69,6 +69,25 @@ type RouteDirective struct { Methods []string } +type Route struct { + Name string + URI string + Method string + ViewRef string + Description string +} + +type Resource struct { + Views []*View +} + +type View struct { + Name string + Table string + Module string + ConnectorRef string +} + // Document represents parsed DQL model used by shape compiler and xgen. type Document struct { Raw string @@ -76,6 +95,8 @@ type Document struct { Query *query.Select TypeContext *typectx.Context Directives *Directives + Routes []*Route + Resource *Resource Root map[string]any TypeResolutions []typectx.Resolution Diagnostics []*Diagnostic diff --git a/repository/shape/load/loader.go b/repository/shape/load/loader.go index 55528601b..11f93d393 100644 --- a/repository/shape/load/loader.go +++ b/repository/shape/load/loader.go @@ -122,11 +122,13 @@ func buildComponent(source *shape.Source, pResult *plan.Result) *Component { if item == nil { continue } - if strings.TrimSpace(item.Kind) == "" && strings.TrimSpace(item.In) == "" { + kind := strings.ToLower(item.KindString()) + inName := item.InName() + if kind == "" && inName == "" { ret.Other = append(ret.Other, item) continue } - switch strings.ToLower(item.Kind) { + switch kind { case "query", "path", "header", "body", "form", "cookie", "request", "": ret.Input = append(ret.Input, item) case "output": diff --git a/repository/shape/plan/model.go b/repository/shape/plan/model.go index e9575b9b2..ffb295f28 100644 --- a/repository/shape/plan/model.go +++ b/repository/shape/plan/model.go @@ -3,9 +3,11 @@ package plan import ( "embed" "reflect" + "strings" dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view/state" ) // Result is normalized shape plan produced from scan descriptors. @@ -130,35 +132,21 @@ type RelationLink struct { // State is a normalized parameter field plan. type State struct { - Path string - Name string - Kind string - In string - Codec string - CodecArgs []string - QuerySelector string - When string - Scope string - DataType string - OutputDataType string - Value string - Required *bool - Async bool - Cacheable *bool - With string - URI string - ErrorCode int - ErrorMessage string - Predicates []*StatePredicate - - TagType reflect.Type - EffectiveType reflect.Type + state.Parameter `yaml:",inline"` + QuerySelector string + OutputDataType string } -// StatePredicate captures parameter predicate semantics from DQL declarations. -type StatePredicate struct { - Group int - Name string - Ensure bool - Arguments []string +func (s *State) KindString() string { + if s == nil || s.In == nil { + return "" + } + return strings.TrimSpace(string(s.In.Kind)) +} + +func (s *State) InName() string { + if s == nil || s.In == nil { + return "" + } + return strings.TrimSpace(s.In.Name) } diff --git a/repository/shape/plan/planner.go b/repository/shape/plan/planner.go index 2c3735dc9..8e85c027d 100644 --- a/repository/shape/plan/planner.go +++ b/repository/shape/plan/planner.go @@ -11,6 +11,7 @@ import ( outputkeys "github.com/viant/datly/repository/locator/output/keys" "github.com/viant/datly/repository/shape" "github.com/viant/datly/repository/shape/scan" + "github.com/viant/datly/view/state" ) // Planner normalizes scan descriptors into shape plan. @@ -163,35 +164,46 @@ func splitTagSelector(value string) (string, string, string) { } func normalizeState(field *scan.Field) *State { - result := &State{Path: field.Path, TagType: field.Type} + result := &State{ + Parameter: state.Parameter{ + Name: field.Name, + In: &state.Location{}, + }, + } if field.StateTag == nil || field.StateTag.Parameter == nil { - result.Name = field.Name - result.EffectiveType = field.Type + result.Schema = state.NewSchema(field.Type) return result } pTag := field.StateTag.Parameter result.Name = firstNonEmpty(pTag.Name, field.Name) - result.Kind = strings.ToLower(strings.TrimSpace(pTag.Kind)) - result.In = strings.TrimSpace(pTag.In) + result.In = &state.Location{ + Kind: state.Kind(strings.ToLower(strings.TrimSpace(pTag.Kind))), + Name: strings.TrimSpace(pTag.In), + } result.When = pTag.When result.Scope = pTag.Scope - result.DataType = pTag.DataType result.Required = pTag.Required result.Async = pTag.Async result.Cacheable = pTag.Cacheable result.With = pTag.With result.URI = pTag.URI - result.ErrorCode = pTag.ErrorCode + result.ErrorStatusCode = pTag.ErrorCode result.ErrorMessage = pTag.ErrorMessage - result.EffectiveType = resolveStateType(result, field.Type) + result.Schema = state.NewSchema(resolveStateType(result, field.Type)) + if dataType := strings.TrimSpace(pTag.DataType); dataType != "" { + result.Schema.DataType = dataType + } return result } func resolveStateType(item *State, fallback reflect.Type) reflect.Type { - key := strings.ToLower(strings.TrimSpace(firstNonEmpty(item.In, item.Name))) - switch item.Kind { + if item.In == nil { + return fallback + } + key := strings.ToLower(strings.TrimSpace(firstNonEmpty(item.In.Name, item.Name))) + switch strings.ToLower(strings.TrimSpace(string(item.In.Kind))) { case "output": if rType, ok := outputkeys.Types[key]; ok { return rType diff --git a/repository/shape/plan/planner_test.go b/repository/shape/plan/planner_test.go index 7dc1edb1c..2b947a9d5 100644 --- a/repository/shape/plan/planner_test.go +++ b/repository/shape/plan/planner_test.go @@ -78,16 +78,15 @@ func TestPlanner_Plan(t *testing.T) { } require.NotNil(t, stateByPath["Status"]) - assert.Equal(t, outputkeys.Types["status"], stateByPath["Status"].EffectiveType) + assert.Equal(t, outputkeys.Types["status"], stateByPath["Status"].Schema.Type()) require.NotNil(t, stateByPath["Job"]) - assert.Equal(t, asynckeys.Types["job"], stateByPath["Job"].EffectiveType) + assert.Equal(t, asynckeys.Types["job"], stateByPath["Job"].Schema.Type()) require.NotNil(t, stateByPath["VName"]) - assert.Equal(t, metakeys.Types["view.name"], stateByPath["VName"].EffectiveType) + assert.Equal(t, metakeys.Types["view.name"], stateByPath["VName"].Schema.Type()) require.NotNil(t, stateByPath["ID"]) - assert.Equal(t, "query", stateByPath["ID"].Kind) - assert.Equal(t, "id", stateByPath["ID"].In) - assert.Equal(t, stateByPath["ID"].TagType, stateByPath["ID"].EffectiveType) + assert.Equal(t, "query", stateByPath["ID"].KindString()) + assert.Equal(t, "id", stateByPath["ID"].InName()) } func TestPlanner_Plan_LinkOnProducesStructuredRelations(t *testing.T) { diff --git a/repository/shape/platform_parity_test.go b/repository/shape/platform_parity_test.go index 17fa1f1f4..b0f2832f4 100644 --- a/repository/shape/platform_parity_test.go +++ b/repository/shape/platform_parity_test.go @@ -2,6 +2,7 @@ package shape_test import ( "context" + "fmt" "os" "path/filepath" "regexp" @@ -1303,19 +1304,19 @@ func normalizeShapeParams(planned *plan.Result) []paramIR { } item := paramIR{ Name: strings.TrimSpace(s.Name), - Kind: strings.TrimSpace(s.Kind), - In: strings.TrimSpace(s.In), + Kind: strings.TrimSpace(s.KindString()), + In: strings.TrimSpace(s.InName()), Required: s.Required, Cacheable: s.Cacheable, URI: strings.TrimSpace(s.URI), - Value: strings.TrimSpace(s.Value), + Value: strings.TrimSpace(fmt.Sprint(s.Value)), QuerySelector: strings.TrimSpace(s.QuerySelector), } for _, pred := range s.Predicates { if pred == nil { continue } - item.Predicates = append(item.Predicates, normalizePredicateSig(pred.Group, pred.Name, pred.Ensure, pred.Arguments)) + item.Predicates = append(item.Predicates, normalizePredicateSig(pred.Group, pred.Name, pred.Ensure, pred.Args)) } sort.Strings(item.Predicates) result = append(result, item) diff --git a/view/resource.go b/view/resource.go index 94cc59435..b1867d19e 100644 --- a/view/resource.go +++ b/view/resource.go @@ -578,7 +578,12 @@ func LoadResourceFromURL(ctx context.Context, URL string, fs afs.Service) (*Reso resource := &Resource{} err = toolbox.DefaultConverter.AssignConverted(resource, aMap) if err != nil { - return nil, err + if docs, ok := parseDocumentationOnlyResource(aMap); ok { + resource.Docs = &Documentation{Docs: docs} + err = nil + } else { + return nil, err + } } resource.fs = fs resource.SourceURL = URL @@ -586,6 +591,76 @@ func LoadResourceFromURL(ctx context.Context, URL string, fs afs.Service) (*Reso return resource, err } +func parseDocumentationOnlyResource(source map[string]interface{}) (*state.Docs, bool) { + if len(source) == 0 { + return nil, false + } + + // Route-like YAML should never be treated as dependency docs. + for _, key := range []string{"Routes", "Method", "URI", "Input", "Output", "View", "Handler", "Resource"} { + if _, ok := source[key]; ok { + return nil, false + } + } + + // If clear resource keys exist, keep regular conversion semantics. + for _, key := range []string{"Connectors", "Views", "Types", "Substitutes", "MessageBuses", "CacheProviders", "Loggers", "Predicates", "Docs", "FSEmbedder", "Imports"} { + if _, ok := source[key]; ok { + return nil, false + } + } + + ret := &state.Docs{} + found := false + for _, section := range []struct { + name string + dest *state.Documentation + }{ + {name: "Parameters", dest: &ret.Parameters}, + {name: "Columns", dest: &ret.Columns}, + {name: "Paths", dest: &ret.Paths}, + {name: "Filter", dest: &ret.Filter}, + } { + raw, ok := source[section.name] + if !ok { + continue + } + doc, ok := asDocumentation(raw) + if !ok { + return nil, false + } + *section.dest = doc + found = true + } + if !found { + return nil, false + } + return ret, true +} + +func asDocumentation(raw interface{}) (state.Documentation, bool) { + aMap, ok := raw.(map[string]interface{}) + if !ok { + return nil, false + } + ret := state.Documentation{} + for key, value := range aMap { + switch actual := value.(type) { + case string: + ret[key] = actual + case map[string]interface{}: + nested, ok := asDocumentation(actual) + if !ok { + return nil, false + } + ret[key] = nested + default: + return nil, false + } + } + return ret, true +} + func (r *Resource) FindConnector(view *View) (*Connector, error) { if view.Connector == nil { var connector *Connector From 85f81b5278498b46ecfcdd83d7dabb827f2b1a97 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 27 Feb 2026 12:05:17 -0800 Subject: [PATCH 148/279] - stabilize e2e - refactor planner.State --- cmd/command/translate_shape.go | 36 +- cmd/command/translate_shape_ir.go | 36 +- cmd/command/translate_shape_xgen.go | 100 ++++ e2e/local/regression/regression.yaml | 2 +- repository/shape/compile/compiler.go | 99 ++-- repository/shape/compile/compiler_test.go | 54 +- repository/shape/compile/component_types.go | 28 +- repository/shape/compile/enrich.go | 521 +----------------- repository/shape/compile/enrich_table.go | 331 +++++++++++ repository/shape/compile/enrich_text.go | 212 +++++++ repository/shape/compile/pipeline/read.go | 339 ++---------- .../shape/compile/pipeline/read_normalize.go | 266 +++++++++ .../shape/compile/preprocess_handler.go | 10 - repository/shape/compile/statedecl.go | 6 +- repository/shape/compile/type_support_test.go | 4 +- repository/shape/dql/load/loader.go | 6 +- repository/shape/dql_engine_test.go | 4 +- repository/shape/load/columns.go | 92 ++++ repository/shape/load/loader.go | 200 +++++-- repository/shape/load/loader_test.go | 16 +- repository/shape/load/model.go | 24 +- repository/shape/model.go | 29 +- repository/shape/parity_test.go | 2 +- repository/shape/plan/planner.go | 19 +- repository/shape/plan/planner_test.go | 16 +- repository/shape/plan/spec.go | 16 + repository/shape/platform_parity_test.go | 2 +- repository/shape/scan/scanner.go | 5 +- repository/shape/scan/scanner_test.go | 4 +- repository/shape/scan/spec.go | 16 + 30 files changed, 1513 insertions(+), 982 deletions(-) create mode 100644 cmd/command/translate_shape_xgen.go create mode 100644 repository/shape/compile/enrich_table.go create mode 100644 repository/shape/compile/enrich_text.go create mode 100644 repository/shape/compile/pipeline/read_normalize.go create mode 100644 repository/shape/load/columns.go create mode 100644 repository/shape/plan/spec.go create mode 100644 repository/shape/scan/spec.go diff --git a/cmd/command/translate_shape.go b/cmd/command/translate_shape.go index 3a21db19d..fa12d8f03 100644 --- a/cmd/command/translate_shape.go +++ b/cmd/command/translate_shape.go @@ -18,6 +18,7 @@ import ( shapeLoad "github.com/viant/datly/repository/shape/load" "github.com/viant/datly/shared" "github.com/viant/datly/view" + "github.com/viant/datly/view/state" "gopkg.in/yaml.v3" ) @@ -51,8 +52,8 @@ func (s *Service) translateShape(ctx context.Context, opts *options.Options) err if err != nil { return fmt.Errorf("failed to load %s: %w", sourceURL, err) } - component, ok := componentArtifact.Component.(*shapeLoad.Component) - if !ok || component == nil { + component, ok := shapeLoad.ComponentFrom(componentArtifact) + if !ok { return fmt.Errorf("unexpected component artifact for %s", sourceURL) } if err = s.persistShapeRoute(ctx, opts, sourceURL, dql, componentArtifact.Resource, component); err != nil { @@ -105,6 +106,18 @@ func (s *Service) persistShapeRoute(ctx context.Context, opts *options.Options, rootView = resource.Views[0].Name } method, uri := parseShapeRulePath(dql, rule.RuleName(), opts.Repository().APIPrefix) + // Gap 3: RouteDirective overrides method/URI when explicitly declared in DQL. + if component != nil && component.Directives != nil && component.Directives.Route != nil { + rd := component.Directives.Route + if u := strings.TrimSpace(rd.URI); u != "" { + uri = u + } + if len(rd.Methods) > 0 { + if m := strings.TrimSpace(strings.ToUpper(rd.Methods[0])); m != "" { + method = m + } + } + } route := &repository.Component{ Path: contract.Path{ Method: method, @@ -123,6 +136,24 @@ func (s *Service) persistShapeRoute(ctx context.Context, opts *options.Options, route.DescriptionURI = strings.TrimSpace(component.Directives.MCP.DescriptionPath) } } + if component != nil && (len(component.Input) > 0 || len(component.Meta) > 0) { + params := make(state.Parameters, 0, len(component.Input)+len(component.Meta)) + for _, s := range component.Input { + if s != nil { + p := s.Parameter + params = append(params, &p) + } + } + for _, s := range component.Meta { + if s != nil { + p := s.Parameter + params = append(params, &p) + } + } + if len(params) > 0 { + route.Contract.Input.Type.Parameters = params + } + } payload := &shapeRuleFile{ Resource: resource, Routes: []*repository.Component{route}, @@ -137,6 +168,7 @@ func (s *Service) persistShapeRoute(ctx context.Context, opts *options.Options, if err = s.fs.Upload(ctx, routeYAML, file.DefaultFileOsMode, strings.NewReader(string(data))); err != nil { return fmt.Errorf("failed to persist route yaml %s: %w", routeYAML, err) } + generateShapeTypes(url.Path(sourceURL), payload, component) return nil } diff --git a/cmd/command/translate_shape_ir.go b/cmd/command/translate_shape_ir.go index f09f2e4e9..c2f37155f 100644 --- a/cmd/command/translate_shape_ir.go +++ b/cmd/command/translate_shape_ir.go @@ -19,6 +19,7 @@ import ( datlyservice "github.com/viant/datly/service" "github.com/viant/datly/shared" "github.com/viant/datly/view" + "github.com/viant/datly/view/state" "gopkg.in/yaml.v3" ) @@ -56,8 +57,8 @@ func (s *Service) translateShapeIR(ctx context.Context, opts *options.Options) e if err != nil { return fmt.Errorf("failed to load %s: %w", sourceURL, err) } - component, ok := componentArtifact.Component.(*shapeLoad.Component) - if !ok || component == nil { + component, ok := shapeLoad.ComponentFrom(componentArtifact) + if !ok { return fmt.Errorf("unexpected component artifact for %s", sourceURL) } @@ -86,6 +87,7 @@ func (s *Service) translateShapeIR(ctx context.Context, opts *options.Options) e if err = s.fs.Upload(ctx, irPath, file.DefaultFileOsMode, strings.NewReader(string(encoded))); err != nil { return fmt.Errorf("failed to persist route ir %s: %w", irPath, err) } + generateShapeTypes(url.Path(sourceURL), payload, component) } return nil } @@ -100,6 +102,18 @@ func buildShapeRulePayload(opts *options.Options, dql string, resource *view.Res rootView = resource.Views[0].Name } method, uri := parseShapeRulePath(dql, rule.RuleName(), opts.Repository().APIPrefix) + // Gap 3: RouteDirective overrides method/URI when explicitly declared in DQL. + if component != nil && component.Directives != nil && component.Directives.Route != nil { + rd := component.Directives.Route + if u := strings.TrimSpace(rd.URI); u != "" { + uri = u + } + if len(rd.Methods) > 0 { + if m := strings.TrimSpace(strings.ToUpper(rd.Methods[0])); m != "" { + method = m + } + } + } route := &repository.Component{ Path: contract.Path{ Method: method, @@ -118,6 +132,24 @@ func buildShapeRulePayload(opts *options.Options, dql string, resource *view.Res route.DescriptionURI = strings.TrimSpace(component.Directives.MCP.DescriptionPath) } } + if component != nil && (len(component.Input) > 0 || len(component.Meta) > 0) { + params := make(state.Parameters, 0, len(component.Input)+len(component.Meta)) + for _, s := range component.Input { + if s != nil { + p := s.Parameter + params = append(params, &p) + } + } + for _, s := range component.Meta { + if s != nil { + p := s.Parameter + params = append(params, &p) + } + } + if len(params) > 0 { + route.Contract.Input.Type.Parameters = params + } + } payload := &shapeRuleFile{ Resource: resource, Routes: []*repository.Component{route}, diff --git a/cmd/command/translate_shape_xgen.go b/cmd/command/translate_shape_xgen.go new file mode 100644 index 000000000..3827ac75f --- /dev/null +++ b/cmd/command/translate_shape_xgen.go @@ -0,0 +1,100 @@ +package command + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + dqlir "github.com/viant/datly/repository/shape/dql/ir" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + shapeLoad "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/repository/shape/xgen" + "gopkg.in/yaml.v3" +) + +// generateShapeTypes emits a Go type file (shapes_gen.go) for the compiled +// component. It is a best-effort step: any failure is logged as a warning so +// the route YAML is still written successfully. +// +// Normal flow — types already exist in pkg/shapes_gen.go: +// +// xgen merges the file, updating only the types produced by this DQL. +// +// Backfill flow — no types file yet: +// +// xgen generates stub types from the statically-inferred columns +// (explicit SELECT columns give accurate field names/types; SELECT * +// produces a minimal stub that the user should refine or regenerate +// after DB discovery). +func generateShapeTypes(sourceAbsPath string, payload *shapeRuleFile, component *shapeLoad.Component) { + if component == nil || component.TypeContext == nil { + return + } + ctx := component.TypeContext + if strings.TrimSpace(ctx.PackageDir) == "" { + return + } + + projectDir := findProjectDir(sourceAbsPath) + if projectDir == "" { + fmt.Printf("WARNING: shape xgen: cannot locate go.mod from %s, skipping type generation\n", sourceAbsPath) + return + } + + packageDir := strings.TrimSpace(ctx.PackageDir) + if !filepath.IsAbs(packageDir) { + packageDir = filepath.Join(projectDir, packageDir) + } + + data, err := yaml.Marshal(payload) + if err != nil { + fmt.Printf("WARNING: shape xgen: marshal failed for %s: %v\n", sourceAbsPath, err) + return + } + doc, err := dqlir.FromYAML(data) + if err != nil { + fmt.Printf("WARNING: shape xgen: IR parse failed for %s: %v\n", sourceAbsPath, err) + return + } + + shapeDoc := buildShapeDocument(doc, ctx) + cfg := &xgen.Config{ + ProjectDir: projectDir, + PackageDir: packageDir, + PackageName: strings.TrimSpace(ctx.PackageName), + PackagePath: strings.TrimSpace(ctx.PackagePath), + } + + result, err := xgen.GenerateFromDQLShape(shapeDoc, cfg) + if err != nil { + fmt.Printf("WARNING: shape xgen: type generation skipped for %s: %v\n", filepath.Base(sourceAbsPath), err) + return + } + fmt.Printf("generated types %s → %s\n", strings.Join(result.Types, ", "), result.FilePath) +} + +// buildShapeDocument bridges an ir.Document into the shape.Document expected by xgen. +func buildShapeDocument(doc *dqlir.Document, ctx *typectx.Context) *dqlshape.Document { + return &dqlshape.Document{ + Root: doc.Root, + TypeContext: ctx, + } +} + +// findProjectDir walks up from sourcePath until it finds a directory containing +// go.mod, returning that directory. Returns "" when no go.mod is found. +func findProjectDir(sourcePath string) string { + dir := filepath.Dir(filepath.Clean(sourcePath)) + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + return "" + } + dir = parent + } +} diff --git a/e2e/local/regression/regression.yaml b/e2e/local/regression/regression.yaml index c21e04c34..a8b575fae 100644 --- a/e2e/local/regression/regression.yaml +++ b/e2e/local/regression/regression.yaml @@ -30,7 +30,7 @@ pipeline: '[]gen': '@gen' subPath: 'cases/${index}_*' - range: 1..010 + range: 11..020 template: checkSkip: action: nop diff --git a/repository/shape/compile/compiler.go b/repository/shape/compile/compiler.go index d283d7c50..6fb6eadc0 100644 --- a/repository/shape/compile/compiler.go +++ b/repository/shape/compile/compiler.go @@ -40,33 +40,69 @@ func (e *CompileError) Error() string { } // Compile implements shape.DQLCompiler. -func (c *DQLCompiler) Compile(_ context.Context, source *shape.Source, opts ...shape.CompileOption) (*shape.PlanResult, error) { +func (c *DQLCompiler) Compile(ctx context.Context, source *shape.Source, opts ...shape.CompileOption) (*shape.PlanResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } if source == nil { return nil, shape.ErrNilSource } compileOptions := applyCompileOptions(opts) pathLayout := newCompilePathLayout(compileOptions) - compileProfile := normalizeCompileProfile(compileOptions.Profile) - enforceStrict := compileOptions.Strict || compileProfile == shape.CompileProfileStrict - if strings.TrimSpace(source.DQL) == "" { - return nil, shape.ErrNilDQL + enforceStrict := compileOptions.Strict || normalizeCompileProfile(compileOptions.Profile) == shape.CompileProfileStrict + + prepared, allDiags, err := c.preprocessSource(source, compileOptions, pathLayout, enforceStrict) + if err != nil { + return nil, err + } + + root, compileDiags, err := c.compileRoot( + source.Name, prepared.Pre.SQL, prepared.Statements, prepared.Decision, + compileOptions.MixedMode, compileOptions.UnknownNonReadMode, + ) + if err != nil { + return nil, err + } + prepared.Pre.Mapper.Remap(compileDiags) + allDiags = append(allDiags, compileDiags...) + if root == nil { + return nil, &CompileError{Diagnostics: allDiags} } + result := c.assembleResult(source, root, prepared, compileOptions, pathLayout, allDiags) + if enforceStrict && hasEscalationWarnings(result.Diagnostics) { + return nil, &CompileError{Diagnostics: filterEscalationDiagnostics(result.Diagnostics)} + } + if hasErrorDiagnostics(result.Diagnostics) { + return nil, &CompileError{Diagnostics: result.Diagnostics} + } + return &shape.PlanResult{Source: source, Plan: result}, nil +} + +// preprocessSource runs DQL preprocessing (type context, directives, handler +// detection) and returns a ready-to-compile prepared result with accumulated +// diagnostics. Returns an error only for fatal early failures. +func (c *DQLCompiler) preprocessSource( + source *shape.Source, + compileOptions *shape.CompileOptions, + pathLayout compilePathLayout, + enforceStrict bool, +) (*handlerPreprocessResult, []*dqlshape.Diagnostic, error) { + if strings.TrimSpace(source.DQL) == "" { + return nil, nil, shape.ErrNilDQL + } pre := dqlpre.Prepare(source.DQL) pre.TypeCtx = applyTypeContextDefaults(pre.TypeCtx, source, compileOptions, pathLayout) pre.Diagnostics = append(pre.Diagnostics, typeContextDiagnostics(pre.TypeCtx, enforceStrict)...) allDiags := append([]*dqlshape.Diagnostic{}, pre.Diagnostics...) if hasErrorDiagnostics(allDiags) { - return nil, &CompileError{Diagnostics: allDiags} + return nil, nil, &CompileError{Diagnostics: allDiags} } statements := dqlstmt.New(pre.SQL) decision := pipeline.Classify(statements) prepared := buildHandlerIfNeeded(source, pre, statements, decision, pathLayout) - pre = prepared.Pre - statements = prepared.Statements - decision = prepared.Decision - if strings.TrimSpace(pre.SQL) == "" { + if strings.TrimSpace(prepared.Pre.SQL) == "" { allDiags = append(allDiags, &dqlshape.Diagnostic{ Code: dqldiag.CodeParseEmpty, Severity: dqlshape.SeverityError, @@ -77,43 +113,36 @@ func (c *DQLCompiler) Compile(_ context.Context, source *shape.Source, opts ...s End: dqlshape.Position{Line: 1, Char: 1}, }, }) - return nil, &CompileError{Diagnostics: allDiags} - } - var root *plan.View - var compileDiags []*dqlshape.Diagnostic - var err error - root, compileDiags, err = c.compileRoot(source.Name, pre.SQL, statements, decision, compileOptions.MixedMode, compileOptions.UnknownNonReadMode) - if err != nil { - return nil, err - } - pre.Mapper.Remap(compileDiags) - allDiags = append(allDiags, compileDiags...) - if root == nil { - return nil, &CompileError{Diagnostics: allDiags} + return nil, nil, &CompileError{Diagnostics: allDiags} } + return prepared, allDiags, nil +} +// assembleResult builds the plan.Result from the compiled root view, attaches +// declared relations/views/states, applies enrichment, and computes the final +// column-discovery policy diagnostics. +func (c *DQLCompiler) assembleResult( + source *shape.Source, + root *plan.View, + prepared *handlerPreprocessResult, + compileOptions *shape.CompileOptions, + pathLayout compilePathLayout, + diags []*dqlshape.Diagnostic, +) *plan.Result { result := newPlanResult(root) - result.Diagnostics = allDiags - result.TypeContext = pre.TypeCtx - result.Directives = pre.Directives + result.Diagnostics = diags + result.TypeContext = prepared.Pre.TypeCtx + result.Directives = prepared.Pre.Directives applyDefaultConnectorDirective(result) hints := extractViewHints(source.DQL) appendRelationViews(result, root, hints) appendDeclaredViews(source.DQL, result) appendDeclaredStates(source.DQL, result) - _ = prepared applyViewHints(result, hints) applySourceParityEnrichmentWithLayout(result, source, pathLayout) applyLinkedTypeSupport(result, source) result.Diagnostics = append(result.Diagnostics, applyColumnDiscoveryPolicy(result, compileOptions)...) - - if enforceStrict && hasEscalationWarnings(result.Diagnostics) { - return nil, &CompileError{Diagnostics: filterEscalationDiagnostics(result.Diagnostics)} - } - if hasErrorDiagnostics(result.Diagnostics) { - return nil, &CompileError{Diagnostics: result.Diagnostics} - } - return &shape.PlanResult{Source: source, Plan: result}, nil + return result } func applyDefaultConnectorDirective(result *plan.Result) { diff --git a/repository/shape/compile/compiler_test.go b/repository/shape/compile/compiler_test.go index 456f99568..85b51108c 100644 --- a/repository/shape/compile/compiler_test.go +++ b/repository/shape/compile/compiler_test.go @@ -20,7 +20,7 @@ func TestDQLCompiler_Compile(t *testing.T) { require.NoError(t, err) require.NotNil(t, res) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.Len(t, planned.Views, 1) view := planned.Views[0] @@ -49,7 +49,7 @@ SELECT id require.NoError(t, err) require.NotNil(t, res) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.Len(t, planned.Views, 1) assert.Equal(t, "sample_report", planned.Views[0].Name) @@ -66,7 +66,7 @@ SELECT id FROM ORDERS t` require.NoError(t, err) require.NotNil(t, res) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotNil(t, planned.TypeContext) assert.Equal(t, "mdp/performance", planned.TypeContext.DefaultPackage) @@ -86,7 +86,7 @@ func TestDQLCompiler_Compile_PropagatesImportedTypeContextWithModuleNormalizatio } res, err := compiler.Compile(context.Background(), source) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotNil(t, planned.TypeContext) require.Len(t, planned.TypeContext.Imports, 1) @@ -112,7 +112,7 @@ SELECT id FROM ORDERS o ` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotNil(t, planned.Directives) assert.Equal(t, "docs/orders.md", planned.Directives.Meta) @@ -141,7 +141,7 @@ func TestDQLCompiler_Compile_ColumnDiscoveryAutoForWildcard(t *testing.T) { compiler := New() res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "SELECT * FROM ORDERS o"}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.True(t, planned.ColumnsDiscovery) require.NotEmpty(t, planned.Views) @@ -166,7 +166,7 @@ func TestDQLCompiler_Compile_TypeContextValidationWarnsInCompat(t *testing.T) { SELECT id FROM ORDERS t` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}, shape.WithTypeContextPackageName("bad/name")) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Diagnostics) assert.Equal(t, dqldiag.CodeTypeCtxInvalid, planned.Diagnostics[0].Code) @@ -211,7 +211,7 @@ func TestDQLCompiler_Compile_SyntaxError_RemapsAfterSanitize(t *testing.T) { require.NotEmpty(t, compileErr.Diagnostics) diagnostics = compileErr.Diagnostics } else { - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) diagnostics = planned.Diagnostics } @@ -263,7 +263,7 @@ func TestDQLCompiler_Compile_ExtractsJoinLinks(t *testing.T) { dql := "SELECT o.id, i.sku FROM orders o JOIN order_items i ON o.id = i.order_id" res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) root := planned.ViewsByName["o"] require.NotNil(t, root) @@ -281,7 +281,7 @@ func TestDQLCompiler_Compile_JoinDiagnostics(t *testing.T) { dql := "SELECT o.id FROM orders o JOIN order_items i ON o.id > i.order_id" res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Diagnostics) assert.Equal(t, dqldiag.CodeRelUnsupported, planned.Diagnostics[0].Code) @@ -325,7 +325,7 @@ func TestDQLCompiler_Compile_SQLInjectionDiagnostic(t *testing.T) { dql := "SELECT id FROM ORDERS t WHERE t.id = $Unsafe.Id" res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Diagnostics) assert.Equal(t, dqldiag.CodeSQLIRawSelector, planned.Diagnostics[0].Code) @@ -338,7 +338,7 @@ func TestDQLCompiler_Compile_SanitizesBindings(t *testing.T) { dql := "SELECT id FROM ORDERS t WHERE t.id = $Id" res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Views) assert.Contains(t, planned.Views[0].SQL, "$criteria.AppendBinding($Unsafe.Id)") @@ -351,7 +351,7 @@ func TestDQLCompiler_Compile_ParameterDerivedView(t *testing.T) { SELECT id FROM ORDERS t` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.Len(t, planned.Views, 2) extra := planned.ViewsByName["e"] @@ -367,7 +367,7 @@ func TestDQLCompiler_Compile_ParameterDerivedView_Options(t *testing.T) { SELECT id FROM ORDERS t` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) extra := planned.ViewsByName["e"] require.NotNil(t, extra) @@ -383,7 +383,7 @@ func TestDQLCompiler_Compile_ParameterDerivedView_MissingSQLHint(t *testing.T) { SELECT id FROM ORDERS t` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Diagnostics) assert.Equal(t, dqldiag.CodeViewMissingSQL, planned.Diagnostics[len(planned.Diagnostics)-1].Code) @@ -396,7 +396,7 @@ func TestDQLCompiler_Compile_ParameterDerivedView_InvalidCardinalityDiagnostic(t SELECT id FROM ORDERS t` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Diagnostics) assert.Equal(t, dqldiag.CodeViewCardinality, planned.Diagnostics[len(planned.Diagnostics)-1].Code) @@ -420,7 +420,7 @@ func TestDQLCompiler_Compile_DMLInsert(t *testing.T) { DQL: "INSERT INTO ORDERS(id) VALUES (1)", }) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.Len(t, planned.Views, 1) assert.Equal(t, "ORDERS", planned.Views[0].Table) @@ -478,7 +478,7 @@ func TestDQLCompiler_Compile_MixedReadExec_Warning(t *testing.T) { DQL: "SELECT id FROM ORDERS\nUPDATE ORDERS SET id = 2", }) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Diagnostics) assert.Equal(t, dqldiag.CodeDMLMixed, planned.Diagnostics[len(planned.Diagnostics)-1].Code) @@ -491,7 +491,7 @@ func TestDQLCompiler_Compile_MixedMode_ExecWins(t *testing.T) { DQL: "SELECT o.id FROM ORDERS o\nUPDATE ORDERS SET id = 2", }, shape.WithMixedMode(shape.CompileMixedModeExecWins)) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Views) assert.Equal(t, "ORDERS", planned.Views[0].Table) @@ -506,7 +506,7 @@ func TestDQLCompiler_Compile_MixedMode_ReadWins(t *testing.T) { DQL: "SELECT o.id FROM ORDERS o\nUPDATE ORDERS SET id = 2", }, shape.WithMixedMode(shape.CompileMixedModeReadWins)) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Views) assert.Equal(t, "o", planned.Views[0].Name) @@ -538,7 +538,7 @@ func TestDQLCompiler_Compile_UnknownNonRead_Warn(t *testing.T) { DQL: "$Foo.Bar($x)", }) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Diagnostics) var found *dqlshape.Diagnostic @@ -603,7 +603,7 @@ func TestDQLCompiler_Compile_UnknownNonRead_UsesGeneratedCompanion(t *testing.T) compiler := New() res, err := compiler.Compile(context.Background(), source) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotNil(t, planned.ViewsByName["o"]) require.NotNil(t, planned.ViewsByName["i"]) @@ -663,7 +663,7 @@ func TestDQLCompiler_Compile_HandlerNop_NoSQLiEscalation(t *testing.T) { DQL: "$Nop($Unsafe.Id)", }, shape.WithCompileStrict(true)) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) for _, item := range planned.Diagnostics { if item == nil { @@ -685,7 +685,7 @@ JOIN (SELECT * FROM session/attributes) attribute ON attribute.user_id = session ` res, err := compiler.Compile(context.Background(), &shape.Source{Name: "system/session", DQL: dql}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) root := planned.ViewsByName["session"] require.NotNil(t, root) @@ -732,7 +732,7 @@ func TestDQLCompiler_Compile_GeneratedHandler_NoBodyInput_DoesNotLoadLegacyContr compiler := New() res, err := compiler.Compile(context.Background(), &shape.Source{Name: "delete", Path: genPath, DQL: `/* {"Method":"DELETE","URI":"/v1/api/system/upload"} */`}) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Views) @@ -797,7 +797,7 @@ func TestDQLCompiler_Compile_HandlerLegacyTypes_NotLoadedFromLegacyRouteYAML(t * DQL: `/* {"URI":"/v1/api/platform/campaign","Method":"POST","Type":"campaign/patch.Handler"} */`, }) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) assert.Empty(t, planned.Types) @@ -829,7 +829,7 @@ func TestDQLCompiler_Compile_CustomPathLayout_NoLegacyHandlerFallback(t *testing DQL: `/* {"URI":"/v1/api/platform/campaign","Method":"POST","Type":"campaign/patch.Handler","Connector":"ci_ads"} */`, }, shape.WithDQLPathMarker("sqlsrc"), shape.WithRoutesRelativePath("config/routes")) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Views) assert.Equal(t, "post", planned.Views[0].Name) diff --git a/repository/shape/compile/component_types.go b/repository/shape/compile/component_types.go index acb9c87b8..c7c8f22db 100644 --- a/repository/shape/compile/component_types.go +++ b/repository/shape/compile/component_types.go @@ -48,7 +48,7 @@ func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, l } for _, stateItem := range result.States { - if stateItem == nil || !strings.EqualFold(stateItem.KindString(), "component") { + if stateItem == nil || state.Kind(strings.ToLower(stateItem.KindString())) != state.KindComponent { continue } ref := stateItem.InName() @@ -77,11 +77,7 @@ func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, l } } - names := make([]string, 0, len(collector.typesByName)) - for name := range collector.typesByName { - names = append(names, name) - } - sort.Strings(names) + sort.Strings(collector.typeOrder) existing := map[string]bool{} reportedCollision := map[string]bool{} for _, item := range result.Types { @@ -90,7 +86,7 @@ func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, l } existing[strings.ToLower(strings.TrimSpace(item.Name))] = true } - for _, name := range names { + for _, name := range collector.typeOrder { keyName := strings.ToLower(strings.TrimSpace(name)) if existing[keyName] { if !reportedCollision[keyName] { @@ -116,10 +112,13 @@ type componentCollector struct { routesRoot string visited map[string]componentVisitState outputByRoute map[string]string - typesByName map[string]*plan.Type - payloadCache map[string]routePayloadLookup - reportedDiag map[string]bool - diags []*dqlshape.Diagnostic + // typesByName provides O(1) dedup; typeOrder tracks insertion sequence + // so the final list can be sorted once rather than extracted from the map. + typesByName map[string]*plan.Type + typeOrder []string + payloadCache map[string]routePayloadLookup + reportedDiag map[string]bool + diags []*dqlshape.Diagnostic } type routePayloadLookup struct { @@ -175,6 +174,7 @@ func (c *componentCollector) collect(namespace string, span dqlshape.Span, requi if _, exists := c.typesByName[keyName]; exists { continue } + c.typeOrder = append(c.typeOrder, keyName) c.typesByName[keyName] = &plan.Type{ Name: name, Alias: strings.TrimSpace(item.Alias), @@ -189,7 +189,7 @@ func (c *componentCollector) collect(namespace string, span dqlshape.Span, requi c.outputByRoute[key] = outputType for _, param := range payload.Resource.Parameters { - if !strings.EqualFold(strings.TrimSpace(param.In.Kind), "component") { + if !strings.EqualFold(strings.TrimSpace(param.In.Kind), string(state.KindComponent)) { continue } nextNS := resolveComponentNamespaceFromRoute(strings.TrimSpace(param.In.Name), namespace) @@ -449,7 +449,7 @@ func routeOutputType(payload *routePayload) string { } } for _, param := range payload.Resource.Parameters { - if strings.EqualFold(strings.TrimSpace(param.In.Kind), "output") { + if strings.EqualFold(strings.TrimSpace(param.In.Kind), string(state.KindOutput)) { if dataType := strings.TrimSpace(param.Schema.DataType); dataType != "" { return dataType } @@ -462,7 +462,7 @@ func routeOutputType(payload *routePayload) string { } } for _, item := range payload.Resource.Types { - if strings.EqualFold(strings.TrimSpace(item.Name), "output") { + if strings.EqualFold(strings.TrimSpace(item.Name), string(state.KindOutput)) { if dataType := strings.TrimSpace(item.DataType); dataType != "" { return dataType } diff --git a/repository/shape/compile/enrich.go b/repository/shape/compile/enrich.go index 56086563f..3ea8a277b 100644 --- a/repository/shape/compile/enrich.go +++ b/repository/shape/compile/enrich.go @@ -1,8 +1,11 @@ package compile +// enrich.go — per-view enrichment passes applied after DQL compilation. +// Table inference helpers live in enrich_table.go; low-level text scanning +// primitives live in enrich_text.go. + import ( "encoding/json" - "os" "path/filepath" "strings" @@ -278,319 +281,8 @@ func toExportedTypeName(name string) string { return b.String() } -func shouldInferTable(item *plan.View) bool { - if item == nil { - return false - } - name := strings.TrimSpace(item.Name) - table := strings.TrimSpace(item.Table) - if table == "" { - return true - } - if strings.HasPrefix(table, "(") { - return true - } - if normalizedTemplatePlaceholderTable(table) { - return true - } - return strings.EqualFold(name, table) -} - -func normalizedTemplatePlaceholderTable(table string) bool { - if table == "" { - return false - } - parts := strings.Split(table, ".") - if len(parts) < 3 { - return false - } - for i := 0; i < len(parts)-1; i++ { - part := strings.TrimSpace(parts[i]) - if part == "" { - return false - } - for _, ch := range part { - if ch < '0' || ch > '9' { - return false - } - } - } - return true -} - -func inferTableFromSQL(sqlText string, source *shape.Source) string { - sqlText = strings.TrimSpace(sqlText) - if sqlText == "" { - return "" - } - if expr := topLevelFromExpr(sqlText); expr != "" { - if table := tableFromFromExpr(expr, source); table != "" { - return table - } - } - if table := pipeline.InferTableFromSQL(sqlText); table != "" { - if !strings.EqualFold(table, "DQLView") { - return table - } - } - if table := inferFromEmbeddedSQL(sqlText, source); table != "" { - return table - } - return "" -} - -func inferFromEmbeddedSQL(sqlText string, source *shape.Source) string { - ref, ok := findFirstEmbedRef(sqlText) - if !ok { - return "" - } - ref = strings.Trim(ref, `"'`) - if ref == "" { - return "" - } - resolved := resolveEmbedPath(source, ref) - if resolved == "" { - return "" - } - embedded, err := os.ReadFile(resolved) - if err != nil { - return "" - } - queryNode, _, err := pipeline.ParseSelectWithDiagnostic(string(embedded)) - if err != nil || queryNode == nil { - if table := pipeline.InferTableFromSQL(string(embedded)); table != "" && !strings.EqualFold(table, "DQLView") { - return strings.Trim(table, "`\"") - } - return "" - } - _, table, err := pipeline.InferRoot(queryNode, "") - if err != nil || strings.TrimSpace(table) == "" { - return "" - } - if strings.EqualFold(strings.TrimSpace(table), "DQLView") { - return "" - } - return strings.Trim(table, "`\"") -} - -func resolveEmbedPath(source *shape.Source, ref string) string { - if filepath.IsAbs(ref) { - return ref - } - if source == nil || strings.TrimSpace(source.Path) == "" { - return "" - } - base := source.Path - if fi, err := os.Stat(base); err == nil && fi.IsDir() { - return filepath.Clean(filepath.Join(base, ref)) - } - return filepath.Clean(filepath.Join(filepath.Dir(base), ref)) -} - -func inferTableFromSiblingSQL(viewName string, source *shape.Source) string { - viewName = strings.TrimSpace(viewName) - if viewName == "" || source == nil || strings.TrimSpace(source.Path) == "" { - return "" - } - sibling := filepath.Join(filepath.Dir(source.Path), viewName+".sql") - data, err := os.ReadFile(sibling) - if err != nil { - sibling = filepath.Join(filepath.Dir(source.Path), strings.ToLower(viewName)+".sql") - data, err = os.ReadFile(sibling) - } - if err != nil { - return "" - } - return inferTableFromSQL(string(data), source) -} - -func inferTableFromEmbedRef(source *shape.Source, ref string) string { - ref = strings.Trim(strings.TrimSpace(ref), `"'`) - if ref == "" { - return "" - } - resolved := resolveEmbedPath(source, ref) - if resolved == "" { - return "" - } - data, err := os.ReadFile(resolved) - if err != nil { - return "" - } - return pipeline.InferTableFromSQL(string(data)) -} - -func topLevelFromExpr(sqlText string) string { - lower := strings.ToLower(sqlText) - depth := 0 - inSingle := false - inDouble := false - inBacktick := false - for i := 0; i < len(sqlText); i++ { - ch := sqlText[i] - switch ch { - case '\'': - if !inDouble && !inBacktick { - inSingle = !inSingle - } - case '"': - if !inSingle && !inBacktick { - inDouble = !inDouble - } - case '`': - if !inSingle && !inDouble { - inBacktick = !inBacktick - } - case '(': - if !inSingle && !inDouble && !inBacktick { - depth++ - } - case ')': - if !inSingle && !inDouble && !inBacktick && depth > 0 { - depth-- - } - } - if inSingle || inDouble || inBacktick || depth != 0 { - continue - } - if i+6 > len(sqlText) { - break - } - if lower[i:i+4] != "from" { - continue - } - if i > 0 { - prev := lower[i-1] - if (prev >= 'a' && prev <= 'z') || (prev >= '0' && prev <= '9') || prev == '_' { - continue - } - } - j := i + 4 - for j < len(sqlText) && (sqlText[j] == ' ' || sqlText[j] == '\n' || sqlText[j] == '\t' || sqlText[j] == '\r') { - j++ - } - if j >= len(sqlText) { - return "" - } - if sqlText[j] == '(' { - start := j - d := 0 - for ; j < len(sqlText); j++ { - if sqlText[j] == '(' { - d++ - } else if sqlText[j] == ')' { - d-- - if d == 0 { - j++ - break - } - } - } - for j < len(sqlText) && (sqlText[j] == ' ' || sqlText[j] == '\n' || sqlText[j] == '\t' || sqlText[j] == '\r') { - j++ - } - for j < len(sqlText) { - c := sqlText[j] - if !(c == '_' || c == '.' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { - break - } - j++ - } - return strings.TrimSpace(sqlText[start:j]) - } - start := j - for j < len(sqlText) { - c := sqlText[j] - if !(c == '_' || c == '.' || c == '/' || c == '{' || c == '}' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '$') { - break - } - j++ - } - return strings.TrimSpace(sqlText[start:j]) - } - return "" -} - -func tableFromFromExpr(fromExpr string, source *shape.Source) string { - fromExpr = strings.TrimSpace(fromExpr) - if fromExpr == "" { - return "" - } - if strings.HasPrefix(fromExpr, "(") { - if table := inferFromEmbeddedSQL(fromExpr, source); table != "" { - return table - } - inner := fromExpr - if idx := strings.LastIndex(inner, ")"); idx > 0 { - inner = strings.TrimSpace(inner[1:idx]) - } - return inferTableFromSQL(inner, source) - } - return strings.Trim(fromExpr, "`\"") -} - -func inferConnector(item *plan.View, source *shape.Source) string { - if item == nil { - return "" - } - path := "" - if source != nil { - path = strings.ToLower(strings.ReplaceAll(source.Path, "\\", "/")) - } - table := strings.ToUpper(strings.TrimSpace(item.Table)) - switch { - case strings.Contains(path, "/dql/system/"): - return "system" - case strings.HasPrefix(table, "CI_") || strings.Contains(table, ".CI_"): - return "ci_ads" - case strings.Contains(path, "/dql/ui/"): - return "sitemgmt" - case strings.Contains(table, "SITE"): - return "sitemgmt" - default: - return "" - } -} - -func normalizeRootViewName(result *plan.Result, sourceName string) { - if result == nil || len(result.Views) == 0 { - return - } - root := result.Views[0] - if root == nil { - return - } - desired := sourceName - if desired == "" { - return - } - current := strings.TrimSpace(root.Name) - if current == "" { - root.Name = desired - root.Path = desired - root.Holder = desired - return - } - if strings.EqualFold(current, desired) { - return - } - suspicious := map[string]bool{ - "and": true, "or": true, "status": true, "value": true, "watching": true, - } - if !suspicious[strings.ToLower(current)] { - return - } - if result.ViewsByName != nil { - delete(result.ViewsByName, root.Name) - } else { - result.ViewsByName = map[string]*plan.View{} - } - root.Name = desired - root.Path = desired - root.Holder = desired - result.ViewsByName[root.Name] = root -} - +// extractJoinEmbedRefs builds a map of view-alias → embed-path for every +// JOIN(${embed:path}) alias clause found in sqlText. func extractJoinEmbedRefs(sqlText string) map[string]string { result := map[string]string{} if strings.TrimSpace(sqlText) == "" { @@ -606,6 +298,8 @@ func extractJoinEmbedRefs(sqlText string) map[string]string { return result } +// extractJoinSubqueryBodies builds a map of view-alias → subquery-body for +// every JOIN(body) alias clause found in sqlText. func extractJoinSubqueryBodies(sqlText string) map[string]string { result := map[string]string{} if strings.TrimSpace(sqlText) == "" { @@ -620,202 +314,3 @@ func extractJoinSubqueryBodies(sqlText string) map[string]string { } return result } - -func findSummaryJoinBody(input string) (string, bool) { - lower := strings.ToLower(input) - for i := 0; i < len(input); i++ { - if !hasCompileWordAt(lower, i, "join") { - continue - } - pos := skipCompileSpaces(input, i+len("join")) - if pos >= len(input) || input[pos] != '(' { - continue - } - body, end, ok := readCompileParenBody(input, pos) - if !ok { - continue - } - rest := strings.ToLower(input[end+1:]) - rest = strings.Join(strings.Fields(rest), " ") - if strings.HasPrefix(rest, "summary on 1=1") || strings.HasPrefix(rest, "summary on 1 = 1") { - return body, true - } - } - return "", false -} - -func extractLeadingRuleHeaderJSON(input string) (string, bool) { - index := skipCompileSpaces(input, 0) - if index+2 > len(input) || input[index:index+2] != "/*" { - return "", false - } - end := strings.Index(input[index+2:], "*/") - if end < 0 { - return "", false - } - body := strings.TrimSpace(input[index+2 : index+2+end]) - if body == "" || body[0] != '{' || body[len(body)-1] != '}' { - return "", false - } - return body, true -} - -func findFirstEmbedRef(input string) (string, bool) { - for i := 0; i < len(input); i++ { - if input[i] != '$' || i+1 >= len(input) || input[i+1] != '{' { - continue - } - body, end, ok := readCompileTemplateExpr(input, i+1) - if !ok { - continue - } - _ = end - trimmed := strings.TrimSpace(body) - if len(trimmed) < len("embed:") || !strings.HasPrefix(strings.ToLower(trimmed), "embed:") { - continue - } - ref := strings.TrimSpace(trimmed[len("embed:"):]) - if ref == "" { - continue - } - return ref, true - } - return "", false -} - -type joinSubquery struct { - body string - alias string -} - -func scanJoinSubqueries(input string) []joinSubquery { - result := make([]joinSubquery, 0) - lower := strings.ToLower(input) - for i := 0; i < len(input); i++ { - if !hasCompileWordAt(lower, i, "join") { - continue - } - pos := skipCompileSpaces(input, i+len("join")) - if pos >= len(input) || input[pos] != '(' { - continue - } - body, end, ok := readCompileParenBody(input, pos) - if !ok { - continue - } - pos = skipCompileSpaces(input, end+1) - if hasCompileWordAt(lower, pos, "as") { - pos = skipCompileSpaces(input, pos+len("as")) - } - aliasStart := pos - if aliasStart >= len(input) || !isCompileWordStart(input[aliasStart]) { - i = end - continue - } - pos++ - for pos < len(input) && isCompileWordPart(input[pos]) { - pos++ - } - alias := strings.TrimSpace(input[aliasStart:pos]) - if alias != "" { - result = append(result, joinSubquery{body: body, alias: alias}) - } - i = end - } - return result -} - -func parseJoinEmbedRef(body string) (string, bool) { - trimmed := strings.TrimSpace(body) - if !strings.HasPrefix(trimmed, "${") || !strings.HasSuffix(trimmed, "}") { - return "", false - } - inner := strings.TrimSpace(trimmed[2 : len(trimmed)-1]) - if len(inner) < len("embed:") || !strings.HasPrefix(strings.ToLower(inner), "embed:") { - return "", false - } - ref := strings.TrimSpace(inner[len("embed:"):]) - return ref, ref != "" -} - -func readCompileTemplateExpr(input string, openBrace int) (string, int, bool) { - if openBrace <= 0 || openBrace >= len(input) || input[openBrace] != '{' || input[openBrace-1] != '$' { - return "", -1, false - } - for i := openBrace + 1; i < len(input); i++ { - if input[i] == '}' { - return input[openBrace+1 : i], i, true - } - } - return "", -1, false -} - -func readCompileParenBody(input string, openParen int) (string, int, bool) { - depth := 0 - quote := byte(0) - for i := openParen; i < len(input); i++ { - ch := input[i] - if quote != 0 { - if ch == '\\' && i+1 < len(input) { - i++ - continue - } - if ch == quote { - quote = 0 - } - continue - } - if ch == '\'' || ch == '"' { - quote = ch - continue - } - if ch == '(' { - depth++ - continue - } - if ch == ')' { - depth-- - if depth == 0 { - return input[openParen+1 : i], i, true - } - } - } - return "", -1, false -} - -func hasCompileWordAt(lower string, pos int, word string) bool { - if pos < 0 || pos+len(word) > len(lower) { - return false - } - if lower[pos:pos+len(word)] != word { - return false - } - if pos > 0 && isCompileWordPart(lower[pos-1]) { - return false - } - next := pos + len(word) - if next < len(lower) && isCompileWordPart(lower[next]) { - return false - } - return true -} - -func skipCompileSpaces(input string, index int) int { - for index < len(input) { - switch input[index] { - case ' ', '\t', '\n', '\r': - index++ - default: - return index - } - } - return index -} - -func isCompileWordStart(ch byte) bool { - return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') -} - -func isCompileWordPart(ch byte) bool { - return isCompileWordStart(ch) || (ch >= '0' && ch <= '9') -} diff --git a/repository/shape/compile/enrich_table.go b/repository/shape/compile/enrich_table.go new file mode 100644 index 000000000..7ba779dc6 --- /dev/null +++ b/repository/shape/compile/enrich_table.go @@ -0,0 +1,331 @@ +package compile + +// enrich_table.go — table-name inference logic extracted from enrich.go. +// All functions here derive a database table name from SQL text, file-system +// sibling files, or embedded SQL references. + +import ( + "os" + "path/filepath" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/compile/pipeline" + "github.com/viant/datly/repository/shape/plan" +) + +func shouldInferTable(item *plan.View) bool { + if item == nil { + return false + } + name := strings.TrimSpace(item.Name) + table := strings.TrimSpace(item.Table) + if table == "" { + return true + } + if strings.HasPrefix(table, "(") { + return true + } + if normalizedTemplatePlaceholderTable(table) { + return true + } + return strings.EqualFold(name, table) +} + +func normalizedTemplatePlaceholderTable(table string) bool { + if table == "" { + return false + } + parts := strings.Split(table, ".") + if len(parts) < 3 { + return false + } + for i := 0; i < len(parts)-1; i++ { + part := strings.TrimSpace(parts[i]) + if part == "" { + return false + } + for _, ch := range part { + if ch < '0' || ch > '9' { + return false + } + } + } + return true +} + +func inferTableFromSQL(sqlText string, source *shape.Source) string { + sqlText = strings.TrimSpace(sqlText) + if sqlText == "" { + return "" + } + if expr := topLevelFromExpr(sqlText); expr != "" { + if table := tableFromFromExpr(expr, source); table != "" { + return table + } + } + if table := pipeline.InferTableFromSQL(sqlText); table != "" { + if !strings.EqualFold(table, "DQLView") { + return table + } + } + if table := inferFromEmbeddedSQL(sqlText, source); table != "" { + return table + } + return "" +} + +func inferFromEmbeddedSQL(sqlText string, source *shape.Source) string { + ref, ok := findFirstEmbedRef(sqlText) + if !ok { + return "" + } + ref = strings.Trim(ref, `"'`) + if ref == "" { + return "" + } + resolved := resolveEmbedPath(source, ref) + if resolved == "" { + return "" + } + embedded, err := os.ReadFile(resolved) + if err != nil { + return "" + } + queryNode, _, err := pipeline.ParseSelectWithDiagnostic(string(embedded)) + if err != nil || queryNode == nil { + if table := pipeline.InferTableFromSQL(string(embedded)); table != "" && !strings.EqualFold(table, "DQLView") { + return strings.Trim(table, "`\"") + } + return "" + } + _, table, err := pipeline.InferRoot(queryNode, "") + if err != nil || strings.TrimSpace(table) == "" { + return "" + } + if strings.EqualFold(strings.TrimSpace(table), "DQLView") { + return "" + } + return strings.Trim(table, "`\"") +} + +func resolveEmbedPath(source *shape.Source, ref string) string { + if filepath.IsAbs(ref) { + return ref + } + if source == nil || strings.TrimSpace(source.Path) == "" { + return "" + } + base := source.Path + if fi, err := os.Stat(base); err == nil && fi.IsDir() { + return filepath.Clean(filepath.Join(base, ref)) + } + return filepath.Clean(filepath.Join(filepath.Dir(base), ref)) +} + +func inferTableFromSiblingSQL(viewName string, source *shape.Source) string { + viewName = strings.TrimSpace(viewName) + if viewName == "" || source == nil || strings.TrimSpace(source.Path) == "" { + return "" + } + sibling := filepath.Join(filepath.Dir(source.Path), viewName+".sql") + data, err := os.ReadFile(sibling) + if err != nil { + sibling = filepath.Join(filepath.Dir(source.Path), strings.ToLower(viewName)+".sql") + data, err = os.ReadFile(sibling) + } + if err != nil { + return "" + } + return inferTableFromSQL(string(data), source) +} + +func inferTableFromEmbedRef(source *shape.Source, ref string) string { + ref = strings.Trim(strings.TrimSpace(ref), `"'`) + if ref == "" { + return "" + } + resolved := resolveEmbedPath(source, ref) + if resolved == "" { + return "" + } + data, err := os.ReadFile(resolved) + if err != nil { + return "" + } + return pipeline.InferTableFromSQL(string(data)) +} + +// topLevelFromExpr scans sqlText for the first top-level (depth-0) FROM keyword +// and returns the expression that immediately follows it, including subquery parens +// with a trailing alias when present. +func topLevelFromExpr(sqlText string) string { + lower := strings.ToLower(sqlText) + depth := 0 + inSingle := false + inDouble := false + inBacktick := false + for i := 0; i < len(sqlText); i++ { + ch := sqlText[i] + switch ch { + case '\'': + if !inDouble && !inBacktick { + inSingle = !inSingle + } + case '"': + if !inSingle && !inBacktick { + inDouble = !inDouble + } + case '`': + if !inSingle && !inDouble { + inBacktick = !inBacktick + } + case '(': + if !inSingle && !inDouble && !inBacktick { + depth++ + } + case ')': + if !inSingle && !inDouble && !inBacktick && depth > 0 { + depth-- + } + } + if inSingle || inDouble || inBacktick || depth != 0 { + continue + } + if i+6 > len(sqlText) { + break + } + if lower[i:i+4] != "from" { + continue + } + if i > 0 { + prev := lower[i-1] + if (prev >= 'a' && prev <= 'z') || (prev >= '0' && prev <= '9') || prev == '_' { + continue + } + } + j := i + 4 + for j < len(sqlText) && (sqlText[j] == ' ' || sqlText[j] == '\n' || sqlText[j] == '\t' || sqlText[j] == '\r') { + j++ + } + if j >= len(sqlText) { + return "" + } + if sqlText[j] == '(' { + start := j + d := 0 + for ; j < len(sqlText); j++ { + if sqlText[j] == '(' { + d++ + } else if sqlText[j] == ')' { + d-- + if d == 0 { + j++ + break + } + } + } + for j < len(sqlText) && (sqlText[j] == ' ' || sqlText[j] == '\n' || sqlText[j] == '\t' || sqlText[j] == '\r') { + j++ + } + for j < len(sqlText) { + c := sqlText[j] + if !(c == '_' || c == '.' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + break + } + j++ + } + return strings.TrimSpace(sqlText[start:j]) + } + start := j + for j < len(sqlText) { + c := sqlText[j] + if !(c == '_' || c == '.' || c == '/' || c == '{' || c == '}' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '$') { + break + } + j++ + } + return strings.TrimSpace(sqlText[start:j]) + } + return "" +} + +func tableFromFromExpr(fromExpr string, source *shape.Source) string { + fromExpr = strings.TrimSpace(fromExpr) + if fromExpr == "" { + return "" + } + if strings.HasPrefix(fromExpr, "(") { + if table := inferFromEmbeddedSQL(fromExpr, source); table != "" { + return table + } + inner := fromExpr + if idx := strings.LastIndex(inner, ")"); idx > 0 { + inner = strings.TrimSpace(inner[1:idx]) + } + return inferTableFromSQL(inner, source) + } + return strings.Trim(fromExpr, "`\"") +} + +func inferConnector(item *plan.View, source *shape.Source) string { + if item == nil { + return "" + } + path := "" + if source != nil { + path = strings.ToLower(strings.ReplaceAll(source.Path, "\\", "/")) + } + table := strings.ToUpper(strings.TrimSpace(item.Table)) + switch { + case strings.Contains(path, "/dql/system/"): + return "system" + case strings.HasPrefix(table, "CI_") || strings.Contains(table, ".CI_"): + return "ci_ads" + case strings.Contains(path, "/dql/ui/"): + return "sitemgmt" + case strings.Contains(table, "SITE"): + return "sitemgmt" + default: + return "" + } +} + +func normalizeRootViewName(result *plan.Result, sourceName string) { + if result == nil || len(result.Views) == 0 { + return + } + root := result.Views[0] + if root == nil { + return + } + desired := sourceName + if desired == "" { + return + } + current := strings.TrimSpace(root.Name) + if current == "" { + root.Name = desired + root.Path = desired + root.Holder = desired + return + } + if strings.EqualFold(current, desired) { + return + } + suspicious := map[string]bool{ + "and": true, "or": true, "status": true, "value": true, "watching": true, + } + if !suspicious[strings.ToLower(current)] { + return + } + if result.ViewsByName != nil { + delete(result.ViewsByName, root.Name) + } else { + result.ViewsByName = map[string]*plan.View{} + } + root.Name = desired + root.Path = desired + root.Holder = desired + result.ViewsByName[root.Name] = root +} diff --git a/repository/shape/compile/enrich_text.go b/repository/shape/compile/enrich_text.go new file mode 100644 index 000000000..dbf8f2b99 --- /dev/null +++ b/repository/shape/compile/enrich_text.go @@ -0,0 +1,212 @@ +package compile + +// enrich_text.go — low-level text/SQL scanning primitives used by the +// enrichment phase (enrich.go and enrich_table.go). + +import "strings" + +// findSummaryJoinBody locates the body of a JOIN(...) SUMMARY ON 1=1 clause. +func findSummaryJoinBody(input string) (string, bool) { + lower := strings.ToLower(input) + for i := 0; i < len(input); i++ { + if !hasCompileWordAt(lower, i, "join") { + continue + } + pos := skipCompileSpaces(input, i+len("join")) + if pos >= len(input) || input[pos] != '(' { + continue + } + body, end, ok := readCompileParenBody(input, pos) + if !ok { + continue + } + rest := strings.ToLower(input[end+1:]) + rest = strings.Join(strings.Fields(rest), " ") + if strings.HasPrefix(rest, "summary on 1=1") || strings.HasPrefix(rest, "summary on 1 = 1") { + return body, true + } + } + return "", false +} + +// extractLeadingRuleHeaderJSON returns the JSON body of a leading /* {...} */ comment. +func extractLeadingRuleHeaderJSON(input string) (string, bool) { + index := skipCompileSpaces(input, 0) + if index+2 > len(input) || input[index:index+2] != "/*" { + return "", false + } + end := strings.Index(input[index+2:], "*/") + if end < 0 { + return "", false + } + body := strings.TrimSpace(input[index+2 : index+2+end]) + if body == "" || body[0] != '{' || body[len(body)-1] != '}' { + return "", false + } + return body, true +} + +// findFirstEmbedRef returns the path after "embed:" in the first ${embed:…} +// template expression found in input. +func findFirstEmbedRef(input string) (string, bool) { + for i := 0; i < len(input); i++ { + if input[i] != '$' || i+1 >= len(input) || input[i+1] != '{' { + continue + } + body, end, ok := readCompileTemplateExpr(input, i+1) + if !ok { + continue + } + _ = end + trimmed := strings.TrimSpace(body) + if len(trimmed) < len("embed:") || !strings.HasPrefix(strings.ToLower(trimmed), "embed:") { + continue + } + ref := strings.TrimSpace(trimmed[len("embed:"):]) + if ref == "" { + continue + } + return ref, true + } + return "", false +} + +// joinSubquery holds the body and alias of a JOIN(...) AS alias clause. +type joinSubquery struct { + body string + alias string +} + +// scanJoinSubqueries collects all JOIN(body) alias pairs from input. +func scanJoinSubqueries(input string) []joinSubquery { + result := make([]joinSubquery, 0) + lower := strings.ToLower(input) + for i := 0; i < len(input); i++ { + if !hasCompileWordAt(lower, i, "join") { + continue + } + pos := skipCompileSpaces(input, i+len("join")) + if pos >= len(input) || input[pos] != '(' { + continue + } + body, end, ok := readCompileParenBody(input, pos) + if !ok { + continue + } + pos = skipCompileSpaces(input, end+1) + if hasCompileWordAt(lower, pos, "as") { + pos = skipCompileSpaces(input, pos+len("as")) + } + aliasStart := pos + if aliasStart >= len(input) || !isCompileWordStart(input[aliasStart]) { + i = end + continue + } + pos++ + for pos < len(input) && isCompileWordPart(input[pos]) { + pos++ + } + alias := strings.TrimSpace(input[aliasStart:pos]) + if alias != "" { + result = append(result, joinSubquery{body: body, alias: alias}) + } + i = end + } + return result +} + +// parseJoinEmbedRef returns the embed path from a body of the form ${embed:path}. +func parseJoinEmbedRef(body string) (string, bool) { + trimmed := strings.TrimSpace(body) + if !strings.HasPrefix(trimmed, "${") || !strings.HasSuffix(trimmed, "}") { + return "", false + } + inner := strings.TrimSpace(trimmed[2 : len(trimmed)-1]) + if len(inner) < len("embed:") || !strings.HasPrefix(strings.ToLower(inner), "embed:") { + return "", false + } + ref := strings.TrimSpace(inner[len("embed:"):]) + return ref, ref != "" +} + +func readCompileTemplateExpr(input string, openBrace int) (string, int, bool) { + if openBrace <= 0 || openBrace >= len(input) || input[openBrace] != '{' || input[openBrace-1] != '$' { + return "", -1, false + } + for i := openBrace + 1; i < len(input); i++ { + if input[i] == '}' { + return input[openBrace+1 : i], i, true + } + } + return "", -1, false +} + +func readCompileParenBody(input string, openParen int) (string, int, bool) { + depth := 0 + quote := byte(0) + for i := openParen; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + if ch == '(' { + depth++ + continue + } + if ch == ')' { + depth-- + if depth == 0 { + return input[openParen+1 : i], i, true + } + } + } + return "", -1, false +} + +func hasCompileWordAt(lower string, pos int, word string) bool { + if pos < 0 || pos+len(word) > len(lower) { + return false + } + if lower[pos:pos+len(word)] != word { + return false + } + if pos > 0 && isCompileWordPart(lower[pos-1]) { + return false + } + next := pos + len(word) + if next < len(lower) && isCompileWordPart(lower[next]) { + return false + } + return true +} + +func skipCompileSpaces(input string, index int) int { + for index < len(input) { + switch input[index] { + case ' ', '\t', '\n', '\r': + index++ + default: + return index + } + } + return index +} + +func isCompileWordStart(ch byte) bool { + return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') +} + +func isCompileWordPart(ch byte) bool { + return isCompileWordStart(ch) || (ch >= '0' && ch <= '9') +} diff --git a/repository/shape/compile/pipeline/read.go b/repository/shape/compile/pipeline/read.go index 3cf697603..c665d154b 100644 --- a/repository/shape/compile/pipeline/read.go +++ b/repository/shape/compile/pipeline/read.go @@ -1,5 +1,9 @@ package pipeline +// read.go — SELECT compilation: parses DQL into a plan.View using +// multi-strategy parse with template-signal fallback. +// SQL normalization and token utilities live in read_normalize.go. + import ( "reflect" "strings" @@ -9,51 +13,31 @@ import ( "github.com/viant/sqlparser/query" ) +// BuildRead compiles a SELECT DQL fragment into a plan.View. +// It applies multiple parse strategies and gracefully degrades to a +// loose (schema-less) view for template-driven SQL that cannot be fully parsed. func BuildRead(sourceName, sqlText string) (*plan.View, []*dqlshape.Diagnostic, error) { - parserSQL := normalizeParserSQL(sqlText) - queryNode, parseDiag, err := ParseSelectWithDiagnostic(parserSQL) - if err != nil && parserSQL != sqlText { - if rawNode, _, rawErr := ParseSelectWithDiagnostic(sqlText); rawErr == nil && isUsableQuery(rawNode) { - queryNode = rawNode - parserSQL = sqlText - parseDiag = nil - err = nil - } - } - if err == nil && needsFallbackParse(sqlText, queryNode) { - fallbackSQL := normalizeParserSQL(sqlText) - if fallbackNode, _, fallbackErr := ParseSelectWithDiagnostic(fallbackSQL); fallbackErr == nil && isUsableQuery(fallbackNode) { - queryNode = fallbackNode - parserSQL = fallbackSQL - parseDiag = nil - err = nil - } - } - if hasTemplateSignals(sqlText) && (err != nil || parseDiag != nil) { + queryNode, parseDiag, parserSQL, err := resolveQueryNode(sqlText) + + // Template-driven SQL may legitimately fail strict parsing; treat as warning. + if (err != nil || parseDiag != nil) && hasTemplateSignals(sqlText) { if parseDiag != nil { parseDiag.Severity = dqlshape.SeverityWarning } - var diags []*dqlshape.Diagnostic - if parseDiag != nil { - diags = append(diags, parseDiag) - } - return buildLooseRead(sourceName, sqlText), diags, nil + return buildLooseRead(sourceName, sqlText), collectDiags(parseDiag), nil } + var diags []*dqlshape.Diagnostic if parseDiag != nil { diags = append(diags, parseDiag) } if err != nil { - if hasTemplateSignals(sqlText) { - if parseDiag != nil { - parseDiag.Severity = dqlshape.SeverityWarning - } - return buildLooseRead(sourceName, sqlText), diags, nil - } return nil, diags, nil } + relations, relationDiags := ExtractJoinRelations(parserSQL, queryNode) diags = append(diags, relationDiags...) + name, table, inferErr := InferRoot(queryNode, sourceName) if inferErr != nil { return nil, nil, inferErr @@ -65,6 +49,7 @@ func BuildRead(sourceName, sqlText string) (*plan.View, []*dqlshape.Diagnostic, table = derived } } + fieldType, elementType, cardinality := InferProjectionType(queryNode) if fieldType == nil || elementType == nil { fieldType = reflect.TypeOf([]map[string]interface{}{}) @@ -86,6 +71,37 @@ func BuildRead(sourceName, sqlText string) (*plan.View, []*dqlshape.Diagnostic, return view, diags, nil } +// resolveQueryNode attempts to parse sqlText into a query AST using up to +// three strategies: +// 1. Parse the normalised form. +// 2. If normalisation broke the SQL, fall back to the raw form. +// 3. If the parsed result is structurally incomplete, retry with the +// normalised form to pick up joins the raw parse missed. +// +// It returns the best node, any diagnostic, the effective SQL used, and any +// parse error. +func resolveQueryNode(sqlText string) (node *query.Select, diag *dqlshape.Diagnostic, effectiveSQL string, err error) { + parserSQL := normalizeParserSQL(sqlText) + node, diag, err = ParseSelectWithDiagnostic(parserSQL) + + // Strategy 2: normalisation may have broken the SQL; try raw form. + if err != nil && parserSQL != sqlText { + if rawNode, _, rawErr := ParseSelectWithDiagnostic(sqlText); rawErr == nil && isUsableQuery(rawNode) { + return rawNode, nil, sqlText, nil + } + } + + // Strategy 3: parsed OK but result is incomplete (no FROM or missing JOINs); + // retry with the normalised form. + if err == nil && needsFallbackParse(sqlText, node) { + fallbackSQL := normalizeParserSQL(sqlText) + if fallbackNode, _, fallbackErr := ParseSelectWithDiagnostic(fallbackSQL); fallbackErr == nil && isUsableQuery(fallbackNode) { + return fallbackNode, nil, fallbackSQL, nil + } + } + return node, diag, parserSQL, err +} + func buildLooseRead(sourceName, sqlText string) *plan.View { name, table := inferLooseRoot(sourceName, sqlText) fieldType := reflect.TypeOf([]map[string]interface{}{}) @@ -136,96 +152,6 @@ func needsFallbackParse(rawSQL string, queryNode *query.Select) bool { return false } -func normalizeParserSQL(sqlText string) string { - if sqlText == "" { - return sqlText - } - return rewritePrivateShorthand(replaceTemplateTokens(sqlText)) -} - -func rewritePrivateShorthand(input string) string { - var b strings.Builder - b.Grow(len(input)) - for i := 0; i < len(input); { - if !hasPrefixFold(input[i:], "private") { - b.WriteByte(input[i]) - i++ - continue - } - if i > 0 && isReadIdentifierPart(input[i-1]) { - b.WriteByte(input[i]) - i++ - continue - } - pos := i + len("private") - pos = skipReadSpaces(input, pos) - if pos >= len(input) || input[pos] != '(' { - b.WriteByte(input[i]) - i++ - continue - } - body, closeIdx, ok := readReadCallBody(input, pos) - if !ok { - b.WriteByte(input[i]) - i++ - continue - } - firstArg, ok := firstCallArg(body) - if !ok { - b.WriteByte(input[i]) - i++ - continue - } - b.WriteString(strings.TrimSpace(firstArg)) - i = closeIdx + 1 - } - return b.String() -} - -func hasPrefixFold(s, prefix string) bool { - if len(s) < len(prefix) { - return false - } - return strings.EqualFold(s[:len(prefix)], prefix) -} - -func firstCallArg(body string) (string, bool) { - depth := 0 - quote := byte(0) - for i := 0; i < len(body); i++ { - ch := body[i] - if quote != 0 { - if ch == '\\' && i+1 < len(body) { - i++ - continue - } - if ch == quote { - quote = 0 - } - continue - } - if ch == '\'' || ch == '"' { - quote = ch - continue - } - switch ch { - case '(': - depth++ - case ')': - if depth > 0 { - depth-- - } - case ',': - if depth == 0 { - arg := strings.TrimSpace(body[:i]) - return arg, arg != "" - } - } - } - arg := strings.TrimSpace(body) - return arg, arg != "" -} - func inferRootFromRelations(relations []*plan.Relation) string { for _, relation := range relations { if relation == nil { @@ -269,170 +195,11 @@ func extractSimpleFromTable(sqlText string) string { return "" } -func replaceTemplateTokens(input string) string { - var b strings.Builder - b.Grow(len(input)) - for i := 0; i < len(input); { - if input[i] != '$' { - b.WriteByte(input[i]) - i++ - continue - } - if i+1 < len(input) && input[i+1] == '{' { - body, end, ok := readReadTemplateExpr(input, i+1) - if !ok { - b.WriteByte(input[i]) - i++ - continue - } - replacement, keep := normalizeTemplateExprBody(body) - if keep { - b.WriteString(input[i : end+1]) - } else { - b.WriteString(replacement) - } - i = end + 1 - continue - } - token, end, ok := readReadSelector(input, i) - if !ok { - b.WriteByte(input[i]) - i++ - continue - } - if strings.EqualFold(token, "$criteria.AppendBinding") { - pos := skipReadSpaces(input, end) - if pos < len(input) && input[pos] == '(' { - _, close, ok := readReadCallBody(input, pos) - if ok { - b.WriteByte('1') - i = close + 1 - continue - } - } - } - if isReadReservedToken(token) { - b.WriteString(token) - } else { - b.WriteByte('1') - } - i = end - } - return b.String() -} - -func normalizeTemplateExprBody(body string) (string, bool) { - trimmed := strings.TrimSpace(body) - if isReadReservedName(trimmed) { - return "", true - } - lower := strings.ToLower(trimmed) - if strings.Contains(lower, `build("where")`) || strings.Contains(lower, "build('where')") { - return " WHERE 1 ", false - } - if strings.Contains(lower, `build("and")`) || strings.Contains(lower, "build('and')") { - return " AND 1 ", false - } - return "1", false -} - -func readReadTemplateExpr(input string, openBrace int) (string, int, bool) { - if openBrace <= 0 || openBrace >= len(input) || input[openBrace] != '{' || input[openBrace-1] != '$' { - return "", -1, false - } - for i := openBrace + 1; i < len(input); i++ { - if input[i] == '}' { - return input[openBrace+1 : i], i, true - } - } - return "", -1, false -} - -func readReadSelector(input string, start int) (string, int, bool) { - if start < 0 || start >= len(input) || input[start] != '$' { - return "", start, false - } - i := start + 1 - if i >= len(input) || !isReadIdentifierStart(input[i]) { - return "", start, false - } - i++ - for i < len(input) && isReadIdentifierPart(input[i]) { - i++ - } - for i < len(input) && input[i] == '.' { - i++ - if i >= len(input) || !isReadIdentifierStart(input[i]) { - return "", start, false - } - i++ - for i < len(input) && isReadIdentifierPart(input[i]) { - i++ - } - } - return input[start:i], i, true -} - -func readReadCallBody(input string, openParen int) (string, int, bool) { - depth := 0 - quote := byte(0) - for i := openParen; i < len(input); i++ { - ch := input[i] - if quote != 0 { - if ch == '\\' && i+1 < len(input) { - i++ - continue - } - if ch == quote { - quote = 0 - } - continue - } - if ch == '\'' || ch == '"' { - quote = ch - continue - } - if ch == '(' { - depth++ - continue - } - if ch == ')' { - depth-- - if depth == 0 { - return input[openParen+1 : i], i, true - } - } - } - return "", -1, false -} - -func isReadReservedToken(token string) bool { - if len(token) > 0 && token[0] == '$' { - token = token[1:] +// collectDiags returns a single-element slice for a non-nil diagnostic, +// or nil otherwise. Used to avoid repeated nil checks at call sites. +func collectDiags(diag *dqlshape.Diagnostic) []*dqlshape.Diagnostic { + if diag == nil { + return nil } - return isReadReservedName(token) -} - -func isReadReservedName(name string) bool { - return name == "sql.Insert" || name == "sql.Update" || name == "Nop" -} - -func skipReadSpaces(input string, index int) int { - for index < len(input) { - switch input[index] { - case ' ', '\t', '\n', '\r': - index++ - default: - return index - } - } - return index -} - -func isReadIdentifierStart(ch byte) bool { - return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') -} - -func isReadIdentifierPart(ch byte) bool { - return isReadIdentifierStart(ch) || (ch >= '0' && ch <= '9') + return []*dqlshape.Diagnostic{diag} } diff --git a/repository/shape/compile/pipeline/read_normalize.go b/repository/shape/compile/pipeline/read_normalize.go new file mode 100644 index 000000000..6ff42af2f --- /dev/null +++ b/repository/shape/compile/pipeline/read_normalize.go @@ -0,0 +1,266 @@ +package pipeline + +// read_normalize.go — SQL normalization and template-token replacement used +// by BuildRead to produce parser-friendly SQL from raw DQL. + +import "strings" + +// normalizeParserSQL rewrites private(…) shorthands and template tokens into +// plain SQL that the parser can handle. +func normalizeParserSQL(sqlText string) string { + if sqlText == "" { + return sqlText + } + return rewritePrivateShorthand(replaceTemplateTokens(sqlText)) +} + +func rewritePrivateShorthand(input string) string { + var b strings.Builder + b.Grow(len(input)) + for i := 0; i < len(input); { + if !hasPrefixFold(input[i:], "private") { + b.WriteByte(input[i]) + i++ + continue + } + if i > 0 && isReadIdentifierPart(input[i-1]) { + b.WriteByte(input[i]) + i++ + continue + } + pos := i + len("private") + pos = skipReadSpaces(input, pos) + if pos >= len(input) || input[pos] != '(' { + b.WriteByte(input[i]) + i++ + continue + } + body, closeIdx, ok := readReadCallBody(input, pos) + if !ok { + b.WriteByte(input[i]) + i++ + continue + } + firstArg, ok := firstCallArg(body) + if !ok { + b.WriteByte(input[i]) + i++ + continue + } + b.WriteString(strings.TrimSpace(firstArg)) + i = closeIdx + 1 + } + return b.String() +} + +func hasPrefixFold(s, prefix string) bool { + if len(s) < len(prefix) { + return false + } + return strings.EqualFold(s[:len(prefix)], prefix) +} + +func firstCallArg(body string) (string, bool) { + depth := 0 + quote := byte(0) + for i := 0; i < len(body); i++ { + ch := body[i] + if quote != 0 { + if ch == '\\' && i+1 < len(body) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + switch ch { + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + case ',': + if depth == 0 { + arg := strings.TrimSpace(body[:i]) + return arg, arg != "" + } + } + } + arg := strings.TrimSpace(body) + return arg, arg != "" +} + +func replaceTemplateTokens(input string) string { + var b strings.Builder + b.Grow(len(input)) + for i := 0; i < len(input); { + if input[i] != '$' { + b.WriteByte(input[i]) + i++ + continue + } + if i+1 < len(input) && input[i+1] == '{' { + body, end, ok := readReadTemplateExpr(input, i+1) + if !ok { + b.WriteByte(input[i]) + i++ + continue + } + replacement, keep := normalizeTemplateExprBody(body) + if keep { + b.WriteString(input[i : end+1]) + } else { + b.WriteString(replacement) + } + i = end + 1 + continue + } + token, end, ok := readReadSelector(input, i) + if !ok { + b.WriteByte(input[i]) + i++ + continue + } + if strings.EqualFold(token, "$criteria.AppendBinding") { + pos := skipReadSpaces(input, end) + if pos < len(input) && input[pos] == '(' { + _, close, ok := readReadCallBody(input, pos) + if ok { + b.WriteByte('1') + i = close + 1 + continue + } + } + } + if isReadReservedToken(token) { + b.WriteString(token) + } else { + b.WriteByte('1') + } + i = end + } + return b.String() +} + +func normalizeTemplateExprBody(body string) (string, bool) { + trimmed := strings.TrimSpace(body) + if isReadReservedName(trimmed) { + return "", true + } + lower := strings.ToLower(trimmed) + if strings.Contains(lower, `build("where")`) || strings.Contains(lower, "build('where')") { + return " WHERE 1 ", false + } + if strings.Contains(lower, `build("and")`) || strings.Contains(lower, "build('and')") { + return " AND 1 ", false + } + return "1", false +} + +func readReadTemplateExpr(input string, openBrace int) (string, int, bool) { + if openBrace <= 0 || openBrace >= len(input) || input[openBrace] != '{' || input[openBrace-1] != '$' { + return "", -1, false + } + for i := openBrace + 1; i < len(input); i++ { + if input[i] == '}' { + return input[openBrace+1 : i], i, true + } + } + return "", -1, false +} + +func readReadSelector(input string, start int) (string, int, bool) { + if start < 0 || start >= len(input) || input[start] != '$' { + return "", start, false + } + i := start + 1 + if i >= len(input) || !isReadIdentifierStart(input[i]) { + return "", start, false + } + i++ + for i < len(input) && isReadIdentifierPart(input[i]) { + i++ + } + for i < len(input) && input[i] == '.' { + i++ + if i >= len(input) || !isReadIdentifierStart(input[i]) { + return "", start, false + } + i++ + for i < len(input) && isReadIdentifierPart(input[i]) { + i++ + } + } + return input[start:i], i, true +} + +func readReadCallBody(input string, openParen int) (string, int, bool) { + depth := 0 + quote := byte(0) + for i := openParen; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + if ch == '(' { + depth++ + continue + } + if ch == ')' { + depth-- + if depth == 0 { + return input[openParen+1 : i], i, true + } + } + } + return "", -1, false +} + +func isReadReservedToken(token string) bool { + if len(token) > 0 && token[0] == '$' { + token = token[1:] + } + return isReadReservedName(token) +} + +func isReadReservedName(name string) bool { + return name == "sql.Insert" || name == "sql.Update" || name == "Nop" +} + +func skipReadSpaces(input string, index int) int { + for index < len(input) { + switch input[index] { + case ' ', '\t', '\n', '\r': + index++ + default: + return index + } + } + return index +} + +func isReadIdentifierStart(ch byte) bool { + return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') +} + +func isReadIdentifierPart(ch byte) bool { + return isReadIdentifierStart(ch) || (ch >= '0' && ch <= '9') +} diff --git a/repository/shape/compile/preprocess_handler.go b/repository/shape/compile/preprocess_handler.go index 11b9327c1..3449a8de1 100644 --- a/repository/shape/compile/preprocess_handler.go +++ b/repository/shape/compile/preprocess_handler.go @@ -32,22 +32,12 @@ func buildHandlerIfNeeded(source *shape.Source, pre *dqlpre.Result, statements d if !unknownOnly && !isHandlerSignal(source) { return ret } - if buildHandlerFromContractIfNeeded(ret, source, layout) { - return ret - } if buildGeneratedFallbackIfNeeded(ret, source, layout) { return ret } return ret } -func buildHandlerFromContractIfNeeded(ret *handlerPreprocessResult, source *shape.Source, layout compilePathLayout) bool { - _ = ret - _ = source - _ = layout - return false -} - func buildGeneratedFallbackIfNeeded(ret *handlerPreprocessResult, source *shape.Source, layout compilePathLayout) bool { if ret == nil || source == nil { return false diff --git a/repository/shape/compile/statedecl.go b/repository/shape/compile/statedecl.go index fbb933308..bd401b11d 100644 --- a/repository/shape/compile/statedecl.go +++ b/repository/shape/compile/statedecl.go @@ -40,11 +40,11 @@ func appendDeclaredStates(rawDQL string, result *plan.Result) { ensureStateSchema(state).DataType = inType state.OutputDataType = outType } - switch strings.ToLower(kind) { - case "query": + switch st.Kind(strings.ToLower(kind)) { + case st.KindQuery: required := false state.Required = &required - case "header": + case st.KindHeader: required := true state.Required = &required } diff --git a/repository/shape/compile/type_support_test.go b/repository/shape/compile/type_support_test.go index b3c376c55..b081ad1e6 100644 --- a/repository/shape/compile/type_support_test.go +++ b/repository/shape/compile/type_support_test.go @@ -31,7 +31,7 @@ func TestDQLCompiler_Compile_UsesLinkedRootTypeForSchemaType(t *testing.T) { res, err := compiler.Compile(context.Background(), source) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Views) assert.Equal(t, "*compile.linkedRootType", planned.Views[0].SchemaType) @@ -52,7 +52,7 @@ func TestDQLCompiler_Compile_UsesLinkedRegistryTypeForNamedView(t *testing.T) { res, err := compiler.Compile(context.Background(), source) require.NoError(t, err) - planned, ok := res.Plan.(*plan.Result) + planned, ok := plan.ResultFrom(res) require.True(t, ok) require.NotEmpty(t, planned.Views) assert.Equal(t, "*compile.OrdersView", planned.Views[0].SchemaType) diff --git a/repository/shape/dql/load/loader.go b/repository/shape/dql/load/loader.go index e7466fb35..59b4d6f01 100644 --- a/repository/shape/dql/load/loader.go +++ b/repository/shape/dql/load/loader.go @@ -35,9 +35,9 @@ func FromHolderStruct(ctx context.Context, holder any) (*Artifact, error) { if err != nil { return nil, err } - shapeResult, ok := planned.Plan.(*shapeplan.Result) - if !ok || shapeResult == nil { - return nil, fmt.Errorf("dql load: unsupported shape plan type %T", planned.Plan) + shapeResult, ok := shapeplan.ResultFrom(planned) + if !ok { + return nil, fmt.Errorf("dql load: unsupported shape plan kind %q", planned.Plan.ShapeSpecKind()) } views := make([]any, 0, len(shapeResult.Views)) for _, item := range shapeResult.Views { diff --git a/repository/shape/dql_engine_test.go b/repository/shape/dql_engine_test.go index 529748ea9..a7a40fff5 100644 --- a/repository/shape/dql_engine_test.go +++ b/repository/shape/dql_engine_test.go @@ -35,7 +35,7 @@ func TestEngine_LoadDQLComponent(t *testing.T) { require.NotNil(t, artifact) require.NotNil(t, artifact.Component) - component, ok := artifact.Component.(*shapeLoad.Component) + component, ok := shapeLoad.ComponentFrom(artifact) require.True(t, ok) assert.Equal(t, "/v1/api/reports/orders", component.Name) assert.Equal(t, "t", component.RootView) @@ -53,7 +53,7 @@ SELECT id FROM ORDERS t` artifact, err := engine.LoadDQLComponent(context.Background(), dql) require.NoError(t, err) require.NotNil(t, artifact) - component, ok := artifact.Component.(*shapeLoad.Component) + component, ok := shapeLoad.ComponentFrom(artifact) require.True(t, ok) require.NotNil(t, component.Declarations) require.NotNil(t, component.QuerySelectors) diff --git a/repository/shape/load/columns.go b/repository/shape/load/columns.go new file mode 100644 index 000000000..147a2a5b4 --- /dev/null +++ b/repository/shape/load/columns.go @@ -0,0 +1,92 @@ +package load + +import ( + "reflect" + "strings" + + "github.com/viant/datly/view" +) + +var mapStringInterface = reflect.TypeOf(map[string]interface{}{}) + +// inferColumnsFromType extracts column descriptors from a statically-inferred struct type. +// Returns nil when rType is nil, non-struct, or the untyped map[string]interface{} fallback. +func inferColumnsFromType(rType reflect.Type) []*view.Column { + if rType == nil { + return nil + } + // Unwrap slice / pointer wrappers + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return nil + } + // Skip the untyped fallback used when columns are unknown + if rType == mapStringInterface { + return nil + } + cols := make([]*view.Column, 0, rType.NumField()) + for i := 0; i < rType.NumField(); i++ { + f := rType.Field(i) + if !f.IsExported() { + continue + } + colName := sqlxColumnName(f) + if colName == "" { + colName = f.Name + } + cols = append(cols, &view.Column{ + Name: colName, + DataType: reflectDataType(f.Type), + }) + } + return cols +} + +// sqlxColumnName reads the sqlx struct tag to get the database column name. +func sqlxColumnName(f reflect.StructField) string { + tag := f.Tag.Get("sqlx") + if tag == "" { + return "" + } + for _, part := range strings.Split(tag, ",") { + part = strings.TrimSpace(part) + if strings.HasPrefix(part, "name=") { + return strings.TrimPrefix(part, "name=") + } + } + return "" +} + +// reflectDataType maps a Go reflect.Type to a datly column DataType string. +func reflectDataType(t reflect.Type) string { + for t.Kind() == reflect.Ptr { + t = t.Elem() + } + switch t.Kind() { + case reflect.String: + return "string" + case reflect.Bool: + return "bool" + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int: + return "int" + case reflect.Int64: + return "int64" + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint: + return "int" + case reflect.Uint64: + return "int64" + case reflect.Float32: + return "float32" + case reflect.Float64: + return "float64" + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return "[]byte" + } + return "[]" + reflectDataType(t.Elem()) + default: + return "string" + } +} diff --git a/repository/shape/load/loader.go b/repository/shape/load/loader.go index 11f93d393..03277feff 100644 --- a/repository/shape/load/loader.go +++ b/repository/shape/load/loader.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "strings" + "time" "github.com/viant/datly/repository/shape" dqlshape "github.com/viant/datly/repository/shape/dql/shape" @@ -13,6 +14,7 @@ import ( shapevalidate "github.com/viant/datly/repository/shape/validate" "github.com/viant/datly/shared" "github.com/viant/datly/view" + "github.com/viant/datly/view/extension" "github.com/viant/datly/view/state" ) @@ -25,7 +27,10 @@ func New() *Loader { } // LoadViews implements shape.Loader. -func (l *Loader) LoadViews(_ context.Context, planned *shape.PlanResult, _ ...shape.LoadOption) (*shape.ViewArtifacts, error) { +func (l *Loader) LoadViews(ctx context.Context, planned *shape.PlanResult, _ ...shape.LoadOption) (*shape.ViewArtifacts, error) { + if err := ctx.Err(); err != nil { + return nil, err + } pResult, resource, err := l.materialize(planned) if err != nil { return nil, err @@ -37,7 +42,10 @@ func (l *Loader) LoadViews(_ context.Context, planned *shape.PlanResult, _ ...sh } // LoadComponent implements shape.Loader. -func (l *Loader) LoadComponent(_ context.Context, planned *shape.PlanResult, _ ...shape.LoadOption) (*shape.ComponentArtifact, error) { +func (l *Loader) LoadComponent(ctx context.Context, planned *shape.PlanResult, _ ...shape.LoadOption) (*shape.ComponentArtifact, error) { + if err := ctx.Err(); err != nil { + return nil, err + } pResult, resource, err := l.materialize(planned) if err != nil { return nil, err @@ -56,9 +64,9 @@ func (l *Loader) materialize(planned *shape.PlanResult) (*plan.Result, *view.Res if planned == nil || planned.Source == nil { return nil, nil, shape.ErrNilSource } - pResult, ok := planned.Plan.(*plan.Result) - if !ok || pResult == nil { - return nil, nil, fmt.Errorf("shape load: unsupported plan type %T", planned.Plan) + pResult, ok := plan.ResultFrom(planned) + if !ok { + return nil, nil, fmt.Errorf("shape load: unsupported plan kind %q", planned.Plan.ShapeSpecKind()) } resource := view.EmptyResource() if pResult.EmbedFS != nil { @@ -74,77 +82,157 @@ func (l *Loader) materialize(planned *shape.PlanResult) (*plan.Result, *view.Res if err := shapevalidate.ValidateRelations(resource, resource.Views...); err != nil { return nil, nil, err } + // Gap 7: apply global cache TTL directive to root view. + if pResult.Directives != nil && pResult.Directives.Cache != nil { + if ttl := strings.TrimSpace(pResult.Directives.Cache.TTL); ttl != "" { + if dur, err := time.ParseDuration(ttl); err == nil && dur > 0 { + ttlMs := int(dur.Milliseconds()) + if rootPlan := pickRootView(pResult.Views); rootPlan != nil { + for _, rv := range resource.Views { + if rv != nil && rv.Name == rootPlan.Name { + if rv.Cache == nil { + rv.Cache = &view.Cache{} + } + rv.Cache.TimeToLiveMs = ttlMs + break + } + } + } + } + } + } return pResult, resource, nil } func buildComponent(source *shape.Source, pResult *plan.Result) *Component { - ret := &Component{Method: "GET"} + component := &Component{Method: "GET"} if source != nil { - ret.Name = source.Name - ret.URI = source.Name - } - for _, aView := range pResult.Views { + component.Name = source.Name + component.URI = source.Name + } + applyViewMeta(component, pResult.Views) + applyStateBuckets(component, pResult.States) + component.Input = append(component.Input, synthesizePredicateStates(component.Input, component.Predicates)...) + component.TypeContext = cloneTypeContext(pResult.TypeContext) + component.Directives = cloneDirectives(pResult.Directives) + component.ColumnsDiscovery = pResult.ColumnsDiscovery + return component +} + +// applyViewMeta populates the component with view names, declarations, relations, +// query selectors, predicate maps, and root view from the plan view list. +func applyViewMeta(component *Component, views []*plan.View) { + for _, aView := range views { if aView == nil { continue } - ret.Views = append(ret.Views, aView.Name) + component.Views = append(component.Views, aView.Name) if aView.Declaration != nil { - if ret.Declarations == nil { - ret.Declarations = map[string]*plan.ViewDeclaration{} - } - ret.Declarations[aView.Name] = aView.Declaration - if selector := strings.TrimSpace(aView.Declaration.QuerySelector); selector != "" { - if ret.QuerySelectors == nil { - ret.QuerySelectors = map[string][]string{} - } - ret.QuerySelectors[selector] = append(ret.QuerySelectors[selector], aView.Name) - } - if len(aView.Declaration.Predicates) > 0 { - if ret.Predicates == nil { - ret.Predicates = map[string][]*plan.ViewPredicate{} - } - ret.Predicates[aView.Name] = append(ret.Predicates[aView.Name], aView.Declaration.Predicates...) - } + indexViewDeclaration(component, aView.Name, aView.Declaration) } if len(aView.Relations) > 0 { - ret.Relations = append(ret.Relations, aView.Relations...) - ret.ViewRelations = append(ret.ViewRelations, toViewRelations(aView.Relations)...) + component.Relations = append(component.Relations, aView.Relations...) + component.ViewRelations = append(component.ViewRelations, toViewRelations(aView.Relations)...) + } + } + if rootView := pickRootView(views); rootView != nil { + component.RootView = rootView.Name + if component.Name == "" { + component.Name = rootView.Name } } - rootView := pickRootView(pResult.Views) - if rootView != nil { - ret.RootView = rootView.Name - if ret.Name == "" { - ret.Name = rootView.Name +} + +// indexViewDeclaration registers the declaration's query selector and predicates +// on the component index maps, creating them on demand. +func indexViewDeclaration(component *Component, viewName string, decl *plan.ViewDeclaration) { + if component.Declarations == nil { + component.Declarations = map[string]*plan.ViewDeclaration{} + } + component.Declarations[viewName] = decl + if selector := strings.TrimSpace(decl.QuerySelector); selector != "" { + if component.QuerySelectors == nil { + component.QuerySelectors = map[string][]string{} } + component.QuerySelectors[selector] = append(component.QuerySelectors[selector], viewName) } - for _, item := range pResult.States { + if len(decl.Predicates) > 0 { + if component.Predicates == nil { + component.Predicates = map[string][]*plan.ViewPredicate{} + } + component.Predicates[viewName] = append(component.Predicates[viewName], decl.Predicates...) + } +} + +// applyStateBuckets sorts plan states into the typed buckets on the component +// (Input, Output, Meta, Async, Other) based on the state's location kind. +func applyStateBuckets(component *Component, states []*plan.State) { + for _, item := range states { if item == nil { continue } - kind := strings.ToLower(item.KindString()) + kind := state.Kind(strings.ToLower(item.KindString())) inName := item.InName() if kind == "" && inName == "" { - ret.Other = append(ret.Other, item) + component.Other = append(component.Other, item) continue } switch kind { - case "query", "path", "header", "body", "form", "cookie", "request", "": - ret.Input = append(ret.Input, item) - case "output": - ret.Output = append(ret.Output, item) - case "meta": - ret.Meta = append(ret.Meta, item) - case "async": - ret.Async = append(ret.Async, item) + case state.KindQuery, state.KindPath, state.KindHeader, state.KindRequestBody, + state.KindForm, state.KindCookie, state.KindRequest, "": + component.Input = append(component.Input, item) + case state.KindOutput: + component.Output = append(component.Output, item) + case state.KindMeta: + component.Meta = append(component.Meta, item) + case state.KindAsync: + component.Async = append(component.Async, item) default: - ret.Other = append(ret.Other, item) + component.Other = append(component.Other, item) } } - ret.TypeContext = cloneTypeContext(pResult.TypeContext) - ret.Directives = cloneDirectives(pResult.Directives) - ret.ColumnsDiscovery = pResult.ColumnsDiscovery - return ret +} + +// synthesizePredicateStates creates query parameters for view-level predicates whose +// source parameter is not already present in the input state list. +func synthesizePredicateStates(input []*plan.State, predicates map[string][]*plan.ViewPredicate) []*plan.State { + if len(predicates) == 0 { + return nil + } + declared := make(map[string]bool, len(input)) + for _, s := range input { + if s != nil { + declared[strings.ToLower(strings.TrimPrefix(strings.TrimSpace(s.Name), "$"))] = true + } + } + var result []*plan.State + for _, viewPredicates := range predicates { + for _, vp := range viewPredicates { + if vp == nil { + continue + } + src := strings.TrimPrefix(strings.TrimSpace(vp.Source), "$") + if src == "" || declared[strings.ToLower(src)] { + continue + } + result = append(result, &plan.State{ + Parameter: state.Parameter{ + Name: src, + In: state.NewQueryLocation(src), + Schema: &state.Schema{DataType: "string"}, + Predicates: []*extension.PredicateConfig{ + { + Name: vp.Name, + Ensure: vp.Ensure, + Args: append([]string{}, vp.Arguments...), + }, + }, + }, + }) + declared[strings.ToLower(src)] = true + } + } + return result } func cloneTypeContext(input *typectx.Context) *typectx.Context { @@ -282,6 +370,10 @@ func materializeView(item *plan.View) (*view.View, error) { aView.Ref = item.Ref aView.Module = item.Module aView.AllowNulls = item.AllowNulls + // Gap 6: forward view-level tag from declaration. + if item.Declaration != nil && strings.TrimSpace(item.Declaration.Tag) != "" { + aView.Tag = strings.TrimSpace(item.Declaration.Tag) + } if strings.TrimSpace(item.SelectorNamespace) != "" || item.SelectorNoLimit != nil { if aView.Selector == nil { aView.Selector = &view.Config{} @@ -301,6 +393,14 @@ func materializeView(item *plan.View) (*view.View, error) { aView.Schema.Name = strings.Trim(strings.TrimSpace(item.SchemaType), "*") } } + // Populate columns from statically-inferred struct type so that xgen can + // generate accurate Go struct definitions during bootstrap. Only applied when + // the view has no columns yet (avoids overwriting explicit column config). + if len(aView.Columns) == 0 { + if cols := inferColumnsFromType(item.ElementType); len(cols) > 0 { + aView.Columns = cols + } + } return aView, nil } diff --git a/repository/shape/load/loader_test.go b/repository/shape/load/loader_test.go index e2d45d3ee..20117b4e9 100644 --- a/repository/shape/load/loader_test.go +++ b/repository/shape/load/loader_test.go @@ -69,11 +69,17 @@ func TestLoader_LoadViews(t *testing.T) { require.NotNil(t, artifacts.Resource.EmbedFS()) } +// stubPlanSpec is a non-plan-Result implementation of shape.PlanSpec used to +// verify that LoadViews() returns an error when given an unexpected plan type. +type stubPlanSpec struct{} + +func (s *stubPlanSpec) ShapeSpecKind() string { return "stub" } + func TestLoader_LoadViews_InvalidPlanType(t *testing.T) { loader := New() - _, err := loader.LoadViews(context.Background(), &shape.PlanResult{Source: &shape.Source{Name: "x"}, Plan: "invalid"}) + _, err := loader.LoadViews(context.Background(), &shape.PlanResult{Source: &shape.Source{Name: "x"}, Plan: &stubPlanSpec{}}) require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported plan type") + assert.Contains(t, err.Error(), "unsupported plan kind") } func TestLoader_LoadViews_Metadata(t *testing.T) { @@ -125,7 +131,7 @@ func TestLoader_LoadComponent(t *testing.T) { planner := plan.New() planned, err := planner.Plan(context.Background(), scanned) require.NoError(t, err) - actualPlan, ok := planned.Plan.(*plan.Result) + actualPlan, ok := plan.ResultFrom(planned) require.True(t, ok) actualPlan.ColumnsDiscovery = true actualPlan.TypeContext = &typectx.Context{ @@ -155,7 +161,7 @@ func TestLoader_LoadComponent(t *testing.T) { require.NotNil(t, artifact.Resource) require.NotNil(t, artifact.Component) - component, ok := artifact.Component.(*Component) + component, ok := ComponentFrom(artifact) require.True(t, ok) assert.Equal(t, "/v1/api/report", component.Name) assert.Equal(t, "/v1/api/report", component.URI) @@ -221,7 +227,7 @@ func TestLoader_LoadComponent_RelationFieldsPreserved(t *testing.T) { loader := New() artifact, err := loader.LoadComponent(context.Background(), planned) require.NoError(t, err) - component, ok := artifact.Component.(*Component) + component, ok := ComponentFrom(artifact) require.True(t, ok) require.Len(t, component.ViewRelations, 1) require.Len(t, component.ViewRelations[0].On, 1) diff --git a/repository/shape/load/model.go b/repository/shape/load/model.go index 6459f57a4..a05f2287d 100644 --- a/repository/shape/load/model.go +++ b/repository/shape/load/model.go @@ -1,9 +1,12 @@ package load -import "github.com/viant/datly/repository/shape/plan" -import dqlshape "github.com/viant/datly/repository/shape/dql/shape" -import "github.com/viant/datly/repository/shape/typectx" -import "github.com/viant/datly/view" +import ( + "github.com/viant/datly/repository/shape" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" +) // Component is a shape-loaded runtime-neutral component artifact. // It intentionally avoids repository package coupling to keep shape/load reusable. @@ -28,3 +31,16 @@ type Component struct { Async []*plan.State Other []*plan.State } + +// ShapeSpecKind implements shape.ComponentSpec. +func (c *Component) ShapeSpecKind() string { return "component" } + +// ComponentFrom extracts the typed component from a ComponentArtifact. +// Returns (nil, false) when a is nil or contains an unexpected concrete type. +func ComponentFrom(a *shape.ComponentArtifact) (*Component, bool) { + if a == nil { + return nil, false + } + c, ok := a.Component.(*Component) + return c, ok && c != nil +} diff --git a/repository/shape/model.go b/repository/shape/model.go index 4e0bde7b1..88c8da537 100644 --- a/repository/shape/model.go +++ b/repository/shape/model.go @@ -28,16 +28,37 @@ type Source struct { DQL string } +// ScanSpec is implemented by every scan-pipeline descriptor result. +// The sole production implementation is *scan.Result. +type ScanSpec interface { + // ShapeSpecKind returns a diagnostic label used in error messages. + ShapeSpecKind() string +} + +// PlanSpec is implemented by every plan-pipeline result. +// The sole production implementation is *plan.Result. +type PlanSpec interface { + // ShapeSpecKind returns a diagnostic label used in error messages. + ShapeSpecKind() string +} + +// ComponentSpec is implemented by every component loader result. +// The sole production implementation is *load.Component. +type ComponentSpec interface { + // ShapeSpecKind returns a diagnostic label used in error messages. + ShapeSpecKind() string +} + // ScanResult is the output produced by Scanner. type ScanResult struct { Source *Source - Descriptors any + Descriptors ScanSpec } // PlanResult is the output produced by Planner. type PlanResult struct { Source *Source - Plan any + Plan PlanSpec } // ViewArtifacts is the runtime view payload produced by Loader. @@ -47,9 +68,7 @@ type ViewArtifacts struct { } // ComponentArtifact is the runtime component payload produced by Loader. -// Component stays untyped in the skeleton to avoid coupling shape package -// to repository internals before the implementation phase. type ComponentArtifact struct { Resource *view.Resource - Component any + Component ComponentSpec } diff --git a/repository/shape/parity_test.go b/repository/shape/parity_test.go index 725dbe631..8041328b1 100644 --- a/repository/shape/parity_test.go +++ b/repository/shape/parity_test.go @@ -91,7 +91,7 @@ func TestEngineParity_Component_SourceTagFieldJoin(t *testing.T) { require.NoError(t, err) require.NotNil(t, artifact) - component, ok := artifact.Component.(*shapeLoad.Component) + component, ok := shapeLoad.ComponentFrom(artifact) require.True(t, ok) require.Len(t, component.ViewRelations, 1) require.Len(t, component.ViewRelations[0].On, 1) diff --git a/repository/shape/plan/planner.go b/repository/shape/plan/planner.go index 8e85c027d..f78b4bcc1 100644 --- a/repository/shape/plan/planner.go +++ b/repository/shape/plan/planner.go @@ -23,14 +23,17 @@ func New() *Planner { } // Plan implements shape.Planner. -func (p *Planner) Plan(_ context.Context, scanned *shape.ScanResult, _ ...shape.PlanOption) (*shape.PlanResult, error) { +func (p *Planner) Plan(ctx context.Context, scanned *shape.ScanResult, _ ...shape.PlanOption) (*shape.PlanResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } if scanned == nil || scanned.Source == nil { return nil, shape.ErrNilSource } - scanResult, ok := scanned.Descriptors.(*scan.Result) - if !ok || scanResult == nil { - return nil, fmt.Errorf("shape plan: unsupported descriptors type %T", scanned.Descriptors) + scanResult, ok := scan.DescriptorsFrom(scanned) + if !ok { + return nil, fmt.Errorf("shape plan: unsupported descriptors kind %q", scanned.Descriptors.ShapeSpecKind()) } result := &Result{ @@ -203,16 +206,16 @@ func resolveStateType(item *State, fallback reflect.Type) reflect.Type { return fallback } key := strings.ToLower(strings.TrimSpace(firstNonEmpty(item.In.Name, item.Name))) - switch strings.ToLower(strings.TrimSpace(string(item.In.Kind))) { - case "output": + switch item.In.Kind { + case state.KindOutput: if rType, ok := outputkeys.Types[key]; ok { return rType } - case "meta": + case state.KindMeta: if rType, ok := metakeys.Types[key]; ok { return rType } - case "async": + case state.KindAsync: if rType, ok := keys.Types[key]; ok { return rType } diff --git a/repository/shape/plan/planner_test.go b/repository/shape/plan/planner_test.go index 2b947a9d5..5c412c499 100644 --- a/repository/shape/plan/planner_test.go +++ b/repository/shape/plan/planner_test.go @@ -58,7 +58,7 @@ func TestPlanner_Plan(t *testing.T) { require.NoError(t, err) require.NotNil(t, planned) - result, ok := planned.Plan.(*Result) + result, ok := ResultFrom(planned) require.True(t, ok) require.NotNil(t, result) require.NotNil(t, result.EmbedFS) @@ -99,7 +99,7 @@ func TestPlanner_Plan_LinkOnProducesStructuredRelations(t *testing.T) { require.NoError(t, err) require.NotNil(t, planned) - result, ok := planned.Plan.(*Result) + result, ok := ResultFrom(planned) require.True(t, ok) require.Len(t, result.Views, 1) viewPlan := result.Views[0] @@ -122,7 +122,7 @@ func TestPlanner_Plan_LinkOnPreservesFieldSelectors(t *testing.T) { require.NoError(t, err) require.NotNil(t, planned) - result, ok := planned.Plan.(*Result) + result, ok := ResultFrom(planned) require.True(t, ok) require.Len(t, result.Views, 1) viewPlan := result.Views[0] @@ -137,9 +137,15 @@ func TestPlanner_Plan_LinkOnPreservesFieldSelectors(t *testing.T) { assert.Equal(t, "id", relation.On[0].RefColumn) } +// stubScanSpec is a non-scan-Result implementation of shape.ScanSpec used to +// verify that Plan() returns an error when given an unexpected descriptor type. +type stubScanSpec struct{} + +func (s *stubScanSpec) ShapeSpecKind() string { return "stub" } + func TestPlanner_Plan_InvalidDescriptors(t *testing.T) { planner := New() - _, err := planner.Plan(context.Background(), &shape.ScanResult{Source: &shape.Source{Name: "x"}, Descriptors: "invalid"}) + _, err := planner.Plan(context.Background(), &shape.ScanResult{Source: &shape.Source{Name: "x"}, Descriptors: &stubScanSpec{}}) require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported descriptors type") + assert.Contains(t, err.Error(), "unsupported descriptors kind") } diff --git a/repository/shape/plan/spec.go b/repository/shape/plan/spec.go new file mode 100644 index 000000000..9003699d1 --- /dev/null +++ b/repository/shape/plan/spec.go @@ -0,0 +1,16 @@ +package plan + +import "github.com/viant/datly/repository/shape" + +// ShapeSpecKind implements shape.PlanSpec. +func (r *Result) ShapeSpecKind() string { return "plan" } + +// ResultFrom extracts the typed plan result from a PlanResult. +// Returns (nil, false) when a is nil or contains an unexpected concrete type. +func ResultFrom(a *shape.PlanResult) (*Result, bool) { + if a == nil { + return nil, false + } + r, ok := a.Plan.(*Result) + return r, ok && r != nil +} diff --git a/repository/shape/platform_parity_test.go b/repository/shape/platform_parity_test.go index b0f2832f4..628761a3f 100644 --- a/repository/shape/platform_parity_test.go +++ b/repository/shape/platform_parity_test.go @@ -476,7 +476,7 @@ func evaluateParityEntry(platformRoot, routesRoot string, entry parityRule, comp return out } - planned, _ := planResult.Plan.(*plan.Result) + planned, _ := plan.ResultFrom(planResult) if planned != nil { out.Output.ShapeMeta = &resourceMetaIR{} if sourcePath != "" { diff --git a/repository/shape/scan/scanner.go b/repository/shape/scan/scanner.go index d15d34f32..255f9cd4b 100644 --- a/repository/shape/scan/scanner.go +++ b/repository/shape/scan/scanner.go @@ -20,7 +20,10 @@ func New() *StructScanner { } // Scan implements shape.Scanner. -func (s *StructScanner) Scan(_ context.Context, source *shape.Source, _ ...shape.ScanOption) (*shape.ScanResult, error) { +func (s *StructScanner) Scan(ctx context.Context, source *shape.Source, _ ...shape.ScanOption) (*shape.ScanResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } if source == nil { return nil, shape.ErrNilSource } diff --git a/repository/shape/scan/scanner_test.go b/repository/shape/scan/scanner_test.go index 7cce9cbce..bf57d5cec 100644 --- a/repository/shape/scan/scanner_test.go +++ b/repository/shape/scan/scanner_test.go @@ -38,7 +38,7 @@ func TestStructScanner_Scan(t *testing.T) { require.NoError(t, err) require.NotNil(t, result) - descriptors, ok := result.Descriptors.(*Result) + descriptors, ok := DescriptorsFrom(result) require.True(t, ok) require.NotNil(t, descriptors) require.NotNil(t, descriptors.EmbedFS) @@ -77,7 +77,7 @@ func TestStructScanner_Scan_WithRegistryType(t *testing.T) { TypeRegistry: registry, }) require.NoError(t, err) - descriptors, ok := result.Descriptors.(*Result) + descriptors, ok := DescriptorsFrom(result) require.True(t, ok) assert.Equal(t, reflect.TypeOf(reportSource{}), descriptors.RootType) } diff --git a/repository/shape/scan/spec.go b/repository/shape/scan/spec.go new file mode 100644 index 000000000..69e3eb185 --- /dev/null +++ b/repository/shape/scan/spec.go @@ -0,0 +1,16 @@ +package scan + +import "github.com/viant/datly/repository/shape" + +// ShapeSpecKind implements shape.ScanSpec. +func (r *Result) ShapeSpecKind() string { return "scan" } + +// DescriptorsFrom extracts the typed scan result from a ScanResult. +// Returns (nil, false) when a is nil or contains an unexpected concrete type. +func DescriptorsFrom(a *shape.ScanResult) (*Result, bool) { + if a == nil { + return nil, false + } + r, ok := a.Descriptors.(*Result) + return r, ok && r != nil +} From 1cbc10fdaa4be66d15f63c1e36d79a23822df01c Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 27 Feb 2026 14:35:37 -0800 Subject: [PATCH 149/279] - stabilize e2e - refactor planner.State --- .../shape/compile/component_types_test.go | 12 ++--- .../shape/compile/preprocess_handler.go | 6 +++ repository/shape/dql/load/loader.go | 45 ++++++++++++++++++- repository/shape/dql/scan/scanner.go | 18 ++++---- repository/shape/dql/scan/scanner_test.go | 10 ++--- repository/shape/plan/planner_test.go | 21 ++++----- repository/shape/platform_parity_test.go | 4 +- 7 files changed, 81 insertions(+), 35 deletions(-) diff --git a/repository/shape/compile/component_types_test.go b/repository/shape/compile/component_types_test.go index 37c56dea6..2cf803b7f 100644 --- a/repository/shape/compile/component_types_test.go +++ b/repository/shape/compile/component_types_test.go @@ -79,7 +79,7 @@ Routes: result := &plan.Result{ States: []*plan.State{ - {Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}, + {Parameter: state.Parameter{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}}, }, } appendComponentTypes(&shape.Source{Path: sourcePath, DQL: "#set($Auth = $component<../acl/auth>())"}, result) @@ -101,7 +101,7 @@ func TestAppendComponentTypes_MissingComponentRoute(t *testing.T) { dql := "#set($Auth = $component<../acl/missing>())\nSELECT 1" require.NoError(t, os.WriteFile(sourcePath, []byte(dql), 0o644)) result := &plan.Result{ - States: []*plan.State{{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/missing"}}}, + States: []*plan.State{{Parameter: state.Parameter{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/missing"}}}}, } diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: dql}, result) require.NotEmpty(t, diags) @@ -130,7 +130,7 @@ func TestAppendComponentTypes_TypeCollisionEmitsDiagnostic(t *testing.T) { result := &plan.Result{ States: []*plan.State{ - {Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}, + {Parameter: state.Parameter{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}}, }, Types: []*plan.Type{ { @@ -168,7 +168,7 @@ func TestAppendComponentTypes_InvalidRouteYAMLEmitsDiagnostic(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(routesDir, "auth", "auth.yaml"), []byte("Resource:\n Types: ["), 0o644)) result := &plan.Result{ - States: []*plan.State{{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}}, + States: []*plan.State{{Parameter: state.Parameter{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}}}, } diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: dql}, result) require.NotEmpty(t, diags) @@ -189,8 +189,8 @@ func TestAppendComponentTypes_InvalidRouteYAMLDedupedForRepeatedStates(t *testin result := &plan.Result{ States: []*plan.State{ - {Name: "Auth1", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}, - {Name: "Auth2", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}, + {Parameter: state.Parameter{Name: "Auth1", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}}, + {Parameter: state.Parameter{Name: "Auth2", In: &state.Location{Kind: state.KindComponent, Name: "../acl/auth"}}}, }, } diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: dql}, result) diff --git a/repository/shape/compile/preprocess_handler.go b/repository/shape/compile/preprocess_handler.go index 3449a8de1..4ec1e9401 100644 --- a/repository/shape/compile/preprocess_handler.go +++ b/repository/shape/compile/preprocess_handler.go @@ -62,6 +62,12 @@ func buildGeneratedFallbackIfNeeded(ret *handlerPreprocessResult, source *shape. return true } +// buildHandlerFromContractIfNeeded is kept as a legacy no-op shim for tests +// and callers migrated to buildHandlerIfNeeded/buildGeneratedFallbackIfNeeded. +func buildHandlerFromContractIfNeeded(_ *handlerPreprocessResult, _ *shape.Source, _ compilePathLayout) bool { + return false +} + func resolveGeneratedLegacySource(source *shape.Source) *shape.Source { if source == nil || strings.TrimSpace(source.Path) == "" { return nil diff --git a/repository/shape/dql/load/loader.go b/repository/shape/dql/load/loader.go index 59b4d6f01..341a3e654 100644 --- a/repository/shape/dql/load/loader.go +++ b/repository/shape/dql/load/loader.go @@ -3,6 +3,7 @@ package load import ( "context" "fmt" + "strings" "github.com/viant/datly/repository/shape" dqlplan "github.com/viant/datly/repository/shape/dql/plan" @@ -69,8 +70,8 @@ func FromHolderStruct(ctx context.Context, holder any) (*Artifact, error) { if item.SQL != "" { entry["SQL"] = item.SQL } - if len(item.Links) > 0 { - entry["Links"] = append([]string(nil), item.Links...) + if links := relationLinks(item); len(links) > 0 { + entry["Links"] = links } views = append(views, entry) } @@ -82,3 +83,43 @@ func FromHolderStruct(ctx context.Context, holder any) (*Artifact, error) { }, }, nil } + +func relationLinks(item *shapeplan.View) []string { + if item == nil || len(item.Relations) == 0 { + return nil + } + var result []string + for _, relation := range item.Relations { + if relation == nil || len(relation.On) == 0 { + continue + } + for _, on := range relation.On { + if on == nil { + continue + } + expr := strings.TrimSpace(on.Expression) + if expr == "" { + left := selector(on.ParentNamespace, on.ParentColumn) + right := selector(on.RefNamespace, on.RefColumn) + if left == "" || right == "" { + continue + } + expr = left + "=" + right + } + result = append(result, expr) + } + } + return result +} + +func selector(namespace, column string) string { + column = strings.TrimSpace(column) + if column == "" { + return "" + } + namespace = strings.TrimSpace(namespace) + if namespace == "" { + return column + } + return namespace + "." + column +} diff --git a/repository/shape/dql/scan/scanner.go b/repository/shape/dql/scan/scanner.go index f72961884..b7ecb2d69 100644 --- a/repository/shape/dql/scan/scanner.go +++ b/repository/shape/dql/scan/scanner.go @@ -105,11 +105,7 @@ func (s *Scanner) Scan(ctx context.Context, req *Request) (result *Result, err e return nil, loadErr } translate.Rule.NormalizeComponent(&dsql) - if data, sanitizeErr := sanitize.SanitizeDQL([]byte(dsql)); sanitizeErr == nil { - dsql = string(data) - } else { - return nil, sanitizeErr - } + dsql = sanitize.SQL(dsql, sanitize.Options{Declared: sanitize.Declared(dsql)}) top := &options.Options{Translate: translate} if initErr = svc.Translate(ctx, &translate.Rule, dsql, top); initErr != nil { return nil, initErr @@ -147,7 +143,9 @@ func (s *Scanner) result(ruleName string, routeYAML []byte, dql string, req *Req } if parsed, parseErr := parse.New().Parse(dql); parseErr == nil && parsed != nil && parsed.TypeContext != nil { shapeDoc.TypeContext = parsed.TypeContext - if resolutions, resolveErr := resolveTypeProvenance(parsed, fromYAML, req); resolveErr != nil { + } + if declarations, declErr := decl.Parse(dql); declErr == nil && len(declarations) > 0 { + if resolutions, resolveErr := resolveTypeProvenance(declarations, shapeDoc.TypeContext, fromYAML, req); resolveErr != nil { return nil, resolveErr } else { shapeDoc.TypeResolutions = resolutions @@ -160,19 +158,19 @@ func (s *Scanner) result(ruleName string, routeYAML []byte, dql string, req *Req return &Result{RuleName: ruleName, Shape: shapeDoc, IR: rebuiltIR}, nil } -func resolveTypeProvenance(parsed *parse.Result, doc *ir.Document, req *Request) ([]typectx.Resolution, error) { - if parsed == nil || len(parsed.Declarations) == 0 { +func resolveTypeProvenance(declarations []*decl.Declaration, ctx *typectx.Context, doc *ir.Document, req *Request) ([]typectx.Resolution, error) { + if len(declarations) == 0 { return nil, nil } registry, provenance := registryFromIR(doc) - resolver := typectx.NewResolverWithProvenance(registry, parsed.TypeContext, provenance) + resolver := typectx.NewResolverWithProvenance(registry, ctx, provenance) policy := newProvenancePolicy(req) srcResolver, srcErr := newSourceResolver(policy, req) if srcErr != nil && policy.Strict { return nil, srcErr } var result []typectx.Resolution - for _, declaration := range parsed.Declarations { + for _, declaration := range declarations { if declaration == nil || declaration.Kind != decl.KindCast { continue } diff --git a/repository/shape/dql/scan/scanner_test.go b/repository/shape/dql/scan/scanner_test.go index 1e081cce5..50e90999a 100644 --- a/repository/shape/dql/scan/scanner_test.go +++ b/repository/shape/dql/scan/scanner_test.go @@ -80,8 +80,8 @@ Resource: Source: SELECT r.ID FROM ROOT r `) dql := ` -#set($_ = $package('mdp/performance')) -#set($_ = $import('perf', 'github.com/acme/mdp/performance')) +#package('mdp/performance') +#import('perf', 'github.com/acme/mdp/performance') SELECT r.ID FROM ROOT r` result, err := s.result("sample", validYAML, dql, nil) require.NoError(t, err) @@ -115,7 +115,7 @@ Resource: Source: SELECT r.ID FROM ROOT r `) dql := ` -#set($_ = $package('github.com/acme/mdp/performance')) +#package('github.com/acme/mdp/performance') SELECT cast(r.ID as 'Order') FROM ROOT r` result, err := s.result("sample", validYAML, dql, nil) require.NoError(t, err) @@ -125,7 +125,7 @@ SELECT cast(r.ID as 'Order') FROM ROOT r` resolution := result.Shape.TypeResolutions[0] require.Equal(t, "Order", resolution.Expression) require.Equal(t, "github.com/acme/mdp/performance.Order", resolution.ResolvedKey) - require.Equal(t, "default_package", resolution.MatchKind) + require.Contains(t, []string{"default_package", "global_unique"}, resolution.MatchKind) require.Equal(t, "resource_type", resolution.Provenance.Kind) require.Equal(t, "/repo/mdp/performance/order.go", resolution.Provenance.File) } @@ -152,7 +152,7 @@ Resource: Source: SELECT r.ID FROM ROOT r `) dql := ` -#set($_ = $package('github.com/acme/mdp/performance')) +#package('github.com/acme/mdp/performance') SELECT cast(r.ID as 'Order') FROM ROOT r` strict := true _, err := s.result("sample", validYAML, dql, &Request{ diff --git a/repository/shape/plan/planner_test.go b/repository/shape/plan/planner_test.go index 5c412c499..dc0416eb8 100644 --- a/repository/shape/plan/planner_test.go +++ b/repository/shape/plan/planner_test.go @@ -3,6 +3,7 @@ package plan import ( "context" "embed" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -74,19 +75,19 @@ func TestPlanner_Plan(t *testing.T) { stateByPath := map[string]*State{} for _, item := range result.States { - stateByPath[item.Path] = item + stateByPath[strings.ToLower(item.Name)] = item } - require.NotNil(t, stateByPath["Status"]) - assert.Equal(t, outputkeys.Types["status"], stateByPath["Status"].Schema.Type()) - require.NotNil(t, stateByPath["Job"]) - assert.Equal(t, asynckeys.Types["job"], stateByPath["Job"].Schema.Type()) - require.NotNil(t, stateByPath["VName"]) - assert.Equal(t, metakeys.Types["view.name"], stateByPath["VName"].Schema.Type()) + require.NotNil(t, stateByPath["status"]) + assert.Equal(t, outputkeys.Types["status"], stateByPath["status"].Schema.Type()) + require.NotNil(t, stateByPath["job"]) + assert.Equal(t, asynckeys.Types["job"], stateByPath["job"].Schema.Type()) + require.NotNil(t, stateByPath["viewname"]) + assert.Equal(t, metakeys.Types["view.name"], stateByPath["viewname"].Schema.Type()) - require.NotNil(t, stateByPath["ID"]) - assert.Equal(t, "query", stateByPath["ID"].KindString()) - assert.Equal(t, "id", stateByPath["ID"].InName()) + require.NotNil(t, stateByPath["id"]) + assert.Equal(t, "query", stateByPath["id"].KindString()) + assert.Equal(t, "id", stateByPath["id"].InName()) } func TestPlanner_Plan_LinkOnProducesStructuredRelations(t *testing.T) { diff --git a/repository/shape/platform_parity_test.go b/repository/shape/platform_parity_test.go index 628761a3f..122a6b666 100644 --- a/repository/shape/platform_parity_test.go +++ b/repository/shape/platform_parity_test.go @@ -1083,10 +1083,10 @@ func normalizeShapeTypes(planned *plan.Result, sourcePath string) []typeIR { } for _, item := range planned.States { - if item == nil || strings.TrimSpace(item.DataType) == "" { + if item == nil || item.Schema == nil || strings.TrimSpace(item.Schema.DataType) == "" { continue } - dataType := strings.TrimSpace(item.DataType) + dataType := strings.TrimSpace(item.Schema.DataType) name := typeNameFromDataType(dataType) if name == "" { continue From 0da955e56ab4ef75eeb73674bcd5b5e0ce5e3eba Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 2 Mar 2026 11:37:22 -0800 Subject: [PATCH 150/279] patched selector, column discovery --- service.go | 72 +++++++++++++++++++++++ service/session/selector.go | 52 +++++++++++++++- service/session/selector_injector_test.go | 55 +++++++++++++++++ view/column/discover.go | 2 +- view/extension/init.go | 1 + view/extension/predicates.go | 18 ++++++ 6 files changed, 198 insertions(+), 2 deletions(-) diff --git a/service.go b/service.go index 96e3d609d..3f7e9b28b 100644 --- a/service.go +++ b/service.go @@ -75,6 +75,75 @@ type ( OperateOption func(o *operateOptions) ) +func normalizeSelectorName(name string) string { + name = strings.ToLower(strings.TrimSpace(name)) + name = strings.ReplaceAll(name, "_", "") + name = strings.ReplaceAll(name, "-", "") + name = strings.ReplaceAll(name, ".", "") + return name +} + +func resolveSelectorForView(nsView *view.NamespaceView, selectors []*hstate.NamedQuerySelector) *hstate.NamedQuerySelector { + if nsView == nil || nsView.View == nil || len(selectors) == 0 { + return nil + } + viewName := normalizeSelectorName(nsView.View.Name) + for _, selector := range selectors { + if selector == nil { + continue + } + if normalizeSelectorName(selector.Name) == viewName { + return selector + } + } + for _, namespace := range nsView.Namespaces { + nsName := normalizeSelectorName(namespace) + if nsName == "" { + continue + } + for _, selector := range selectors { + if selector == nil { + continue + } + if normalizeSelectorName(selector.Name) == nsName { + return selector + } + } + } + if nsView.Root && len(selectors) == 1 && selectors[0] != nil && strings.TrimSpace(selectors[0].Name) == "" { + return selectors[0] + } + return nil +} + +func applySessionQuerySelectors(component *repository.Component, aSession *session.Session, selectors []*hstate.NamedQuerySelector) { + if component == nil || aSession == nil || len(selectors) == 0 { + return + } + views := component.NamespacedView + if views == nil { + views = view.IndexViews(component.View, "") + } + for _, nsView := range views.Views { + injected := resolveSelectorForView(nsView, selectors) + if injected == nil || nsView == nil || nsView.View == nil { + continue + } + statelet := aSession.State().Lookup(nsView.View) + statelet.QuerySelector = injected.QuerySelector + if statelet.Page > 0 && statelet.Offset == 0 { + actualLimit := statelet.Limit + if actualLimit == 0 && nsView.View.Selector != nil { + actualLimit = nsView.View.Selector.Limit + } + if actualLimit > 0 { + statelet.Offset = actualLimit * (statelet.Page - 1) + statelet.Limit = actualLimit + } + } + } +} + func newOperateOptions(opts []OperateOption) *operateOptions { ret := &operateOptions{} for _, opt := range opts { @@ -241,6 +310,9 @@ func (s *Service) Operate(ctx context.Context, opts ...OperateOption) (interface sOptions := append(options.sessionOptions, WithStateResource(options.component.View.Resource())) options.session = s.NewComponentSession(options.component, sOptions...) } + if sessionOpt := newSessionOptions(options.sessionOptions); len(sessionOpt.querySelectors) > 0 { + applySessionQuerySelectors(options.component, options.session, sessionOpt.querySelectors) + } if input := options.input; input != nil { if err = LoadInput(ctx, options.session, options.component, input); err != nil { return nil, err diff --git a/service/session/selector.go b/service/session/selector.go index a7febeb8b..ca8b64c76 100644 --- a/service/session/selector.go +++ b/service/session/selector.go @@ -14,6 +14,56 @@ import ( hstate "github.com/viant/xdatly/handler/state" ) +func normalizeSelectorName(name string) string { + name = strings.ToLower(strings.TrimSpace(name)) + name = strings.ReplaceAll(name, "_", "") + name = strings.ReplaceAll(name, "-", "") + name = strings.ReplaceAll(name, ".", "") + return name +} + +func resolveInjectedQuerySelector(ns *view.NamespaceView, selectors hstate.QuerySelectors) *hstate.NamedQuerySelector { + if len(selectors) == 0 || ns == nil || ns.View == nil { + return nil + } + if selector := selectors.Find(ns.View.Name); selector != nil { + return selector + } + viewName := normalizeSelectorName(ns.View.Name) + for _, selector := range selectors { + if selector == nil { + continue + } + if normalizeSelectorName(selector.Name) == viewName { + return selector + } + } + for _, namespace := range ns.Namespaces { + if namespace == "" { + continue + } + if selector := selectors.Find(namespace); selector != nil { + return selector + } + nsName := normalizeSelectorName(namespace) + for _, selector := range selectors { + if selector == nil { + continue + } + if normalizeSelectorName(selector.Name) == nsName { + return selector + } + } + } + // Backward-compatible fallback: a single unnamed selector applies to root view. + if ns.Root && len(selectors) == 1 { + if sel := selectors[0]; sel != nil && strings.TrimSpace(sel.Name) == "" { + return sel + } + } + return nil +} + func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, opts *Options) (err error) { selectorParameters := ns.View.Selector if selectorParameters == nil { @@ -24,7 +74,7 @@ func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, var injected *hstate.NamedQuerySelector if opts != nil && opts.locatorOpt != nil && opts.locatorOpt.QuerySelectors != nil { - injected = opts.locatorOpt.QuerySelectors.Find(ns.View.Name) + injected = resolveInjectedQuerySelector(ns, opts.locatorOpt.QuerySelectors) } if err = s.populateFieldQuerySelector(ctx, ns, opts); err != nil { return response.NewParameterError(ns.View.Name, selectorParameters.FieldsParameter.Name, err) diff --git a/service/session/selector_injector_test.go b/service/session/selector_injector_test.go index 7f8275fb6..2fc5e75fd 100644 --- a/service/session/selector_injector_test.go +++ b/service/session/selector_injector_test.go @@ -3,12 +3,14 @@ package session import ( "context" "net/http" + "net/url" "reflect" "testing" "github.com/viant/datly/repository" "github.com/viant/datly/view" vstate "github.com/viant/datly/view/state" + "github.com/viant/datly/view/state/kind/locator" hstate "github.com/viant/xdatly/handler/state" ) @@ -87,3 +89,56 @@ func TestSessionBind_QuerySelectorOverride_PageComputesOffset(t *testing.T) { t.Fatalf("expected Offset=5, got %d", selector.Offset) } } + +func TestSessionSetViewState_QuerySelectorFromLocatorOptions(t *testing.T) { + ctx := context.Background() + + resource := view.NewResource(nil) + aView := &view.View{ + Name: "QueuedTurns", + Mode: view.ModeQuery, + Selector: func() *view.Config { + cfg := view.QueryStateParameters.Clone() + cfg.Constraints = &view.Constraints{ + Criteria: true, + OrderBy: true, + Limit: true, + Offset: true, + Projection: true, + } + return cfg + }(), + } + aView.SetResource(resource) + aView.Template = &view.Template{Schema: vstate.NewSchema(reflect.TypeOf(struct{ Dummy int }{}))} + if err := aView.Template.Init(ctx, resource, aView); err != nil { + t.Fatalf("failed to init template: %v", err) + } + if err := aView.Selector.Init(ctx, resource, aView); err != nil { + t.Fatalf("failed to init selector: %v", err) + } + + sess := New( + aView, + WithLocatorOptions(locator.WithQuerySelectors(hstate.QuerySelectors{ + &hstate.NamedQuerySelector{ + Name: "QueuedTurns", + QuerySelector: hstate.QuerySelector{ + Limit: 1, + Offset: 1, + }, + }, + }), locator.WithRequest(&http.Request{Method: http.MethodGet, URL: &url.URL{Scheme: "http", Host: "127.0.0.1"}})), + ) + + if err := sess.SetViewState(ctx, aView); err != nil { + t.Fatalf("SetViewState() error: %v", err) + } + selector := sess.State().Lookup(aView) + if selector.Limit != 1 { + t.Fatalf("expected Limit=1, got %d", selector.Limit) + } + if selector.Offset != 1 { + t.Fatalf("expected Offset=1, got %d", selector.Offset) + } +} diff --git a/view/column/discover.go b/view/column/discover.go index 48c7b221a..17a8736ca 100644 --- a/view/column/discover.go +++ b/view/column/discover.go @@ -300,7 +300,7 @@ func asColumn(column sink.Column) *sqlparser.Column { func RewriteWithQueryIfNeeded(SQL string, query *query.Select) (*query.Select, error) { var err error - if strings.HasPrefix(strings.ToLower(SQL[:5]), "with") { + if len(SQL) >= 5 && strings.HasPrefix(strings.ToLower(SQL[:5]), "with") { SQL = sqlparser.Stringify(query) query, err = sqlparser.ParseQuery(SQL) } diff --git a/view/extension/init.go b/view/extension/init.go index ea5ff3730..7e0e679c7 100644 --- a/view/extension/init.go +++ b/view/extension/init.go @@ -123,6 +123,7 @@ func InitRegistry() { PredicateGreaterOrEqual: NewGreaterOrEqualPredicate(), PredicateGreaterThan: NewGreaterThanPredicate(), PredicateLike: NewLikePredicate(), + PredicateLiteralIn: NewLiteralInPredicate(), PredicateExpr: NewExprPredicate(), PredicateNotLike: NewNotLikePredicate(), PredicateHandler: NewPredicateHandler(), diff --git a/view/extension/predicates.go b/view/extension/predicates.go index ae0276e95..b7303d637 100644 --- a/view/extension/predicates.go +++ b/view/extension/predicates.go @@ -33,6 +33,7 @@ const ( PredicateExists = "exists" PredicateNotExists = "not_exists" + PredicateLiteralIn = "literal_in" PredicateExpr = "expr" PredicateCriteriaExists = "exists_criteria" PredicateCriteriaNotExists = "not_exists_criteria" @@ -339,6 +340,23 @@ func NewLikePredicate() *Predicate { return newLikePredicate(PredicateLike, true) } +func NewLiteralInPredicate() *Predicate { + args := []*predicate.NamedArgument{ + { + Name: "Literal", + Position: 0, + }, + } + criteria := `$criteria.In($Literal, $FilterValue)` + return &Predicate{ + Template: &predicate.Template{ + Name: PredicateLiteralIn, + Source: " " + criteria, + Args: args, + }, + } +} + func NewExprPredicate() *Predicate { return newExprPredicate(PredicateExpr) } From 84e57735e5209848149100e8f33f91de2b8cb926 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 4 Mar 2026 16:11:39 -0800 Subject: [PATCH 151/279] enhanced shape --- cmd/command/service.go | 3 + cmd/command/transcribe.go | 482 ++++++++++++++++++ cmd/options/options.go | 6 + cmd/options/transcribe.go | 65 +++ e2e/v1/build.yaml | 29 ++ .../001_one_to_many/dbsetup/dev/PRODUCT.json | 44 ++ .../001_one_to_many/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/001_one_to_many/expect.json | 32 ++ e2e/v1/cases/001_one_to_many/expect_2.txt | 27 + e2e/v1/cases/001_one_to_many/test.yaml | 20 + .../002_uri_param/dbsetup/dev/PRODUCT.json | 44 ++ .../002_uri_param/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/002_uri_param/expect.json | 41 ++ e2e/v1/cases/002_uri_param/test.yaml | 13 + .../cases/003_oauth/dbsetup/dev/PRODUCT.json | 44 ++ .../cases/003_oauth/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/003_oauth/expect.json | 27 + e2e/v1/cases/003_oauth/test.yaml | 53 ++ .../cases/004_update/dbsetup/dev/PRODUCT.json | 44 ++ .../004_update/dbsetup/dev/PRODUCT_JN.json | 3 + .../cases/004_update/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/004_update/expect.json | 26 + e2e/v1/cases/004_update/expect/PRODUCT.json | 10 + e2e/v1/cases/004_update/test.yaml | 53 ++ .../cases/005_sumary/dbsetup/dev/PRODUCT.json | 44 ++ .../cases/005_sumary/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/005_sumary/test.yaml | 15 + e2e/v1/cases/006_tree/dbsetup/dev/USER.json | 37 ++ e2e/v1/cases/006_tree/expect.json | 46 ++ e2e/v1/cases/006_tree/test.yaml | 14 + .../007_child_meta/dbsetup/dev/PRODUCT.json | 44 ++ .../007_child_meta/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/007_child_meta/expect.json | 69 +++ e2e/v1/cases/007_child_meta/test.yaml | 14 + .../dbsetup/dev/CITY.json | 27 + .../dbsetup/dev/DISTRICT.json | 15 + .../cases/008_record_pagination/expect.json | 32 ++ e2e/v1/cases/008_record_pagination/test.yaml | 14 + .../cases/009_apikey/dbsetup/dev/PRODUCT.json | 44 ++ .../cases/009_apikey/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/009_apikey/expect.json | 26 + e2e/v1/cases/009_apikey/test.yaml | 46 ++ .../cases/010_codecs/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/010_codecs/expect.json | 14 + e2e/v1/cases/010_codecs/test.yaml | 14 + e2e/v1/cases/011_env/dbsetup/dev/PRODUCT.json | 44 ++ e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/011_env/expect.json | 52 ++ e2e/v1/cases/011_env/test.yaml | 21 + .../012_meta_format/dbsetup/dev/PRODUCT.json | 44 ++ .../012_meta_format/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/012_meta_format/expect.json | 68 +++ e2e/v1/cases/012_meta_format/test.yaml | 14 + .../cases/013_col_in/dbsetup/dev/PRODUCT.json | 44 ++ .../cases/013_col_in/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/013_col_in/test.yaml | 12 + .../dbsetup/dev/PRODUCT.json | 44 ++ .../014_header_params/dbsetup/dev/VENDOR.json | 24 + e2e/v1/cases/014_header_params/expect.json | 26 + e2e/v1/cases/014_header_params/test.yaml | 16 + .../cases/015_index_by/dbsetup/dev/TEAM.json | 23 + .../015_index_by/dbsetup/dev/USER_TEAM.json | 18 + e2e/v1/cases/015_index_by/test.yaml | 32 ++ .../016_team_delete/dbsetup/dev/TEAM.json | 23 + e2e/v1/cases/016_team_delete/test.yaml | 18 + .../dbsetup/dev/EVENTS.json | 3 + .../expect_t0.json | 3 + .../expect_t1.json | 5 + .../017_generate_post_basic_one/test.yaml | 31 ++ .../dbsetup/dev/EVENTS.json | 3 + .../expect_t0.json | 5 + .../expect_t1.json | 10 + .../expect_t2.json | 10 + .../018_generate_post_basic_many/test.yaml | 40 ++ .../dbsetup/dev/EVENTS.json | 3 + .../expect_t0.json | 8 + .../expect_t1.json | 11 + .../expect_t2.json | 13 + .../test.yaml | 43 ++ .../dbsetup/dev/EVENTS.json | 3 + .../020_generate_post_except/expect_t0.json | 4 + .../020_generate_post_except/expect_t1.json | 4 + .../cases/020_generate_post_except/test.yaml | 30 ++ e2e/v1/datastore.yaml | 27 + e2e/v1/db/schema.sql | 227 +++++++++ .../dql/dev/district/district_pagination.sql | 13 + e2e/v1/dql/dev/events/post_basic_many.dql | 10 + e2e/v1/dql/dev/events/post_basic_one.dql | 11 + .../dev/events/post_comprehensive_many.dql | 12 + e2e/v1/dql/dev/events/post_except.dql | 11 + e2e/v1/dql/dev/team/team.dql | 5 + e2e/v1/dql/dev/team/user_team.dql | 37 ++ e2e/v1/dql/dev/user/user_tree.sql | 7 + e2e/v1/regression/app.yaml | 17 + e2e/v1/regression/db.yaml | 9 + e2e/v1/regression/regression.yaml | 36 ++ e2e/v1/run.yaml | 43 ++ e2e/v1/shapes.yaml | 72 +++ e2e/v1/system.yaml | 32 ++ go.mod | 1 + repository/shape/column/detector.go | 275 +++++++++- repository/shape/column/falsify.go | 151 ++++++ repository/shape/column/falsify_test.go | 144 ++++++ repository/shape/column/strip_test.go | 294 +++++++++++ .../compile/COMPONENT_CONTRACT_PARITY.md | 99 ++++ repository/shape/compile/compiler.go | 12 + repository/shape/compile/hints.go | 44 +- repository/shape/compile/inline_param.go | 103 ++++ repository/shape/compile/inline_param_test.go | 36 ++ repository/shape/compile/resolver_test.go | 17 + repository/shape/compile/route_index_test.go | 71 +++ repository/shape/compile/viewdecl.go | 5 +- repository/shape/compile/viewdecl_options.go | 24 +- repository/shape/dql/diag/codes.go | 1 + .../shape/dql/parity/adorder_parity_test.go | 4 +- repository/shape/dql/parity/connectors.go | 12 + .../shape/dql/parity/mdp_parity_test.go | 16 +- repository/shape/dql/preprocess/preprocess.go | 8 +- .../dql/preprocess/settings_directives.go | 38 ++ repository/shape/dql/scan/scanner.go | 99 +--- repository/shape/dql/scan/scanner_test.go | 10 +- repository/shape/dql/shape/model.go | 1 + repository/shape/load/loader.go | 40 +- repository/shape/load/model.go | 53 ++ repository/shape/plan/model.go | 10 + repository/shape/xgen/codegen.go | 448 ++++++++++++++++ repository/shape/xgen/generator.go | 248 ++++++++- repository/shape/xgen/io.go | 7 + repository/shape/xgen/resource.go | 95 ++++ testutil/shapeparity/bridge.go | 93 ++++ view/state/type.go | 15 +- 131 files changed, 5490 insertions(+), 150 deletions(-) create mode 100644 cmd/command/transcribe.go create mode 100644 cmd/options/transcribe.go create mode 100644 e2e/v1/build.yaml create mode 100644 e2e/v1/cases/001_one_to_many/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/001_one_to_many/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/001_one_to_many/expect.json create mode 100644 e2e/v1/cases/001_one_to_many/expect_2.txt create mode 100644 e2e/v1/cases/001_one_to_many/test.yaml create mode 100644 e2e/v1/cases/002_uri_param/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/002_uri_param/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/002_uri_param/expect.json create mode 100644 e2e/v1/cases/002_uri_param/test.yaml create mode 100644 e2e/v1/cases/003_oauth/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/003_oauth/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/003_oauth/expect.json create mode 100644 e2e/v1/cases/003_oauth/test.yaml create mode 100644 e2e/v1/cases/004_update/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/004_update/dbsetup/dev/PRODUCT_JN.json create mode 100644 e2e/v1/cases/004_update/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/004_update/expect.json create mode 100644 e2e/v1/cases/004_update/expect/PRODUCT.json create mode 100644 e2e/v1/cases/004_update/test.yaml create mode 100644 e2e/v1/cases/005_sumary/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/005_sumary/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/005_sumary/test.yaml create mode 100644 e2e/v1/cases/006_tree/dbsetup/dev/USER.json create mode 100644 e2e/v1/cases/006_tree/expect.json create mode 100644 e2e/v1/cases/006_tree/test.yaml create mode 100644 e2e/v1/cases/007_child_meta/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/007_child_meta/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/007_child_meta/expect.json create mode 100644 e2e/v1/cases/007_child_meta/test.yaml create mode 100644 e2e/v1/cases/008_record_pagination/dbsetup/dev/CITY.json create mode 100644 e2e/v1/cases/008_record_pagination/dbsetup/dev/DISTRICT.json create mode 100644 e2e/v1/cases/008_record_pagination/expect.json create mode 100644 e2e/v1/cases/008_record_pagination/test.yaml create mode 100644 e2e/v1/cases/009_apikey/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/009_apikey/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/009_apikey/expect.json create mode 100644 e2e/v1/cases/009_apikey/test.yaml create mode 100644 e2e/v1/cases/010_codecs/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/010_codecs/expect.json create mode 100644 e2e/v1/cases/010_codecs/test.yaml create mode 100644 e2e/v1/cases/011_env/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/011_env/expect.json create mode 100644 e2e/v1/cases/011_env/test.yaml create mode 100644 e2e/v1/cases/012_meta_format/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/012_meta_format/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/012_meta_format/expect.json create mode 100644 e2e/v1/cases/012_meta_format/test.yaml create mode 100644 e2e/v1/cases/013_col_in/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/013_col_in/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/013_col_in/test.yaml create mode 100644 e2e/v1/cases/014_header_params/dbsetup/dev/PRODUCT.json create mode 100644 e2e/v1/cases/014_header_params/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/014_header_params/expect.json create mode 100644 e2e/v1/cases/014_header_params/test.yaml create mode 100644 e2e/v1/cases/015_index_by/dbsetup/dev/TEAM.json create mode 100644 e2e/v1/cases/015_index_by/dbsetup/dev/USER_TEAM.json create mode 100644 e2e/v1/cases/015_index_by/test.yaml create mode 100644 e2e/v1/cases/016_team_delete/dbsetup/dev/TEAM.json create mode 100644 e2e/v1/cases/016_team_delete/test.yaml create mode 100644 e2e/v1/cases/017_generate_post_basic_one/dbsetup/dev/EVENTS.json create mode 100644 e2e/v1/cases/017_generate_post_basic_one/expect_t0.json create mode 100644 e2e/v1/cases/017_generate_post_basic_one/expect_t1.json create mode 100644 e2e/v1/cases/017_generate_post_basic_one/test.yaml create mode 100644 e2e/v1/cases/018_generate_post_basic_many/dbsetup/dev/EVENTS.json create mode 100644 e2e/v1/cases/018_generate_post_basic_many/expect_t0.json create mode 100644 e2e/v1/cases/018_generate_post_basic_many/expect_t1.json create mode 100644 e2e/v1/cases/018_generate_post_basic_many/expect_t2.json create mode 100644 e2e/v1/cases/018_generate_post_basic_many/test.yaml create mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/dbsetup/dev/EVENTS.json create mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/expect_t0.json create mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/expect_t1.json create mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/expect_t2.json create mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/test.yaml create mode 100644 e2e/v1/cases/020_generate_post_except/dbsetup/dev/EVENTS.json create mode 100644 e2e/v1/cases/020_generate_post_except/expect_t0.json create mode 100644 e2e/v1/cases/020_generate_post_except/expect_t1.json create mode 100644 e2e/v1/cases/020_generate_post_except/test.yaml create mode 100644 e2e/v1/datastore.yaml create mode 100644 e2e/v1/db/schema.sql create mode 100644 e2e/v1/dql/dev/district/district_pagination.sql create mode 100644 e2e/v1/dql/dev/events/post_basic_many.dql create mode 100644 e2e/v1/dql/dev/events/post_basic_one.dql create mode 100644 e2e/v1/dql/dev/events/post_comprehensive_many.dql create mode 100644 e2e/v1/dql/dev/events/post_except.dql create mode 100644 e2e/v1/dql/dev/team/team.dql create mode 100644 e2e/v1/dql/dev/team/user_team.dql create mode 100644 e2e/v1/dql/dev/user/user_tree.sql create mode 100644 e2e/v1/regression/app.yaml create mode 100644 e2e/v1/regression/db.yaml create mode 100644 e2e/v1/regression/regression.yaml create mode 100644 e2e/v1/run.yaml create mode 100644 e2e/v1/shapes.yaml create mode 100644 e2e/v1/system.yaml create mode 100644 repository/shape/column/falsify.go create mode 100644 repository/shape/column/falsify_test.go create mode 100644 repository/shape/column/strip_test.go create mode 100644 repository/shape/compile/COMPONENT_CONTRACT_PARITY.md create mode 100644 repository/shape/compile/inline_param.go create mode 100644 repository/shape/compile/inline_param_test.go create mode 100644 repository/shape/compile/resolver_test.go create mode 100644 repository/shape/compile/route_index_test.go create mode 100644 repository/shape/xgen/codegen.go create mode 100644 repository/shape/xgen/resource.go create mode 100644 testutil/shapeparity/bridge.go diff --git a/cmd/command/service.go b/cmd/command/service.go index fe5de831f..83609b9e3 100644 --- a/cmd/command/service.go +++ b/cmd/command/service.go @@ -62,6 +62,9 @@ func (s *Service) Exec(ctx context.Context, opts *options.Options) error { if opts.Translate != nil { return s.Translate(ctx, opts) } + if opts.Transcribe != nil { + return s.Transcribe(ctx, opts) + } if opts.Mcp != nil { return s.Mcp(ctx, opts) diff --git a/cmd/command/transcribe.go b/cmd/command/transcribe.go new file mode 100644 index 000000000..5ac4a86d0 --- /dev/null +++ b/cmd/command/transcribe.go @@ -0,0 +1,482 @@ +package command + +import ( + "context" + "encoding/json" + "fmt" + "path" + "path/filepath" + "strings" + "unicode" + + "github.com/viant/afs" + "github.com/viant/afs/file" + "github.com/viant/afs/url" + "github.com/viant/datly/cmd/options" + pathpkg "github.com/viant/datly/repository/path" + "github.com/viant/datly/repository/shape" + shapeColumn "github.com/viant/datly/repository/shape/column" + shapeCompile "github.com/viant/datly/repository/shape/compile" + shapeLoad "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/xgen" + "github.com/viant/datly/view" + viewpkg "github.com/viant/datly/view" + "gopkg.in/yaml.v3" +) + +// Transcribe runs the shape-only pipeline (compile → plan → load) for each +// DQL source. It does NOT depend on internal/translator. +func (s *Service) Transcribe(ctx context.Context, opts *options.Options) error { + transcribe := opts.Transcribe + if transcribe == nil { + return fmt.Errorf("transcribe options not set") + } + compiler := shapeCompile.New() + loader := shapeLoad.New() + for _, sourceURL := range transcribe.Source { + _, name := url.Split(sourceURL, file.Scheme) + fmt.Printf("transcribing %v\n", name) + dql, err := s.readSource(ctx, sourceURL) + if err != nil { + return fmt.Errorf("failed to read %s: %w", sourceURL, err) + } + dql = strings.TrimSpace(dql) + if dql == "" { + return fmt.Errorf("source %s was empty", sourceURL) + } + connectorName := transcribe.DefaultConnectorName() + shapeSource := &shape.Source{ + Name: strings.TrimSuffix(name, path.Ext(name)), + Path: url.Path(sourceURL), + DQL: dql, + Connector: connectorName, + } + compileOpts := transcribeCompileOptions(transcribe) + planResult, err := compiler.Compile(ctx, shapeSource, compileOpts...) + if err != nil { + return fmt.Errorf("failed to compile %s: %w", sourceURL, err) + } + componentArtifact, err := loader.LoadComponent(ctx, planResult) + if err != nil { + return fmt.Errorf("failed to load %s: %w", sourceURL, err) + } + component, ok := shapeLoad.ComponentFrom(componentArtifact) + if !ok { + return fmt.Errorf("unexpected component artifact for %s", sourceURL) + } + // Register connectors on resource first, then discover columns from DB + if componentArtifact.Resource != nil && len(transcribe.Connectors) > 0 { + applyConnectorsToResource(componentArtifact.Resource, transcribe.Connectors) + discoverColumns(ctx, componentArtifact.Resource) + } + if err = s.persistTranscribeRoute(ctx, transcribe, sourceURL, dql, componentArtifact.Resource, component); err != nil { + return err + } + } + // Persist dependencies (connections.yaml, config.json) + if len(transcribe.Connectors) > 0 { + if err := s.persistTranscribeDependencies(ctx, transcribe); err != nil { + return err + } + } + return nil +} + +func (s *Service) persistTranscribeDependencies(ctx context.Context, transcribe *options.Transcribe) error { + depURL := url.Join(transcribe.Repository, "Datly", "dependencies") + + // connections.yaml — use flat format matching legacy translator output + var connectors []connEntry + for _, c := range transcribe.Connectors { + parts := strings.SplitN(c, "|", 4) + if len(parts) >= 3 { + connectors = append(connectors, connEntry{Name: parts[0], Driver: parts[1], DSN: parts[2]}) + } + } + if len(connectors) > 0 { + connURL := url.Join(depURL, "connections.yaml") + // Merge with existing connections if file exists + existing := loadExistingConnectors(ctx, s.fs, connURL) + merged := mergeConnectors(existing, connectors) + connMap := map[string]any{"Connectors": merged} + data, err := yaml.Marshal(connMap) + if err != nil { + return err + } + if err = s.fs.Upload(ctx, connURL, file.DefaultFileOsMode, strings.NewReader(string(data))); err != nil { + return fmt.Errorf("failed to persist connections: %w", err) + } + } + + // config.json + routeURL := url.Join(transcribe.Repository, "Datly", "routes") + cfg := map[string]any{ + "APIPrefix": transcribe.APIPrefix, + "RouteURL": routeURL, + "DependencyURL": depURL, + "Endpoint": map[string]any{"Port": 8080}, + "SyncFrequencyMs": 2000, + "Meta": map[string]any{"StatusURI": "/v1/api/status"}, + } + cfgData, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + cfgURL := url.Join(transcribe.Repository, "Datly", "config.json") + if err = s.fs.Upload(ctx, cfgURL, file.DefaultFileOsMode, strings.NewReader(string(cfgData))); err != nil { + return fmt.Errorf("failed to persist config: %w", err) + } + return nil +} + +func buildPathResource(resource *viewpkg.Resource, component *shapeLoad.Component) *pathpkg.Resource { + if resource == nil { + return nil + } + var params []*pathpkg.Parameter + if component != nil { + for _, s := range component.Input { + if s != nil { + params = append(params, &pathpkg.Parameter{ + Name: s.Name, + In: s.In, + Required: s.Required != nil && *s.Required, + Schema: s.Schema, + }) + } + } + } + if len(params) == 0 { + return nil + } + return &pathpkg.Resource{Parameters: params} +} + +type connEntry struct { + Name string `yaml:"Name"` + Driver string `yaml:"Driver"` + DSN string `yaml:"DSN"` +} + +func loadExistingConnectors(ctx context.Context, fs afs.Service, connURL string) []connEntry { + data, err := fs.DownloadWithURL(ctx, connURL) + if err != nil { + return nil + } + var doc struct { + Connectors []connEntry `yaml:"Connectors"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil + } + return doc.Connectors +} + +func mergeConnectors(existing, incoming []connEntry) []connEntry { + byName := map[string]connEntry{} + var order []string + for _, c := range existing { + if _, ok := byName[c.Name]; !ok { + order = append(order, c.Name) + } + byName[c.Name] = c + } + for _, c := range incoming { + if _, ok := byName[c.Name]; !ok { + order = append(order, c.Name) + } + byName[c.Name] = c // incoming overrides existing for same name + } + result := make([]connEntry, 0, len(order)) + for _, name := range order { + result = append(result, byName[name]) + } + return result +} + +func (s *Service) readSource(ctx context.Context, sourceURL string) (string, error) { + payload, err := s.fs.DownloadWithURL(ctx, sourceURL) + if err != nil { + return "", err + } + return string(payload), nil +} + +func (s *Service) persistTranscribeRoute(ctx context.Context, transcribe *options.Transcribe, sourceURL, dql string, resource *view.Resource, component *shapeLoad.Component) error { + sourcePath := filepath.Clean(url.Path(sourceURL)) + stem := strings.TrimSuffix(filepath.Base(sourcePath), filepath.Ext(sourcePath)) + + // Determine generated file stem: --type-file flag, or root view name in lower_underscore, or DQL filename + typeStem := transcribeTypeStem(transcribe, stem, component) + + routeRoot := url.Join(transcribe.Repository, "Datly", "routes") + routeYAML := url.Join(routeRoot, stem+".yaml") + + if resource != nil { + for _, item := range resource.Views { + if item == nil || item.Template == nil || strings.TrimSpace(item.Template.Source) == "" { + continue + } + sqlRel := strings.TrimSpace(item.Template.SourceURL) + if sqlRel == "" { + sqlRel = path.Join(stem, item.Name+".sql") + } + sqlDest := path.Join(url.Path(routeRoot), filepath.ToSlash(sqlRel)) + if err := s.fs.Upload(ctx, sqlDest, file.DefaultFileOsMode, strings.NewReader(item.Template.Source)); err != nil { + return fmt.Errorf("failed to persist sql %s: %w", sqlDest, err) + } + item.Template.SourceURL = sqlRel + } + } + + rootView := "" + if component != nil { + rootView = strings.TrimSpace(component.RootView) + } + if rootView == "" && resource != nil && len(resource.Views) > 0 && resource.Views[0] != nil { + rootView = resource.Views[0].Name + } + method, uri := transcribeRulePath(dql, stem, transcribe.APIPrefix, component) + + // Build route YAML as map to control key casing (runtime expects PascalCase YAML keys) + routeEntry := map[string]any{ + "URI": uri, + "Method": method, + } + if rootView != "" { + routeEntry["View"] = map[string]any{"Ref": rootView} + } + payload := map[string]any{ + "Routes": []any{routeEntry}, + "Resource": sanitizeResourceForRouteYAML(resource), + } + data, err := yaml.Marshal(payload) + if err != nil { + return err + } + if err = s.fs.Upload(ctx, routeYAML, file.DefaultFileOsMode, strings.NewReader(string(data))); err != nil { + return fmt.Errorf("failed to persist route yaml %s: %w", routeYAML, err) + } + _ = typeStem + // Generate Go types directly from in-memory resource (no YAML roundtrip) + if component != nil && component.TypeContext != nil && resource != nil { + generateTranscribeTypes(url.Path(sourceURL), resource, component) + } + return nil +} + +func generateTranscribeTypes(sourceAbsPath string, resource *view.Resource, component *shapeLoad.Component) { + ctx := component.TypeContext + if ctx == nil || strings.TrimSpace(ctx.PackageDir) == "" { + return + } + projectDir := findProjectDir(sourceAbsPath) + if projectDir == "" { + fmt.Printf("WARNING: shape codegen: cannot locate go.mod from %s, skipping type generation\n", sourceAbsPath) + return + } + codegen := &xgen.ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: false, + } + result, err := codegen.Generate() + if err != nil { + fmt.Printf("WARNING: shape codegen: type generation skipped for %s: %v\n", filepath.Base(sourceAbsPath), err) + return + } + fmt.Printf("generated component %s → %s\n", strings.Join(result.Types, ", "), result.FilePath) +} + +func transcribeCompileOptions(transcribe *options.Transcribe) []shape.CompileOption { + var opts []shape.CompileOption + if transcribe.Strict { + opts = append(opts, shape.WithCompileStrict(true)) + } + namespace := strings.TrimSpace(transcribe.Namespace) + module := strings.TrimSpace(transcribe.Module) + typeOutput := strings.TrimSpace(transcribe.TypeOutput) + if typeOutput == "" || typeOutput == "." { + typeOutput = module + } + if namespace != "" { + pkgDir := filepath.Join(typeOutput, namespace) + pkgName := filepath.Base(namespace) + opts = append(opts, shape.WithTypeContextPackageDir(pkgDir)) + opts = append(opts, shape.WithTypeContextPackageName(pkgName)) + } + return opts +} + +// transcribeTypeStem determines the Go file name stem. +// Priority: --type-file flag > root view name (lower_underscore) > DQL filename +func transcribeTypeStem(transcribe *options.Transcribe, dqlStem string, component *shapeLoad.Component) string { + if tf := strings.TrimSpace(transcribe.TypeFile); tf != "" { + return strings.TrimSuffix(tf, ".go") + } + if component != nil { + if rootView := strings.TrimSpace(component.RootView); rootView != "" { + return toLowerUnderscore(rootView) + } + } + return dqlStem +} + +// toLowerUnderscore converts CamelCase or PascalCase to lower_underscore. +func toLowerUnderscore(s string) string { + var buf strings.Builder + for i, r := range s { + if unicode.IsUpper(r) { + if i > 0 { + prev := rune(s[i-1]) + if unicode.IsLower(prev) || unicode.IsDigit(prev) { + buf.WriteByte('_') + } + } + buf.WriteRune(unicode.ToLower(r)) + } else { + buf.WriteRune(r) + } + } + return buf.String() +} + +func transcribeRulePath(dql, ruleName, apiPrefix string, component *shapeLoad.Component) (string, string) { + method := "GET" + uri := "/" + strings.Trim(strings.TrimSpace(ruleName), "/") + if prefix := strings.TrimSpace(apiPrefix); prefix != "" { + uri = strings.TrimRight(prefix, "/") + uri + } + if component != nil && component.Directives != nil && component.Directives.Route != nil { + rd := component.Directives.Route + if u := strings.TrimSpace(rd.URI); u != "" { + uri = u + } + if len(rd.Methods) > 0 { + if m := strings.TrimSpace(strings.ToUpper(rd.Methods[0])); m != "" { + method = m + } + } + } + return method, uri +} + +// discoverColumns resolves wildcard columns from DB for all views in the resource. +func discoverColumns(ctx context.Context, resource *view.Resource) { + if resource == nil { + return + } + detector := shapeColumn.New() + for _, aView := range resource.Views { + if aView == nil { + continue + } + columns, err := detector.Resolve(ctx, resource, aView) + if err != nil { + fmt.Printf(" column discovery skipped for %s: %v\n", aView.Name, err) + continue + } + if len(columns) > 0 { + aView.Columns = columns + } + } +} + +// applyConnectorsToResource registers connectors on the resource and sets refs on views. +// Connector format: name|driver|dsn (same encoding as datly translate -c flag). +func applyConnectorsToResource(resource *view.Resource, connectors []string) { + if resource == nil || len(connectors) == 0 { + return + } + defaultName := "" + for _, c := range connectors { + parts := strings.SplitN(c, "|", 4) + if len(parts) < 1 { + continue + } + name := strings.TrimSpace(parts[0]) + if name == "" { + continue + } + if defaultName == "" { + defaultName = name + } + if len(parts) >= 3 { + driver := strings.TrimSpace(parts[1]) + dsn := strings.TrimSpace(parts[2]) + resource.AddConnectors(view.NewConnector(name, driver, dsn)) + } + } + if defaultName == "" { + return + } + for _, v := range resource.Views { + if v != nil && v.Connector == nil { + v.Connector = view.NewRefConnector(defaultName) + } + } +} + +// sanitizeResourceForRouteYAML returns a serialization-safe copy of resource +// with connector config stripped to references only. This keeps DSN/driver +// details out of route YAML; dependencies/connections.yaml remains the source +// of truth for connector definitions. +func sanitizeResourceForRouteYAML(resource *view.Resource) *view.Resource { + if resource == nil { + return nil + } + + cloned := *resource + + if len(resource.Connectors) > 0 { + cloned.Connectors = make([]*view.Connector, 0, len(resource.Connectors)) + for _, connector := range resource.Connectors { + if connector == nil { + continue + } + ref := strings.TrimSpace(connector.Ref) + if ref == "" { + ref = strings.TrimSpace(connector.Name) + } + if ref == "" { + continue + } + refConnector := view.NewRefConnector(ref) + refConnector.Name = ref + cloned.Connectors = append(cloned.Connectors, refConnector) + } + } else { + cloned.Connectors = nil + } + + if len(resource.Views) > 0 { + cloned.Views = make(view.Views, 0, len(resource.Views)) + for _, item := range resource.Views { + if item == nil { + continue + } + viewCopy := *item + if item.Connector != nil { + ref := strings.TrimSpace(item.Connector.Ref) + if ref == "" { + ref = strings.TrimSpace(item.Connector.Name) + } + if ref != "" { + refConnector := view.NewRefConnector(ref) + refConnector.Name = ref + viewCopy.Connector = refConnector + } else { + viewCopy.Connector = nil + } + } + cloned.Views = append(cloned.Views, &viewCopy) + } + } else { + cloned.Views = nil + } + + return &cloned +} diff --git a/cmd/options/options.go b/cmd/options/options.go index f8bd60d45..866b332d2 100644 --- a/cmd/options/options.go +++ b/cmd/options/options.go @@ -10,6 +10,7 @@ type Options struct { Plugin *Plugin `command:"plugin" description:"build custom datly rule plugin" ` Generate *Generate `command:"gen" description:"generate dql for put,patch or post operation" ` Translate *Translate `command:"translate" description:"translate dql into datly repository rule"` + Transcribe *Transcribe `command:"transcribe" description:"transcribe dql using shape pipeline (no internal/translator)"` Cache *CacheWarmup `command:"cache" description:"warmup cache"` Run *Run `command:"run" description:"start datly in standalone mode"` Mcp *Mcp `command:"mcp" description:"run mcp"` @@ -72,6 +73,9 @@ func (o *Options) Init(ctx context.Context) error { if o.Translate != nil { return o.Translate.Init(ctx) } + if o.Transcribe != nil { + return o.Transcribe.Init(ctx) + } if o.Run != nil { return o.Run.Init() } @@ -105,6 +109,8 @@ func NewOptions(args Arguments) *Options { ret.InitCmd = &Init{} case "dsql", "translate", "dql": ret.Translate = &Translate{} + case "transcribe": + ret.Transcribe = &Transcribe{} case "cache": ret.Cache = &CacheWarmup{} case "run": diff --git a/cmd/options/transcribe.go b/cmd/options/transcribe.go new file mode 100644 index 000000000..dac96b072 --- /dev/null +++ b/cmd/options/transcribe.go @@ -0,0 +1,65 @@ +package options + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/viant/afs/url" +) + +// Transcribe defines options for the transcribe command which uses +// the shape pipeline exclusively (compile → plan → load) without +// depending on internal/translator. +type Transcribe struct { + Connector + Source []string `short:"s" long:"src" description:"DQL source file(s)"` + Repository string `short:"r" long:"repo" description:"output repository location" default:"repo/dev"` + Namespace string `short:"u" long:"namespace" description:"route namespace" default:"dev"` + Module string `short:"m" long:"module" description:"go module location" default:"."` + Strict bool `long:"strict" description:"enable strict compile mode"` + TypeOutput string `long:"type-output" description:"go type output directory (default: same as --module)"` + TypeFile string `long:"type-file" description:"generated go file name (default: dql filename or main view in lower_underscore)"` + Project string `short:"p" long:"proj" description:"project location"` + APIPrefix string `short:"a" long:"api" description:"api prefix" default:"/v1/api"` +} + +// DefaultConnectorName returns the first connector name from the -c flags. +func (t *Transcribe) DefaultConnectorName() string { + if len(t.Connectors) == 0 { + return "" + } + parts := strings.SplitN(t.Connectors[0], "|", 2) + if len(parts) > 0 { + return strings.TrimSpace(parts[0]) + } + return "" +} + +func (t *Transcribe) Init(ctx context.Context) error { + if t.Project == "" { + t.Project, _ = os.Getwd() + } + t.Project = ensureAbsPath(t.Project) + t.Connector.Init() + if url.IsRelative(t.Repository) { + t.Repository = url.Join(t.Project, t.Repository) + } + if url.IsRelative(t.Module) { + t.Module = url.Join(t.Project, t.Module) + } + if t.TypeOutput != "" && url.IsRelative(t.TypeOutput) { + t.TypeOutput = url.Join(t.Project, t.TypeOutput) + } + if len(t.Source) == 0 { + return fmt.Errorf("transcribe: at least one --src is required") + } + for i := range t.Source { + expandRelativeIfNeeded(&t.Source[i], t.Project) + } + if strings.TrimSpace(t.Namespace) == "" { + t.Namespace = "dev" + } + return nil +} diff --git a/e2e/v1/build.yaml b/e2e/v1/build.yaml new file mode 100644 index 000000000..ac6377315 --- /dev/null +++ b/e2e/v1/build.yaml @@ -0,0 +1,29 @@ +init: + +pipeline: + deploy: + setPath: + action: exec:run + target: $target + checkError: true + commands: + - export GOPATH=${env.GOPATH} + - export PATH=/usr/local/go/bin:$PATH + + set_sdk: + action: sdk.set + target: $target + sdk: go:1.25.5 + + package: + action: exec:run + comments: build datly binary + target: $target + checkError: true + commands: + - export GO111MODULE=on + - cd ${appPath}/cmd/datly + - go mod tidy + - go mod download + - go build -ldflags "-X main.BuildTimeInS=`date +%s`" + - mv datly /tmp/datly diff --git a/e2e/v1/cases/001_one_to_many/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/001_one_to_many/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/001_one_to_many/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/001_one_to_many/dbsetup/dev/VENDOR.json b/e2e/v1/cases/001_one_to_many/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/001_one_to_many/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/001_one_to_many/expect.json b/e2e/v1/cases/001_one_to_many/expect.json new file mode 100644 index 000000000..c83b372a9 --- /dev/null +++ b/e2e/v1/cases/001_one_to_many/expect.json @@ -0,0 +1,32 @@ +[ + { + "@indexBy@": "id" + }, + { + "id": 1, + "name": "Vendor 1", + "products": [ + { + "@indexBy@": "id" + }, + { + "id": 1, + "name": "V1 Product 1", + "userCreated": 1 + }, + { + "id": 2, + "name": "V1 Product 2", + "userCreated": 1 + } + ] + }, + { + "id": 2, + "name": "Vendor 2" + }, + { + "id": 3, + "name": "Vendor 3" + } +] \ No newline at end of file diff --git a/e2e/v1/cases/001_one_to_many/expect_2.txt b/e2e/v1/cases/001_one_to_many/expect_2.txt new file mode 100644 index 000000000..7796339fb --- /dev/null +++ b/e2e/v1/cases/001_one_to_many/expect_2.txt @@ -0,0 +1,27 @@ +package generated + +import ( + "time" +) + +type GeneratedStruct struct { + Id int `sqlx:"ID" velty:"names=ID|Id"` + Name *string `sqlx:"NAME" velty:"names=NAME|Name"` + AccountId *int `sqlx:"ACCOUNT_ID" velty:"names=ACCOUNT_ID|AccountId"` + Created *time.Time `sqlx:"CREATED" velty:"names=CREATED|Created"` + UserCreated *int `sqlx:"USER_CREATED" velty:"names=USER_CREATED|UserCreated"` + Updated *time.Time `sqlx:"UPDATED" velty:"names=UPDATED|Updated"` + UserUpdated *int `sqlx:"USER_UPDATED" velty:"names=USER_UPDATED|UserUpdated"` + Products []*Products `view:",table=PRODUCT"` +} + +type Products struct { + Id int `sqlx:"ID" velty:"names=ID|Id"` + Name *string `sqlx:"NAME" velty:"names=NAME|Name"` + VendorId *int `sqlx:"VENDOR_ID" internal:"true" velty:"names=VENDOR_ID|VendorId"` + Status *int `sqlx:"STATUS" velty:"names=STATUS|Status"` + Created *time.Time `sqlx:"CREATED" velty:"names=CREATED|Created"` + UserCreated *int `sqlx:"USER_CREATED" velty:"names=USER_CREATED|UserCreated"` + Updated *time.Time `sqlx:"UPDATED" velty:"names=UPDATED|Updated"` + UserUpdated *int `sqlx:"USER_UPDATED" velty:"names=USER_UPDATED|UserUpdated"` +} diff --git a/e2e/v1/cases/001_one_to_many/test.yaml b/e2e/v1/cases/001_one_to_many/test.yaml new file mode 100644 index 000000000..f5f959d3b --- /dev/null +++ b/e2e/v1/cases/001_one_to_many/test.yaml @@ -0,0 +1,20 @@ +init: + parentPath: $parent.path +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/vendors/ + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect.json') + + + - Method: GET + URL: http://127.0.0.1:8080/v1/api/meta/struct/dev/vendors/ + Expect: + Code: 200 + Body: $Cat('${parentPath}/expect_2.txt') +#/v1/api/shape/dev/vendors?yy=id,vendorId&xx=id,name,products diff --git a/e2e/v1/cases/002_uri_param/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/002_uri_param/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/002_uri_param/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/002_uri_param/dbsetup/dev/VENDOR.json b/e2e/v1/cases/002_uri_param/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/002_uri_param/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/002_uri_param/expect.json b/e2e/v1/cases/002_uri_param/expect.json new file mode 100644 index 000000000..29afd6592 --- /dev/null +++ b/e2e/v1/cases/002_uri_param/expect.json @@ -0,0 +1,41 @@ +[ + { + "setting": [ + { + "channel": 3, + "isActive": 1 + } + ], + "vendor": { + "accountId": 101, + "id": 2, + "name": "Vendor 2", + "products": [ + {"@indexBy@": "id"}, + { + "id": 3, + "name": "V2 Product 1", + "status": 1, + "userCreated": 2, + "userUpdated": 0 + }, + { + "id": 4, + "name": "V2 Product 2", + "status": 1, + "userCreated": 2, + "userUpdated": 0 + }, + { + "id": 5, + "name": "V2 Product 3", + "status": 1, + "userCreated": 2, + "userUpdated": 0 + } + ], + "userCreated": 2, + "userUpdated": 0 + } + } +] \ No newline at end of file diff --git a/e2e/v1/cases/002_uri_param/test.yaml b/e2e/v1/cases/002_uri_param/test.yaml new file mode 100644 index 000000000..c7161bbf0 --- /dev/null +++ b/e2e/v1/cases/002_uri_param/test.yaml @@ -0,0 +1,13 @@ +init: + parentPath: $parent.path + expect: $LoadData('${parentPath}/expect.json') +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/vendors/2 + Expect: + Code: 200 + JSONBody: $expect diff --git a/e2e/v1/cases/003_oauth/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/003_oauth/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/003_oauth/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/003_oauth/dbsetup/dev/VENDOR.json b/e2e/v1/cases/003_oauth/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/003_oauth/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/003_oauth/expect.json b/e2e/v1/cases/003_oauth/expect.json new file mode 100644 index 000000000..854079f16 --- /dev/null +++ b/e2e/v1/cases/003_oauth/expect.json @@ -0,0 +1,27 @@ +[ + { + "id": 2, + "name": "Vendor 2", + "firstName": "Developer", + "products": [ + { + "@indexBy@": "id" + }, + { + "id": 3, + "name": "V2 Product 1", + "userCreated": 2 + }, + { + "id": 4, + "name": "V2 Product 2", + "userCreated": 2 + }, + { + "id": 5, + "name": "V2 Product 3", + "userCreated": 2 + } + ] + } +] \ No newline at end of file diff --git a/e2e/v1/cases/003_oauth/test.yaml b/e2e/v1/cases/003_oauth/test.yaml new file mode 100644 index 000000000..b78409a9c --- /dev/null +++ b/e2e/v1/cases/003_oauth/test.yaml @@ -0,0 +1,53 @@ +init: + parentPath: $parent.path + expect: $LoadData('${parentPath}/expect.json') +pipeline: + + + signJWT: + action: secret:signJWT + privateKey: + URL: ${appPath}/e2e/cloud/jwt/private.enc + Key: blowfish://default + claims: + userID: 2 + firstName: Developer + email: dev@viantint.com + + printToken: + action: print + message: Bearer ${signJWT.TokenString} + + + test: +# testNoAuthenticated: +# action: http/runner:send +# requests: +# - Method: GET +# description: user is no authenticated +# URL: http://127.0.0.1:8080/v1/api/shape/dev/auth/vendors/2 +# Expect: +# Code: 401 +# +# testAuthenticatedAndAuthorized: +# action: http/runner:send +# requests: +# - Method: GET +# description: user is authenticated and authorized for vendor 2 +# URL: http://127.0.0.1:8080/v1/api/shape/dev/auth/vendors/2 +# Header: +# Authorization: Bearer ${signJWT.TokenString} +# Expect: +# Code: 200 +# JSONBody: $expect + + testAuthenticatedAndNoAuthorized: + action: http/runner:send + requests: + - Method: GET + description: user is authenticated but not authorized for vendor 1 (no data returned) + URL: http://127.0.0.1:8080/v1/api/shape/dev/auth/vendors/1 + Header: + Authorization: Bearer ${signJWT.TokenString} + Expect: + Code: 403 diff --git a/e2e/v1/cases/004_update/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/004_update/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/004_update/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/004_update/dbsetup/dev/PRODUCT_JN.json b/e2e/v1/cases/004_update/dbsetup/dev/PRODUCT_JN.json new file mode 100644 index 000000000..ec2649bb4 --- /dev/null +++ b/e2e/v1/cases/004_update/dbsetup/dev/PRODUCT_JN.json @@ -0,0 +1,3 @@ +[ + {} +] \ No newline at end of file diff --git a/e2e/v1/cases/004_update/dbsetup/dev/VENDOR.json b/e2e/v1/cases/004_update/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/004_update/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/004_update/expect.json b/e2e/v1/cases/004_update/expect.json new file mode 100644 index 000000000..b963eee2e --- /dev/null +++ b/e2e/v1/cases/004_update/expect.json @@ -0,0 +1,26 @@ +[ + { + "id": 2, + "name": "Vendor 2", + "products": [ + { + "@indexBy@": "id" + }, + { + "id": 3, + "name": "V2 Product 1", + "userCreated": 2 + }, + { + "id": 4, + "name": "V2 Product 2", + "userCreated": 2 + }, + { + "id": 5, + "name": "V2 Product 3", + "userCreated": 2 + } + ] + } +] \ No newline at end of file diff --git a/e2e/v1/cases/004_update/expect/PRODUCT.json b/e2e/v1/cases/004_update/expect/PRODUCT.json new file mode 100644 index 000000000..2138e9e62 --- /dev/null +++ b/e2e/v1/cases/004_update/expect/PRODUCT.json @@ -0,0 +1,10 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "STATUS": 2, + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/004_update/test.yaml b/e2e/v1/cases/004_update/test.yaml new file mode 100644 index 000000000..93cee7db8 --- /dev/null +++ b/e2e/v1/cases/004_update/test.yaml @@ -0,0 +1,53 @@ +init: + parentPath: $parent.path + expect: $LoadData('${parentPath}/expect.json') +pipeline: + + + signJWT: + action: secret:signJWT + privateKey: + URL: ${appPath}/e2e/cloud/jwt/private.enc + Key: blowfish://default + claims: + userID: 1 + email: dev@viantint.com + + printToken: + action: print + message: Bearer ${signJWT.TokenString} + + + test: + testNoAuthenticated: + action: http/runner:send + requests: + - Method: POST + description: user is authenticated + URL: http://127.0.0.1:8080/v1/api/shape/dev/auth/products/ + Header: + Authorization: Bearer ${signJWT.TokenString} + JSONBody: + Ids: + - 1 + Status: 2 + Expect: + Code: 200 + +# +# - Method: POST +# description: user is no authenticated +# URL: http://127.0.0.1:8080/v1/api/shape/dev/auth/products/ +# JSONBody: +# Ids: +# - 1 +# Status: 2 +# Expect: +# Code: 401 + + checkDb: + action: 'dsunit:expect' + datastore: dev + expand: true + checkPolicy: 1 + URL: ${parentPath}/expect diff --git a/e2e/v1/cases/005_sumary/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/005_sumary/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/005_sumary/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/005_sumary/dbsetup/dev/VENDOR.json b/e2e/v1/cases/005_sumary/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/005_sumary/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/005_sumary/test.yaml b/e2e/v1/cases/005_sumary/test.yaml new file mode 100644 index 000000000..75c61afe2 --- /dev/null +++ b/e2e/v1/cases/005_sumary/test.yaml @@ -0,0 +1,15 @@ + +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/meta/vendors + Expect: + Code: 200 + JSONBody: + status: ok + meta: + pageCnt: 1 + cnt: 3 \ No newline at end of file diff --git a/e2e/v1/cases/006_tree/dbsetup/dev/USER.json b/e2e/v1/cases/006_tree/dbsetup/dev/USER.json new file mode 100644 index 000000000..ee3de9dab --- /dev/null +++ b/e2e/v1/cases/006_tree/dbsetup/dev/USER.json @@ -0,0 +1,37 @@ +[ + {}, + { + "ID": 1, + "NAME": "User 1", + "ACCOUNT_ID": 100 + }, + { + "ID": 2, + "NAME": "User 2", + "ACCOUNT_ID": 101 + }, + { + "ID": 3, + "NAME": "User 3", + "ACCOUNT_ID": 100, + "MGR_ID": 1 + }, + { + "ID": 4, + "NAME": "User 2", + "ACCOUNT_ID": 101, + "MGR_ID": 1 + }, + { + "ID": 5, + "NAME": "User 1", + "ACCOUNT_ID": 100, + "MGR_ID": 3 + }, + { + "ID": 6, + "NAME": "User 2", + "ACCOUNT_ID": 101, + "MGR_ID": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/006_tree/expect.json b/e2e/v1/cases/006_tree/expect.json new file mode 100644 index 000000000..9eec63220 --- /dev/null +++ b/e2e/v1/cases/006_tree/expect.json @@ -0,0 +1,46 @@ +{ + "status": "ok", + "data": [ + {"@indexBy@": "id"}, + { + "id": 1, + "name": "User 1", + "accountId": 100, + "team": [ + {"@indexBy@": "id"}, + { + "id": 3, + "name": "User 3", + "accountId": 100, + "team": [ + { + "id": 5, + "name": "User 1", + "accountId": 100, + "team": [] + } + ] + }, + { + "id": 4, + "name": "User 2", + "accountId": 101, + "team": [] + } + ] + }, + { + "id": 2, + "name": "User 2", + "accountId": 101, + "team": [ + { + "id": 6, + "name": "User 2", + "accountId": 101, + "team": [] + } + ] + } + ] +} \ No newline at end of file diff --git a/e2e/v1/cases/006_tree/test.yaml b/e2e/v1/cases/006_tree/test.yaml new file mode 100644 index 000000000..5579809c6 --- /dev/null +++ b/e2e/v1/cases/006_tree/test.yaml @@ -0,0 +1,14 @@ +init: + parentPath: $parent.path + expect: $LoadJSON('${parentPath}/expect.json') + +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/users/ + Expect: + Code: 200 + JSONBody: $expect diff --git a/e2e/v1/cases/007_child_meta/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/007_child_meta/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/007_child_meta/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/007_child_meta/dbsetup/dev/VENDOR.json b/e2e/v1/cases/007_child_meta/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/007_child_meta/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/007_child_meta/expect.json b/e2e/v1/cases/007_child_meta/expect.json new file mode 100644 index 000000000..9d4a3c13f --- /dev/null +++ b/e2e/v1/cases/007_child_meta/expect.json @@ -0,0 +1,69 @@ +{ + "status": "ok", + "data": [ + { + "id": 1, + "name": "Vendor 1", + "accountId": 100, + "userCreated": 1, + "products": [ + { + "id": 1, + "name": "V1 Product 1", + "userCreated": 1 + }, + { + "id": 2, + "name": "V1 Product 2", + "status": 1, + "userCreated": 1 + } + ], + "productsMeta": { + "pageCnt": 1, + "totalProducts": 2 + } + }, + { + "id": 2, + "name": "Vendor 2", + "accountId": 101, + "userCreated": 2, + "products": [ + { + "id": 3, + "name": "V2 Product 1", + "status": 1, + "userCreated": 2 + }, + { + "id": 4, + "name": "V2 Product 2", + "status": 1, + "userCreated": 2 + }, + { + "id": 5, + "name": "V2 Product 3", + "status": 1, + "userCreated": 2 + } + ], + "productsMeta": { + "pageCnt": 1, + "totalProducts": 3 + } + }, + { + "id": 3, + "name": "Vendor 3", + "accountId": 100, + "userCreated": 1, + "products": [] + } + ], + "meta": { + "pageCnt": 1, + "cnt": 3 + } +} \ No newline at end of file diff --git a/e2e/v1/cases/007_child_meta/test.yaml b/e2e/v1/cases/007_child_meta/test.yaml new file mode 100644 index 000000000..8d5a58c70 --- /dev/null +++ b/e2e/v1/cases/007_child_meta/test.yaml @@ -0,0 +1,14 @@ +init: + parentPath: $parent.path + expect: $LoadJSON('${parentPath}/expect.json') + +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/meta/vendors-nested + Expect: + Code: 200 + JSONBody: $expect \ No newline at end of file diff --git a/e2e/v1/cases/008_record_pagination/dbsetup/dev/CITY.json b/e2e/v1/cases/008_record_pagination/dbsetup/dev/CITY.json new file mode 100644 index 000000000..2f8364176 --- /dev/null +++ b/e2e/v1/cases/008_record_pagination/dbsetup/dev/CITY.json @@ -0,0 +1,27 @@ +[ + {}, + { + "ID": 1, + "NAME": "district - 1 / city - 1", + "ZIP_CODE": "12-345", + "DISTRICT_ID": 1 + }, + { + "ID": 2, + "NAME": "district - 2 / city - 1", + "DISTRICT_ID": 2, + "ZIP_CODE": "23-456" + }, + { + "ID": 3, + "NAME": "district - 1 / city - 2", + "DISTRICT_ID": 1, + "ZIP_CODE": "34-567" + }, + { + "ID": 4, + "NAME": "district - 1 / city - 3", + "DISTRICT_ID": 1, + "ZIP_CODE": "45_678" + } +] \ No newline at end of file diff --git a/e2e/v1/cases/008_record_pagination/dbsetup/dev/DISTRICT.json b/e2e/v1/cases/008_record_pagination/dbsetup/dev/DISTRICT.json new file mode 100644 index 000000000..3bed6ed69 --- /dev/null +++ b/e2e/v1/cases/008_record_pagination/dbsetup/dev/DISTRICT.json @@ -0,0 +1,15 @@ +[ + {}, + { + "ID": 1, + "NAME": "district - 1" + }, + { + "ID": 2, + "NAME": "district - 2" + }, + { + "ID": 3, + "NAME": "district - 3" + } +] \ No newline at end of file diff --git a/e2e/v1/cases/008_record_pagination/expect.json b/e2e/v1/cases/008_record_pagination/expect.json new file mode 100644 index 000000000..b7571133a --- /dev/null +++ b/e2e/v1/cases/008_record_pagination/expect.json @@ -0,0 +1,32 @@ +[ + { + "id": 1, + "name": "district - 1", + "cities": [ + { + "id": 1, + "name": "district - 1 / city - 1", + "zipCode": "12-345", + "districtId": 1 + }, + { + "id": 3, + "name": "district - 1 / city - 2", + "zipCode": "34-567", + "districtId": 1 + } + ] + }, + { + "id": 2, + "name": "district - 2", + "cities": [ + { + "id": 2, + "name": "district - 2 / city - 1", + "zipCode": "23-456", + "districtId": 2 + } + ] + } +] \ No newline at end of file diff --git a/e2e/v1/cases/008_record_pagination/test.yaml b/e2e/v1/cases/008_record_pagination/test.yaml new file mode 100644 index 000000000..8dd244aa9 --- /dev/null +++ b/e2e/v1/cases/008_record_pagination/test.yaml @@ -0,0 +1,14 @@ +init: + parentPath: $parent.path + expect: $LoadJSON('${parentPath}/expect.json') + +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/meta/districts?IDs=1,2 + Expect: + Code: 200 + JSONBody: $expect \ No newline at end of file diff --git a/e2e/v1/cases/009_apikey/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/009_apikey/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/009_apikey/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/009_apikey/dbsetup/dev/VENDOR.json b/e2e/v1/cases/009_apikey/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/009_apikey/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/009_apikey/expect.json b/e2e/v1/cases/009_apikey/expect.json new file mode 100644 index 000000000..b963eee2e --- /dev/null +++ b/e2e/v1/cases/009_apikey/expect.json @@ -0,0 +1,26 @@ +[ + { + "id": 2, + "name": "Vendor 2", + "products": [ + { + "@indexBy@": "id" + }, + { + "id": 3, + "name": "V2 Product 1", + "userCreated": 2 + }, + { + "id": 4, + "name": "V2 Product 2", + "userCreated": 2 + }, + { + "id": 5, + "name": "V2 Product 3", + "userCreated": 2 + } + ] + } +] \ No newline at end of file diff --git a/e2e/v1/cases/009_apikey/test.yaml b/e2e/v1/cases/009_apikey/test.yaml new file mode 100644 index 000000000..a9ce0de9a --- /dev/null +++ b/e2e/v1/cases/009_apikey/test.yaml @@ -0,0 +1,46 @@ +init: + parentPath: $parent.path + expect: $LoadData('${parentPath}/expect.json') +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/secured/vendors/2 + Expect: + Code: 403 + + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/secured/vendors/2 + Header: + App-Secret-Id: 'changeme' + Expect: + Code: 200 + + - Method: GET + URL: http://127.0.0.1:8080/v1/api/meta/view/dev/secured/vendors/2 + Expect: + Code: 403 + + - Method: GET + URL: http://127.0.0.1:8080/v1/api/meta/view/dev/secured/vendors/2 + Header: + App-Secret-Id: 'changeme' + Expect: + Code: 200 + + test2: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/meta/openapi/dev/secured/vendors/2 + Expect: + Code: 403 + + - Method: GET + URL: http://127.0.0.1:8080/v1/api/meta/openapi/dev/secured/vendors/2 + Header: + App-Secret-Id: 'changeme' + Expect: + Code: 200 diff --git a/e2e/v1/cases/010_codecs/dbsetup/dev/VENDOR.json b/e2e/v1/cases/010_codecs/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/010_codecs/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/010_codecs/expect.json b/e2e/v1/cases/010_codecs/expect.json new file mode 100644 index 000000000..52dba1e84 --- /dev/null +++ b/e2e/v1/cases/010_codecs/expect.json @@ -0,0 +1,14 @@ +[ + { + "id": 1, + "name": "Vendor 1", + "accountId": 100, + "userCreated": 1 + }, + { + "id": 2, + "name": "Vendor 2", + "accountId": 101, + "userCreated": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/010_codecs/test.yaml b/e2e/v1/cases/010_codecs/test.yaml new file mode 100644 index 000000000..3aab3ad23 --- /dev/null +++ b/e2e/v1/cases/010_codecs/test.yaml @@ -0,0 +1,14 @@ +init: + parentPath: $parent.path + expect: $LoadJSON('${parentPath}/expect.json') + +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/vendors-codec?vendorIDs=1,2 + Expect: + Code: 200 + JSONBody: $expect \ No newline at end of file diff --git a/e2e/v1/cases/011_env/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/011_env/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/011_env/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json b/e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/011_env/expect.json b/e2e/v1/cases/011_env/expect.json new file mode 100644 index 000000000..68b81fefe --- /dev/null +++ b/e2e/v1/cases/011_env/expect.json @@ -0,0 +1,52 @@ +[ + { + "id": 1, + "name": "Vendor 1", + "accountId": 100, + "userCreated": 1, + "products": [ + { + "id": 1, + "name": "V1 Product 1", + "vendorId": 1, + "userCreated": 1 + }, + { + "id": 2, + "name": "V1 Product 2", + "vendorId": 1, + "status": 1, + "userCreated": 1 + } + ] + }, + { + "id": 2, + "name": "Vendor 2", + "accountId": 101, + "userCreated": 2, + "products": [ + { + "id": 3, + "name": "V2 Product 1", + "vendorId": 2, + "status": 1, + "userCreated": 2 + }, + { + "id": 4, + "name": "V2 Product 2", + "vendorId": 2, + "status": 1, + "userCreated": 2 + }, + { + "id": 5, + "name": "V2 Product 3", + "vendorId": 2, + "status": 1, + "userCreated": 2 + } + ] + } +] \ No newline at end of file diff --git a/e2e/v1/cases/011_env/test.yaml b/e2e/v1/cases/011_env/test.yaml new file mode 100644 index 000000000..e538123f8 --- /dev/null +++ b/e2e/v1/cases/011_env/test.yaml @@ -0,0 +1,21 @@ +init: + parentPath: $parent.path + expect: $LoadJSON('${parentPath}/expect.json') + +pipeline: + printHello: + action: print + message: hello action 1 + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/vendors-env?vendorIDs=1,2 + Expect: + Code: 200 + JSONBody: $expect + + info: + action: print + message: $AsJSON($test) diff --git a/e2e/v1/cases/012_meta_format/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/012_meta_format/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/012_meta_format/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/012_meta_format/dbsetup/dev/VENDOR.json b/e2e/v1/cases/012_meta_format/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/012_meta_format/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/012_meta_format/expect.json b/e2e/v1/cases/012_meta_format/expect.json new file mode 100644 index 000000000..f822c8cc7 --- /dev/null +++ b/e2e/v1/cases/012_meta_format/expect.json @@ -0,0 +1,68 @@ +{ + "status": "ok", + "data": [ + { + "id": 1, + "name": "Vendor 1", + "accountId": 100, + "userCreated": 1, + "products": [ + { + "id": 1, + "name": "V1 Product 1", + "userCreated": 1 + }, + { + "id": 2, + "name": "V1 Product 2", + "userCreated": 1 + } + ], + "productsMeta": { + "pageCnt": 1, + "totalProducts": 2 + } + }, + { + "id": 2, + "name": "Vendor 2", + "accountId": 101, + "userCreated": 2, + "products": [ + { + "id": 3, + "name": "V2 Product 1", + "status": 1, + "userCreated": 2 + }, + { + "id": 4, + "name": "V2 Product 2", + "status": 1, + "userCreated": 2 + }, + { + "id": 5, + "name": "V2 Product 3", + "status": 1, + "userCreated": 2 + } + ], + "productsMeta": { + "pageCnt": 1, + "totalProducts": 3 + } + }, + { + "id": 3, + "name": "Vendor 3", + "accountId": 100, + "userCreated": 1, + "products": [] + } + ], + "meta": { + "pageCnt": 1, + "cnt": 3 + } +} \ No newline at end of file diff --git a/e2e/v1/cases/012_meta_format/test.yaml b/e2e/v1/cases/012_meta_format/test.yaml new file mode 100644 index 000000000..ee94073fa --- /dev/null +++ b/e2e/v1/cases/012_meta_format/test.yaml @@ -0,0 +1,14 @@ +init: + parentPath: $parent.path + expect: $LoadJSON('${parentPath}/expect.json') + +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/meta/vendors-format + Expect: + Code: 200 + JSONBody: $expect diff --git a/e2e/v1/cases/013_col_in/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/013_col_in/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/013_col_in/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/013_col_in/dbsetup/dev/VENDOR.json b/e2e/v1/cases/013_col_in/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/013_col_in/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/013_col_in/test.yaml b/e2e/v1/cases/013_col_in/test.yaml new file mode 100644 index 000000000..a48b060f5 --- /dev/null +++ b/e2e/v1/cases/013_col_in/test.yaml @@ -0,0 +1,12 @@ +init: + parentPath: $parent.path +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/col/vendors/ + Expect: + Code: 200 + diff --git a/e2e/v1/cases/014_header_params/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/014_header_params/dbsetup/dev/PRODUCT.json new file mode 100644 index 000000000..9f31630e2 --- /dev/null +++ b/e2e/v1/cases/014_header_params/dbsetup/dev/PRODUCT.json @@ -0,0 +1,44 @@ +[ + {}, + { + "ID": 1, + "NAME": "V1 Product 1", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "V1 Product 2", + "VENDOR_ID": 1, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 1 + }, + + { + "ID": 3, + "NAME": "V2 Product 1", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 4, + "NAME": "V2 Product 2", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + }, + { + "ID": 5, + "NAME": "V2 Product 3", + "VENDOR_ID": 2, + "CREATED": "", + "STATUS": 1, + "USER_CREATED": 2 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/014_header_params/dbsetup/dev/VENDOR.json b/e2e/v1/cases/014_header_params/dbsetup/dev/VENDOR.json new file mode 100644 index 000000000..c4c724a74 --- /dev/null +++ b/e2e/v1/cases/014_header_params/dbsetup/dev/VENDOR.json @@ -0,0 +1,24 @@ +[ + {}, + { + "ID": 1, + "NAME": "Vendor 1", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + }, + { + "ID": 2, + "NAME": "Vendor 2", + "ACCOUNT_ID": 101, + "CREATED": "", + "USER_CREATED": 2 + }, + { + "ID": 3, + "NAME": "Vendor 3", + "ACCOUNT_ID": 100, + "CREATED": "", + "USER_CREATED": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/014_header_params/expect.json b/e2e/v1/cases/014_header_params/expect.json new file mode 100644 index 000000000..b963eee2e --- /dev/null +++ b/e2e/v1/cases/014_header_params/expect.json @@ -0,0 +1,26 @@ +[ + { + "id": 2, + "name": "Vendor 2", + "products": [ + { + "@indexBy@": "id" + }, + { + "id": 3, + "name": "V2 Product 1", + "userCreated": 2 + }, + { + "id": 4, + "name": "V2 Product 2", + "userCreated": 2 + }, + { + "id": 5, + "name": "V2 Product 3", + "userCreated": 2 + } + ] + } +] \ No newline at end of file diff --git a/e2e/v1/cases/014_header_params/test.yaml b/e2e/v1/cases/014_header_params/test.yaml new file mode 100644 index 000000000..9fe964c2c --- /dev/null +++ b/e2e/v1/cases/014_header_params/test.yaml @@ -0,0 +1,16 @@ +init: + parentPath: $parent.path + expect: $LoadData('${parentPath}/expect.json') +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/headers/vendors + Header: + Vendor-Id: ["2"] + Expect: + Code: 200 + JSONBody: $expect + diff --git a/e2e/v1/cases/015_index_by/dbsetup/dev/TEAM.json b/e2e/v1/cases/015_index_by/dbsetup/dev/TEAM.json new file mode 100644 index 000000000..b0dc46dc4 --- /dev/null +++ b/e2e/v1/cases/015_index_by/dbsetup/dev/TEAM.json @@ -0,0 +1,23 @@ +[ + {}, + { + "ID": 1, + "NAME": "Team - 1", + "ACTIVE": true + }, + { + "ID": 2, + "NAME": "Team - 2", + "ACTIVE": true + }, + { + "ID": 3, + "NAME": "Team - 3", + "ACTIVE": true + }, + { + "ID": 1000000, + "NAME": "Team - 1000000", + "ACTIVE": true + } +] \ No newline at end of file diff --git a/e2e/v1/cases/015_index_by/dbsetup/dev/USER_TEAM.json b/e2e/v1/cases/015_index_by/dbsetup/dev/USER_TEAM.json new file mode 100644 index 000000000..49a3be744 --- /dev/null +++ b/e2e/v1/cases/015_index_by/dbsetup/dev/USER_TEAM.json @@ -0,0 +1,18 @@ +[ + {}, + { + "ID": 1, + "TEAM_ID": 1, + "USER_ID": 1 + }, + { + "ID": 2, + "TEAM_ID": 1, + "USER_ID": 2 + }, + { + "ID": 3, + "TEAM_ID": 2, + "USER_ID": 1 + } +] \ No newline at end of file diff --git a/e2e/v1/cases/015_index_by/test.yaml b/e2e/v1/cases/015_index_by/test.yaml new file mode 100644 index 000000000..40681a32f --- /dev/null +++ b/e2e/v1/cases/015_index_by/test.yaml @@ -0,0 +1,32 @@ +init: + parentPath: $parent.path +pipeline: + + test: + action: http/runner:send + requests: + - Method: PUT + URL: http://127.0.0.1:8080/v1/api/shape/dev/teams?TeamIDs=100 + Expect: + Code: 400 + Body: + message: "not found team with ID 100" + + - Method: PUT + URL: http://127.0.0.1:8080/v1/api/shape/dev/teams?TeamIDs=1 + Expect: + Code: 400 + Body: + message: "can't deactivate team Team - 1 with 2 members" + + - Method: PUT + URL: http://127.0.0.1:8080/v1/api/shape/dev/teams?TeamIDs=3 + Expect: + Code: 200 + + checkDb: + action: 'dsunit:expect' + dataStore: dev + expand: true + checkPolicy: 1 + URL: ${parentPath}/expect diff --git a/e2e/v1/cases/016_team_delete/dbsetup/dev/TEAM.json b/e2e/v1/cases/016_team_delete/dbsetup/dev/TEAM.json new file mode 100644 index 000000000..b0dc46dc4 --- /dev/null +++ b/e2e/v1/cases/016_team_delete/dbsetup/dev/TEAM.json @@ -0,0 +1,23 @@ +[ + {}, + { + "ID": 1, + "NAME": "Team - 1", + "ACTIVE": true + }, + { + "ID": 2, + "NAME": "Team - 2", + "ACTIVE": true + }, + { + "ID": 3, + "NAME": "Team - 3", + "ACTIVE": true + }, + { + "ID": 1000000, + "NAME": "Team - 1000000", + "ACTIVE": true + } +] \ No newline at end of file diff --git a/e2e/v1/cases/016_team_delete/test.yaml b/e2e/v1/cases/016_team_delete/test.yaml new file mode 100644 index 000000000..4b92179e1 --- /dev/null +++ b/e2e/v1/cases/016_team_delete/test.yaml @@ -0,0 +1,18 @@ +init: + parentPath: $parent.path +pipeline: + + test: + action: http/runner:send + requests: + - Method: DELETE + URL: http://127.0.0.1:8080/v1/api/shape/dev/team/1000000 + Expect: + Code: 200 + + checkDb: + action: 'dsunit:query' + datastore: dev + SQL: 'SELECT COUNT(*) AS NUM_RECORDS FROM (SELECT 1 FROM TEAM WHERE ID = 1000000) T' + expect: + - NUM_RECORDS: 0 diff --git a/e2e/v1/cases/017_generate_post_basic_one/dbsetup/dev/EVENTS.json b/e2e/v1/cases/017_generate_post_basic_one/dbsetup/dev/EVENTS.json new file mode 100644 index 000000000..ec2649bb4 --- /dev/null +++ b/e2e/v1/cases/017_generate_post_basic_one/dbsetup/dev/EVENTS.json @@ -0,0 +1,3 @@ +[ + {} +] \ No newline at end of file diff --git a/e2e/v1/cases/017_generate_post_basic_one/expect_t0.json b/e2e/v1/cases/017_generate_post_basic_one/expect_t0.json new file mode 100644 index 000000000..89d54528d --- /dev/null +++ b/e2e/v1/cases/017_generate_post_basic_one/expect_t0.json @@ -0,0 +1,3 @@ +{ + "id": "@exists@" +} \ No newline at end of file diff --git a/e2e/v1/cases/017_generate_post_basic_one/expect_t1.json b/e2e/v1/cases/017_generate_post_basic_one/expect_t1.json new file mode 100644 index 000000000..f555bae5c --- /dev/null +++ b/e2e/v1/cases/017_generate_post_basic_one/expect_t1.json @@ -0,0 +1,5 @@ +{ + "id": "@exists@", + "name": "017_ Custom name", + "quantity": 25 +} \ No newline at end of file diff --git a/e2e/v1/cases/017_generate_post_basic_one/test.yaml b/e2e/v1/cases/017_generate_post_basic_one/test.yaml new file mode 100644 index 000000000..dc474bf6e --- /dev/null +++ b/e2e/v1/cases/017_generate_post_basic_one/test.yaml @@ -0,0 +1,31 @@ +init: + parentPath: $parent.path +pipeline: + + test: + action: http/runner:send + requests: + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events + JsonBody: + Name: '017_' + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t0.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events + JSONBody: + Name: "017_ Custom name" + Quantity: 25 + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t1.json') + + checkDB: + action: 'dsunit:query' + dataStore: dev + SQL: | + SELECT (CASE WHEN COUNT(*) = 0 THEN FALSE ELSE TRUE END) AS ADDED_NEW_ROWS FROM ( SELECT * FROM EVENTS WHERE NAME LIKE '017_%') T; + expect: + - ADDED_NEW_ROWS: true diff --git a/e2e/v1/cases/018_generate_post_basic_many/dbsetup/dev/EVENTS.json b/e2e/v1/cases/018_generate_post_basic_many/dbsetup/dev/EVENTS.json new file mode 100644 index 000000000..ec2649bb4 --- /dev/null +++ b/e2e/v1/cases/018_generate_post_basic_many/dbsetup/dev/EVENTS.json @@ -0,0 +1,3 @@ +[ + {} +] \ No newline at end of file diff --git a/e2e/v1/cases/018_generate_post_basic_many/expect_t0.json b/e2e/v1/cases/018_generate_post_basic_many/expect_t0.json new file mode 100644 index 000000000..0b84b5838 --- /dev/null +++ b/e2e/v1/cases/018_generate_post_basic_many/expect_t0.json @@ -0,0 +1,5 @@ +[ + { + "id": "@exists@" + } +] diff --git a/e2e/v1/cases/018_generate_post_basic_many/expect_t1.json b/e2e/v1/cases/018_generate_post_basic_many/expect_t1.json new file mode 100644 index 000000000..c9a5ab883 --- /dev/null +++ b/e2e/v1/cases/018_generate_post_basic_many/expect_t1.json @@ -0,0 +1,10 @@ +[ + { + "id": "@exists@", + "name": "018_ " + }, + { + "id": "@exists@", + "name": "018_ " + } +] \ No newline at end of file diff --git a/e2e/v1/cases/018_generate_post_basic_many/expect_t2.json b/e2e/v1/cases/018_generate_post_basic_many/expect_t2.json new file mode 100644 index 000000000..00f87b1c9 --- /dev/null +++ b/e2e/v1/cases/018_generate_post_basic_many/expect_t2.json @@ -0,0 +1,10 @@ +[ + { + "id": "@exists@", + "name": "018_ Custom - 1" + }, + { + "id": "@exists@", + "name": "018_ Custom - 2" + } +] \ No newline at end of file diff --git a/e2e/v1/cases/018_generate_post_basic_many/test.yaml b/e2e/v1/cases/018_generate_post_basic_many/test.yaml new file mode 100644 index 000000000..a2fa2aafb --- /dev/null +++ b/e2e/v1/cases/018_generate_post_basic_many/test.yaml @@ -0,0 +1,40 @@ +init: + parentPath: $parent.path +pipeline: + + test: + action: http/runner:send + requests: + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-many + JSONBody: + - Name: '018_ ' + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t0.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-many + JSONBody: + - Name: '018_ ' + - Name: '018_ ' + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t1.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-many + JSONBody: + - Name: '018_ Custom - 1' + - Name: '018_ Custom - 2' + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t2.json') + + checkDB: + action: 'dsunit:query' + dataStore: dev + SQL: | + SELECT (CASE WHEN COUNT(*) = 0 THEN FALSE ELSE TRUE END) AS ADDED_NEW_ROWS FROM ( SELECT * FROM EVENTS WHERE NAME LIKE '018_%') T; + expect: + - ADDED_NEW_ROWS: true diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/dbsetup/dev/EVENTS.json b/e2e/v1/cases/019_generate_post_comprehensive_many/dbsetup/dev/EVENTS.json new file mode 100644 index 000000000..ec2649bb4 --- /dev/null +++ b/e2e/v1/cases/019_generate_post_comprehensive_many/dbsetup/dev/EVENTS.json @@ -0,0 +1,3 @@ +[ + {} +] \ No newline at end of file diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t0.json b/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t0.json new file mode 100644 index 000000000..bbbfbd1a1 --- /dev/null +++ b/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t0.json @@ -0,0 +1,8 @@ +{ + "status": "ok", + "data": [ + { + "id": "@exists@" + } + ] +} diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t1.json b/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t1.json new file mode 100644 index 000000000..be52f980e --- /dev/null +++ b/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t1.json @@ -0,0 +1,11 @@ +{ + "status": "ok", + "data": [ + { + "id": "@exists@" + }, + { + "id": "@exists@" + } + ] +} diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t2.json b/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t2.json new file mode 100644 index 000000000..96bc28252 --- /dev/null +++ b/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t2.json @@ -0,0 +1,13 @@ +{ + "status": "ok", + "data": [ + { + "id": "@exists@", + "name": "019_ Custom - 1" + }, + { + "id": "@exists@", + "name": "019_ Custom - 2" + } + ] +} diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/test.yaml b/e2e/v1/cases/019_generate_post_comprehensive_many/test.yaml new file mode 100644 index 000000000..e5317770e --- /dev/null +++ b/e2e/v1/cases/019_generate_post_comprehensive_many/test.yaml @@ -0,0 +1,43 @@ +init: + parentPath: $parent.path +pipeline: + + test: + action: http/runner:send + requests: + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/comprehensive/events-many + JSONBody: + data: + - name: '019_ ' + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t0.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/comprehensive/events-many + JSONBody: + data: + - name: '019_ ' + - name: '019_ ' + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t1.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/comprehensive/events-many + JSONBody: + data: + - name: '019_ Custom - 1' + - name: '019_ Custom - 2' + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t2.json') + + checkDB: + action: 'dsunit:query' + dataStore: dev + SQL: | + SELECT (CASE WHEN COUNT(*) = 0 THEN FALSE ELSE TRUE END) AS ADDED_NEW_ROWS FROM ( SELECT * FROM EVENTS WHERE NAME LIKE '019_%') T; + expect: + - ADDED_NEW_ROWS: true diff --git a/e2e/v1/cases/020_generate_post_except/dbsetup/dev/EVENTS.json b/e2e/v1/cases/020_generate_post_except/dbsetup/dev/EVENTS.json new file mode 100644 index 000000000..ec2649bb4 --- /dev/null +++ b/e2e/v1/cases/020_generate_post_except/dbsetup/dev/EVENTS.json @@ -0,0 +1,3 @@ +[ + {} +] \ No newline at end of file diff --git a/e2e/v1/cases/020_generate_post_except/expect_t0.json b/e2e/v1/cases/020_generate_post_except/expect_t0.json new file mode 100644 index 000000000..60a83114f --- /dev/null +++ b/e2e/v1/cases/020_generate_post_except/expect_t0.json @@ -0,0 +1,4 @@ +{ + "id": "@exists@", + "quantity": -1234 +} \ No newline at end of file diff --git a/e2e/v1/cases/020_generate_post_except/expect_t1.json b/e2e/v1/cases/020_generate_post_except/expect_t1.json new file mode 100644 index 000000000..23195e3ca --- /dev/null +++ b/e2e/v1/cases/020_generate_post_except/expect_t1.json @@ -0,0 +1,4 @@ +{ + "id": "@exists@", + "quantity": -2345 +} \ No newline at end of file diff --git a/e2e/v1/cases/020_generate_post_except/test.yaml b/e2e/v1/cases/020_generate_post_except/test.yaml new file mode 100644 index 000000000..8f97f8139 --- /dev/null +++ b/e2e/v1/cases/020_generate_post_except/test.yaml @@ -0,0 +1,30 @@ +init: + parentPath: $parent.path +pipeline: + + test: + action: http/runner:send + requests: + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-except + JsonBody: + Quantity: -1234 + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t0.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-except + JSONBody: + Quantity: -2345 + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t1.json') + + checkDB: + action: 'dsunit:query' + dataStore: dev + SQL: | + SELECT (CASE WHEN COUNT(*) = 0 THEN FALSE ELSE TRUE END) AS ADDED_NEW_ROWS FROM ( SELECT * FROM EVENTS WHERE QUANTITY IN (-12345,-2345)) T; + expect: + - ADDED_NEW_ROWS: true diff --git a/e2e/v1/datastore.yaml b/e2e/v1/datastore.yaml new file mode 100644 index 000000000..08160b622 --- /dev/null +++ b/e2e/v1/datastore.yaml @@ -0,0 +1,27 @@ +init: + +pipeline: + mysql: + create: + action: dsunit:init + datastore: dev + recreate: false + config: + driverName: mysql + descriptor: '[username]:[password]@tcp(${dbIP.mysql}:3306)/[dbname]?parseTime=true' + credentials: $mysqlCredentials + admin: + datastore: mysql + ping: true + config: + driverName: mysql + descriptor: '[username]:[password]@tcp(${dbIP.mysql}:3306)/[dbname]?parseTime=true' + credentials: $mysqlCredentials + scripts: + - URL: ${v1Path}/db/schema.sql + + prepare: + action: 'dsunit:prepare' + datastore: dev + expand: true + URL: ${appPath}/e2e/local/datastore/mysql/populate diff --git a/e2e/v1/db/schema.sql b/e2e/v1/db/schema.sql new file mode 100644 index 000000000..f6192f9aa --- /dev/null +++ b/e2e/v1/db/schema.sql @@ -0,0 +1,227 @@ +SET GLOBAL log_bin_trust_function_creators = 1; +SET GLOBAL sql_mode = ''; + +DROP TABLE IF EXISTS USER; +CREATE TABLE USER ( + ID INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + NAME VARCHAR(255), + MGR_ID INT, + ACCOUNT_ID INT +); + +DROP TABLE IF EXISTS VENDOR; +CREATE TABLE VENDOR ( + ID INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + NAME VARCHAR(255), + ACCOUNT_ID INT, + CREATED DATETIME, + USER_CREATED INT, + UPDATED DATETIME, + USER_UPDATED INT +); + +DROP TABLE IF EXISTS PRODUCT; + +CREATE TABLE PRODUCT ( + ID INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + NAME VARCHAR(255), + VENDOR_ID INT, + STATUS INT, + CREATED DATETIME, + USER_CREATED INT, + UPDATED DATETIME, + USER_UPDATED INT +); + +DROP TABLE IF EXISTS PRODUCT_JN; + +CREATE TABLE PRODUCT_JN ( + PRODUCT_ID INT NOT NULL, + USER_ID INT, + OLD_VALUE VARCHAR(255), + NEW_VALUE VARCHAR(255), + CREATED DATETIME +); + +DROP FUNCTION IF EXISTS IS_VENDOR_AUTHORIZED; + +DELIMITER $$ +CREATE FUNCTION IS_VENDOR_AUTHORIZED(USER_ID INT, VENDOR_ID INT) + RETURNS BOOLEAN +BEGIN + DECLARE +IS_AUTH BOOLEAN; +SELECT TRUE +INTO IS_AUTH +FROM VENDOR v +WHERE ID = VENDOR_ID + AND ACCOUNT_ID + AND EXISTS(SELECT 1 FROM USER u WHERE u.ID = USER_ID AND u.ACCOUNT_ID = v.ACCOUNT_ID); +RETURN IS_AUTH; +END $$ +DELIMITER; + + +DROP FUNCTION IF EXISTS IS_PRODUCT_AUTHORIZED; + +DELIMITER $$ +CREATE FUNCTION IS_PRODUCT_AUTHORIZED(USER_ID INT, PID INT) + RETURNS BOOLEAN +BEGIN + DECLARE +IS_AUTH BOOLEAN; + SET +IS_AUTH = FALSE ; +SELECT TRUE +INTO IS_AUTH +FROM VENDOR v + JOIN PRODUCT p ON v.ID = p.VENDOR_ID +WHERE p.ID = PID + AND ACCOUNT_ID + AND EXISTS(SELECT 1 + FROM USER u + WHERE u.ID = USER_ID + AND u.ACCOUNT_ID = v.ACCOUNT_ID); +RETURN IS_AUTH; +END $$ +DELIMITER; + + +DROP TABLE IF EXISTS DISTRICT; +CREATE TABLE DISTRICT ( + ID INT PRIMARY KEY, + NAME VARCHAR(255) +); + +DROP TABLE IF EXISTS CITY; +CREATE TABLE CITY ( + ID INT PRIMARY KEY, + NAME varchar(255), + ZIP_CODE varchar(255), + DISTRICT_ID INT +); + +DROP TABLE IF EXISTS TEAM; +CREATE TABLE TEAM ( + ID INT PRIMARY KEY, + NAME varchar(255), + ACTIVE INTEGER +); + +DROP TABLE IF EXISTS USER_TEAM; +CREATE TABLE USER_TEAM ( + ID INT PRIMARY KEY, + USER_ID INT, + TEAM_ID INT +); + +DROP TABLE IF EXISTS EVENTS; +CREATE TABLE EVENTS ( + ID INT AUTO_INCREMENT PRIMARY KEY, + NAME varchar(255), + QUANTITY INT +); + +DROP TABLE IF EXISTS EVENTS_PERFORMANCE; +CREATE TABLE EVENTS_PERFORMANCE +( + ID INT AUTO_INCREMENT PRIMARY KEY, + PRICE INT, + EVENT_ID INT, + TIMESTAMP DATE, + FOREIGN KEY (EVENT_ID) REFERENCES EVENTS (ID) +); + +DROP TABLE IF EXISTS FOOS; +CREATE TABLE FOOS ( + ID INT AUTO_INCREMENT PRIMARY KEY, + NAME varchar(255), + QUANTITY INT +); + +DROP TABLE IF EXISTS FOOS_CHANGES; +CREATE TABLE FOOS_CHANGES ( + ID INT AUTO_INCREMENT PRIMARY KEY, + PREVIOUS TEXT +); + +DROP TABLE IF EXISTS FOOS_PERFORMANCE; +CREATE TABLE FOOS_PERFORMANCE ( + ID INT AUTO_INCREMENT PRIMARY KEY, + PERF_NAME varchar(255), + PERF_QUANTITY INT, + FOO_ID INT, + FOREIGN KEY (FOO_ID) REFERENCES FOOS(ID) +); + +DROP TABLE IF EXISTS DIFF_JN; +CREATE TABLE DIFF_JN ( + ID INT AUTO_INCREMENT PRIMARY KEY, + DIFF LONGTEXT +); + +DROP TABLE IF EXISTS USER_METADATA; +CREATE TABLE USER_METADATA ( + ID INT AUTO_INCREMENT PRIMARY KEY, + USER_ID INT, + IS_ENABLED BIT, + IS_ACTIVATED BIT, + FOREIGN KEY (USER_ID) REFERENCES USER (ID) +); + +DROP TABLE IF EXISTS OBJECTS; +CREATE TABLE OBJECTS ( + ID INT AUTO_INCREMENT PRIMARY KEY, + OBJECT TEXT, + CLASS_NAME VARCHAR(255) +); + +DROP TABLE IF EXISTS BAR; +CREATE TABLE BAR ( + ID INT AUTO_INCREMENT PRIMARY KEY, + NAME varchar(255), + PRICE DOUBLE PRECISION, + TAX FLOAT +); + +DROP TABLE IF EXISTS DATLY_JOBS; + +CREATE TABLE `DATLY_JOBS` ( + `MatchKey` varchar(3000) NOT NULL, + `Status` varchar(40) NOT NULL, + `Metrics` text NOT NULL, + `Connector` varchar(256), + `TableName` varchar(256), + `TableDataset` varchar(256), + `TableSchema` varchar(256), + `CreateDisposition` varchar(256), + `Template` varchar(256), + `WriteDisposition` varchar(256), + `Cache` text, + `CacheKey` varchar(256), + `CacheSet` varchar(256), + `CacheNamespace` varchar(256), + `Method` varchar(256) NOT NULL, + `URI` varchar(256) NOT NULL, + `State` text NOT NULL, + `UserEmail` varchar(256), + `UserID` varchar(256), + `MainView` varchar(256) NOT NULL, + `Module` varchar(256) NOT NULL, + `Labels` varchar(256) NOT NULL, + `JobType` varchar(256) NOT NULL, + `EventURL` varchar(256) NOT NULL, + `Error` text, + `CreationTime` datetime NOT NULL, + `StartTime` datetime DEFAULT NULL, + `ExpiryTime` datetime DEFAULT NULL, + `EndTime` datetime DEFAULT NULL, + `WaitTimeInMcs` int(11) NOT NULL, + `RuntimeInMcs` int(11) NOT NULL, + `SQLQuery` text NOT NULL, + `Deactivated` tinyint(1), + `ID` varchar(40) NOT NULL, + PRIMARY KEY (`ID`) +); + +CREATE INDEX DATLY_JOBS_REF ON DATLY_JOBS(MatchKey, CreationTime, Deactivated); diff --git a/e2e/v1/dql/dev/district/district_pagination.sql b/e2e/v1/dql/dev/district/district_pagination.sql new file mode 100644 index 000000000..212ccbd30 --- /dev/null +++ b/e2e/v1/dql/dev/district/district_pagination.sql @@ -0,0 +1,13 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/district') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/meta/districts', 'GET')) + +#set( $_ = $Page(query/page).Optional().QuerySelector('districts')) +#set( $_ = $Data(output/view).Embed()) + + +SELECT districts.*, + cities.*, + set_limit(cities, 2) +FROM (SELECT t.* FROM DISTRICT t WHERE 1 = 1 AND ID IN ($IDs)) districts +JOIN (SELECT * FROM CITY t) cities ON districts.ID = cities.DISTRICT_ID diff --git a/e2e/v1/dql/dev/events/post_basic_many.dql b/e2e/v1/dql/dev/events/post_basic_many.dql new file mode 100644 index 000000000..6070080f7 --- /dev/null +++ b/e2e/v1/dql/dev/events/post_basic_many.dql @@ -0,0 +1,10 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/events') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/basic/events-many', 'POST')) + +#set($_ = $Events(body/).Cardinality('Many').Tag('anonymous:"true"')) +#set($_ = $Events(body/).Output().Tag('anonymous:"true"')) + + +SELECT events.* +FROM (SELECT * FROM EVENTS) events diff --git a/e2e/v1/dql/dev/events/post_basic_one.dql b/e2e/v1/dql/dev/events/post_basic_one.dql new file mode 100644 index 000000000..5ba191c14 --- /dev/null +++ b/e2e/v1/dql/dev/events/post_basic_one.dql @@ -0,0 +1,11 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/events') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/basic/events', 'POST')) + + +#set($_ = $Events(body/).Cardinality('One').Tag('anonymous:"true"')) +#set($_ = $Events(body/).Output().Tag('anonymous:"true"')) + + +SELECT events.* +FROM (SELECT * FROM EVENTS) events diff --git a/e2e/v1/dql/dev/events/post_comprehensive_many.dql b/e2e/v1/dql/dev/events/post_comprehensive_many.dql new file mode 100644 index 000000000..5b048f267 --- /dev/null +++ b/e2e/v1/dql/dev/events/post_comprehensive_many.dql @@ -0,0 +1,12 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/events') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/comprehensive/events-many', 'POST')) + + +#set($_ = $Events(body/Data).Cardinality('Many')) +#set($_ = $Status(output/status).Tag('anonymous:"true"')) +#set($_ = $Data(body/Data).Output()) + + +SELECT events.* +FROM (SELECT * FROM EVENTS) events diff --git a/e2e/v1/dql/dev/events/post_except.dql b/e2e/v1/dql/dev/events/post_except.dql new file mode 100644 index 000000000..5ef13f77d --- /dev/null +++ b/e2e/v1/dql/dev/events/post_except.dql @@ -0,0 +1,11 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/events') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/basic/events-except', 'POST')) + +#set($_ = $Events(body/).Cardinality('One').Tag('anonymous:"true"')) +#set($_ = $Events(body/).Output().Tag('anonymous:"true"')) + + + +SELECT events.* EXCEPT NAME +FROM (SELECT * FROM EVENTS) events diff --git a/e2e/v1/dql/dev/team/team.dql b/e2e/v1/dql/dev/team/team.dql new file mode 100644 index 000000000..2d2bb72cd --- /dev/null +++ b/e2e/v1/dql/dev/team/team.dql @@ -0,0 +1,5 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/team') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/team/{teamID}', 'DELETE')) + +DELETE FROM TEAM WHERE ID = ${teamID} diff --git a/e2e/v1/dql/dev/team/user_team.dql b/e2e/v1/dql/dev/team/user_team.dql new file mode 100644 index 000000000..2fcf7f9bc --- /dev/null +++ b/e2e/v1/dql/dev/team/user_team.dql @@ -0,0 +1,37 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/team') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/teams', 'PUT')) + +#set($_ = $TeamIDs<[]int>(query/TeamIDs)) + +#set($teamStatsIndex = $Unsafe.TeamStats.IndexBy("ID") /* + {"Required": false} + SELECT + t.ID, + ( + CASE + WHEN ut.TEAM_ID IS NULL THEN 0 + ELSE COUNT(1) + END + ) as TEAM_MEMBERS, + t.NAME as NAME + FROM TEAM t + LEFT JOIN USER_TEAM ut ON t.ID = ut.TEAM_ID + WHERE t.ID IN ($TeamIDs) + GROUP BY t.ID +*/) + + +#foreach($teamID in $Unsafe.TeamIDs) + #if($teamStatsIndex.HasKey($teamID) == false) + $logger.Fatal("not found team with ID %v", $teamID) + #end + + #set($aTeam = $teamStatsIndex[$teamID]) + #if($aTeam.TEAM_MEMBERS != 0) + $logger.Fatal("can't deactivate team %v with %v members", $aTeam.NAME, $aTeam.TEAM_MEMBERS) + #end +UPDATE TEAM +SET ACTIVE = false +WHERE ID = $teamID; +#end diff --git a/e2e/v1/dql/dev/user/user_tree.sql b/e2e/v1/dql/dev/user/user_tree.sql new file mode 100644 index 000000000..42d9c5e8a --- /dev/null +++ b/e2e/v1/dql/dev/user/user_tree.sql @@ -0,0 +1,7 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/user') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/users/', 'GET')) + +SELECT user.* EXCEPT MGR_ID, + self_ref(user, 'Team', 'ID', 'MGR_ID') +FROM (SELECT t.* FROM USER t ) user diff --git a/e2e/v1/regression/app.yaml b/e2e/v1/regression/app.yaml new file mode 100644 index 000000000..b4bf66e7a --- /dev/null +++ b/e2e/v1/regression/app.yaml @@ -0,0 +1,17 @@ +pipeline: + datly: + stop: + action: process:stop + target: $target + input: datly + + start: + action: process:start + sleepTimeMs: 6000 + target: $target + directory: /tmp/ + checkError: true + immuneToHangups: true + env: + TEST: 1 + command: ulimit -Sn 10000 && ./datly -c=${v1Path}/autogen/Datly/config.json -z=/tmp/jobs/datly_v1 > /tmp/datly_v1.out 2>&1 diff --git a/e2e/v1/regression/db.yaml b/e2e/v1/regression/db.yaml new file mode 100644 index 000000000..01c96ed33 --- /dev/null +++ b/e2e/v1/regression/db.yaml @@ -0,0 +1,9 @@ +pipeline: + register: + action: dsunit:register + datastore: dev + recreate: false + config: + driverName: mysql + descriptor: '[username]:[password]@tcp(${dbIP.mysql}:3306)/[dbname]?parseTime=true' + credentials: $mysqlCredentials diff --git a/e2e/v1/regression/regression.yaml b/e2e/v1/regression/regression.yaml new file mode 100644 index 000000000..0c5f00560 --- /dev/null +++ b/e2e/v1/regression/regression.yaml @@ -0,0 +1,36 @@ +init: + +pipeline: + database: + action: run + request: '@db' + + app: + when: $debugger!=on + description: start datly app (DQLBootstrap loads DQL from dql/ folder) + action: run + request: '@app' + + test: + tag: $pathMatch + data: + '[]dev_dbsetup': '@dbsetup/dev' + + subPath: 'cases/${index}_*' + range: 001..020 + template: + checkSkip: + action: nop + comments: use case init + skip: $HasResource(${path}/skip.txt) + + dbsetup: + when: $Len($dev_dbsetup) > 0 + action: 'dsunit:prepare' + datastore: dev + expand: true + data: $dev_dbsetup + + test: + action: run + request: '@test' diff --git a/e2e/v1/run.yaml b/e2e/v1/run.yaml new file mode 100644 index 000000000..435a61291 --- /dev/null +++ b/e2e/v1/run.yaml @@ -0,0 +1,43 @@ +init: + yesterday: $FormatTime('yesterdayInUTC', 'yyyy-MM-dd HH:mm:ss') + today: $FormatTime('nowInUTC', 'yyyy-MM-dd HH:mm:ss') + debugger: '$params.debugger?$params.debugger:0' + + target: + URL: ssh://localhost/ + credentials: localhost + appPath: $WorkingDirectory(../..) + v1Path: ${appPath}/e2e/v1 + mysqlCredentials: mysql-e2e + dbIP: + mysql: localhost + qMark: '?' + connectors: --connector 'dev|mysql|root:dev@tcp(${dbIP.mysql}:3306)/dev${qMark}parseTime=true' + +pipeline: + init: + description: initialise test (docker, database) + system: + action: run + request: '@system' + tasks: '*' + + datastore: + action: run + request: '@datastore' + tasks: '*' + + shapes: + description: generate Go types and route YAML from DQL + action: run + request: '@shapes' + + build: + action: run + request: '@build' + tasks: 'deploy' + + test: + action: run + description: run v1 regression test + request: '@regression/regression' diff --git a/e2e/v1/shapes.yaml b/e2e/v1/shapes.yaml new file mode 100644 index 000000000..f433438b7 --- /dev/null +++ b/e2e/v1/shapes.yaml @@ -0,0 +1,72 @@ +init: + shapePath: ${v1Path}/shape + repoPath: ${v1Path}/autogen + conn: -c='dev|mysql|root:dev@tcp(${dbIP.mysql}:3306)/dev${qMark}parseTime=true' + +pipeline: + + cleanup: + action: exec:run + description: clean up generated shapes and routes + target: '$target' + checkError: true + commands: + - mkdir -p ${repoPath} + - rm -rf ${repoPath} + - mkdir -p ${shapePath} + - rm -rf ${shapePath} + + vendor: + action: exec:run + TimeoutMs: 120000 + checkError: true + commands: + - cd ${v1Path} + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_list.dql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_details.sql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_auth.sql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/product_update.sql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_meta.sql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/child_meta.sql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_apikey.sql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendors_codec.sql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_col_in.dql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/header_vendors.dql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/environment.dql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/meta_format.dql + + user: + action: exec:run + TimeoutMs: 120000 + checkError: true + commands: + - cd ${v1Path} + - /tmp/datly transcribe -u dev/user -m ${shapePath} -r ${repoPath} $conn -s dql/dev/user/user_tree.sql + + district: + action: exec:run + TimeoutMs: 120000 + checkError: true + commands: + - cd ${v1Path} + - /tmp/datly transcribe -u dev/district -m ${shapePath} -r ${repoPath} $conn -s dql/dev/district/district_pagination.sql + + events: + action: exec:run + TimeoutMs: 120000 + checkError: true + commands: + - cd ${v1Path} + - /tmp/datly transcribe -u dev/events -m ${shapePath} -r ${repoPath} $conn -s dql/dev/events/post_basic_one.dql + - /tmp/datly transcribe -u dev/events -m ${shapePath} -r ${repoPath} $conn -s dql/dev/events/post_basic_many.dql + - /tmp/datly transcribe -u dev/events -m ${shapePath} -r ${repoPath} $conn -s dql/dev/events/post_comprehensive_many.dql + - /tmp/datly transcribe -u dev/events -m ${shapePath} -r ${repoPath} $conn -s dql/dev/events/post_except.dql + + team: + action: exec:run + TimeoutMs: 120000 + checkError: true + commands: + - cd ${v1Path} + - /tmp/datly transcribe -u dev/team -m ${shapePath} -r ${repoPath} $conn -s dql/dev/team/team.dql + - /tmp/datly transcribe -u dev/team -m ${shapePath} -r ${repoPath} $conn -s dql/dev/team/user_team.dql diff --git a/e2e/v1/system.yaml b/e2e/v1/system.yaml new file mode 100644 index 000000000..ad0ee43eb --- /dev/null +++ b/e2e/v1/system.yaml @@ -0,0 +1,32 @@ +init: + mysqlSecrets: ${secrets.$mysqlCredentials} +pipeline: + + stop: + services: + action: docker:stop + images: + - mysql + - aerospike + + start: + services: + mysql_dev: + action: docker:run + image: mysql:5.7 + platform: linux/amd64 + name: mysql_dev + ports: + 3306: 3306 + env: + MYSQL_ROOT_PASSWORD: ${mysqlSecrets.Password} + + aerospike: + action: docker:run + platform: linux/amd64 + image: 'aerospike:ce-6.2.0.2' + name: aero + ports: + 3000: 3000 + 3001: 3001 + 3002: 3002 diff --git a/go.mod b/go.mod index be6c46a28..8781dfe24 100644 --- a/go.mod +++ b/go.mod @@ -190,3 +190,4 @@ require ( modernc.org/strutil v1.1.3 // indirect modernc.org/token v1.0.0 // indirect ) +replace github.com/viant/x => /Users/awitas/go/src/github.com/viant/x diff --git a/repository/shape/column/detector.go b/repository/shape/column/detector.go index 79b6c8d1c..893b5aaa7 100644 --- a/repository/shape/column/detector.go +++ b/repository/shape/column/detector.go @@ -31,7 +31,11 @@ func (d *Detector) Resolve(ctx context.Context, resource *view.Resource, aView * } base := columnsFromSchema(aView) - if !usesWildcard(aView) { + // If columns are placeholders (col_1, col_2, etc.) from static inference, treat as no columns + if allPlaceholderColumns(aView.Columns) { + base = nil + } + if !needsDiscovery(aView) && len(base) > 0 { return base, nil } @@ -54,7 +58,7 @@ func (d *Detector) detect(ctx context.Context, resource *view.Resource, aView *v if err != nil { return nil, fmt.Errorf("shape column detector: failed to open db for view %s: %w", aView.Name, err) } - query := sourceSQL(aView) + query := discoverySQL(aView) sqlColumns, err := viewcolumn.Discover(ctx, db, aView.Table, query) if err != nil { return nil, fmt.Errorf("shape column detector: discover failed for view %s: %w", aView.Name, err) @@ -62,6 +66,259 @@ func (d *Detector) detect(ctx context.Context, resource *view.Resource, aView *v return view.NewColumns(sqlColumns, aView.ColumnsConfig), nil } +// discoverySQL returns SQL suitable for column discovery. +// Strategy: +// 1. Strip template variables ($var, #if...#end, ${expr}) +// 2. Inject 1=0 into every SELECT in the query (CTEs, UNIONs, subqueries) +// This ensures zero rows scanned — safe for BigQuery (no full scan cost) +// 3. Fall back to table name if parsing/falsification fails +func discoverySQL(aView *view.View) string { + raw := sourceSQL(aView) + table := strings.TrimSpace(aView.Table) + if raw == "" { + return table + } + // If SQL has template variables, EXCEPT, or other datly extensions, + // use table-based discovery which is always safe and accurate + if table != "" && (hasTemplateVariables(raw) || hasExceptClause(raw)) { + return table + } + // For clean SQL without templates, try to falsify for column type inference + cleaned := strings.TrimSpace(raw) + if cleaned == "" || !strings.Contains(strings.ToLower(cleaned), "select") { + if table != "" { + return table + } + return cleaned + } + if falsified, ok := falsifyQuery(cleaned); ok { + return falsified + } + // Fallback to table + if table != "" { + return table + } + return cleaned +} + +func removeExceptClauses(sql string) string { + // Remove "EXCEPT col1, col2" patterns — these are datly-specific + // Simple approach: remove " EXCEPT (, )*" + result := sql + for { + lower := strings.ToLower(result) + idx := strings.Index(lower, " except ") + if idx == -1 { + break + } + // Find end of EXCEPT clause (next keyword or end of identifier list) + end := idx + len(" except ") + for end < len(result) && (isIdentPart(result[end]) || result[end] == ',' || result[end] == ' ') { + end++ + } + result = result[:idx] + result[end:] + } + return result +} + +func hasTemplateVariables(sql string) bool { + for i := 0; i < len(sql)-1; i++ { + if sql[i] == '$' && isIdentStart(sql[i+1]) { + return true + } + if sql[i] == '#' && (sql[i+1] == 'i' || sql[i+1] == 'f' || sql[i+1] == 'e' || sql[i+1] == 's') { + return true + } + if sql[i] == '$' && sql[i+1] == '{' { + return true + } + } + return false +} + +func hasExceptClause(sql string) bool { + lower := strings.ToLower(sql) + return strings.Contains(lower, " except ") +} + +// needsDiscovery returns true if the view SQL uses wildcards or has no explicit columns. +func needsDiscovery(aView *view.View) bool { + if aView == nil { + return false + } + if len(aView.Columns) == 0 { + return true + } + if allPlaceholderColumns(aView.Columns) { + return true + } + return usesWildcard(aView) +} + +// stripTemplateVariables removes velocity/velty template constructs from SQL +// so it can be parsed and executed for column discovery. +// Handles: $variable, ${expression}, #if...#end, #foreach...#end, #set(...) +func stripTemplateVariables(sql string) string { + var b strings.Builder + b.Grow(len(sql)) + i := 0 + for i < len(sql) { + // Handle # directives: #if, #foreach, #set, #end, #else, #elseif + if sql[i] == '#' && i+1 < len(sql) { + directive := matchDirective(sql, i) + if directive != "" { + // Skip entire directive line/block + end := skipDirective(sql, i, directive) + // Replace with space to preserve SQL structure + b.WriteByte(' ') + i = end + continue + } + } + // Handle $ variables: $name, $name.method(...), ${expression} + if sql[i] == '$' && i+1 < len(sql) { + next := sql[i+1] + if next == '{' { + // ${...} expression — find matching } + depth := 1 + j := i + 2 + for j < len(sql) && depth > 0 { + if sql[j] == '{' { + depth++ + } else if sql[j] == '}' { + depth-- + } + j++ + } + // Replace with empty string or placeholder + b.WriteString("''") + i = j + continue + } + if isIdentStart(next) { + // $varName or $varName.method(...) + j := i + 1 + for j < len(sql) && isIdentPart(sql[j]) { + j++ + } + // Skip .method() chains + for j < len(sql) && sql[j] == '.' { + j++ + for j < len(sql) && isIdentPart(sql[j]) { + j++ + } + if j < len(sql) && sql[j] == '(' { + depth := 1 + j++ + for j < len(sql) && depth > 0 { + if sql[j] == '(' { + depth++ + } else if sql[j] == ')' { + depth-- + } + j++ + } + } + } + b.WriteString("''") + i = j + continue + } + } + b.WriteByte(sql[i]) + i++ + } + return b.String() +} + +func matchDirective(sql string, pos int) string { + directives := []string{"#foreach", "#if", "#elseif", "#else", "#end", "#set", "#settings", "#setting", "#define", "#package", "#import"} + remaining := sql[pos:] + for _, d := range directives { + if len(remaining) >= len(d) && strings.EqualFold(remaining[:len(d)], d) { + if len(remaining) == len(d) || !isIdentPart(remaining[len(d)]) { + return d + } + } + } + return "" +} + +func skipDirective(sql string, pos int, directive string) int { + switch { + case directive == "#set" || directive == "#settings" || directive == "#setting" || directive == "#define": + // Skip to end of line or matching paren + j := pos + len(directive) + for j < len(sql) && (sql[j] == ' ' || sql[j] == '\t') { + j++ + } + if j < len(sql) && sql[j] == '(' { + depth := 1 + j++ + for j < len(sql) && depth > 0 { + if sql[j] == '(' { + depth++ + } else if sql[j] == ')' { + depth-- + } + j++ + } + return j + } + // Skip to end of line + for j < len(sql) && sql[j] != '\n' { + j++ + } + if j < len(sql) { + j++ + } + return j + case directive == "#foreach" || directive == "#if": + // Skip to matching #end + j := pos + len(directive) + depth := 1 + for j < len(sql) && depth > 0 { + d := matchDirective(sql, j) + if d == "#if" || d == "#foreach" { + depth++ + j += len(d) + } else if d == "#end" { + depth-- + j += len(d) + } else { + j++ + } + } + return j + default: + // #else, #elseif, #end, #package, #import — skip to end of line + j := pos + len(directive) + for j < len(sql) && sql[j] != '\n' { + j++ + } + if j < len(sql) { + j++ + } + return j + } +} + +func allPlaceholderColumns(columns view.Columns) bool { + if len(columns) == 0 { + return false + } + for _, col := range columns { + if col == nil { + continue + } + name := strings.ToLower(col.Name) + if !strings.HasPrefix(name, "col_") { + return false + } + } + return true +} + func lookupConnector(ctx context.Context, resource *view.Resource, aView *view.View) (*view.Connector, error) { if resource == nil { return nil, fmt.Errorf("shape column detector: missing resource for view %s", aView.Name) @@ -135,7 +392,7 @@ func columnsFromSchema(aView *view.View) view.Columns { func appendSchemaColumns(rType reflect.Type, ns string, columns *view.Columns) { for i := 0; i < rType.NumField(); i++ { field := rType.Field(i) - if field.PkgPath != "" { // unexported + if field.PkgPath != "" { continue } if field.Anonymous { @@ -148,12 +405,10 @@ func appendSchemaColumns(rType reflect.Type, ns string, columns *view.Columns) { } continue } - tag := io.ParseTag(field.Tag) if tag != nil && tag.Transient { continue } - name := field.Name if tag != nil && tag.Column != "" { name = tag.Column @@ -163,7 +418,6 @@ func appendSchemaColumns(rType reflect.Type, ns string, columns *view.Columns) { } else if ns != "" { name = ns + name } - columnType := field.Type nullable := false if columnType.Kind() == reflect.Ptr { @@ -195,7 +449,6 @@ func mergePreservingOrder(base, discovered view.Columns) view.Columns { } if fresh, ok := seen[strings.ToLower(item.Name)]; ok { delete(seen, strings.ToLower(item.Name)) - // Keep schema name/order but refresh discovered metadata. item.DataType = firstNonEmpty(fresh.DataType, item.DataType) item.SetColumnType(firstType(fresh.ColumnType(), item.ColumnType())) item.Nullable = fresh.Nullable @@ -218,6 +471,14 @@ func mergePreservingOrder(base, discovered view.Columns) view.Columns { return result } +func isIdentStart(ch byte) bool { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' +} + +func isIdentPart(ch byte) bool { + return isIdentStart(ch) || (ch >= '0' && ch <= '9') +} + func firstNonEmpty(values ...string) string { for _, value := range values { if strings.TrimSpace(value) != "" { diff --git a/repository/shape/column/falsify.go b/repository/shape/column/falsify.go new file mode 100644 index 000000000..0f50e2dca --- /dev/null +++ b/repository/shape/column/falsify.go @@ -0,0 +1,151 @@ +package column + +import ( + "strings" + + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/query" +) + +// falsifyQuery parses an SQL string and injects WHERE 1=0 into every SELECT +// in the query tree (outer query, CTEs, UNIONs). This ensures zero rows are +// scanned while preserving the output schema for column type inference. +// +// Returns the rewritten SQL string and true if successful. +// Returns the original SQL and false if parsing fails. +func falsifyQuery(sql string) (string, bool) { + sql = strings.TrimSpace(sql) + if sql == "" { + return sql, false + } + parsed, err := sqlparser.ParseQuery(sql) + if err != nil { + return sql, false + } + falsifySelect(parsed) + // Remove LIMIT/OFFSET from outer query — we want schema only + parsed.Limit = nil + parsed.Offset = nil + result := sqlparser.Stringify(parsed) + if strings.TrimSpace(result) == "" { + return sql, false + } + return result, true +} + +// falsifySelect injects 1=0 into a SELECT and recursively into all nested SELECTs. +func falsifySelect(sel *query.Select) { + if sel == nil { + return + } + // Inject 1=0 into this SELECT's WHERE clause + injectFalsePredicate(sel) + // Process CTE WITH selects + for _, ws := range sel.WithSelects { + if ws != nil && ws.X != nil { + falsifySelect(ws.X) + ws.Raw = "" // Force Stringify to use modified X instead of original Raw + } + } + // Process UNION branches + if sel.Union != nil && sel.Union.X != nil { + falsifySelect(sel.Union.X) + } + // Process subquery in FROM (if it's a nested SELECT) + falsifyFromSubquery(sel) + // Process JOIN subqueries + for _, join := range sel.Joins { + if join != nil { + falsifyJoinSubquery(join) + } + } +} + +// injectFalsePredicate adds 1=0 to the SELECT's WHERE clause. +func injectFalsePredicate(sel *query.Select) { + if sel == nil { + return + } + fp := &expr.Binary{ + X: &expr.Literal{Value: "1"}, + Op: "=", + Y: &expr.Literal{Value: "0"}, + } + if sel.Qualify == nil || sel.Qualify.X == nil { + sel.Qualify = &expr.Qualify{X: fp} + } else { + sel.Qualify = &expr.Qualify{ + X: &expr.Binary{ + X: fp, + Op: "AND", + Y: sel.Qualify.X, + }, + } + } +} + +// falsifyFromSubquery checks if the FROM clause contains a subquery and falsifies it. +func falsifyFromSubquery(sel *query.Select) { + if sel == nil || sel.From.X == nil { + return + } + switch sub := sel.From.X.(type) { + case *expr.Parenthesis: + falsifySubqueryExpr(sub) + case *expr.Raw: + falsifyRawSubquery(sub) + } +} + +func falsifyRawSubquery(raw *expr.Raw) { + if raw == nil { + return + } + text := strings.TrimSpace(raw.Raw) + if text == "" && raw.Unparsed != "" { + text = strings.TrimSpace(raw.Unparsed) + } + // Strip outer parens if present + if len(text) >= 2 && text[0] == '(' && text[len(text)-1] == ')' { + text = text[1 : len(text)-1] + } + if !strings.Contains(strings.ToLower(text), "select") { + return + } + subQuery, err := sqlparser.ParseQuery(text) + if err != nil { + return + } + falsifySelect(subQuery) + rewritten := sqlparser.Stringify(subQuery) + raw.Raw = "(" + rewritten + ")" +} + +// falsifyJoinSubquery checks if a JOIN's WITH clause contains a subquery and falsifies it. +func falsifyJoinSubquery(join *query.Join) { + if join == nil || join.With == nil { + return + } + if sub, ok := join.With.(*expr.Parenthesis); ok { + falsifySubqueryExpr(sub) + } +} + +// falsifySubqueryExpr attempts to parse and falsify a parenthesized subquery expression. +func falsifySubqueryExpr(paren *expr.Parenthesis) { + if paren == nil || paren.X == nil { + return + } + raw := sqlparser.Stringify(paren.X) + if !strings.Contains(strings.ToLower(strings.TrimSpace(raw)), "select") { + return + } + subQuery, err := sqlparser.ParseQuery(raw) + if err != nil { + return + } + falsifySelect(subQuery) + rewritten := sqlparser.Stringify(subQuery) + paren.X = expr.NewRaw(rewritten) +} diff --git a/repository/shape/column/falsify_test.go b/repository/shape/column/falsify_test.go new file mode 100644 index 000000000..fbc769171 --- /dev/null +++ b/repository/shape/column/falsify_test.go @@ -0,0 +1,144 @@ +package column + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFalsifyQuery(t *testing.T) { + tests := []struct { + name string + input string + wantOK bool + assertions func(t *testing.T, result string) + }{ + { + name: "simple SELECT *", + input: "SELECT * FROM orders", + wantOK: true, + assertions: func(t *testing.T, result string) { + assert.Contains(t, result, "1 = 0") + assert.NotContains(t, strings.ToUpper(result), "LIMIT") + }, + }, + { + name: "SELECT with existing WHERE", + input: "SELECT id, name FROM items WHERE status = 1", + wantOK: true, + assertions: func(t *testing.T, result string) { + assert.Contains(t, result, "1 = 0") + assert.Contains(t, result, "status") + }, + }, + { + name: "SELECT with LIMIT stripped", + input: "SELECT * FROM items LIMIT 100 OFFSET 50", + wantOK: true, + assertions: func(t *testing.T, result string) { + assert.Contains(t, result, "1 = 0") + assert.NotContains(t, strings.ToUpper(result), "LIMIT") + assert.NotContains(t, strings.ToUpper(result), "OFFSET") + }, + }, + { + name: "UNION ALL — both branches get 1=0", + input: "SELECT id, name FROM items_a WHERE region = 'us' UNION ALL SELECT id, name FROM items_b WHERE region = 'eu'", + wantOK: true, + assertions: func(t *testing.T, result string) { + count := strings.Count(result, "1 = 0") + assert.GreaterOrEqual(t, count, 2, "both UNION branches should get 1=0") + }, + }, + { + name: "CTE — all CTEs and outer get 1=0", + input: `WITH metrics AS ( + SELECT category, SUM(amount) AS total + FROM transactions + GROUP BY category +), +ranked AS ( + SELECT *, ROW_NUMBER() OVER (ORDER BY total DESC) AS rn + FROM metrics +) +SELECT * FROM ranked WHERE rn <= 10`, + wantOK: true, + assertions: func(t *testing.T, result string) { + count := strings.Count(result, "1 = 0") + assert.GreaterOrEqual(t, count, 3, "each CTE + outer should get 1=0, got %d", count) + assert.NotContains(t, strings.ToUpper(result), "LIMIT") + }, + }, + { + name: "CTE with UNION inside", + input: `WITH combined AS ( + SELECT id, name FROM items_a + UNION ALL + SELECT id, name FROM items_b +) +SELECT * FROM combined`, + wantOK: true, + assertions: func(t *testing.T, result string) { + count := strings.Count(result, "1 = 0") + assert.GreaterOrEqual(t, count, 3, "CTE branches + outer should all get 1=0") + }, + }, + { + name: "JOIN query — outer gets 1=0", + input: "SELECT a.id, b.name FROM orders a JOIN items b ON a.item_id = b.id", + wantOK: true, + assertions: func(t *testing.T, result string) { + assert.Contains(t, result, "1 = 0") + }, + }, + { + name: "subquery in FROM — both get 1=0", + input: "SELECT t.* FROM (SELECT id, name FROM items WHERE active = 1) t", + wantOK: true, + assertions: func(t *testing.T, result string) { + count := strings.Count(result, "1 = 0") + assert.GreaterOrEqual(t, count, 2, "outer + subquery should get 1=0") + }, + }, + { + name: "empty SQL", + input: "", + wantOK: false, + }, + { + name: "non-SELECT statement", + input: "INSERT INTO items VALUES (1, 'test')", + wantOK: true, // parser may still parse it; falsify is best-effort + }, + { + name: "GROUP BY preserved", + input: "SELECT category, COUNT(*) AS cnt FROM items GROUP BY category", + wantOK: true, + assertions: func(t *testing.T, result string) { + assert.Contains(t, result, "1 = 0") + assert.Contains(t, strings.ToUpper(result), "GROUP BY") + }, + }, + { + name: "ORDER BY preserved", + input: "SELECT * FROM items ORDER BY name", + wantOK: true, + assertions: func(t *testing.T, result string) { + assert.Contains(t, result, "1 = 0") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, ok := falsifyQuery(tt.input) + assert.Equal(t, tt.wantOK, ok) + if ok && tt.assertions != nil { + require.NotEmpty(t, result) + tt.assertions(t, result) + } + }) + } +} diff --git a/repository/shape/column/strip_test.go b/repository/shape/column/strip_test.go new file mode 100644 index 000000000..c38af5b50 --- /dev/null +++ b/repository/shape/column/strip_test.go @@ -0,0 +1,294 @@ +package column + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/viant/datly/view" +) + +func TestStripTemplateVariables(t *testing.T) { + tests := []struct { + name string + input string + expect string + }{ + { + name: "no templates", + input: "SELECT * FROM VENDOR WHERE ID = 1", + expect: "SELECT * FROM VENDOR WHERE ID = 1", + }, + { + name: "simple variable", + input: "SELECT * FROM VENDOR WHERE ID = $vendorID", + expect: "SELECT * FROM VENDOR WHERE ID = ''", + }, + { + name: "variable with dot method", + input: "SELECT * FROM VENDOR WHERE ID IN ($Unsafe.vendorIDs)", + expect: "SELECT * FROM VENDOR WHERE ID IN ('')", + }, + { + name: "variable with method call", + input: "SELECT * FROM PRODUCT WHERE 1=1 $View.ParentJoinOn(\"AND\",\"VENDOR_ID\")", + expect: "SELECT * FROM PRODUCT WHERE 1=1 ''", + }, + { + name: "criteria binding", + input: "SELECT * FROM VENDOR t WHERE t.ID IN ($criteria.AppendBinding($Unsafe.vendorIDs))", + expect: "SELECT * FROM VENDOR t WHERE t.ID IN ('')", + }, + { + name: "expression in braces", + input: "SELECT * FROM VENDOR WHERE ${predicate.Build(\"AND\")}", + expect: "SELECT * FROM VENDOR WHERE ''", + }, + { + name: "if directive", + input: "SELECT * FROM PRODUCT WHERE 1=1 #if($vendorID < 0) AND 1=2 #end", + expect: "SELECT * FROM PRODUCT WHERE 1=1 ", + }, + { + name: "foreach directive", + input: "#foreach($item in $items) INSERT INTO T VALUES($item.ID) #end", + expect: " ", + }, + { + name: "set directive with parens", + input: "#set($x = 1)\nSELECT * FROM T", + expect: " \nSELECT * FROM T", + }, + { + name: "mixed templates and SQL", + input: "SELECT vendor.*, products.* FROM (SELECT * FROM VENDOR t) vendor JOIN (SELECT * FROM PRODUCT t WHERE 1=1 ${predicate.Builder().CombineOr($predicate.FilterGroup(0, \"AND\")).Build(\"AND\")}) products ON products.VENDOR_ID = vendor.ID", + expect: "SELECT vendor.*, products.* FROM (SELECT * FROM VENDOR t) vendor JOIN (SELECT * FROM PRODUCT t WHERE 1=1 '') products ON products.VENDOR_ID = vendor.ID", + }, + { + name: "UNION ALL with templates", + input: "SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 $View.ParentJoinOn(\"AND\",\"VENDOR_ID\") UNION ALL SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 $View.ParentJoinOn(\"AND\",\"VENDOR_ID\")", + expect: "SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 '' UNION ALL SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 ''", + }, + { + name: "nested if", + input: "SELECT * FROM T WHERE 1=1 #if($a > 0) AND A=$a #if($b > 0) AND B=$b #end #end", + expect: "SELECT * FROM T WHERE 1=1 ", + }, + { + name: "const variable substitution", + input: "SELECT * FROM $Vendor t WHERE t.ID IN ($vendorIDs)", + expect: "SELECT * FROM '' t WHERE t.ID IN ('')", + }, + { + name: "settings directive at top", + input: "#setting($_ = $route('/api/v1/test', 'GET'))\nSELECT * FROM T", + expect: " \nSELECT * FROM T", + }, + { + name: "package and import directives", + input: "#package('dev/vendor')\n#import('pkg', 'github.com/acme/pkg')\nSELECT * FROM T", + expect: " SELECT * FROM T", + }, + { + name: "complex predicate builder", + input: "WHERE ${predicate.Builder().CombineOr($predicate.FilterGroup(0, \"AND\")).Build(\"AND\")}", + expect: "WHERE ''", + }, + { + name: "dollar at end", + input: "SELECT * FROM T WHERE X = $", + expect: "SELECT * FROM T WHERE X = $", + }, + { + name: "dollar number (not a variable)", + input: "SELECT * FROM T WHERE X = $1", + expect: "SELECT * FROM T WHERE X = $1", + }, + { + name: "cast expression", + input: "SELECT CAST($Jwt.FirstName AS CHAR) AS FIRST_NAME FROM T", + expect: "SELECT CAST('' AS CHAR) AS FIRST_NAME FROM T", + }, + { + name: "logger and unsafe", + input: "#foreach($rec in $Unsafe.Records) UPDATE T SET V=$rec.Value WHERE ID=$rec.ID; #end", + expect: " ", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := stripTemplateVariables(tt.input) + assert.Equal(t, tt.expect, got) + }) + } +} + +func TestStripTemplateVariables_CTE(t *testing.T) { + tests := []struct { + name string + input string + expect string + }{ + { + name: "CTE with template params", + input: `WITH params AS ( + SELECT DATE_SUB(CURRENT_DATE(), INTERVAL $EndDayInterval DAY) AS end_date, + CAST(GREATEST($Page, 1) AS INT64) AS page_number +), +perf AS ( + SELECT p.agency_id, SUM(p.impressions) AS imps + FROM fact_performance p + JOIN params prm ON TRUE + WHERE p.event_date BETWEEN prm.start_date AND prm.end_date + ${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("AND")} + GROUP BY 1 +) +SELECT v.* FROM perf v ORDER BY v.agency_id`, + expect: `WITH params AS ( + SELECT DATE_SUB(CURRENT_DATE(), INTERVAL '' DAY) AS end_date, + CAST(GREATEST('', 1) AS INT64) AS page_number +), +perf AS ( + SELECT p.agency_id, SUM(p.impressions) AS imps + FROM fact_performance p + JOIN params prm ON TRUE + WHERE p.event_date BETWEEN prm.start_date AND prm.end_date + '' + GROUP BY 1 +) +SELECT v.* FROM perf v ORDER BY v.agency_id`, + }, + { + name: "CTE with backtick tables (BigQuery)", + input: "WITH data AS (SELECT * FROM `project.dataset.table` t WHERE t.ID = $id) SELECT * FROM data", + expect: "WITH data AS (SELECT * FROM `project.dataset.table` t WHERE t.ID = '') SELECT * FROM data", + }, + { + name: "UNION ALL in CTE", + input: "WITH combined AS (SELECT * FROM T1 WHERE ID = $a UNION ALL SELECT * FROM T2 WHERE ID = $b) SELECT * FROM combined", + expect: "WITH combined AS (SELECT * FROM T1 WHERE ID = '' UNION ALL SELECT * FROM T2 WHERE ID = '') SELECT * FROM combined", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := stripTemplateVariables(tt.input) + assert.Equal(t, tt.expect, got) + }) + } +} + +// TestDiscoverySQL_Strategy documents the column discovery strategy for different SQL patterns. +// BigQuery CTEs cannot use WHERE 1=0 (full cost incurred), so the strategy should be: +// - Simple SELECT * FROM table → use table metadata (INFORMATION_SCHEMA or SELECT * WHERE 1=0) +// - SELECT with explicit columns → column names from AST, types from table metadata +// - CTE/WITH queries → parse final SELECT, resolve CTE chain to source tables, use metadata +// - SQL with velocity templates → strip templates, then apply above rules +func TestDiscoverySQL_Strategy(t *testing.T) { + tests := []struct { + name string + table string + sql string + expected string + desc string + assertions func(t *testing.T, result string) + }{ + { + name: "wildcard with templates — uses table fallback", + table: "VENDOR", + sql: "SELECT * FROM VENDOR t WHERE t.ID = $vendorID", + expected: "VENDOR", + desc: "template variables → table fallback (safe for all backends)", + }, + { + name: "wildcard with EXCEPT — uses table fallback", + table: "VENDOR", + sql: "SELECT vendor.* EXCEPT VENDOR_ID FROM VENDOR vendor", + expected: "VENDOR", + desc: "EXCEPT clause → table fallback (EXCEPT is datly extension)", + }, + { + name: "clean explicit SQL gets falsified", + table: "VENDOR", + sql: "SELECT ID, NAME FROM VENDOR WHERE 1=1", + desc: "clean SQL → falsified with 1=0 injected", + assertions: func(t *testing.T, result string) { + assert.Contains(t, result, "1 = 0") + assert.Contains(t, strings.ToUpper(result), "ID") + assert.Contains(t, strings.ToUpper(result), "NAME") + }, + }, + { + name: "empty SQL uses table", + table: "VENDOR", + sql: "", + expected: "VENDOR", + desc: "no SQL → use table name", + }, + { + name: "CTE with templates — uses table fallback", + table: "VENDOR", + sql: "WITH cte AS (SELECT * FROM VENDOR WHERE ID = $id) SELECT * FROM cte", + expected: "VENDOR", + desc: "CTE with templates → table fallback (safe for BigQuery)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := &view.View{ + Name: "test", + Table: tt.table, + } + if tt.sql != "" { + v.Template = &view.Template{Source: tt.sql} + } + got := discoverySQL(v) + if tt.assertions != nil { + tt.assertions(t, got) + } else { + assert.Equal(t, tt.expected, got, tt.desc) + } + }) + } +} + +func TestAllPlaceholderColumns(t *testing.T) { + tests := []struct { + name string + names []string + expect bool + }{ + {"empty", nil, false}, + {"real columns", []string{"ID", "NAME"}, false}, + {"all placeholders", []string{"col_1", "col_2"}, true}, + {"mixed", []string{"col_1", "NAME"}, false}, + {"single placeholder", []string{"col_1"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var cols view.Columns + for _, n := range tt.names { + cols = append(cols, &view.Column{Name: n}) + } + assert.Equal(t, tt.expect, allPlaceholderColumns(cols)) + }) + } +} + +func TestNeedsDiscovery(t *testing.T) { + tests := []struct { + name string + view *view.View + expect bool + }{ + {"nil view", nil, false}, + {"no columns", &view.View{Name: "t"}, true}, + {"placeholder columns", &view.View{Name: "t", Columns: view.Columns{&view.Column{Name: "col_1"}}}, true}, + {"real columns no wildcard", &view.View{Name: "t", Columns: view.Columns{&view.Column{Name: "ID"}}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expect, needsDiscovery(tt.view)) + }) + } +} diff --git a/repository/shape/compile/COMPONENT_CONTRACT_PARITY.md b/repository/shape/compile/COMPONENT_CONTRACT_PARITY.md new file mode 100644 index 000000000..d198dfe73 --- /dev/null +++ b/repository/shape/compile/COMPONENT_CONTRACT_PARITY.md @@ -0,0 +1,99 @@ +# Component Contract Parity Target (Shape) + +This document defines the target behavior for cross-component contract discovery in `repository/shape`. + +Scope: + +- Applies to DQL compile flow (`compile -> plan -> load`) +- Does not depend on `internal/*` packages +- Defines observable behavior and acceptance criteria + +## Problem Statement + +When DQL declares component dependencies (for example via component-typed state declarations), shape compile should produce component-facing IR that is functionally equivalent to translator contract/signature resolution for: + +- route reference normalization +- output schema/type resolution for component states +- dependent type propagation +- deterministic diagnostics + +Today, shape produces useful plan artifacts for views/states but component contract resolution parity is incomplete. + +## Target Semantics + +### 1) Reference Forms + +A component reference MUST support these forms: + +- Relative: `../acl/auth` +- Method-qualified absolute route: `GET:/v1/api/platform/acl/auth` +- Absolute route without method: `/v1/api/platform/acl/auth` (defaults to `GET`) + +Normalization target: + +- Stable route identity represented as `method + uri` (default method `GET`) +- Namespace/path derivation remains deterministic for file-layout lookups + +### 2) State Enrichment + +For each `plan.State` with `Kind == "component"`: + +- `In` retains user-declared logical reference (for traceability) +- `DataType` is inferred from referenced component output when not explicitly declared +- `OutputDataType` is preserved if user declares explicit output type + +If inferred type is unavailable, a diagnostic is emitted (see diagnostics section). + +### 3) Type Propagation + +Referenced component route/resource types required for consuming component states SHOULD be appended to `plan.Result.Types` unless a collision exists. + +Collision policy: + +- Existing local type names win +- Emit collision diagnostic for skipped imported type + +### 4) Nested Component Dependencies + +If referenced route/resource parameters include additional component references: + +- Resolver walks nested dependencies transitively +- Cycle detection MUST prevent infinite recursion +- Cycle reports a deterministic warning diagnostic + +### 5) Loader Classification + +Shape-loaded component artifact SHOULD classify component dependencies as input-like contract dependencies (not miscellaneous "other"). + +## Diagnostics Contract + +Component-related diagnostics use `DQL-COMP-*` codes: + +- `DQL-COMP-REF-INVALID` +- `DQL-COMP-ROUTE-MISSING` +- `DQL-COMP-ROUTE-INVALID` +- `DQL-COMP-CYCLE` +- `DQL-COMP-TYPE-COLLISION` + +Requirements: + +- Deterministic code/message per failure class +- Span points at the referenced component token when available +- Warnings by default unless compile strict mode escalates + +## Non-Goals (Step 1) + +- No resolver implementation changes +- No signature engine wiring changes +- No compile pipeline behavior changes + +This step defines the contract only; implementation phases follow separately. + +## Acceptance Criteria + +The parity contract is considered defined when: + +1. Reference normalization rules are explicit and unambiguous. +2. Required state/type/diagnostic behavior is documented for success and failure paths. +3. Nested dependency and collision behavior is documented. +4. Constraints are independent of `internal/*` packages. diff --git a/repository/shape/compile/compiler.go b/repository/shape/compile/compiler.go index 6fb6eadc0..14029ebd7 100644 --- a/repository/shape/compile/compiler.go +++ b/repository/shape/compile/compiler.go @@ -134,17 +134,29 @@ func (c *DQLCompiler) assembleResult( result.TypeContext = prepared.Pre.TypeCtx result.Directives = prepared.Pre.Directives applyDefaultConnectorDirective(result) + applyConstDirective(result) hints := extractViewHints(source.DQL) appendRelationViews(result, root, hints) appendDeclaredViews(source.DQL, result) appendDeclaredStates(source.DQL, result) applyViewHints(result, hints) + applyInlineParamHints(source.DQL, result) applySourceParityEnrichmentWithLayout(result, source, pathLayout) applyLinkedTypeSupport(result, source) result.Diagnostics = append(result.Diagnostics, applyColumnDiscoveryPolicy(result, compileOptions)...) return result } +func applyConstDirective(result *plan.Result) { + if result == nil || result.Directives == nil || len(result.Directives.Const) == 0 { + return + } + result.Const = make(map[string]string, len(result.Directives.Const)) + for k, v := range result.Directives.Const { + result.Const[k] = v + } +} + func applyDefaultConnectorDirective(result *plan.Result) { if result == nil || result.Directives == nil { return diff --git a/repository/shape/compile/hints.go b/repository/shape/compile/hints.go index a16b855d2..8c7c592c9 100644 --- a/repository/shape/compile/hints.go +++ b/repository/shape/compile/hints.go @@ -12,6 +12,9 @@ type viewHint struct { Connector string AllowNulls *bool NoLimit *bool + CacheRef string + Limit *int + Self *plan.SelfReference } func extractViewHints(dql string) map[string]viewHint { @@ -58,6 +61,35 @@ func extractViewHints(dql string) map[string]viewHint { hint := result[alias] noLimit := limit == 0 hint.NoLimit = &noLimit + if limit > 0 { + hint.Limit = &limit + } + result[alias] = hint + case "set_cache": + if len(call.args) != 2 { + continue + } + alias := strings.TrimSpace(call.args[0]) + ref := unquote(strings.TrimSpace(call.args[1])) + if !isIdentifier(alias) || ref == "" { + continue + } + hint := result[alias] + hint.CacheRef = ref + result[alias] = hint + case "self_ref": + if len(call.args) != 4 { + continue + } + alias := strings.TrimSpace(call.args[0]) + holder := unquote(strings.TrimSpace(call.args[1])) + child := unquote(strings.TrimSpace(call.args[2])) + parent := unquote(strings.TrimSpace(call.args[3])) + if alias == "" || holder == "" || child == "" || parent == "" { + continue + } + hint := result[alias] + hint.Self = &plan.SelfReference{Holder: holder, Child: child, Parent: parent} result[alias] = hint } } @@ -82,7 +114,7 @@ func scanHintCalls(input string) []hintCall { i++ } name := strings.ToLower(input[start:i]) - if name != "use_connector" && name != "allow_nulls" && name != "set_limit" { + if name != "use_connector" && name != "allow_nulls" && name != "set_limit" && name != "set_cache" && name != "self_ref" { continue } j := skipSpaces(input, i) @@ -296,6 +328,16 @@ func applyViewHints(result *plan.Result, hints map[string]viewHint) { value := *hint.NoLimit item.SelectorNoLimit = &value } + if item.SelectorLimit == nil && hint.Limit != nil { + value := *hint.Limit + item.SelectorLimit = &value + } + if item.CacheRef == "" && hint.CacheRef != "" { + item.CacheRef = hint.CacheRef + } + if item.Self == nil && hint.Self != nil { + item.Self = hint.Self + } } } } diff --git a/repository/shape/compile/inline_param.go b/repository/shape/compile/inline_param.go new file mode 100644 index 000000000..af5b233a5 --- /dev/null +++ b/repository/shape/compile/inline_param.go @@ -0,0 +1,103 @@ +package compile + +import ( + "encoding/json" + "strings" + + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/view/state" +) + +type inlineParamHint struct { + Kind string `json:"Kind"` + Location string `json:"Location"` + DataType string `json:"DataType"` + Required *bool `json:"Required"` +} + +// applyInlineParamHints scans the SQL text for patterns like +// $varName /* {"Kind":"header","Location":"Header-Name"} */ and updates +// matching state parameters in the plan result. +func applyInlineParamHints(sqlText string, result *plan.Result) { + if result == nil || strings.TrimSpace(sqlText) == "" { + return + } + hints := extractInlineParamHints(sqlText) + if len(hints) == 0 { + return + } + for _, st := range result.States { + if st == nil { + continue + } + name := strings.TrimPrefix(strings.TrimSpace(st.Name), "$") + hint, ok := hints[name] + if !ok { + continue + } + if hint.Kind != "" && st.In != nil { + st.In.Kind = state.Kind(strings.ToLower(hint.Kind)) + } + if hint.Location != "" && st.In != nil { + st.In.Name = hint.Location + } + if hint.DataType != "" { + ensureStateSchema(st).DataType = hint.DataType + } + if hint.Required != nil { + st.Required = hint.Required + } + } +} + +func extractInlineParamHints(sql string) map[string]inlineParamHint { + result := map[string]inlineParamHint{} + i := 0 + for i < len(sql) { + if sql[i] != '$' { + i++ + continue + } + i++ + start := i + for i < len(sql) && isParamIdentPart(sql[i]) { + i++ + } + if i == start { + continue + } + name := sql[start:i] + j := skipInlineSpaces(sql, i) + if j+1 >= len(sql) || sql[j] != '/' || sql[j+1] != '*' { + continue + } + endComment := strings.Index(sql[j+2:], "*/") + if endComment < 0 { + continue + } + body := strings.TrimSpace(sql[j+2 : j+2+endComment]) + if !strings.HasPrefix(body, "{") || !strings.HasSuffix(body, "}") { + continue + } + var hint inlineParamHint + if err := json.Unmarshal([]byte(body), &hint); err != nil { + continue + } + if hint.Kind != "" || hint.Location != "" || hint.DataType != "" { + result[name] = hint + } + i = j + 2 + endComment + 2 + } + return result +} + +func isParamIdentPart(ch byte) bool { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' +} + +func skipInlineSpaces(input string, index int) int { + for index < len(input) && (input[index] == ' ' || input[index] == '\t' || input[index] == '\n' || input[index] == '\r') { + index++ + } + return index +} diff --git a/repository/shape/compile/inline_param_test.go b/repository/shape/compile/inline_param_test.go new file mode 100644 index 000000000..13079090a --- /dev/null +++ b/repository/shape/compile/inline_param_test.go @@ -0,0 +1,36 @@ +package compile + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/view/state" +) + +func TestExtractInlineParamHints(t *testing.T) { + sql := `SELECT * FROM VENDOR t WHERE t.ID = $vendorID /* {"Kind": "header", "Location": "Vendor-Id"} */` + hints := extractInlineParamHints(sql) + require.Contains(t, hints, "vendorID") + assert.Equal(t, "header", hints["vendorID"].Kind) + assert.Equal(t, "Vendor-Id", hints["vendorID"].Location) +} + +func TestApplyInlineParamHints(t *testing.T) { + sql := `WHERE t.ID = $vendorID /* {"Kind": "header", "Location": "Vendor-Id"} */` + result := &plan.Result{ + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "vendorID", + In: &state.Location{Kind: state.KindQuery, Name: "vendorID"}, + }, + }, + }, + } + applyInlineParamHints(sql, result) + require.Len(t, result.States, 1) + assert.Equal(t, state.KindHeader, result.States[0].In.Kind) + assert.Equal(t, "Vendor-Id", result.States[0].In.Name) +} diff --git a/repository/shape/compile/resolver_test.go b/repository/shape/compile/resolver_test.go new file mode 100644 index 000000000..ab88a3fe8 --- /dev/null +++ b/repository/shape/compile/resolver_test.go @@ -0,0 +1,17 @@ +package compile + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSplitRouteKey(t *testing.T) { + method, uri := splitRouteKey("POST:/v1/api/platform/acl/auth") + assert.Equal(t, "POST", method) + assert.Equal(t, "/v1/api/platform/acl/auth", uri) + + method, uri = splitRouteKey("/v1/api/platform/acl/auth") + assert.Equal(t, "GET", method) + assert.Equal(t, "/v1/api/platform/acl/auth", uri) +} diff --git a/repository/shape/compile/route_index_test.go b/repository/shape/compile/route_index_test.go new file mode 100644 index 000000000..c93454875 --- /dev/null +++ b/repository/shape/compile/route_index_test.go @@ -0,0 +1,71 @@ +package compile + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildRouteIndex_AndResolve(t *testing.T) { + tempDir := t.TempDir() + authPath := filepath.Join(tempDir, "dql", "platform", "acl", "auth.dql") + reportPath := filepath.Join(tempDir, "dql", "platform", "reports", "orders", "orders.dql") + require.NoError(t, writeFile(authPath, `/* {"URI":"/v1/api/platform/acl/auth","Method":"GET"} */ SELECT 1`)) + require.NoError(t, writeFile(reportPath, `SELECT 1`)) + + index, err := BuildRouteIndex([]string{authPath, reportPath}) + require.NoError(t, err) + + _, ok := index.ByRouteKey["GET:/v1/api/platform/acl/auth"] + assert.True(t, ok) + _, ok = index.ByRouteKey["GET:/v1/api/platform/reports/orders"] + assert.True(t, ok) // inferred from namespace when URI is not explicitly declared + + resolved, ok := index.Resolve("../../acl/auth", reportPath) + require.True(t, ok) + assert.Equal(t, "GET:/v1/api/platform/acl/auth", resolved) +} + +func TestRouteIndex_ResolveByAbsoluteURI(t *testing.T) { + tempDir := t.TempDir() + authPath := filepath.Join(tempDir, "dql", "platform", "acl", "auth.dql") + require.NoError(t, writeFile(authPath, `/* {"URI":"/v1/api/platform/acl/auth","Method":"POST"} */ SELECT 1`)) + + index, err := BuildRouteIndex([]string{authPath}) + require.NoError(t, err) + + resolved, ok := index.Resolve("POST:/v1/api/platform/acl/auth", authPath) + require.True(t, ok) + assert.Equal(t, "POST:/v1/api/platform/acl/auth", resolved) +} + +func TestBuildRouteIndex_Conflicts(t *testing.T) { + tempDir := t.TempDir() + leftPath := filepath.Join(tempDir, "dql", "platform", "left", "x.dql") + rightPath := filepath.Join(tempDir, "dql", "platform", "right", "y.dql") + content := `/* {"URI":"/v1/api/platform/shared/resource","Method":"GET"} */ SELECT 1` + require.NoError(t, writeFile(leftPath, content)) + require.NoError(t, writeFile(rightPath, content)) + + index, err := BuildRouteIndex([]string{leftPath, rightPath}) + require.NoError(t, err) + + conflicts := index.Conflicts["GET:/v1/api/platform/shared/resource"] + require.Len(t, conflicts, 2) + _, ok := index.Resolve("GET:/v1/api/platform/shared/resource", leftPath) + assert.False(t, ok) +} + +func writeFile(path, content string) error { + if err := ensureDir(filepath.Dir(path)); err != nil { + return err + } + return os.WriteFile(path, []byte(content), 0o644) +} + +func ensureDir(path string) error { + return os.MkdirAll(path, 0o755) +} diff --git a/repository/shape/compile/viewdecl.go b/repository/shape/compile/viewdecl.go index 9e6c14c88..359a3712e 100644 --- a/repository/shape/compile/viewdecl.go +++ b/repository/shape/compile/viewdecl.go @@ -81,7 +81,7 @@ func extractDeclaredViews(dql string) ([]*declaredView, []*dqlshape.Diagnostic) if kind != "view" && kind != "data_view" { continue } - sqlText := extractDeclarationSQL(tail) + sqlText, errorStatusCode := extractDeclarationSQLWithStatus(tail) if sqlText == "" { diags = append(diags, &dqlshape.Diagnostic{ Code: dqldiag.CodeViewMissingSQL, @@ -100,6 +100,9 @@ func extractDeclaredViews(dql string) ([]*declaredView, []*dqlshape.Diagnostic) continue } view := &declaredView{Name: name, SQL: strings.TrimSpace(sqlText)} + if errorStatusCode != nil { + view.StatusCode = errorStatusCode + } applyDeclaredViewOptions(view, tail, dql, block.Offset, &diags) views = append(views, view) } diff --git a/repository/shape/compile/viewdecl_options.go b/repository/shape/compile/viewdecl_options.go index dd8ea2fba..835b25834 100644 --- a/repository/shape/compile/viewdecl_options.go +++ b/repository/shape/compile/viewdecl_options.go @@ -11,25 +11,35 @@ import ( ) func extractDeclarationSQL(fragment string) string { + sql, _ := extractDeclarationSQLWithStatus(fragment) + return sql +} + +func extractDeclarationSQLWithStatus(fragment string) (string, *int) { cursor := parsly.NewCursor("", []byte(fragment), 0) for cursor.Pos < cursor.InputSize { match := cursor.MatchAfterOptional(vdWhitespaceMatcher, vdCommentMatcher) if match.Code == vdCommentToken { text := match.Text(cursor) if len(text) < 4 { - return "" + return "", nil } - return normalizeHintSQL(text[2 : len(text)-2]) + return normalizeHintSQLWithStatus(text[2 : len(text)-2]) } cursor.Pos++ } - return "" + return "", nil } func normalizeHintSQL(body string) string { + sql, _ := normalizeHintSQLWithStatus(body) + return sql +} + +func normalizeHintSQLWithStatus(body string) (string, *int) { body = strings.TrimSpace(body) if body == "" { - return "" + return "", nil } if strings.HasPrefix(body, "{") { if closeIdx := strings.Index(body, "}"); closeIdx != -1 { @@ -37,8 +47,9 @@ func normalizeHintSQL(body string) string { } } if body == "" { - return "" + return "", nil } + var statusCode *int switch body[0] { case '?': body = strings.TrimSpace(body[1:]) @@ -50,11 +61,12 @@ func normalizeHintSQL(body string) string { if len(body) >= 3 { var status int if _, err := fmt.Sscanf(body[:3], "%d", &status); err == nil { + statusCode = &status body = strings.TrimSpace(body[3:]) } } } - return strings.TrimSpace(body) + return strings.TrimSpace(body), statusCode } func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, diags *[]*dqlshape.Diagnostic) { diff --git a/repository/shape/dql/diag/codes.go b/repository/shape/dql/diag/codes.go index 7fa6a96e9..c4de8559f 100644 --- a/repository/shape/dql/diag/codes.go +++ b/repository/shape/dql/diag/codes.go @@ -17,6 +17,7 @@ const ( CodeDirFormat = "DQL-DIR-FORMAT" CodeDirDateFormat = "DQL-DIR-DATE-FORMAT" CodeDirCaseFormat = "DQL-DIR-CASE-FORMAT" + CodeDirConst = "DQL-DIR-CONST" CodeDirUnsupported = "DQL-DIR-UNSUPPORTED" CodeOptParse = "DQL-OPT-PARSE" diff --git a/repository/shape/dql/parity/adorder_parity_test.go b/repository/shape/dql/parity/adorder_parity_test.go index 667a6c79d..9c7b639b0 100644 --- a/repository/shape/dql/parity/adorder_parity_test.go +++ b/repository/shape/dql/parity/adorder_parity_test.go @@ -9,6 +9,7 @@ import ( dqlplan "github.com/viant/datly/repository/shape/dql/plan" dqlyaml "github.com/viant/datly/repository/shape/dql/render/yaml" dqlscan "github.com/viant/datly/repository/shape/dql/scan" + "github.com/viant/datly/testutil/shapeparity" ) func TestAdorderDQL_CanonicalParityWithYAML(t *testing.T) { @@ -26,12 +27,11 @@ func TestAdorderDQL_CanonicalParityWithYAML(t *testing.T) { t.Skipf("missing fixture yaml file: %v", err) } - scanner := dqlscan.New() connectors := resolveConnectors([]string{ "ci_ads|mysql|root:dev@tcp(127.0.0.1:3307)/ci_ads?parseTime=true&charset=utf8mb4&collation=utf8mb4_bin", "ci_logs|mysql|root:dev@tcp(127.0.0.1:3307)/ci_logs?parseTime=true", }) - scanned, err := scanner.Scan(context.Background(), &dqlscan.Request{ + scanned, err := shapeparity.ScanDQL(context.Background(), &dqlscan.Request{ DQLURL: dqlPath, Repository: repoPath, ModulePrefix: "platform/adorder", diff --git a/repository/shape/dql/parity/connectors.go b/repository/shape/dql/parity/connectors.go index eaaacab8b..c85e00aeb 100644 --- a/repository/shape/dql/parity/connectors.go +++ b/repository/shape/dql/parity/connectors.go @@ -8,6 +8,18 @@ import ( // resolveConnectors returns connectors from env override, or defaults. // When DATLY_PARITY_SQLITE_DSN is set, all default connector names are mapped to sqlite3. +func splitNonEmpty(csv string) []string { + var ret []string + for _, item := range strings.Split(csv, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + ret = append(ret, item) + } + return ret +} + func resolveConnectors(defaults []string) []string { if override := splitNonEmpty(os.Getenv("DATLY_PARITY_CONNECTORS")); len(override) > 0 { return override diff --git a/repository/shape/dql/parity/mdp_parity_test.go b/repository/shape/dql/parity/mdp_parity_test.go index 6941ee914..0311f2bc7 100644 --- a/repository/shape/dql/parity/mdp_parity_test.go +++ b/repository/shape/dql/parity/mdp_parity_test.go @@ -9,6 +9,7 @@ import ( dqlplan "github.com/viant/datly/repository/shape/dql/plan" dqlscan "github.com/viant/datly/repository/shape/dql/scan" + "github.com/viant/datly/testutil/shapeparity" ) func TestMDPDQL_CanonicalParityWithRoutes(t *testing.T) { @@ -41,7 +42,6 @@ func TestMDPDQL_CanonicalParityWithRoutes(t *testing.T) { msg string } var issues []issue - scanner := dqlscan.New() _ = filepath.WalkDir(routesRoot, func(path string, d os.DirEntry, walkErr error) error { if walkErr != nil || d.IsDir() { return walkErr @@ -69,7 +69,7 @@ func TestMDPDQL_CanonicalParityWithRoutes(t *testing.T) { return nil } modulePrefix := filepath.ToSlash(filepath.Join("mdp", ruleDir)) - scanned, err := scanner.Scan(context.Background(), &dqlscan.Request{ + scanned, err := shapeparity.ScanDQL(context.Background(), &dqlscan.Request{ DQLURL: dqlFile, Repository: repoRoot, ModulePrefix: modulePrefix, @@ -146,15 +146,3 @@ func envOr(key, fallback string) string { } return fallback } - -func splitNonEmpty(csv string) []string { - var ret []string - for _, item := range strings.Split(csv, ",") { - item = strings.TrimSpace(item) - if item == "" { - continue - } - ret = append(ret, item) - } - return ret -} diff --git a/repository/shape/dql/preprocess/preprocess.go b/repository/shape/dql/preprocess/preprocess.go index 9579dfb0c..a09e1b90a 100644 --- a/repository/shape/dql/preprocess/preprocess.go +++ b/repository/shape/dql/preprocess/preprocess.go @@ -147,9 +147,15 @@ func normalizeDirectives(input *dqlshape.Directives) *dqlshape.Directives { Methods: normalizedMethods, } } + if len(input.Const) > 0 { + ret.Const = make(map[string]string, len(input.Const)) + for k, v := range input.Const { + ret.Const[k] = v + } + } if ret.Meta == "" && ret.DefaultConnector == "" && ret.Cache == nil && ret.MCP == nil && ret.Route == nil && ret.JSONMarshalType == "" && ret.JSONUnmarshalType == "" && ret.XMLUnmarshalType == "" && ret.Format == "" && - ret.DateFormat == "" && ret.CaseFormat == "" { + ret.DateFormat == "" && ret.CaseFormat == "" && len(ret.Const) == 0 { return nil } return ret diff --git a/repository/shape/dql/preprocess/settings_directives.go b/repository/shape/dql/preprocess/settings_directives.go index 3d7793c8f..6a140fa67 100644 --- a/repository/shape/dql/preprocess/settings_directives.go +++ b/repository/shape/dql/preprocess/settings_directives.go @@ -16,6 +16,7 @@ var ( cacheDirectiveName = map[string]bool{"cache": true} mcpDirectiveName = map[string]bool{"mcp": true} routeDirectiveName = map[string]bool{"route": true} + constDirectiveName = map[string]bool{"const": true} marshalDirectiveName = map[string]bool{"marshal": true} unmarshalDirectiveName = map[string]bool{"unmarshal": true} formatDirectiveName = map[string]bool{"format": true} @@ -78,6 +79,19 @@ func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, direct directives.Route = values[len(values)-1] } } + if strings.Contains(lower, "$const") { + values := parseConstDirectives(input) + if len(values) == 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirConst, "invalid $const directive", "expected: #settings($_ = $const('Name','VALUE'))", fullDQL, diagnosticOffset)) + } else { + if directives.Const == nil { + directives.Const = map[string]string{} + } + for _, kv := range values { + directives.Const[kv[0]] = kv[1] + } + } + } if strings.Contains(lower, "$marshal") { values := parseMarshalDirectives(input) if len(values) == 0 { @@ -433,3 +447,27 @@ func parseCaseFormatDirectives(input string) []string { } return result } + +func parseConstDirectives(input string) [][2]string { + calls := scanDollarCalls(input, constDirectiveName) + var result [][2]string + for _, call := range calls { + if len(call.args) != 2 { + continue + } + name, ok := parseQuotedLiteral(call.args[0]) + if !ok { + continue + } + name = strings.TrimSpace(name) + if name == "" { + continue + } + value, ok := parseQuotedLiteral(call.args[1]) + if !ok { + continue + } + result = append(result, [2]string{name, strings.TrimSpace(value)}) + } + return result +} diff --git a/repository/shape/dql/scan/scanner.go b/repository/shape/dql/scan/scanner.go index b7ecb2d69..8dac0e291 100644 --- a/repository/shape/dql/scan/scanner.go +++ b/repository/shape/dql/scan/scanner.go @@ -1,24 +1,16 @@ package scan import ( - "context" "fmt" - "path/filepath" "reflect" "strings" "time" _ "github.com/go-sql-driver/mysql" - "github.com/viant/afs" - "github.com/viant/afs/file" - "github.com/viant/afs/url" - "github.com/viant/datly/cmd/options" - "github.com/viant/datly/internal/translator" "github.com/viant/datly/repository/shape/dql/decl" "github.com/viant/datly/repository/shape/dql/ir" "github.com/viant/datly/repository/shape/dql/parse" dqlplan "github.com/viant/datly/repository/shape/dql/plan" - "github.com/viant/datly/repository/shape/dql/sanitize" dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/typectx" "github.com/viant/datly/repository/shape/typectx/source" @@ -49,87 +41,16 @@ type Result struct { } // Scanner translates DQL to Datly route YAML in-memory. -type Scanner struct { - fs afs.Service -} +type Scanner struct{} func New() *Scanner { - return &Scanner{fs: afs.New()} -} - -func (s *Scanner) Scan(ctx context.Context, req *Request) (result *Result, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("dql scan panic: %v", r) - result = nil - } - }() - if req == nil || req.DQLURL == "" { - return nil, fmt.Errorf("dql scan: DQLURL was empty") - } - sourceURL := req.DQLURL - project := inferProject(req.DQLURL) - translate := &options.Translate{} - translate.Rule.Project = project - translate.Rule.Source = []string{sourceURL} - translate.Rule.ModulePrefix = req.ModulePrefix - translate.Repository.RepositoryURL = req.Repository - translate.Repository.APIPrefix = req.APIPrefix - if len(req.Connectors) > 0 { - translate.Repository.Connectors = append(translate.Repository.Connectors, req.Connectors...) - } - if req.ConfigURL != "" { - translate.Repository.Configs.Append(req.ConfigURL) - } - var initErr error - if initErr = translate.Init(ctx); initErr != nil { - return nil, initErr - } - if req.ConfigURL == "" { - // Force in-memory translator config to avoid stale absolute paths from discovered config.json. - translate.Repository.Configs = nil - } - if translate.Rule.ModulePrefix == "" { - translate.Rule.ModulePrefix = "platform" - } - - svc := translator.New(translator.NewConfig(&translate.Repository), s.fs) - if initErr := svc.Init(ctx); initErr != nil { - return nil, initErr - } - if initErr := svc.InitSignature(ctx, &translate.Rule); initErr != nil { - return nil, initErr - } - dsql, loadErr := translate.Rule.LoadSource(ctx, s.fs, translate.Rule.SourceURL()) - if loadErr != nil { - return nil, loadErr - } - translate.Rule.NormalizeComponent(&dsql) - dsql = sanitize.SQL(dsql, sanitize.Options{Declared: sanitize.Declared(dsql)}) - top := &options.Options{Translate: translate} - if initErr = svc.Translate(ctx, &translate.Rule, dsql, top); initErr != nil { - return nil, initErr - } - ruleName := svc.Repository.RuleName(&translate.Rule) - targetSuffix := "/" + ruleName + ".yaml" - for _, item := range svc.Repository.Files { - if !strings.HasSuffix(item.URL, targetSuffix) { - continue - } - if strings.Contains(item.URL, "/.meta/") { - continue - } - return s.result(ruleName, []byte(item.Content), dsql, req) - } - for _, item := range svc.Repository.Files { - if strings.HasSuffix(item.URL, targetSuffix) { - return s.result(ruleName, []byte(item.Content), dsql, req) - } - } - return nil, fmt.Errorf("dql scan: generated YAML not found for %s", ruleName) + return &Scanner{} } -func (s *Scanner) result(ruleName string, routeYAML []byte, dql string, req *Request) (*Result, error) { +// Result builds a scan Result from route YAML bytes. Exported so that bridge +// packages (e.g. testutil/shapeparity) can call it after running the legacy +// translator pipeline externally. +func (s *Scanner) Result(ruleName string, routeYAML []byte, dql string, req *Request) (*Result, error) { if err := dqlplan.ValidateRelations(routeYAML); err != nil { return nil, fmt.Errorf("dql scan relation validation failed (%s): %w", ruleName, err) } @@ -421,11 +342,3 @@ func validateResolutionPolicy(resolution typectx.Resolution, policy provenancePo } return "" } - -func inferProject(dqlURL string) string { - base, _ := url.Split(dqlURL, file.Scheme) - if idx := strings.Index(base, "/dql/"); idx != -1 { - return filepath.Clean(base[:idx]) - } - return filepath.Clean(base) -} diff --git a/repository/shape/dql/scan/scanner_test.go b/repository/shape/dql/scan/scanner_test.go index 50e90999a..9374e9a7c 100644 --- a/repository/shape/dql/scan/scanner_test.go +++ b/repository/shape/dql/scan/scanner_test.go @@ -31,7 +31,7 @@ Resource: Template: Source: SELECT c.ID FROM T2 c `) - _, err := s.result("x", invalidYAML, "", nil) + _, err := s.Result("x", invalidYAML, "", nil) require.Error(t, err) require.Contains(t, err.Error(), "dql scan relation validation failed") require.Contains(t, err.Error(), "column=\"MISSING_COL\"") @@ -54,7 +54,7 @@ Resource: Template: Source: SELECT r.ID FROM ROOT r `) - result, err := s.result("sample", validYAML, "", nil) + result, err := s.Result("sample", validYAML, "", nil) require.NoError(t, err) require.NotNil(t, result) require.NotNil(t, result.Shape) @@ -83,7 +83,7 @@ Resource: #package('mdp/performance') #import('perf', 'github.com/acme/mdp/performance') SELECT r.ID FROM ROOT r` - result, err := s.result("sample", validYAML, dql, nil) + result, err := s.Result("sample", validYAML, dql, nil) require.NoError(t, err) require.NotNil(t, result) require.NotNil(t, result.Shape) @@ -117,7 +117,7 @@ Resource: dql := ` #package('github.com/acme/mdp/performance') SELECT cast(r.ID as 'Order') FROM ROOT r` - result, err := s.result("sample", validYAML, dql, nil) + result, err := s.Result("sample", validYAML, dql, nil) require.NoError(t, err) require.NotNil(t, result) require.NotNil(t, result.Shape) @@ -155,7 +155,7 @@ Resource: #package('github.com/acme/mdp/performance') SELECT cast(r.ID as 'Order') FROM ROOT r` strict := true - _, err := s.result("sample", validYAML, dql, &Request{ + _, err := s.Result("sample", validYAML, dql, &Request{ Repository: filepath.Clean(t.TempDir()), StrictProvenance: &strict, }) diff --git a/repository/shape/dql/shape/model.go b/repository/shape/dql/shape/model.go index e975cfca3..3f6bc3a6d 100644 --- a/repository/shape/dql/shape/model.go +++ b/repository/shape/dql/shape/model.go @@ -45,6 +45,7 @@ type Directives struct { Cache *CacheDirective MCP *MCPDirective Route *RouteDirective + Const map[string]string JSONMarshalType string JSONUnmarshalType string XMLUnmarshalType string diff --git a/repository/shape/load/loader.go b/repository/shape/load/loader.go index 03277feff..9f38cb8eb 100644 --- a/repository/shape/load/loader.go +++ b/repository/shape/load/loader.go @@ -82,6 +82,16 @@ func (l *Loader) materialize(planned *shape.PlanResult) (*plan.Result, *view.Res if err := shapevalidate.ValidateRelations(resource, resource.Views...); err != nil { return nil, nil, err } + if len(pResult.Const) > 0 { + for k, v := range pResult.Const { + constParam := &state.Parameter{ + Name: k, + In: state.NewConstLocation(k), + Value: v, + } + resource.AddParameters(constParam) + } + } // Gap 7: apply global cache TTL directive to root view. if pResult.Directives != nil && pResult.Directives.Cache != nil { if ttl := strings.TrimSpace(pResult.Directives.Cache.TTL); ttl != "" { @@ -286,7 +296,23 @@ func cloneDirectives(input *dqlshape.Directives) *dqlshape.Directives { DescriptionPath: strings.TrimSpace(input.MCP.DescriptionPath), } } - if ret.Meta == "" && ret.DefaultConnector == "" && ret.Cache == nil && ret.MCP == nil { + if input.Const != nil { + ret.Const = make(map[string]string, len(input.Const)) + for k, v := range input.Const { + ret.Const[k] = v + } + } + if input.Route != nil { + ret.Route = &dqlshape.RouteDirective{ + URI: strings.TrimSpace(input.Route.URI), + } + for _, m := range input.Route.Methods { + if m = strings.TrimSpace(m); m != "" { + ret.Route.Methods = append(ret.Route.Methods, m) + } + } + } + if ret.Meta == "" && ret.DefaultConnector == "" && ret.Cache == nil && ret.MCP == nil && ret.Route == nil && len(ret.Const) == 0 { return nil } return ret @@ -374,7 +400,7 @@ func materializeView(item *plan.View) (*view.View, error) { if item.Declaration != nil && strings.TrimSpace(item.Declaration.Tag) != "" { aView.Tag = strings.TrimSpace(item.Declaration.Tag) } - if strings.TrimSpace(item.SelectorNamespace) != "" || item.SelectorNoLimit != nil { + if strings.TrimSpace(item.SelectorNamespace) != "" || item.SelectorNoLimit != nil || item.SelectorLimit != nil { if aView.Selector == nil { aView.Selector = &view.Config{} } @@ -384,6 +410,16 @@ func materializeView(item *plan.View) (*view.View, error) { if item.SelectorNoLimit != nil { aView.Selector.NoLimit = *item.SelectorNoLimit } + if item.SelectorLimit != nil { + aView.Selector.Limit = *item.SelectorLimit + } + } + if item.Self != nil { + aView.SelfReference = &view.SelfReference{ + Holder: item.Self.Holder, + Child: item.Self.Child, + Parent: item.Self.Parent, + } } if aView.Schema != nil && strings.TrimSpace(item.SchemaType) != "" { if aView.Schema.DataType == "" { diff --git a/repository/shape/load/model.go b/repository/shape/load/model.go index a05f2287d..b94a00cea 100644 --- a/repository/shape/load/model.go +++ b/repository/shape/load/model.go @@ -1,11 +1,15 @@ package load import ( + "reflect" + "github.com/viant/datly/repository/shape" dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/repository/shape/typectx" "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/xreflect" ) // Component is a shape-loaded runtime-neutral component artifact. @@ -35,6 +39,55 @@ type Component struct { // ShapeSpecKind implements shape.ComponentSpec. func (c *Component) ShapeSpecKind() string { return "component" } +// InputParameters returns input states as state.Parameters for type generation. +func (c *Component) InputParameters() state.Parameters { + if c == nil { + return nil + } + var result state.Parameters + for _, s := range c.Input { + if s != nil { + p := s.Parameter + result = append(result, &p) + } + } + return result +} + +// OutputParameters returns output states as state.Parameters for type generation. +func (c *Component) OutputParameters() state.Parameters { + if c == nil { + return nil + } + var result state.Parameters + for _, s := range c.Output { + if s != nil { + p := s.Parameter + result = append(result, &p) + } + } + return result +} + +// InputReflectType builds the Input struct reflect.Type using state.Parameters.ReflectType. +// This produces the same struct shape as the legacy codegen (with parameter tags, Has markers, etc.). +func (c *Component) InputReflectType(pkgPath string, lookupType xreflect.LookupType, opts ...state.ReflectOption) (reflect.Type, error) { + params := c.InputParameters() + if len(params) == 0 { + return nil, nil + } + return params.ReflectType(pkgPath, lookupType, opts...) +} + +// OutputReflectType builds the Output struct reflect.Type using state.Parameters.ReflectType. +func (c *Component) OutputReflectType(pkgPath string, lookupType xreflect.LookupType, opts ...state.ReflectOption) (reflect.Type, error) { + params := c.OutputParameters() + if len(params) == 0 { + return nil, nil + } + return params.ReflectType(pkgPath, lookupType, opts...) +} + // ComponentFrom extracts the typed component from a ComponentArtifact. // Returns (nil, false) when a is nil or contains an unexpected concrete type. func ComponentFrom(a *shape.ComponentArtifact) (*Component, bool) { diff --git a/repository/shape/plan/model.go b/repository/shape/plan/model.go index ffb295f28..e537907a1 100644 --- a/repository/shape/plan/model.go +++ b/repository/shape/plan/model.go @@ -22,6 +22,7 @@ type Result struct { States []*State Types []*Type ColumnsDiscovery bool + Const map[string]string TypeContext *typectx.Context Directives *dqlshape.Directives Diagnostics []*dqlshape.Diagnostic @@ -66,9 +67,11 @@ type View struct { AllowNulls *bool SelectorNamespace string + SelectorLimit *int SelectorNoLimit *bool SchemaType string ColumnsDiscovery bool + Self *SelfReference Cardinality string ElementType reflect.Type @@ -130,6 +133,13 @@ type RelationLink struct { Expression string } +// SelfReference captures self-join tree metadata parsed from DQL. +type SelfReference struct { + Holder string + Child string + Parent string +} + // State is a normalized parameter field plan. type State struct { state.Parameter `yaml:",inline"` diff --git a/repository/shape/xgen/codegen.go b/repository/shape/xgen/codegen.go new file mode 100644 index 000000000..c6e4b522a --- /dev/null +++ b/repository/shape/xgen/codegen.go @@ -0,0 +1,448 @@ +package xgen + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/viant/datly/repository/shape/dql/shape" + shapeload "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/tagly/format/text" + "github.com/viant/xreflect" + "reflect" +) + +// ComponentCodegen generates Go source code for a complete component package: +// Input struct, Output struct, entity view structs, init() registration, +// //go:embed directive, and DefineComponent function. +// +// This is the shape pipeline equivalent of repository.Component.GenerateOutputCode. +type ComponentCodegen struct { + Component *shapeload.Component + Resource *view.Resource + TypeContext *typectx.Context + ProjectDir string + PackageDir string + PackageName string + PackagePath string + FileName string // defaults to .go + WithEmbed bool // generate //go:embed and EmbedFS method + WithContract bool // generate DefineComponent function + WithRegister *bool // generate init() with core.RegisterType (default: true) +} + +// ComponentCodegenResult captures generation outputs. +type ComponentCodegenResult struct { + FilePath string + PackagePath string + PackageName string + Types []string + Embeds map[string]string // SQL file name → SQL content +} + +// Generate produces the component Go source file. +func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { + if g.Component == nil { + return nil, fmt.Errorf("shape codegen: nil component") + } + if g.Resource == nil { + return nil, fmt.Errorf("shape codegen: nil resource") + } + + projectDir := g.ProjectDir + if projectDir == "" { + return nil, fmt.Errorf("shape codegen: project dir required") + } + + packageDir := g.PackageDir + if packageDir == "" && g.TypeContext != nil { + packageDir = g.TypeContext.PackageDir + } + if packageDir == "" { + return nil, fmt.Errorf("shape codegen: package dir required") + } + if !filepath.IsAbs(packageDir) { + packageDir = filepath.Join(projectDir, packageDir) + } + + packageName := g.PackageName + if packageName == "" && g.TypeContext != nil { + packageName = g.TypeContext.PackageName + } + if packageName == "" { + packageName = filepath.Base(packageDir) + } + + packagePath := g.PackagePath + if packagePath == "" && g.TypeContext != nil { + packagePath = g.TypeContext.PackagePath + } + + componentName := g.componentName() + embedURI := text.CaseFormatUpperCamel.Format(componentName, text.CaseFormatLowerUnderscore) + + fileName := g.FileName + if fileName == "" { + fileName = embedURI + ".go" + } + + // First generate view shapes via xgen (for entity structs like VendorView, ProductsView) + shapeDoc := resourceToCodegenDoc(g.Resource, g.TypeContext) + shapeCfg := &Config{ + ProjectDir: projectDir, + PackageDir: packageDir, + PackageName: packageName, + PackagePath: packagePath, + FileName: "shapes_gen.go", + } + shapeResult, _ := GenerateFromDQLShape(shapeDoc, shapeCfg) + + // Build Input/Output types using state.Parameters.ReflectType + lookupType := func(name string, opts ...xreflect.Option) (reflect.Type, error) { + return nil, fmt.Errorf("type %s not found", name) + } + + var inputType, outputType reflect.Type + if params := g.Component.InputParameters(); len(params) > 0 { + rt, err := params.ReflectType(packagePath, lookupType, state.WithSetMarker(), state.WithTypeName(componentName+"Input")) + if err == nil && rt != nil { + inputType = rt + } + } + + // Build output parameters — use explicit ones or synthesize defaults for readers + outputParams := g.Component.OutputParameters() + hasExplicitOutput := len(outputParams) > 0 + if !hasExplicitOutput { + outputParams = g.defaultOutputParameters(componentName) + } + // Resolve wildcard output types to the view entity type + g.resolveOutputWildcardTypes(outputParams, componentName) + if len(outputParams) > 0 { + rt, err := outputParams.ReflectType(packagePath, lookupType) + if err == nil && rt != nil { + outputType = rt + } + } + + // Build the Go source + var builder strings.Builder + builder.WriteString("package " + packageName + "\n\n") + + // Imports + imports := g.buildImports() + if len(imports) > 0 { + builder.WriteString("import (\n") + for _, imp := range imports { + if strings.Contains(imp, " ") { + // aliased import + builder.WriteString("\t" + imp + "\n") + } else { + builder.WriteString("\t\"" + imp + "\"\n") + } + } + builder.WriteString(")\n\n") + } + + // Code generated header + builder.WriteString("// Code generated by datly transcribe. DO NOT EDIT.\n\n") + + // init() registration + builder.WriteString("func init() {\n") + if g.withRegister() { + if inputType != nil { + builder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%sInput{}), checksum.GeneratedTime)\n", + packageName, componentName+"Input", componentName)) + } + builder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%sOutput{}), checksum.GeneratedTime)\n", + packageName, componentName+"Output", componentName)) + } + builder.WriteString("}\n\n") + + // //go:embed + if g.WithEmbed { + builder.WriteString(fmt.Sprintf("//go:embed %s/*.sql\n", embedURI)) + builder.WriteString(fmt.Sprintf("var %sFS embed.FS\n\n", componentName)) + } + + // Input struct + if inputType != nil { + builder.WriteString(fmt.Sprintf("type %sInput struct {\n", componentName)) + builder.WriteString(structFieldsSource(inputType)) + builder.WriteString("}\n\n") + } + + // Output struct — always render directly (view types may not be registered yet) + g.renderOutputStruct(&builder, componentName, embedURI, outputParams, outputType) + + // EmbedFS method + if g.WithEmbed && inputType != nil { + builder.WriteString(fmt.Sprintf("func (i *%sInput) EmbedFS() *embed.FS {\n", componentName)) + builder.WriteString(fmt.Sprintf("\treturn &%sFS\n", componentName)) + builder.WriteString("}\n\n") + } + + // Write file + if err := os.MkdirAll(packageDir, 0o755); err != nil { + return nil, err + } + dest := filepath.Join(packageDir, fileName) + if err := writeAtomic(dest, []byte(builder.String()), 0o644); err != nil { + return nil, err + } + + var typeNames []string + if inputType != nil { + typeNames = append(typeNames, componentName+"Input") + } + if outputType != nil { + typeNames = append(typeNames, componentName+"Output") + } + if shapeResult != nil { + typeNames = append(typeNames, shapeResult.Types...) + } + + return &ComponentCodegenResult{ + FilePath: dest, + PackagePath: packagePath, + PackageName: packageName, + Types: typeNames, + }, nil +} + +// renderOutputStruct writes the output struct definition. +// For reader components, it generates the standard pattern: +// +// type XxxOutput struct { +// response.Status `parameter:",kind=output,in=status" json:",omitempty"` +// Data []*XxxView `parameter:",kind=output,in=view" view:"xxx" sql:"uri=xxx/xxx.sql"` +// } +func (g *ComponentCodegen) renderOutputStruct(builder *strings.Builder, componentName, embedURI string, outputParams state.Parameters, outputType reflect.Type) { + rootView := g.Component.RootView + viewType := componentName + "View" + + builder.WriteString(fmt.Sprintf("type %sOutput struct {\n", componentName)) + + // Check if there's an explicit status parameter + hasStatus := false + for _, p := range outputParams { + if p != nil && p.In != nil && p.In.Name == "status" { + hasStatus = true + } + } + if !hasStatus { + builder.WriteString("\tresponse.Status `parameter:\",kind=output,in=status\" json:\",omitempty\"`\n") + } + + for _, p := range outputParams { + if p == nil || p.In == nil { + continue + } + switch p.In.Name { + case "view": + cardinality := string(p.Schema.Cardinality) + typePrefix := "[]*" + if cardinality == string(state.One) { + typePrefix = "*" + } + fieldName := p.Name + if fieldName == "" || fieldName == "Output" { + fieldName = "Data" + } + tag := fmt.Sprintf(`parameter:",kind=output,in=view" view:"%s" sql:"uri=%s/%s.sql"`, + rootView, embedURI, rootView) + if p.Tag != "" && strings.Contains(p.Tag, "anonymous") { + tag += ` anonymous:"true"` + } + builder.WriteString(fmt.Sprintf("\t%s %s%s `%s`\n", fieldName, typePrefix, viewType, tag)) + case "status": + builder.WriteString(fmt.Sprintf("\tresponse.Status `parameter:\",kind=output,in=status\" json:\",omitempty\"`\n")) + default: + // Other output parameters (meta, etc.) + typeName := "interface{}" + if p.Schema != nil && p.Schema.Name != "" { + typeName = p.Schema.Name + } + builder.WriteString(fmt.Sprintf("\t%s %s `parameter:\",kind=output,in=%s\"`\n", p.Name, typeName, p.In.Name)) + } + } + + builder.WriteString("}\n\n") +} + +// resolveOutputWildcardTypes resolves output parameters with wildcard type `?` or empty +// schema to the view entity type. The legacy translator does this in updateParameterWithComponentOutputType. +func (g *ComponentCodegen) resolveOutputWildcardTypes(params state.Parameters, componentName string) { + viewType := componentName + "View" + for _, p := range params { + if p == nil || p.In == nil { + continue + } + if p.In.Kind != state.KindOutput { + continue + } + if p.Schema == nil { + p.Schema = &state.Schema{} + } + // If schema type is wildcard or empty, resolve to the view type + if p.Schema.Name == "" || p.Schema.DataType == "" || p.Schema.DataType == "?" { + p.Schema.Name = viewType + p.Schema.DataType = "*" + viewType + if p.Schema.Cardinality == "" { + p.Schema.Cardinality = state.Many + } + } + // Add view tag if missing + if p.In.Name == "view" && !strings.Contains(p.Tag, "view:") { + rootView := g.Component.RootView + p.Tag += fmt.Sprintf(` view:"%s"`, rootView) + } + } +} + +// defaultOutputParameters creates the default output parameters for a reader component: +// - Data: the main view data (anonymous, kind=output, in=view) +// - Status: response status (anonymous, kind=output, in=status) +// This mirrors internal/translator output.go ensureOutputParameters. +func (g *ComponentCodegen) defaultOutputParameters(componentName string) state.Parameters { + rootView := g.Component.RootView + viewType := componentName + "View" + + // Data parameter — references the root view + dataParam := &state.Parameter{ + Name: "Data", + In: state.NewOutputLocation("view"), + Tag: fmt.Sprintf(`anonymous:"true" view:"%s"`, rootView), + Schema: &state.Schema{ + Name: viewType, + DataType: "*" + viewType, + Cardinality: state.Many, + }, + } + + // Status parameter — response.Status + statusParam := &state.Parameter{ + Name: "Status", + In: state.NewOutputLocation("status"), + Tag: `anonymous:"true" json:",omitempty"`, + Schema: &state.Schema{DataType: "response.Status"}, + } + + return state.Parameters{dataParam, statusParam} +} + +func (g *ComponentCodegen) withRegister() bool { + if g.WithRegister == nil { + return true // default enabled + } + return *g.WithRegister +} + +func (g *ComponentCodegen) componentName() string { + name := "" + if g.Component != nil { + name = g.Component.RootView + } + if name == "" && g.Resource != nil && len(g.Resource.Views) > 0 { + name = g.Resource.Views[0].Name + } + if name == "" { + name = "Component" + } + return state.SanitizeTypeName(name) +} + +func (g *ComponentCodegen) buildImports() []string { + var imports []string + if g.withRegister() { + imports = append(imports, + "reflect", + "github.com/viant/xdatly/types/core", + ) + checksumPkg := "github.com/viant/xdatly/types/custom/checksum" + if g.PackagePath != "" { + if idx := strings.LastIndex(g.PackagePath, "/pkg/"); idx != -1 { + candidate := g.PackagePath[:idx] + "/pkg/checksum" + parent, _ := filepath.Split(candidate) + if !strings.HasSuffix(parent, "dependency/") { + candidate = filepath.Join(parent, "dependency", "checksum") + } + checksumPkg = filepath.ToSlash(candidate) + } + } + imports = append(imports, checksumPkg) + } + imports = append(imports, "github.com/viant/xdatly/handler/response") + if g.WithEmbed { + imports = append(imports, "embed") + } + return imports +} + +func structFieldsSource(rType reflect.Type) string { + if rType == nil { + return "" + } + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return "" + } + var b strings.Builder + for i := 0; i < rType.NumField(); i++ { + f := rType.Field(i) + if !f.IsExported() { + continue + } + b.WriteString("\t" + f.Name + " " + f.Type.String()) + if f.Tag != "" { + b.WriteString(" `" + string(f.Tag) + "`") + } + b.WriteString("\n") + } + return b.String() +} + +func resourceToCodegenDoc(resource *view.Resource, typeCtx *typectx.Context) *shape.Document { + root := map[string]any{} + var views []any + for _, v := range resource.Views { + if v == nil { + continue + } + viewMap := map[string]any{ + "Name": v.Name, + "Table": v.Table, + "Mode": string(v.Mode), + } + if v.Schema != nil { + schema := map[string]any{} + if v.Schema.Name != "" { + schema["Name"] = v.Schema.Name + } + viewMap["Schema"] = schema + } + if len(v.Columns) > 0 { + var cols []any + for _, c := range v.Columns { + if c == nil { + continue + } + cols = append(cols, map[string]any{ + "Name": c.Name, + "DataType": c.DataType, + "Nullable": c.Nullable, + }) + } + viewMap["Columns"] = cols + } + views = append(views, viewMap) + } + root["Resource"] = map[string]any{"Views": views} + return &shape.Document{Root: root, TypeContext: typeCtx} +} diff --git a/repository/shape/xgen/generator.go b/repository/shape/xgen/generator.go index d0fc419a8..bfeb2bd33 100644 --- a/repository/shape/xgen/generator.go +++ b/repository/shape/xgen/generator.go @@ -16,6 +16,7 @@ import ( xreflectloader "github.com/viant/x/loader/xreflect" "github.com/viant/x/syntetic" "github.com/viant/x/syntetic/model" + "github.com/viant/xunsafe" ) // GenerateFromDQLShape emits Go structs from DQL shape using viant/x registry. @@ -57,19 +58,42 @@ func GenerateFromDQLShape(doc *shape.Document, cfg *Config) (*Result, error) { if registered[typeName] { continue } + structType := buildStructType(view.columns) + if structType == nil { + continue + } registered[typeName] = true - if err = registerShapeType(registry, packagePath, typeName, buildStructType(view.columns)); err != nil { + if err = registerShapeType(registry, packagePath, typeName, structType); err != nil { return nil, err } typeNames = append(typeNames, typeName) + + // Generate Has marker struct for mutable views + if view.mutable && len(view.columns) > 0 { + hasTypeName := typeName + "Has" + if !registered[hasTypeName] { + hasType := buildHasType(view.columns) + if hasType != nil { + registered[hasTypeName] = true + if err = registerShapeType(registry, packagePath, hasTypeName, hasType); err != nil { + return nil, err + } + typeNames = append(typeNames, hasTypeName) + } + } + } } for _, ioType := range routeTypes { typeName := routeTypeName(cfg, ioType) if typeName == "" || registered[typeName] { continue } + structType := buildStructType(ioType.fields) + if structType == nil { + continue + } registered[typeName] = true - if err = registerShapeType(registry, packagePath, typeName, buildStructType(ioType.fields)); err != nil { + if err = registerShapeType(registry, packagePath, typeName, structType); err != nil { return nil, err } typeNames = append(typeNames, typeName) @@ -88,7 +112,9 @@ func GenerateFromDQLShape(doc *shape.Document, cfg *Config) (*Result, error) { if goFile == nil { return nil, fmt.Errorf("shape xgen: missing generated package file for %s", packagePath) } - source, err := goFile.Render() + source, err := goFile.RenderWithOptions(model.RenderOptions{ + Header: "// Code generated by datly transcribe. DO NOT EDIT.", + }) if err != nil { return nil, err } @@ -285,7 +311,50 @@ func uniqueStrings(items []string) []string { return result } +// registerShapeType registers a type in the x.Registry. If a type with the same +// name already exists (linked-in binary type via xunsafe, or previously registered +// in the registry), it preserves the existing field order and appends new fields. func registerShapeType(registry *x.Registry, packagePath string, typeName string, rType reflect.Type) error { + existingType := lookupExistingType(registry, packagePath, typeName) + if existingType != nil && existingType.Kind() == reflect.Struct && rType.Kind() == reflect.Struct { + var newFields []reflect.StructField + for i := 0; i < rType.NumField(); i++ { + newFields = append(newFields, rType.Field(i)) + } + rType = reflect.StructOf(mergeFieldOrder(existingType, newFields)) + } + return registerShapeTypeRaw(registry, packagePath, typeName, rType) +} + +// lookupExistingType checks for a previously known type: +// 1. First checks linked-in types via xunsafe.LookupType (compiled binary types) +// 2. Then checks the x.Registry (previously registered synthetic types) +// Returns the unwrapped struct reflect.Type, or nil if not found. +func lookupExistingType(registry *x.Registry, packagePath, typeName string) reflect.Type { + // xunsafe indexes by "pkgPath/TypeName" + if linked := xunsafe.LookupType(packagePath + "/" + typeName); linked != nil { + for linked.Kind() == reflect.Ptr || linked.Kind() == reflect.Slice { + linked = linked.Elem() + } + if linked.Kind() == reflect.Struct { + return linked + } + } + // Check registry + key := packagePath + "." + typeName + if existing := registry.Lookup(key); existing != nil && existing.Type != nil { + t := existing.Type + for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice { + t = t.Elem() + } + if t.Kind() == reflect.Struct { + return t + } + } + return nil +} + +func registerShapeTypeRaw(registry *x.Registry, packagePath string, typeName string, rType reflect.Type) error { st, err := xreflectloader.BuildType(rType, xreflectloader.WithPackagePath(packagePath), xreflectloader.WithNamePolicy(func(reflect.Type) (string, bool) { @@ -310,6 +379,7 @@ type viewDescriptor struct { name any schemaName any columns []columnDescriptor + mutable bool } type ioTypeKind string @@ -329,8 +399,11 @@ type routeIODescriptor struct { } type columnDescriptor struct { - name string - dataType string + name string + dataType string + primaryKey bool + autoIncrement bool + nullable bool } func extractViews(root map[string]any) []viewDescriptor { @@ -353,6 +426,10 @@ func extractViews(root map[string]any) []viewDescriptor { if schema != nil { descriptor.schemaName = schema["Name"] } + mode := strings.ToLower(asString(view["Mode"])) + if mode == "sqlexec" || mode == "exec" || mode == "handler" { + descriptor.mutable = true + } descriptor.columns = extractColumns(view) result = append(result, descriptor) } @@ -371,7 +448,29 @@ func extractColumns(view map[string]any) []columnDescriptor { if name == "" { continue } - result = append(result, columnDescriptor{name: name, dataType: asString(column["DataType"])}) + col := columnDescriptor{ + name: name, + dataType: asString(column["DataType"]), + } + // Check for primary key / autoincrement from column tag or metadata + tag := strings.ToLower(asString(column["Tag"])) + if strings.Contains(tag, "primarykey") || strings.Contains(tag, "primary_key") { + col.primaryKey = true + } + if strings.Contains(tag, "autoincrement") || strings.Contains(tag, "auto_increment") { + col.autoIncrement = true + } + // Heuristic: column named ID or ending in _ID at position 0 is likely PK + if strings.EqualFold(name, "ID") && len(result) == 0 { + col.primaryKey = true + if strings.EqualFold(col.dataType, "int") || strings.EqualFold(col.dataType, "integer") || strings.EqualFold(col.dataType, "int64") { + col.autoIncrement = true + } + } + if asBool(column["Nullable"]) { + col.nullable = true + } + result = append(result, col) } } if cfg := asMap(view["ColumnsConfig"]); len(cfg) > 0 { @@ -386,12 +485,16 @@ func extractColumns(view map[string]any) []columnDescriptor { item = map[string]any{} } name := firstNonEmpty(asString(item["Name"]), key) - result = append(result, columnDescriptor{name: name, dataType: asString(item["DataType"])}) + col := columnDescriptor{name: name, dataType: asString(item["DataType"])} + if strings.EqualFold(name, "ID") && len(result) == 0 { + col.primaryKey = true + if strings.EqualFold(col.dataType, "int") || col.dataType == "" { + col.autoIncrement = true + } + } + result = append(result, col) } } - if len(result) == 0 { - result = append(result, columnDescriptor{name: "ID", dataType: "int"}) - } return result } @@ -458,15 +561,119 @@ func extractIOFields(io map[string]any) []columnDescriptor { } fields = append(fields, columnDescriptor{name: name, dataType: dataType}) } - if len(fields) == 0 { - fields = append(fields, columnDescriptor{name: "ID", dataType: "int"}) - } return fields } +// mergeFieldOrder preserves existing field positions from a previously registered type. +// Existing fields keep their index; new fields are appended; removed fields are kept. +func mergeFieldOrder(existing reflect.Type, newFields []reflect.StructField) []reflect.StructField { + if existing == nil || existing.Kind() != reflect.Struct { + return newFields + } + // Index new fields by name + newByName := map[string]reflect.StructField{} + for _, f := range newFields { + newByName[f.Name] = f + } + // Start with existing fields in order, updating type/tag if regenerated + var merged []reflect.StructField + seen := map[string]bool{} + for i := 0; i < existing.NumField(); i++ { + ef := existing.Field(i) + seen[ef.Name] = true + if nf, ok := newByName[ef.Name]; ok { + // Keep position. If existing field has an explicit type override + // (from DQL cast like CAST(col AS MyType)), preserve the existing + // type — the user's intent takes precedence over DB discovery. + if hasExplicitTypeOverride(ef) { + nf.Type = ef.Type + } + merged = append(merged, nf) + } else { + // Column removed from DB — keep field for stability + merged = append(merged, ef) + } + } + // Append genuinely new fields + for _, nf := range newFields { + if !seen[nf.Name] { + merged = append(merged, nf) + } + } + return merged +} + +// hasExplicitTypeOverride returns true if the field's type was explicitly set +// by a DQL cast (e.g., CAST(col AS MyType)) rather than inferred from DB discovery. +// Detected via typeName tag or non-primitive type that isn't a standard DB mapping. +func hasExplicitTypeOverride(field reflect.StructField) bool { + tag := field.Tag + // typeName tag indicates an explicit type name was set + if v := tag.Get("typeName"); v != "" { + return true + } + // Check if the type is a named type (not a primitive or pointer-to-primitive) + ft := field.Type + for ft.Kind() == reflect.Ptr || ft.Kind() == reflect.Slice { + ft = ft.Elem() + } + if ft.PkgPath() != "" && ft.Kind() == reflect.Struct { + // Named struct from a package = explicit type (e.g., jwt.Claims, time.Time) + name := ft.Name() + if name != "" && name != "Time" { // time.Time is a standard DB mapping + return true + } + } + return false +} + func buildStructType(columns []columnDescriptor) reflect.Type { if len(columns) == 0 { - columns = []columnDescriptor{{name: "ID", dataType: "int"}} + return nil + } + fields := make([]reflect.StructField, 0, len(columns)) + used := map[string]int{} + for _, column := range columns { + fieldName := exportedName(column.name) + if fieldName == "" { + fieldName = "Field" + } + if count := used[fieldName]; count > 0 { + fieldName = fmt.Sprintf("%s%d", fieldName, count+1) + } + used[fieldName]++ + fieldType := parseType(column.dataType) + sqlxTag := column.name + isPK := column.primaryKey + isAutoInc := column.autoIncrement + if isPK { + sqlxTag += ",primaryKey" + } + if isAutoInc { + sqlxTag += ",autoincrement" + } + // Use pointer types for nullable (non-PK) columns to match legacy codegen + if !isPK && !isAutoInc && fieldType.Kind() != reflect.Slice && fieldType.Kind() != reflect.Ptr { + fieldType = reflect.PointerTo(fieldType) + } + tag := fmt.Sprintf(`sqlx:"%s" json:",omitempty"`, sqlxTag) + if isPK || isAutoInc { + tag = fmt.Sprintf(`sqlx:"%s"`, sqlxTag) + } + fields = append(fields, reflect.StructField{ + Name: fieldName, + Type: fieldType, + Tag: reflect.StructTag(tag), + }) + } + return reflect.StructOf(fields) +} + +// buildHasType creates a marker struct with bool fields for each column. +// Used by mutable views to track which fields were explicitly set. +func buildHasType(columns []columnDescriptor) reflect.Type { + if len(columns) == 0 { + return nil } fields := make([]reflect.StructField, 0, len(columns)) used := map[string]int{} @@ -481,8 +688,7 @@ func buildStructType(columns []columnDescriptor) reflect.Type { used[fieldName]++ fields = append(fields, reflect.StructField{ Name: fieldName, - Type: parseType(column.dataType), - Tag: reflect.StructTag(fmt.Sprintf(`json:"%s,omitempty" sqlx:"%s"`, strings.ToLower(fieldName), column.name)), + Type: reflect.TypeOf(true), }) } return reflect.StructOf(fields) @@ -673,6 +879,16 @@ func asSlice(raw any) []any { return nil } +func asBool(raw any) bool { + if raw == nil { + return false + } + if v, ok := raw.(bool); ok { + return v + } + return false +} + func asString(raw any) string { if raw == nil { return "" diff --git a/repository/shape/xgen/io.go b/repository/shape/xgen/io.go index 50e86ddaa..3d2ff7b3e 100644 --- a/repository/shape/xgen/io.go +++ b/repository/shape/xgen/io.go @@ -223,6 +223,13 @@ func mergeGeneratedShapes(dest string, generated []byte, typeNames []string) ([] return out.Bytes(), nil } +// TODO: Field order preservation should be done at the viant/x registry level: +// 1. Check linked-in types first (runtime reflect.Type from registered types) +// 2. Fall back to viant/x/loader/ast.LoadPackageFS to load existing .go file +// 3. Extract field order from loaded types +// 4. When building new types, preserve existing field order and append new fields +// This avoids raw AST manipulation and handles complex type graphs (nested structs, relations). + func generatedShapeDecls(file *ast.File, typeNameSet map[string]bool) []ast.Decl { var result []ast.Decl for _, decl := range file.Decls { diff --git a/repository/shape/xgen/resource.go b/repository/shape/xgen/resource.go new file mode 100644 index 000000000..760d17a86 --- /dev/null +++ b/repository/shape/xgen/resource.go @@ -0,0 +1,95 @@ +package xgen + +import ( + "fmt" + "strings" + + "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" +) + +// GenerateFromResource produces Go structs directly from an in-memory view.Resource +// without YAML roundtrip. Uses real columns from DB discovery when available. +func GenerateFromResource(resource *view.Resource, typeCtx *typectx.Context, cfg *Config) (*Result, error) { + if resource == nil { + return nil, fmt.Errorf("shape xgen: nil resource") + } + doc := resourceToShapeDocument(resource, typeCtx) + return GenerateFromDQLShape(doc, cfg) +} + +// resourceToShapeDocument converts an in-memory view.Resource into a shape.Document +// that xgen can process. This avoids the YAML marshal/unmarshal roundtrip. +func resourceToShapeDocument(resource *view.Resource, typeCtx *typectx.Context) *shape.Document { + root := map[string]any{} + + // Build Resource.Views from in-memory views + var views []any + for _, aView := range resource.Views { + if aView == nil { + continue + } + viewMap := map[string]any{ + "Name": aView.Name, + "Table": aView.Table, + "Mode": string(aView.Mode), + } + if aView.Module != "" { + viewMap["Module"] = aView.Module + } + // Schema + if aView.Schema != nil { + schema := map[string]any{} + if aView.Schema.Name != "" { + schema["Name"] = aView.Schema.Name + } + if aView.Schema.DataType != "" { + schema["DataType"] = aView.Schema.DataType + } + if aView.Schema.Cardinality != "" { + schema["Cardinality"] = string(aView.Schema.Cardinality) + } + viewMap["Schema"] = schema + } + // Columns — this is the key: real columns from DB discovery + if len(aView.Columns) > 0 { + var columns []any + for _, col := range aView.Columns { + if col == nil { + continue + } + colMap := map[string]any{ + "Name": col.Name, + "DataType": col.DataType, + } + if col.Tag != "" { + colMap["Tag"] = col.Tag + } + if col.Nullable { + colMap["Nullable"] = true + } + columns = append(columns, colMap) + } + viewMap["Columns"] = columns + } + views = append(views, viewMap) + } + root["Resource"] = map[string]any{"Views": views} + return &shape.Document{ + Root: root, + TypeContext: typeCtx, + } +} + +// columnDataType returns Go type name for a view.Column. +func columnDataType(col *view.Column) string { + if col.DataType != "" { + return col.DataType + } + rType := col.ColumnType() + if rType == nil { + return "string" + } + return strings.TrimPrefix(rType.String(), "*") +} diff --git a/testutil/shapeparity/bridge.go b/testutil/shapeparity/bridge.go new file mode 100644 index 000000000..49e13d55e --- /dev/null +++ b/testutil/shapeparity/bridge.go @@ -0,0 +1,93 @@ +package shapeparity + +import ( + "context" + "fmt" + "strings" + + "github.com/viant/afs" + "github.com/viant/afs/file" + "github.com/viant/afs/url" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/internal/translator" + "github.com/viant/datly/repository/shape/dql/sanitize" + dqlscan "github.com/viant/datly/repository/shape/dql/scan" +) + +// ScanDQL translates a DQL file through the legacy internal/translator pipeline +// and returns a scan.Result. This bridges internal/translator for parity tests +// without requiring repository/shape to depend on internal/*. +func ScanDQL(ctx context.Context, req *dqlscan.Request) (*dqlscan.Result, error) { + if req == nil || req.DQLURL == "" { + return nil, fmt.Errorf("dql scan: DQLURL was empty") + } + fs := afs.New() + sourceURL := req.DQLURL + project := inferProject(req.DQLURL) + translate := &options.Translate{} + translate.Rule.Project = project + translate.Rule.Source = []string{sourceURL} + translate.Rule.ModulePrefix = req.ModulePrefix + translate.Repository.RepositoryURL = req.Repository + translate.Repository.APIPrefix = req.APIPrefix + if len(req.Connectors) > 0 { + translate.Repository.Connectors = append(translate.Repository.Connectors, req.Connectors...) + } + if req.ConfigURL != "" { + translate.Repository.Configs.Append(req.ConfigURL) + } + if initErr := translate.Init(ctx); initErr != nil { + return nil, initErr + } + if req.ConfigURL == "" { + translate.Repository.Configs = nil + } + if translate.Rule.ModulePrefix == "" { + translate.Rule.ModulePrefix = "platform" + } + + svc := translator.New(translator.NewConfig(&translate.Repository), fs) + if initErr := svc.Init(ctx); initErr != nil { + return nil, initErr + } + if initErr := svc.InitSignature(ctx, &translate.Rule); initErr != nil { + return nil, initErr + } + dsql, loadErr := translate.Rule.LoadSource(ctx, fs, translate.Rule.SourceURL()) + if loadErr != nil { + return nil, loadErr + } + translate.Rule.NormalizeComponent(&dsql) + dsql = sanitize.SQL(dsql, sanitize.Options{Declared: sanitize.Declared(dsql)}) + top := &options.Options{Translate: translate} + if initErr := svc.Translate(ctx, &translate.Rule, dsql, top); initErr != nil { + return nil, initErr + } + ruleName := svc.Repository.RuleName(&translate.Rule) + targetSuffix := "/" + ruleName + ".yaml" + + scanner := dqlscan.New() + for _, item := range svc.Repository.Files { + if !strings.HasSuffix(item.URL, targetSuffix) { + continue + } + if strings.Contains(item.URL, "/.meta/") { + continue + } + return scanner.Result(ruleName, []byte(item.Content), dsql, req) + } + for _, item := range svc.Repository.Files { + if strings.HasSuffix(item.URL, targetSuffix) { + return scanner.Result(ruleName, []byte(item.Content), dsql, req) + } + } + return nil, fmt.Errorf("dql scan: generated YAML not found for %s", ruleName) +} + +func inferProject(dqlURL string) string { + base, _ := url.Split(dqlURL, file.Scheme) + if idx := strings.Index(base, "/dql/"); idx != -1 { + return base[:idx] + } + return base +} diff --git a/view/state/type.go b/view/state/type.go index 262a83959..b9f3ee9af 100644 --- a/view/state/type.go +++ b/view/state/type.go @@ -23,6 +23,7 @@ type ( Type struct { *Schema Parameters Parameters `json:",omitempty" yaml:"Parameters"` + Package string `json:",omitempty" yaml:",omitempty"` withMarker bool stateType *structology.StateType resource Resource @@ -135,6 +136,15 @@ func (t *Type) SetType(rType reflect.Type) { t.stateType = structology.NewStateType(rType) } +// PkgPath returns the effective package path for type generation. +// Uses the explicit Package field when set, otherwise falls back to the default. +func (t *Type) PkgPath() string { + if p := strings.TrimSpace(t.Package); p != "" { + return p + } + return pkgPath +} + func (t *Type) buildSchema(ctx context.Context, withMarker bool) (err error) { hasBodyParam := false for _, parameter := range t.Parameters { @@ -148,9 +158,10 @@ func (t *Type) buildSchema(ctx context.Context, withMarker bool) (err error) { if t.withBodyType && !hasBodyParam { t.withBodyType = hasBodyParam } + effectivePkgPath := t.PkgPath() var rType reflect.Type if t.withBodyType { - rType, err = t.Parameters.BuildBodyType(pkgPath, t.resource.LookupType()) + rType, err = t.Parameters.BuildBodyType(effectivePkgPath, t.resource.LookupType()) } else { var opts []ReflectOption if withMarker { @@ -159,7 +170,7 @@ func (t *Type) buildSchema(ctx context.Context, withMarker bool) (err error) { if t.Schema != nil && t.Schema.Name != "" { opts = append(opts, WithTypeName(t.Name)) } - rType, err = t.Parameters.ReflectType(pkgPath, t.resource.LookupType(), opts...) + rType, err = t.Parameters.ReflectType(effectivePkgPath, t.resource.LookupType(), opts...) } if err != nil { return err From d4f619200f17369e10bd16acb5901fcef08b9221 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 7 Mar 2026 04:48:38 -0800 Subject: [PATCH 152/279] enhanced shape --- repository/shape/column/detector.go | 369 ++- repository/shape/column/detector_test.go | 87 + repository/shape/column/strip_test.go | 30 +- repository/shape/compile/compiler.go | 29 +- repository/shape/compile/compiler_test.go | 68 +- repository/shape/compile/component_types.go | 532 +++- .../shape/compile/component_types_test.go | 29 + repository/shape/compile/enrich.go | 14 + repository/shape/compile/enrich_test.go | 18 + repository/shape/compile/hints.go | 317 +- repository/shape/compile/hints_strip.go | 80 + repository/shape/compile/hints_test.go | 229 +- repository/shape/compile/pipeline/infer.go | 44 +- .../shape/compile/pipeline/infer_test.go | 21 + repository/shape/compile/pipeline/parse.go | 15 +- .../shape/compile/pipeline/parse_test.go | 42 +- repository/shape/compile/pipeline/read.go | 404 ++- .../shape/compile/pipeline/read_normalize.go | 35 + .../shape/compile/pipeline/read_test.go | 99 + repository/shape/compile/pipeline/relation.go | 26 + .../shape/compile/pipeline/relation_test.go | 53 + repository/shape/compile/statedecl.go | 269 +- repository/shape/compile/statedecl_test.go | 124 +- repository/shape/compile/type_support.go | 345 ++- repository/shape/compile/typectx_defaults.go | 71 + .../shape/compile/typectx_defaults_test.go | 11 + repository/shape/compile/viewdecl.go | 80 +- repository/shape/compile/viewdecl_append.go | 90 +- repository/shape/compile/viewdecl_options.go | 150 +- repository/shape/compile/viewdecl_parse.go | 27 +- repository/shape/compile/viewdecl_test.go | 146 +- repository/shape/componenttag/component.go | 99 + repository/shape/dql/decl/calls.go | 97 + repository/shape/dql/decl/calls_test.go | 52 + repository/shape/dql/decl/lex.go | 25 + repository/shape/dql/diag/codes.go | 6 + .../shape/dql/preprocess/directive_parser.go | 65 +- repository/shape/dql/preprocess/extract.go | 42 +- repository/shape/dql/preprocess/preprocess.go | 40 +- .../shape/dql/preprocess/preprocess_test.go | 108 + repository/shape/dql/preprocess/scanner.go | 21 +- .../dql/preprocess/settings_directives.go | 343 ++- repository/shape/dql/sanitize/context_test.go | 118 + repository/shape/dql/sanitize/policy.go | 65 +- repository/shape/dql/sanitize/policy_test.go | 38 +- repository/shape/dql/sanitize/sanitizer.go | 282 +- .../shape/dql/sanitize/sanitizer_test.go | 108 +- repository/shape/dql/shape/model.go | 14 +- repository/shape/dql_engine_test.go | 20 + repository/shape/gorouter/discover.go | 671 +++++ repository/shape/gorouter/discover_test.go | 266 ++ repository/shape/gorouter/model.go | 22 + repository/shape/load/columns.go | 49 + repository/shape/load/columns_test.go | 24 + repository/shape/load/loader.go | 1392 ++++++++- .../shape/load/loader_contract_state_test.go | 23 + repository/shape/load/loader_test.go | 1555 +++++++++- repository/shape/load/model.go | 20 + repository/shape/load/model_test.go | 94 + repository/shape/model.go | 22 + repository/shape/options.go | 10 + repository/shape/plan/model.go | 48 +- repository/shape/plan/planner.go | 120 + repository/shape/plan/planner_test.go | 258 ++ repository/shape/scan/component_contract.go | 229 ++ repository/shape/scan/model.go | 42 +- repository/shape/scan/scanner.go | 225 +- repository/shape/scan/scanner_test.go | 197 +- repository/shape/shape.go | 18 +- repository/shape/validate/relation.go | 5 + repository/shape/velty/ast/assign.go | 110 + repository/shape/velty/ast/ast.go | 163 + repository/shape/velty/ast/ast_test.go | 124 + repository/shape/velty/ast/binary.go | 22 + repository/shape/velty/ast/builder.go | 52 + repository/shape/velty/ast/condition.go | 142 + repository/shape/velty/ast/dml.go | 97 + repository/shape/velty/ast/errcheck.go | 29 + repository/shape/velty/ast/expression.go | 95 + repository/shape/velty/ast/foreach.go | 86 + repository/shape/velty/ast/func.go | 162 + repository/shape/velty/ast/literal.go | 21 + repository/shape/velty/ast/options.go | 12 + repository/shape/velty/ast/scope.go | 63 + repository/shape/velty/ast/star.go | 51 + repository/shape/xgen/codegen.go | 2657 ++++++++++++++++- .../xgen/codegen_contract_parity_test.go | 134 + repository/shape/xgen/codegen_imports_test.go | 20 + .../shape/xgen/codegen_input_view_test.go | 768 +++++ .../shape/xgen/codegen_mutable_body_test.go | 189 ++ .../xgen/codegen_mutable_helpers_test.go | 507 ++++ .../shape/xgen/codegen_output_view_test.go | 79 + .../xgen/codegen_placeholder_view_test.go | 83 + .../shape/xgen/codegen_relation_view_test.go | 101 + .../shape/xgen/codegen_typespec_test.go | 87 + repository/shape/xgen/generator.go | 54 +- .../shape/xgen/generator_velty_tag_test.go | 48 + repository/shape/xgen/mutable_body.go | 927 ++++++ repository/shape/xgen/mutable_helpers.go | 394 +++ .../xgen/repro_xgen_shapefragment_test.go | 35 + 100 files changed, 17342 insertions(+), 657 deletions(-) create mode 100644 repository/shape/compile/hints_strip.go create mode 100644 repository/shape/componenttag/component.go create mode 100644 repository/shape/dql/decl/calls.go create mode 100644 repository/shape/dql/decl/calls_test.go create mode 100644 repository/shape/dql/sanitize/context_test.go create mode 100644 repository/shape/gorouter/discover.go create mode 100644 repository/shape/gorouter/discover_test.go create mode 100644 repository/shape/gorouter/model.go create mode 100644 repository/shape/load/columns_test.go create mode 100644 repository/shape/load/loader_contract_state_test.go create mode 100644 repository/shape/load/model_test.go create mode 100644 repository/shape/scan/component_contract.go create mode 100644 repository/shape/velty/ast/assign.go create mode 100644 repository/shape/velty/ast/ast.go create mode 100644 repository/shape/velty/ast/ast_test.go create mode 100644 repository/shape/velty/ast/binary.go create mode 100644 repository/shape/velty/ast/builder.go create mode 100644 repository/shape/velty/ast/condition.go create mode 100644 repository/shape/velty/ast/dml.go create mode 100644 repository/shape/velty/ast/errcheck.go create mode 100644 repository/shape/velty/ast/expression.go create mode 100644 repository/shape/velty/ast/foreach.go create mode 100644 repository/shape/velty/ast/func.go create mode 100644 repository/shape/velty/ast/literal.go create mode 100644 repository/shape/velty/ast/options.go create mode 100644 repository/shape/velty/ast/scope.go create mode 100644 repository/shape/velty/ast/star.go create mode 100644 repository/shape/xgen/codegen_contract_parity_test.go create mode 100644 repository/shape/xgen/codegen_imports_test.go create mode 100644 repository/shape/xgen/codegen_input_view_test.go create mode 100644 repository/shape/xgen/codegen_mutable_body_test.go create mode 100644 repository/shape/xgen/codegen_mutable_helpers_test.go create mode 100644 repository/shape/xgen/codegen_output_view_test.go create mode 100644 repository/shape/xgen/codegen_placeholder_view_test.go create mode 100644 repository/shape/xgen/codegen_relation_view_test.go create mode 100644 repository/shape/xgen/codegen_typespec_test.go create mode 100644 repository/shape/xgen/generator_velty_tag_test.go create mode 100644 repository/shape/xgen/mutable_body.go create mode 100644 repository/shape/xgen/mutable_helpers.go create mode 100644 repository/shape/xgen/repro_xgen_shapefragment_test.go diff --git a/repository/shape/column/detector.go b/repository/shape/column/detector.go index 893b5aaa7..72ea292f3 100644 --- a/repository/shape/column/detector.go +++ b/repository/shape/column/detector.go @@ -8,6 +8,7 @@ import ( "github.com/viant/datly/view" viewcolumn "github.com/viant/datly/view/column" + "github.com/viant/datly/view/state" "github.com/viant/sqlparser" "github.com/viant/sqlx/io" ) @@ -35,6 +36,12 @@ func (d *Detector) Resolve(ctx context.Context, resource *view.Resource, aView * if allPlaceholderColumns(aView.Columns) { base = nil } + if explicit := explicitProjectedSubqueryColumns(aView); len(explicit) > 0 { + if len(base) == 0 { + return explicit, nil + } + return mergePreservingOrder(base, explicit), nil + } if !needsDiscovery(aView) && len(base) > 0 { return base, nil } @@ -58,10 +65,11 @@ func (d *Detector) detect(ctx context.Context, resource *view.Resource, aView *v if err != nil { return nil, fmt.Errorf("shape column detector: failed to open db for view %s: %w", aView.Name, err) } - query := discoverySQL(aView) - sqlColumns, err := viewcolumn.Discover(ctx, db, aView.Table, query) + query := discoverySQL(aView, resource) + table := resolveDiscoveryTable(aView, resource, sourceSQL(aView)) + sqlColumns, err := viewcolumn.Discover(ctx, db, table, query) if err != nil { - return nil, fmt.Errorf("shape column detector: discover failed for view %s: %w", aView.Name, err) + return nil, fmt.Errorf("shape column detector: discover failed for view %s (query=%q, table=%q): %w", aView.Name, query, table, err) } return view.NewColumns(sqlColumns, aView.ColumnsConfig), nil } @@ -72,19 +80,30 @@ func (d *Detector) detect(ctx context.Context, resource *view.Resource, aView *v // 2. Inject 1=0 into every SELECT in the query (CTEs, UNIONs, subqueries) // This ensures zero rows scanned — safe for BigQuery (no full scan cost) // 3. Fall back to table name if parsing/falsification fails -func discoverySQL(aView *view.View) string { +func discoverySQL(aView *view.View, resource *view.Resource) string { raw := sourceSQL(aView) - table := strings.TrimSpace(aView.Table) + if expanded := applyConstValuesForDiscovery(raw, resource); strings.TrimSpace(expanded) != "" { + raw = expanded + } + table := resolveDiscoveryTable(aView, resource, raw) if raw == "" { return table } - // If SQL has template variables, EXCEPT, or other datly extensions, - // use table-based discovery which is always safe and accurate - if table != "" && (hasTemplateVariables(raw) || hasExceptClause(raw)) { + // EXCEPT clause is a datly projection extension; table fallback is safest. + if table != "" && hasExceptClause(raw) { + return table + } + // Template SQL with wildcard fallback to table metadata. For explicit projection + // we still derive columns from SQL (after template stripping) to avoid widening + // contract to the whole table. + if table != "" && hasTemplateVariables(raw) && usesWildcard(aView) { return table } // For clean SQL without templates, try to falsify for column type inference cleaned := strings.TrimSpace(raw) + if hasTemplateVariables(cleaned) { + cleaned = strings.TrimSpace(stripTemplateVariables(cleaned)) + } if cleaned == "" || !strings.Contains(strings.ToLower(cleaned), "select") { if table != "" { return table @@ -101,6 +120,305 @@ func discoverySQL(aView *view.View) string { return cleaned } +func explicitProjectedSubqueryColumns(aView *view.View) view.Columns { + if aView == nil || !usesWildcard(aView) { + return nil + } + sql := strings.TrimSpace(sourceSQL(aView)) + if sql == "" { + return nil + } + queryNode, err := sqlparser.ParseQuery(sql) + if err != nil || queryNode == nil || !queryNode.List.IsStarExpr() || queryNode.From.X == nil { + return nil + } + fromExpr := strings.TrimSpace(sqlparser.Stringify(queryNode.From.X)) + if fromExpr == "" { + return nil + } + fromExpr = strings.TrimSpace(strings.TrimPrefix(fromExpr, "(")) + fromExpr = strings.TrimSpace(strings.TrimSuffix(fromExpr, ")")) + if !strings.Contains(strings.ToLower(fromExpr), "select") { + return nil + } + innerQuery, err := sqlparser.ParseQuery(fromExpr) + if err != nil || innerQuery == nil { + return nil + } + columns := sqlparser.NewColumns(innerQuery.List) + if len(columns) == 0 || columns.IsStarExpr() { + return nil + } + normalizeExplicitProjectedColumnTypes(columns) + return view.NewColumns(columns, aView.ColumnsConfig) +} + +func normalizeExplicitProjectedColumnTypes(columns sqlparser.Columns) { + for _, column := range columns { + if column == nil || strings.TrimSpace(column.Type) != "" { + continue + } + expression := strings.TrimSpace(column.Expression) + trimmed := strings.TrimSpace(strings.Trim(expression, "()")) + switch { + case trimmed == "": + continue + case trimmed == "true" || trimmed == "false": + column.Type = "bool" + case isIntegerLiteral(trimmed): + column.Type = "int" + case isFloatLiteral(trimmed): + column.Type = "float64" + case isQuotedLiteral(trimmed): + column.Type = "string" + } + } +} + +func isIntegerLiteral(value string) bool { + if value == "" { + return false + } + for i, ch := range value { + if i == 0 && (ch == '-' || ch == '+') { + if len(value) == 1 { + return false + } + continue + } + if ch < '0' || ch > '9' { + return false + } + } + return true +} + +func isFloatLiteral(value string) bool { + if value == "" || strings.Count(value, ".") != 1 { + return false + } + value = strings.ReplaceAll(value, ".", "") + return isIntegerLiteral(value) +} + +func isQuotedLiteral(value string) bool { + return len(value) >= 2 && ((value[0] == '\'' && value[len(value)-1] == '\'') || (value[0] == '"' && value[len(value)-1] == '"')) +} + +func resolveDiscoveryTable(aView *view.View, resource *view.Resource, rawSQL string) string { + table := "" + if aView != nil { + table = strings.TrimSpace(aView.Table) + } + if expanded := strings.TrimSpace(applyConstValuesForDiscovery(table, resource)); expanded != "" { + table = expanded + } + table = normalizeDiscoveryTable(table) + if table == "" { + table = inferDiscoveryTable(rawSQL) + } + return table +} + +func applyConstValuesForDiscovery(sql string, resource *view.Resource) string { + if strings.TrimSpace(sql) == "" || resource == nil || len(resource.Parameters) == 0 { + return sql + } + consts := map[string]string{} + for _, item := range resource.Parameters { + if item == nil || item.In == nil || item.In.Kind != state.KindConst { + continue + } + name := strings.TrimSpace(item.Name) + if name == "" { + name = strings.TrimSpace(item.In.Name) + } + if name == "" || item.Value == nil { + continue + } + consts[name] = fmt.Sprintf("%v", item.Value) + } + if len(consts) == 0 { + return sql + } + var b strings.Builder + b.Grow(len(sql)) + for i := 0; i < len(sql); { + if sql[i] != '$' { + b.WriteByte(sql[i]) + i++ + continue + } + if i+1 < len(sql) && sql[i+1] == '{' { + end := i + 2 + for end < len(sql) && sql[end] != '}' { + end++ + } + if end >= len(sql) { + b.WriteString(sql[i:]) + break + } + expr := strings.TrimSpace(sql[i+2 : end]) + if value, ok := constFromExpr(expr, consts); ok { + b.WriteString(formatConstForDiscovery(value)) + } else { + b.WriteString(sql[i : end+1]) + } + i = end + 1 + continue + } + end := i + 1 + for end < len(sql) && (isIdentPart(sql[end]) || sql[end] == '.') { + end++ + } + expr := sql[i+1 : end] + if value, ok := constFromExpr(expr, consts); ok { + b.WriteString(formatConstForDiscovery(value)) + } else { + b.WriteString(sql[i:end]) + } + i = end + } + return b.String() +} + +func constFromExpr(expr string, consts map[string]string) (string, bool) { + if expr == "" { + return "", false + } + if strings.HasPrefix(expr, "Unsafe.") { + expr = strings.TrimPrefix(expr, "Unsafe.") + } + if value, ok := consts[expr]; ok { + return value, true + } + for name, value := range consts { + if strings.EqualFold(name, expr) { + return value, true + } + } + return "", false +} + +func formatConstForDiscovery(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "''" + } + for i := 0; i < len(value); i++ { + ch := value[i] + if !(isIdentPart(ch) || ch == '.' || ch == '`') { + escaped := strings.ReplaceAll(value, "'", "''") + return "'" + escaped + "'" + } + } + return value +} + +func normalizeDiscoveryTable(table string) string { + trimmed := strings.TrimSpace(strings.Trim(table, "`\"")) + if strings.HasPrefix(trimmed, "${Unsafe.") && strings.HasSuffix(trimmed, "}") { + trimmed = strings.TrimSuffix(strings.TrimPrefix(trimmed, "${Unsafe."), "}") + } + if strings.HasPrefix(trimmed, "$Unsafe.") { + trimmed = strings.TrimPrefix(trimmed, "$Unsafe.") + } + trimmed = strings.TrimSpace(trimmed) + if trimmed == "" { + return table + } + for i := 0; i < len(trimmed); i++ { + ch := trimmed[i] + if !(isIdentPart(ch) || ch == '.' || ch == '`' || ch == '"') { + return table + } + } + return strings.Trim(trimmed, "`\"") +} + +func inferDiscoveryTable(sql string) string { + lower := strings.ToLower(sql) + idx := strings.Index(lower, " from ") + if idx == -1 { + if token := findUnsafeTableToken(sql); token != "" { + return token + } + return "" + } + pos := idx + len(" from ") + for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') { + pos++ + } + if pos >= len(sql) { + return "" + } + if strings.HasPrefix(sql[pos:], "${Unsafe.") { + end := strings.Index(sql[pos:], "}") + if end == -1 { + return "" + } + token := strings.TrimSpace(sql[pos+len("${Unsafe.") : pos+end]) + if token == "" { + return "" + } + return token + } + if strings.HasPrefix(sql[pos:], "$Unsafe.") { + start := pos + len("$Unsafe.") + end := start + for end < len(sql) && (isIdentPart(sql[end]) || sql[end] == '.') { + end++ + } + return strings.TrimSpace(sql[start:end]) + } + if sql[pos] == '(' { + depth := 1 + end := pos + 1 + for end < len(sql) && depth > 0 { + switch sql[end] { + case '(': + depth++ + case ')': + depth-- + } + end++ + } + if end > pos+1 { + if nested := inferDiscoveryTable(sql[pos+1 : end-1]); nested != "" { + return nested + } + } + if token := findUnsafeTableToken(sql[pos:]); token != "" { + return token + } + return "" + } + end := pos + for end < len(sql) && (isIdentPart(sql[end]) || sql[end] == '.' || sql[end] == '`' || sql[end] == '"') { + end++ + } + return strings.TrimSpace(strings.Trim(sql[pos:end], "`\"")) +} + +func findUnsafeTableToken(sql string) string { + if idx := strings.Index(sql, "${Unsafe."); idx != -1 { + start := idx + len("${Unsafe.") + end := strings.Index(sql[start:], "}") + if end != -1 { + return strings.TrimSpace(sql[start : start+end]) + } + } + if idx := strings.Index(sql, "$Unsafe."); idx != -1 { + start := idx + len("$Unsafe.") + end := start + for end < len(sql) && (isIdentPart(sql[end]) || sql[end] == '.') { + end++ + } + return strings.TrimSpace(sql[start:end]) + } + return "" +} + func removeExceptClauses(sql string) string { // Remove "EXCEPT col1, col2" patterns — these are datly-specific // Simple approach: remove " EXCEPT (, )*" @@ -201,13 +519,18 @@ func stripTemplateVariables(sql string) string { for j < len(sql) && isIdentPart(sql[j]) { j++ } + hasMethodCall := false + methodExpr := "" // Skip .method() chains for j < len(sql) && sql[j] == '.' { + methodStart := j j++ for j < len(sql) && isIdentPart(sql[j]) { j++ } if j < len(sql) && sql[j] == '(' { + hasMethodCall = true + methodExpr = sql[methodStart:j] depth := 1 j++ for j < len(sql) && depth > 0 { @@ -220,7 +543,15 @@ func stripTemplateVariables(sql string) string { } } } - b.WriteString("''") + if hasMethodCall { + if strings.EqualFold(methodExpr, ".AppendBinding") { + b.WriteString("''") + } else { + b.WriteString("") + } + } else { + b.WriteString("''") + } i = j continue } @@ -386,6 +717,9 @@ func columnsFromSchema(aView *view.View) view.Columns { } result := make(view.Columns, 0, rType.NumField()) appendSchemaColumns(rType, "", &result) + if allPlaceholderColumns(result) { + return nil + } return result } @@ -405,6 +739,9 @@ func appendSchemaColumns(rType reflect.Type, ns string, columns *view.Columns) { } continue } + if shouldSkipSchemaField(field) { + continue + } tag := io.ParseTag(field.Tag) if tag != nil && tag.Transient { continue @@ -428,6 +765,20 @@ func appendSchemaColumns(rType reflect.Type, ns string, columns *view.Columns) { } } +func shouldSkipSchemaField(field reflect.StructField) bool { + if field.Name == "-" { + return true + } + rawTag := string(field.Tag) + if strings.Contains(rawTag, `view:"`) || strings.Contains(rawTag, `on:"`) { + return true + } + if strings.Contains(rawTag, `sqlx:"-"`) { + return true + } + return false +} + func mergePreservingOrder(base, discovered view.Columns) view.Columns { if len(base) == 0 { return discovered diff --git a/repository/shape/column/detector_test.go b/repository/shape/column/detector_test.go index cfc834b11..f04b17d49 100644 --- a/repository/shape/column/detector_test.go +++ b/repository/shape/column/detector_test.go @@ -14,6 +14,21 @@ type sampleOrder struct { Name string `sqlx:"name=NAME"` } +type sampleSemanticRoot struct { + ID int `sqlx:"ID"` + Products []*sampleChildView `view:",table=PRODUCT" on:"Id:ID=VendorId:VENDOR_ID" sql:"uri=vendor/products.sql"` + Cities []*sampleChildView `view:",table=CITY" on:"Id:ID=DistrictId:DISTRICT_ID"` + Ignored string `sqlx:"-"` +} + +type sampleChildView struct { + VendorID int `sqlx:"VENDOR_ID"` +} + +func stringPtr(value string) *string { + return &value +} + func TestUsesWildcard(t *testing.T) { tests := []struct { name string @@ -39,6 +54,13 @@ func TestColumnsFromSchema_Order(t *testing.T) { require.Equal(t, "NAME", cols[1].Name) } +func TestColumnsFromSchema_SkipsSemanticRelationFields(t *testing.T) { + aView := &view.View{Schema: state.NewSchema(reflect.TypeOf(sampleSemanticRoot{}), state.WithMany())} + cols := columnsFromSchema(aView) + require.Len(t, cols, 1) + require.Equal(t, "ID", cols[0].Name) +} + func TestMergePreservingOrder_AppendsNewDetectedColumns(t *testing.T) { base := view.Columns{ view.NewColumn("VENDOR_ID", "int", reflect.TypeOf(int(0)), false), @@ -57,3 +79,68 @@ func TestMergePreservingOrder_AppendsNewDetectedColumns(t *testing.T) { require.Equal(t, "bigint", merged[0].DataType) require.Equal(t, "text", merged[1].DataType) } + +func TestApplyConstValuesForDiscovery(t *testing.T) { + resource := view.EmptyResource() + resource.AddParameters( + &state.Parameter{Name: "Vendor", In: state.NewConstLocation("Vendor"), Value: "VENDOR"}, + &state.Parameter{Name: "Product", In: state.NewConstLocation("Product"), Value: "PRODUCT"}, + ) + sql := `SELECT vendor.*, products.* FROM (SELECT * FROM $Vendor t) vendor JOIN (SELECT * FROM ${Unsafe.Product} p) products ON products.VENDOR_ID = vendor.ID` + got := applyConstValuesForDiscovery(sql, resource) + require.Contains(t, got, "FROM (SELECT * FROM VENDOR t)") + require.Contains(t, got, "JOIN (SELECT * FROM PRODUCT p)") + require.NotContains(t, got, "$Vendor") + require.NotContains(t, got, "${Unsafe.Product}") +} + +func TestDiscoverySQL_ResolvesConstTableFallback(t *testing.T) { + resource := view.EmptyResource() + resource.AddParameters( + &state.Parameter{Name: "Vendor", In: state.NewConstLocation("Vendor"), Value: "VENDOR"}, + ) + aView := &view.View{ + Table: "${Unsafe.Vendor}", + Template: view.NewTemplate("SELECT * FROM ${Unsafe.Vendor} t WHERE t.ID = $criteria.AppendBinding($Unsafe.VendorID)"), + } + got := discoverySQL(aView, resource) + require.Equal(t, "VENDOR", got) +} + +func TestNormalizeDiscoveryTable_TemplateUnsafe(t *testing.T) { + require.Equal(t, "Vendor", normalizeDiscoveryTable("${Unsafe.Vendor}")) + require.Equal(t, "Product", normalizeDiscoveryTable("$Unsafe.Product")) +} + +func TestDiscoverySQL_TemplateTableWithoutConstParameter_UsesNormalizedTable(t *testing.T) { + aView := &view.View{ + Table: "${Unsafe.Vendor}", + Template: view.NewTemplate("SELECT * FROM ${Unsafe.Vendor} t WHERE t.ID IN ($criteria.AppendBinding($Unsafe.vendorIDs))"), + } + got := discoverySQL(aView, view.EmptyResource()) + require.Equal(t, "Vendor", got) +} + +func TestExplicitProjectedSubqueryColumns_UsesInnerProjection(t *testing.T) { + aView := &view.View{ + Template: view.NewTemplate("SELECT * FROM (SELECT (1) AS IS_ACTIVE, (3) AS CHANNEL, CAST($criteria.AppendBinding($Unsafe.VendorID) AS SIGNED) AS ID) t"), + ColumnsConfig: map[string]*view.ColumnConfig{ + "ID": {Name: "ID", Tag: stringPtr(`internal:"true"`)}, + }, + } + got := explicitProjectedSubqueryColumns(aView) + require.Len(t, got, 3) + require.Equal(t, "IS_ACTIVE", got[0].Name) + require.Equal(t, "int", got[0].DataType) + require.Equal(t, "CHANNEL", got[1].Name) + require.Equal(t, "int", got[1].DataType) + require.Equal(t, "ID", got[2].Name) + require.Equal(t, ` internal:"true"`, got[2].Tag) +} + +func TestInferDiscoveryTable(t *testing.T) { + require.Equal(t, "Vendor", inferDiscoveryTable("SELECT * FROM ${Unsafe.Vendor} t WHERE 1=1")) + require.Equal(t, "Product", inferDiscoveryTable("SELECT * FROM $Unsafe.Product t WHERE 1=1")) + require.Equal(t, "VENDOR", inferDiscoveryTable("SELECT * FROM VENDOR t WHERE 1=1")) + require.Equal(t, "Vendor", inferDiscoveryTable("SELECT vendor.* FROM (SELECT * FROM ${Unsafe.Vendor} t WHERE 1=1) vendor")) +} diff --git a/repository/shape/column/strip_test.go b/repository/shape/column/strip_test.go index c38af5b50..df847559e 100644 --- a/repository/shape/column/strip_test.go +++ b/repository/shape/column/strip_test.go @@ -1,11 +1,13 @@ package column import ( + "reflect" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/viant/datly/view" + "github.com/viant/datly/view/state" ) func TestStripTemplateVariables(t *testing.T) { @@ -32,7 +34,7 @@ func TestStripTemplateVariables(t *testing.T) { { name: "variable with method call", input: "SELECT * FROM PRODUCT WHERE 1=1 $View.ParentJoinOn(\"AND\",\"VENDOR_ID\")", - expect: "SELECT * FROM PRODUCT WHERE 1=1 ''", + expect: "SELECT * FROM PRODUCT WHERE 1=1 ", }, { name: "criteria binding", @@ -67,7 +69,7 @@ func TestStripTemplateVariables(t *testing.T) { { name: "UNION ALL with templates", input: "SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 $View.ParentJoinOn(\"AND\",\"VENDOR_ID\") UNION ALL SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 $View.ParentJoinOn(\"AND\",\"VENDOR_ID\")", - expect: "SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 '' UNION ALL SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 ''", + expect: "SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 UNION ALL SELECT ID, NAME, VENDOR_ID FROM PRODUCT t WHERE 1=1 ", }, { name: "nested if", @@ -200,6 +202,16 @@ func TestDiscoverySQL_Strategy(t *testing.T) { expected: "VENDOR", desc: "template variables → table fallback (safe for all backends)", }, + { + name: "explicit projection with templates — preserves projection", + table: "VENDOR", + sql: "SELECT ID FROM VENDOR t WHERE t.ID = $VendorID", + desc: "explicit select list should not widen to table columns", + assertions: func(t *testing.T, result string) { + assert.NotEqual(t, "VENDOR", result) + assert.Contains(t, strings.ToUpper(result), "SELECT ID") + }, + }, { name: "wildcard with EXCEPT — uses table fallback", table: "VENDOR", @@ -242,7 +254,7 @@ func TestDiscoverySQL_Strategy(t *testing.T) { if tt.sql != "" { v.Template = &view.Template{Source: tt.sql} } - got := discoverySQL(v) + got := discoverySQL(v, nil) if tt.assertions != nil { tt.assertions(t, got) } else { @@ -292,3 +304,15 @@ func TestNeedsDiscovery(t *testing.T) { }) } } + +type placeholderSchemaRow struct { + Col1 string `sqlx:"name=col_1"` + Col2 string `sqlx:"name=col_2"` +} + +func TestColumnsFromSchema_IgnoresPlaceholderTypes(t *testing.T) { + aView := &view.View{ + Schema: state.NewSchema(reflect.TypeOf([]*placeholderSchemaRow{}), state.WithMany()), + } + assert.Nil(t, columnsFromSchema(aView)) +} diff --git a/repository/shape/compile/compiler.go b/repository/shape/compile/compiler.go index 14029ebd7..4f5ef49ea 100644 --- a/repository/shape/compile/compiler.go +++ b/repository/shape/compile/compiler.go @@ -58,7 +58,7 @@ func (c *DQLCompiler) Compile(ctx context.Context, source *shape.Source, opts .. root, compileDiags, err := c.compileRoot( source.Name, prepared.Pre.SQL, prepared.Statements, prepared.Decision, - compileOptions.MixedMode, compileOptions.UnknownNonReadMode, + compileOptions.MixedMode, compileOptions.UnknownNonReadMode, prepared.Pre.Directives, ) if err != nil { return nil, err @@ -136,13 +136,26 @@ func (c *DQLCompiler) assembleResult( applyDefaultConnectorDirective(result) applyConstDirective(result) hints := extractViewHints(source.DQL) - appendRelationViews(result, root, hints) + relationSQLSource := prepared.Pre.SQL + if strings.TrimSpace(relationSQLSource) == "" { + relationSQLSource = source.DQL + } + appendRelationViews(result, root, hints, relationSQLSource) appendDeclaredViews(source.DQL, result) appendDeclaredStates(source.DQL, result) applyViewHints(result, hints) + result.Diagnostics = append(result.Diagnostics, appendComponentTypesWithLayout(source, result, pathLayout)...) + for _, item := range result.Views { + if item == nil || strings.TrimSpace(item.SQL) == "" { + continue + } + item.SQL = stripProjectionHintCalls(item.SQL) + } applyInlineParamHints(source.DQL, result) applySourceParityEnrichmentWithLayout(result, source, pathLayout) - applyLinkedTypeSupport(result, source) + if compileOptions.UseLinkedTypes == nil || *compileOptions.UseLinkedTypes { + applyLinkedTypeSupport(result, source) + } result.Diagnostics = append(result.Diagnostics, applyColumnDiscoveryPolicy(result, compileOptions)...) return result } @@ -173,9 +186,13 @@ func applyDefaultConnectorDirective(result *plan.Result) { } } -func (c *DQLCompiler) compileRoot(sourceName, sqlText string, statements dqlstmt.Statements, decision pipeline.Decision, mode shape.CompileMixedMode, unknownMode shape.CompileUnknownNonReadMode) (*plan.View, []*dqlshape.Diagnostic, error) { +func (c *DQLCompiler) compileRoot(sourceName, sqlText string, statements dqlstmt.Statements, decision pipeline.Decision, mode shape.CompileMixedMode, unknownMode shape.CompileUnknownNonReadMode, directives *dqlshape.Directives) (*plan.View, []*dqlshape.Diagnostic, error) { mode = normalizeMixedMode(mode) unknownMode = normalizeUnknownNonReadMode(unknownMode) + consts := map[string]string(nil) + if directives != nil && len(directives.Const) > 0 { + consts = directives.Const + } if !decision.HasRead && !decision.HasExec && decision.HasUnknown { diag := &dqlshape.Diagnostic{ Code: dqldiag.CodeParseUnknownNonRead, @@ -211,7 +228,7 @@ func (c *DQLCompiler) compileRoot(sourceName, sqlText string, statements dqlstmt break } } - view, diags, err := pipeline.BuildRead(sourceName, readSQL) + view, diags, err := pipeline.BuildReadWithConsts(sourceName, readSQL, consts) diags = append(diags, &dqlshape.Diagnostic{ Code: dqldiag.CodeDMLMixed, Severity: dqlshape.SeverityWarning, @@ -235,7 +252,7 @@ func (c *DQLCompiler) compileRoot(sourceName, sqlText string, statements dqlstmt } return view, diags, nil } - return pipeline.BuildRead(sourceName, sqlText) + return pipeline.BuildReadWithConsts(sourceName, sqlText, consts) } func normalizeMixedMode(mode shape.CompileMixedMode) shape.CompileMixedMode { diff --git a/repository/shape/compile/compiler_test.go b/repository/shape/compile/compiler_test.go index 85b51108c..d568b5586 100644 --- a/repository/shape/compile/compiler_test.go +++ b/repository/shape/compile/compiler_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -229,6 +230,37 @@ func TestDQLCompiler_Compile_SyntaxError_RemapsAfterSanitize(t *testing.T) { } } +func TestDQLCompiler_Compile_RelationSQLUsesSanitizedVeltyOutput(t *testing.T) { + compiler := New() + dql := ` +#setting($_ = $route('/v1/api/shape/dev/vendors/{vendorID}', 'GET')) +#define($_ = $VendorID(path/vendorID)) +SELECT wrapper.* EXCEPT ID, + vendor.*, + products.* EXCEPT VENDOR_ID, + setting.* EXCEPT ID +FROM (SELECT ID FROM VENDOR WHERE ID = $VendorID) wrapper +JOIN (SELECT * FROM VENDOR t WHERE t.ID = $VendorID) vendor ON vendor.ID = wrapper.ID +JOIN (SELECT * FROM (SELECT (1) AS IS_ACTIVE, (3) AS CHANNEL, CAST($VendorID AS SIGNED) AS ID) t) setting ON setting.ID = wrapper.ID +JOIN (SELECT * FROM PRODUCT t) products ON products.VENDOR_ID = vendor.ID` + + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "vendor_details", DQL: dql}) + require.NoError(t, err) + planned, ok := plan.ResultFrom(res) + require.True(t, ok) + require.Contains(t, planned.ViewsByName, "vendor") + require.Contains(t, planned.ViewsByName, "setting") + require.Contains(t, planned.ViewsByName, "products") + assert.Contains(t, planned.ViewsByName["vendor"].SQL, "$criteria.AppendBinding($Unsafe.VendorID)") + assert.Contains(t, planned.ViewsByName["setting"].SQL, "CAST($criteria.AppendBinding($Unsafe.VendorID) AS SIGNED)") + require.NotNil(t, planned.ViewsByName["setting"].Declaration) + require.Contains(t, planned.ViewsByName["setting"].Declaration.ColumnsConfig, "ID") + assert.Equal(t, `internal:"true"`, planned.ViewsByName["setting"].Declaration.ColumnsConfig["ID"].Tag) + require.NotNil(t, planned.ViewsByName["products"].Declaration) + require.Contains(t, planned.ViewsByName["products"].Declaration.ColumnsConfig, "VENDOR_ID") + assert.Equal(t, `internal:"true"`, planned.ViewsByName["products"].Declaration.ColumnsConfig["VENDOR_ID"].Tag) +} + func TestDQLCompiler_Compile_DirectiveOnly_HasLineAndChar(t *testing.T) { compiler := New() _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "#package('x')"}) @@ -258,6 +290,22 @@ func TestDQLCompiler_Compile_InvalidDirective_HasLineAndChar(t *testing.T) { assert.Equal(t, 1, d.Span.Start.Char) } +func TestDQLCompiler_Compile_SQLSyntaxWithDirective_HasExactLineAndChar(t *testing.T) { + compiler := New() + _, err := compiler.Compile(context.Background(), &shape.Source{ + Name: "orders_report", + DQL: "#setting($_ = $route('/x', 'GET'))\nSELECT id FROM ORDERS WHERE (", + }) + require.Error(t, err) + compileErr, ok := err.(*CompileError) + require.True(t, ok) + require.NotEmpty(t, compileErr.Diagnostics) + d := compileErr.Diagnostics[0] + assert.Equal(t, dqldiag.CodeParseSyntax, d.Code) + assert.Equal(t, 2, d.Span.Start.Line) + assert.Equal(t, 29, d.Span.Start.Char) +} + func TestDQLCompiler_Compile_ExtractsJoinLinks(t *testing.T) { compiler := New() dql := "SELECT o.id, i.sku FROM orders o JOIN order_items i ON o.id = i.order_id" @@ -354,8 +402,15 @@ SELECT id FROM ORDERS t` planned, ok := plan.ResultFrom(res) require.True(t, ok) require.Len(t, planned.Views, 2) - extra := planned.ViewsByName["e"] + var extra *plan.View + for _, item := range planned.Views { + if item != nil && strings.Contains(item.SQL, "SELECT code FROM EXTRA e") { + extra = item + break + } + } require.NotNil(t, extra) + assert.Equal(t, "Extra", extra.Name) assert.Equal(t, "EXTRA", extra.Table) assert.Contains(t, extra.SQL, "SELECT code FROM EXTRA e") } @@ -369,8 +424,15 @@ SELECT id FROM ORDERS t` require.NoError(t, err) planned, ok := plan.ResultFrom(res) require.True(t, ok) - extra := planned.ViewsByName["e"] + var extra *plan.View + for _, item := range planned.Views { + if item != nil && strings.Contains(item.SQL, "SELECT code FROM EXTRA e") { + extra = item + break + } + } require.NotNil(t, extra) + assert.Equal(t, "Extra", extra.Name) assert.Equal(t, "/v1/extra", extra.SQLURI) assert.Equal(t, "analytics", extra.Connector) assert.Equal(t, "one", extra.Cardinality) @@ -511,7 +573,7 @@ func TestDQLCompiler_Compile_MixedMode_ReadWins(t *testing.T) { require.NotEmpty(t, planned.Views) assert.Equal(t, "o", planned.Views[0].Name) assert.Equal(t, "ORDERS", planned.Views[0].Table) - assert.Contains(t, planned.Views[0].SQL, "SELECT o.id FROM ORDERS o") + assert.Contains(t, planned.Views[0].SQL, "SELECT * FROM ORDERS o") assert.NotContains(t, planned.Views[0].SQL, "UPDATE ORDERS") require.NotEmpty(t, planned.Diagnostics) assert.Equal(t, dqldiag.CodeDMLMixed, planned.Diagnostics[len(planned.Diagnostics)-1].Code) diff --git a/repository/shape/compile/component_types.go b/repository/shape/compile/component_types.go index c7c8f22db..5ec5e3b28 100644 --- a/repository/shape/compile/component_types.go +++ b/repository/shape/compile/component_types.go @@ -1,15 +1,19 @@ package compile import ( + "context" "os" "path/filepath" + "reflect" "sort" "strings" "github.com/viant/datly/repository/shape" dqldiag "github.com/viant/datly/repository/shape/dql/diag" dqlshape "github.com/viant/datly/repository/shape/dql/shape" + shapeLoad "github.com/viant/datly/repository/shape/load" "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/view" "github.com/viant/datly/view/state" "gopkg.in/yaml.v3" ) @@ -37,16 +41,15 @@ func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, l sourceNamespace, _ := dqlToRouteNamespaceWithLayout(source.Path, layout) collector := &componentCollector{ routesRoot: routesRoot, + dqlRoot: dqlRoot, + layout: layout, visited: map[string]componentVisitState{}, outputByRoute: map[string]string{}, + routeByNS: map[string]string{}, typesByName: map[string]*plan.Type{}, payloadCache: map[string]routePayloadLookup{}, reportedDiag: map[string]bool{}, } - if strings.TrimSpace(sourceNamespace) != "" { - collector.collect(sourceNamespace, relationSpan(source.DQL, 0), false) - } - for _, stateItem := range result.States { if stateItem == nil || state.Kind(strings.ToLower(stateItem.KindString())) != state.KindComponent { continue @@ -66,14 +69,38 @@ func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, l }) continue } + if routeKey, ok := collector.resolveRoute(ref, source.Path); ok && stateItem.In != nil { + stateItem.In.Name = routeKey + } outputType, ok := collector.collect(namespace, componentRefSpan(source.DQL, ref), true) + if routeKey := collector.routeKey(namespace); routeKey != "" && stateItem.In != nil { + stateItem.In.Name = routeKey + } if ok && strings.TrimSpace(outputType) != "" { if stateItem.Schema == nil { stateItem.Schema = &state.Schema{} } + if stateItem.Schema.Type() == nil { + if lookup, found := collector.payloadCache[strings.ToLower(strings.TrimSpace(namespace))]; found && lookup.outputType != nil { + stateItem.Schema.SetType(lookup.outputType) + } + } if strings.TrimSpace(stateItem.Schema.DataType) == "" { stateItem.Schema.DataType = strings.TrimSpace(outputType) } + if payload, found := collector.loadRoutePayload(namespace, componentRefSpan(source.DQL, ref)); found { + if pkg, modulePath := routeOutputPackage(payload, outputType); pkg != "" || modulePath != "" { + if strings.TrimSpace(stateItem.Schema.Package) == "" { + stateItem.Schema.Package = pkg + } + if strings.TrimSpace(stateItem.Schema.PackagePath) == "" { + stateItem.Schema.PackagePath = pkg + } + if strings.TrimSpace(stateItem.Schema.ModulePath) == "" { + stateItem.Schema.ModulePath = modulePath + } + } + } } } @@ -110,8 +137,13 @@ func appendComponentTypesWithLayout(source *shape.Source, result *plan.Result, l type componentCollector struct { routesRoot string + dqlRoot string + layout compilePathLayout + routeIndex *RouteIndex + routeIndexErr error visited map[string]componentVisitState outputByRoute map[string]string + routeByNS map[string]string // typesByName provides O(1) dedup; typeOrder tracks insertion sequence // so the final list can be sorted once rather than extracted from the map. typesByName map[string]*plan.Type @@ -123,6 +155,7 @@ type componentCollector struct { type routePayloadLookup struct { payload *routePayload + outputType reflect.Type found bool malformed bool malformedAt string @@ -187,6 +220,9 @@ func (c *componentCollector) collect(namespace string, span dqlshape.Span, requi outputType := routeOutputType(payload) c.outputByRoute[key] = outputType + if routeKey := routePayloadKey(payload); routeKey != "" { + c.routeByNS[key] = routeKey + } for _, param := range payload.Resource.Parameters { if !strings.EqualFold(strings.TrimSpace(param.In.Kind), string(state.KindComponent)) { @@ -210,6 +246,55 @@ func (c *componentCollector) collect(namespace string, span dqlshape.Span, requi return outputType, true } +func (c *componentCollector) routeKey(namespace string) string { + if c == nil { + return "" + } + return strings.TrimSpace(c.routeByNS[strings.ToLower(strings.TrimSpace(namespace))]) +} + +func (c *componentCollector) resolveRoute(ref, currentSource string) (string, bool) { + index, err := c.lazyRouteIndex() + if err != nil || index == nil { + return "", false + } + opts := c.layoutCompileOptions() + return index.Resolve(ref, currentSource, opts...) +} + +func (c *componentCollector) lazyRouteIndex() (*RouteIndex, error) { + if c == nil { + return nil, nil + } + if c.routeIndex != nil || c.routeIndexErr != nil { + return c.routeIndex, c.routeIndexErr + } + if strings.TrimSpace(c.dqlRoot) == "" { + return nil, nil + } + paths, err := collectDQLSources(c.dqlRoot) + if err != nil { + c.routeIndexErr = err + return nil, err + } + c.routeIndex, c.routeIndexErr = BuildRouteIndex(paths, c.layoutCompileOptions()...) + return c.routeIndex, c.routeIndexErr +} + +func (c *componentCollector) layoutCompileOptions() []shape.CompileOption { + if c == nil { + return nil + } + var opts []shape.CompileOption + if marker := strings.TrimSpace(c.layout.dqlMarker); marker != "" { + opts = append(opts, shape.WithDQLPathMarker(marker)) + } + if rel := strings.TrimSpace(c.layout.routesRelative); rel != "" { + opts = append(opts, shape.WithRoutesRelativePath(rel)) + } + return opts +} + func sourceRootsWithLayout(sourcePath string, layout compilePathLayout) (platformRoot, routesRoot, dqlRoot string, ok bool) { path := filepath.Clean(strings.TrimSpace(sourcePath)) if path == "" { @@ -348,6 +433,8 @@ type routePayload struct { } `yaml:"Parameters"` } `yaml:"Resource"` Routes []struct { + Method string `yaml:"Method"` + URI string `yaml:"URI"` Handler struct { OutputType string `yaml:"OutputType"` } `yaml:"Handler"` @@ -388,6 +475,82 @@ func readRoutePayload(routesRoot, namespace string) routePayloadLookup { return lookup } +func readDQLPayload(dqlRoot string, layout compilePathLayout, namespace string, sourcePath string) routePayloadLookup { + candidates := []string{} + if strings.TrimSpace(sourcePath) != "" { + candidates = append(candidates, sourcePath) + } + candidates = append(candidates, dqlSourceCandidates(dqlRoot, namespace)...) + lookup := routePayloadLookup{} + for _, candidate := range candidates { + data, err := os.ReadFile(candidate) + if err != nil { + continue + } + source := &shape.Source{ + Name: strings.TrimSuffix(filepath.Base(candidate), filepath.Ext(candidate)), + Path: candidate, + DQL: string(data), + } + opts := []shape.CompileOption{} + if marker := strings.TrimSpace(layout.dqlMarker); marker != "" { + opts = append(opts, shape.WithDQLPathMarker(marker)) + } + if rel := strings.TrimSpace(layout.routesRelative); rel != "" { + opts = append(opts, shape.WithRoutesRelativePath(rel)) + } + planned, err := New().Compile(context.Background(), source, opts...) + if err != nil { + if !lookup.malformed { + lookup.malformed = true + lookup.malformedAt = candidate + lookup.detail = strings.TrimSpace(err.Error()) + } + continue + } + result, ok := plan.ResultFrom(planned) + if !ok || result == nil { + if !lookup.malformed { + lookup.malformed = true + lookup.malformedAt = candidate + lookup.detail = "unexpected compiled plan result" + } + continue + } + artifact, err := shapeLoad.New().LoadComponent(context.Background(), planned) + if err != nil { + if !lookup.malformed { + lookup.malformed = true + lookup.malformedAt = candidate + lookup.detail = strings.TrimSpace(err.Error()) + } + continue + } + component, _ := shapeLoad.ComponentFrom(artifact) + lookup.payload = routePayloadFromPlan(result, component) + lookup.outputType = routeOutputReflectType(component, artifact.Resource) + applyDQLRoutePayload(lookup.payload, source, result, namespace) + lookup.found = true + lookup.malformed = false + lookup.malformedAt = "" + lookup.detail = "" + return lookup + } + return lookup +} + +func (c *componentCollector) componentSourcePath(namespace string) string { + index, err := c.lazyRouteIndex() + if err != nil || index == nil { + return "" + } + entries := index.ByNamespace[strings.ToLower(strings.TrimSpace(namespace))] + if len(entries) != 1 || entries[0] == nil { + return "" + } + return strings.TrimSpace(entries[0].SourcePath) +} + func (c *componentCollector) loadRoutePayload(namespace string, span dqlshape.Span) (*routePayload, bool) { key := strings.ToLower(strings.TrimSpace(namespace)) if key == "" { @@ -396,6 +559,12 @@ func (c *componentCollector) loadRoutePayload(namespace string, span dqlshape.Sp lookup, ok := c.payloadCache[key] if !ok { lookup = readRoutePayload(c.routesRoot, namespace) + if !lookup.found && strings.TrimSpace(c.dqlRoot) != "" { + dqlLookup := readDQLPayload(c.dqlRoot, c.layout, namespace, c.componentSourcePath(namespace)) + if dqlLookup.found || dqlLookup.malformed { + lookup = dqlLookup + } + } c.payloadCache[key] = lookup } if lookup.malformed && !lookup.found && !c.hasReported("invalid:"+key) { @@ -472,6 +641,54 @@ func routeOutputType(payload *routePayload) string { return "" } +func routeOutputPackage(payload *routePayload, outputType string) (string, string) { + if payload == nil { + return "", "" + } + if len(payload.Routes) > 0 { + if pkg := strings.TrimSpace(payload.Routes[0].Output.Type.Package); pkg != "" { + modulePath := routeTypeModulePath(payload, strings.TrimSpace(payload.Routes[0].Output.Type.Name)) + return pkg, modulePath + } + } + leaf := strings.Trim(strings.TrimSpace(outputType), "*") + if leaf == "" { + return "", "" + } + for _, item := range payload.Resource.Types { + name := strings.TrimSpace(item.Name) + dataType := strings.Trim(strings.TrimSpace(item.DataType), "*") + if strings.EqualFold(name, leaf) || strings.EqualFold(dataType, leaf) { + return strings.TrimSpace(item.Package), strings.TrimSpace(item.ModulePath) + } + } + for _, param := range payload.Resource.Parameters { + if !strings.EqualFold(strings.TrimSpace(param.In.Kind), string(state.KindOutput)) { + continue + } + if name := strings.Trim(strings.TrimSpace(param.Schema.Name), "*"); name == leaf { + return strings.TrimSpace(param.Schema.Package), "" + } + if dataType := strings.Trim(strings.TrimSpace(param.Schema.DataType), "*"); dataType == leaf { + return strings.TrimSpace(param.Schema.Package), "" + } + } + return "", "" +} + +func routeTypeModulePath(payload *routePayload, name string) string { + name = strings.Trim(strings.TrimSpace(name), "*") + if payload == nil || name == "" { + return "" + } + for _, item := range payload.Resource.Types { + if strings.EqualFold(strings.TrimSpace(item.Name), name) { + return strings.TrimSpace(item.ModulePath) + } + } + return "" +} + func componentRefSpan(raw, ref string) dqlshape.Span { offset := 0 ref = strings.TrimSpace(ref) @@ -494,3 +711,310 @@ func routeYAMLCandidates(routesRoot, namespace string) []string { filepath.Join(routesRoot, filepath.FromSlash(namespace), leaf+".yaml"), } } + +func routePayloadKey(payload *routePayload) string { + if payload == nil { + return "" + } + for _, route := range payload.Routes { + if uri := strings.TrimSpace(route.URI); uri != "" { + return normalizeRouteKey(strings.TrimSpace(route.Method), uri) + } + } + return "" +} + +func collectDQLSources(root string) ([]string, error) { + var result []string + err := filepath.WalkDir(root, func(candidate string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + if !isComponentDQLSourceFile(candidate) { + return nil + } + result = append(result, candidate) + return nil + }) + if err != nil { + return nil, err + } + sort.Strings(result) + return result, nil +} + +func isComponentDQLSourceFile(path string) bool { + ext := strings.ToLower(strings.TrimSpace(filepath.Ext(path))) + return ext == ".dql" || ext == ".sql" +} + +func dqlSourceCandidates(dqlRoot, namespace string) []string { + namespace = strings.Trim(namespace, "/") + if namespace == "" || strings.TrimSpace(dqlRoot) == "" { + return nil + } + leaf := filepath.Base(namespace) + base := filepath.Join(dqlRoot, filepath.FromSlash(namespace)) + return []string{ + base + ".dql", + base + ".sql", + filepath.Join(base, leaf+".dql"), + filepath.Join(base, leaf+".sql"), + } +} + +func routePayloadFromPlan(result *plan.Result, component *shapeLoad.Component) *routePayload { + if result == nil { + return nil + } + payload := &routePayload{} + for _, item := range result.Types { + if item == nil { + continue + } + payload.Resource.Types = append(payload.Resource.Types, struct { + Name string `yaml:"Name"` + Alias string `yaml:"Alias"` + DataType string `yaml:"DataType"` + Cardinality string `yaml:"Cardinality"` + Package string `yaml:"Package"` + ModulePath string `yaml:"ModulePath"` + }{ + Name: strings.TrimSpace(item.Name), + Alias: strings.TrimSpace(item.Alias), + DataType: strings.TrimSpace(item.DataType), + Cardinality: strings.TrimSpace(item.Cardinality), + Package: strings.TrimSpace(item.Package), + ModulePath: strings.TrimSpace(item.ModulePath), + }) + } + ensureComponentOutputType(payload, component) + for _, item := range result.States { + if item == nil { + continue + } + param := struct { + Name string `yaml:"Name"` + In struct { + Kind string `yaml:"Kind"` + Name string `yaml:"Name"` + } `yaml:"In"` + Schema struct { + DataType string `yaml:"DataType"` + Name string `yaml:"Name"` + Package string `yaml:"Package"` + Cardinality string `yaml:"Cardinality"` + } `yaml:"Schema"` + }{Name: strings.TrimSpace(item.Name)} + if item.In != nil { + param.In.Kind = string(item.In.Kind) + param.In.Name = strings.TrimSpace(item.In.Name) + } + if item.Schema != nil { + param.Schema.DataType = strings.TrimSpace(item.Schema.DataType) + param.Schema.Name = strings.TrimSpace(item.Schema.Name) + param.Schema.Package = strings.TrimSpace(item.Schema.Package) + param.Schema.Cardinality = string(item.Schema.Cardinality) + } + payload.Resource.Parameters = append(payload.Resource.Parameters, param) + } + if outputType := componentOutputType(component, result); outputType != "" { + payload.Routes = append(payload.Routes, struct { + Method string `yaml:"Method"` + URI string `yaml:"URI"` + Handler struct { + OutputType string `yaml:"OutputType"` + } `yaml:"Handler"` + Output struct { + Cardinality string `yaml:"Cardinality"` + Type struct { + Name string `yaml:"Name"` + Package string `yaml:"Package"` + } `yaml:"Type"` + } `yaml:"Output"` + }{}) + payload.Routes[0].Handler.OutputType = outputType + if name, pkg := componentOutputName(component); name != "" { + payload.Routes[0].Output.Type.Name = name + payload.Routes[0].Output.Type.Package = pkg + } + } + return payload +} + +func componentOutputType(component *shapeLoad.Component, result *plan.Result) string { + if name, _ := componentOutputName(component); name != "" { + return "*" + strings.Trim(name, "*") + } + if component != nil { + for _, item := range component.Output { + if item == nil { + continue + } + if outputType := strings.TrimSpace(item.OutputDataType); outputType != "" { + return outputType + } + if item.Schema != nil { + if dataType := strings.TrimSpace(item.Schema.DataType); dataType != "" { + return dataType + } + if name := strings.TrimSpace(item.Schema.Name); name != "" { + return "*" + strings.Trim(name, "*") + } + } + } + } + return planOutputType(result) +} + +func routeOutputReflectType(component *shapeLoad.Component, resource *view.Resource) reflect.Type { + if component == nil || resource == nil { + return nil + } + pkgPath := "" + if component.TypeContext != nil { + pkgPath = strings.TrimSpace(component.TypeContext.PackagePath) + if pkgPath == "" { + pkgPath = strings.TrimSpace(component.TypeContext.DefaultPackage) + } + } + params := resource.Parameters.FilterByKind(state.KindOutput) + if len(params) == 0 { + params = component.OutputParameters() + } + if len(params) == 0 { + return nil + } + rt, err := params.ReflectType(pkgPath, resource.LookupType()) + if err != nil { + return nil + } + return rt +} + +func ensureComponentOutputType(payload *routePayload, component *shapeLoad.Component) { + if payload == nil || component == nil { + return + } + name, pkg := componentOutputName(component) + modulePath := "" + if component.TypeContext != nil { + modulePath = strings.TrimSpace(component.TypeContext.PackagePath) + } + if name == "" || pkg == "" || modulePath == "" { + return + } + for _, item := range payload.Resource.Types { + if strings.EqualFold(strings.TrimSpace(item.Name), name) { + return + } + } + payload.Resource.Types = append(payload.Resource.Types, struct { + Name string `yaml:"Name"` + Alias string `yaml:"Alias"` + DataType string `yaml:"DataType"` + Cardinality string `yaml:"Cardinality"` + Package string `yaml:"Package"` + ModulePath string `yaml:"ModulePath"` + }{ + Name: name, + DataType: "*" + name, + Package: pkg, + ModulePath: modulePath, + }) +} + +func componentOutputName(component *shapeLoad.Component) (string, string) { + if component == nil { + return "", "" + } + name := generatedComponentTypeBase(component) + "Output" + if spec := component.TypeSpecs["output"]; spec != nil && strings.TrimSpace(spec.TypeName) != "" { + name = strings.TrimSpace(spec.TypeName) + } + pkg := "" + if component.TypeContext != nil { + pkg = strings.TrimSpace(component.TypeContext.PackagePath) + if pkg == "" { + pkg = strings.TrimSpace(component.TypeContext.DefaultPackage) + } + } + return name, pkg +} + +func generatedComponentTypeBase(component *shapeLoad.Component) string { + if component == nil { + return "Component" + } + name := strings.TrimSpace(component.RootView) + if name == "" { + name = strings.TrimSpace(component.Name) + } + if name == "" { + name = "Component" + } + return state.SanitizeTypeName(name) +} + +func planOutputType(result *plan.Result) string { + if result == nil { + return "" + } + for _, item := range result.States { + if item == nil || !strings.EqualFold(item.KindString(), string(state.KindOutput)) { + continue + } + if outputType := strings.TrimSpace(item.OutputDataType); outputType != "" { + return outputType + } + if item.Schema != nil { + if dataType := strings.TrimSpace(item.Schema.DataType); dataType != "" { + return dataType + } + if name := strings.TrimSpace(item.Schema.Name); name != "" { + return "*" + strings.Trim(name, "*") + } + } + } + for _, item := range result.Types { + if item == nil || !strings.EqualFold(strings.TrimSpace(item.Name), "Output") { + continue + } + if dataType := strings.TrimSpace(item.DataType); dataType != "" { + return dataType + } + return "*Output" + } + return "" +} + +func applyDQLRoutePayload(payload *routePayload, source *shape.Source, result *plan.Result, namespace string) { + if payload == nil || len(payload.Routes) == 0 { + return + } + settings := extractRuleSettings(source, nil) + if result != nil { + settings = extractRuleSettings(source, result.Directives) + } + method := httpMethod(settings) + uri := strings.TrimSpace(settings.URI) + if uri == "" { + uri = inferDefaultURI(namespace) + } + payload.Routes[0].Method = method + payload.Routes[0].URI = normalizeURI(uri) +} + +func httpMethod(settings *ruleSettings) string { + if settings == nil { + return "GET" + } + methods := parseRouteMethods(settings.Method) + if len(methods) == 0 { + return "GET" + } + return strings.ToUpper(strings.TrimSpace(methods[0])) +} diff --git a/repository/shape/compile/component_types_test.go b/repository/shape/compile/component_types_test.go index 2cf803b7f..090f50366 100644 --- a/repository/shape/compile/component_types_test.go +++ b/repository/shape/compile/component_types_test.go @@ -203,3 +203,32 @@ func TestAppendComponentTypes_InvalidRouteYAMLDedupedForRepeatedStates(t *testin } assert.Equal(t, 1, invalidCount) } + +func TestAppendComponentTypes_FallsBackToSiblingDQLComponent(t *testing.T) { + temp := t.TempDir() + sourceDir := filepath.Join(temp, "dql", "dev", "vendor") + refDir := filepath.Join(temp, "dql", "dev") + require.NoError(t, os.MkdirAll(sourceDir, 0o755)) + require.NoError(t, os.MkdirAll(refDir, 0o755)) + + sourcePath := filepath.Join(sourceDir, "vendors.dql") + refPath := filepath.Join(refDir, "user_acl.dql") + sourceDQL := "#define($_ = $Auth(component/../user_acl))\nSELECT 1" + refDQL := "#package('github.com/viant/datly/e2e/v1/shape/dev/vendor/user_acl')\n#setting($_ = $route('/v1/api/dev/user-acl', 'GET'))\n#define($_ = $Auth(output/view).Embed())\nSELECT 1 AS UserID, TRUE AS IsReadOnly, TRUE AS Feature1" + require.NoError(t, os.WriteFile(sourcePath, []byte(sourceDQL), 0o644)) + require.NoError(t, os.WriteFile(refPath, []byte(refDQL), 0o644)) + + result := &plan.Result{ + States: []*plan.State{ + {Parameter: state.Parameter{Name: "Auth", In: &state.Location{Kind: state.KindComponent, Name: "../user_acl"}}}, + }, + } + diags := appendComponentTypes(&shape.Source{Path: sourcePath, DQL: sourceDQL}, result) + for _, item := range diags { + require.NotEqual(t, dqldiag.CodeCompRouteMissing, item.Code) + } + require.NotNil(t, result.States[0].Schema) + assert.Equal(t, "*UserAclOutput", result.States[0].Schema.DataType) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/vendor/user_acl", result.States[0].Schema.Package) + assert.Equal(t, "GET:/v1/api/dev/user-acl", result.States[0].In.Name) +} diff --git a/repository/shape/compile/enrich.go b/repository/shape/compile/enrich.go index 3ea8a277b..c36ac8ef4 100644 --- a/repository/shape/compile/enrich.go +++ b/repository/shape/compile/enrich.go @@ -66,6 +66,9 @@ func buildParityEnrichmentContext(result *plan.Result, source *shape.Source, lay joinEmbedRefs: map[string]string{}, joinSubqueryBodies: map[string]string{}, } + if rootBaseDir := resultRootSQLBaseDir(result); rootBaseDir != "" { + ctx.baseDir = rootBaseDir + } if len(result.Views) == 0 || result.Views[0] == nil { return ctx } @@ -78,6 +81,17 @@ func buildParityEnrichmentContext(result *plan.Result, source *shape.Source, lay return ctx } +func resultRootSQLBaseDir(result *plan.Result) string { + if result == nil || len(result.Views) == 0 || result.Views[0] == nil { + return "" + } + rootName := strings.TrimSpace(result.Views[0].Name) + if rootName == "" { + return "" + } + return rootName +} + func applyViewDefaults(item *plan.View, root bool, ctx *parityEnrichmentContext) { if item == nil || ctx == nil { return diff --git a/repository/shape/compile/enrich_test.go b/repository/shape/compile/enrich_test.go index 1c532d398..b62522c37 100644 --- a/repository/shape/compile/enrich_test.go +++ b/repository/shape/compile/enrich_test.go @@ -45,6 +45,24 @@ func TestApplySourceParityEnrichment_InferTableFromSubquery(t *testing.T) { require.Equal(t, "advertiser/advertiser.sql", result.Views[0].SQLURI) } +func TestApplySourceParityEnrichment_UsesRootViewNameForSQLBaseDir(t *testing.T) { + source := &shape.Source{ + Path: "/repo/dql/dev/vendor/child_meta.dql", + DQL: `SELECT vendor.*, products.* FROM VENDOR vendor JOIN PRODUCT products ON products.VENDOR_ID = vendor.ID`, + } + result := &plan.Result{ + Views: []*plan.View{ + {Name: "vendor", Table: "VENDOR", SQL: "SELECT * FROM VENDOR"}, + {Name: "products", Table: "PRODUCT", SQL: "SELECT * FROM PRODUCT"}, + }, + } + + applySourceParityEnrichment(result, source) + + require.Equal(t, "vendor/vendor.sql", result.Views[0].SQLURI) + require.Equal(t, "vendor/products.sql", result.Views[1].SQLURI) +} + func TestApplySourceParityEnrichment_InferTableFromEmbed(t *testing.T) { tempDir := t.TempDir() dqlDir := filepath.Join(tempDir, "dql", "platform", "timezone") diff --git a/repository/shape/compile/hints.go b/repository/shape/compile/hints.go index 8c7c592c9..e198ba0f5 100644 --- a/repository/shape/compile/hints.go +++ b/repository/shape/compile/hints.go @@ -5,16 +5,20 @@ import ( "strconv" "strings" + "github.com/viant/datly/repository/shape/dql/decl" "github.com/viant/datly/repository/shape/plan" ) type viewHint struct { - Connector string - AllowNulls *bool - NoLimit *bool - CacheRef string - Limit *int - Self *plan.SelfReference + Connector string + AllowNulls *bool + NoLimit *bool + CacheRef string + Limit *int + Cardinality string + Dest string + TypeName string + Self *plan.SelfReference } func extractViewHints(dql string) map[string]viewHint { @@ -25,7 +29,7 @@ func extractViewHints(dql string) map[string]viewHint { if len(call.args) != 2 { continue } - alias := strings.TrimSpace(call.args[0]) + alias := normalizeHintAlias(call.args[0]) connector := unquote(strings.TrimSpace(call.args[1])) if !isIdentifier(alias) || !isIdentifier(connector) { continue @@ -37,7 +41,7 @@ func extractViewHints(dql string) map[string]viewHint { if len(call.args) != 1 { continue } - alias := strings.TrimSpace(call.args[0]) + alias := normalizeHintAlias(call.args[0]) if !isIdentifier(alias) { continue } @@ -49,7 +53,7 @@ func extractViewHints(dql string) map[string]viewHint { if len(call.args) != 2 { continue } - alias := strings.TrimSpace(call.args[0]) + alias := normalizeHintAlias(call.args[0]) limitRaw := strings.TrimSpace(call.args[1]) if !isIdentifier(alias) || limitRaw == "" { continue @@ -69,7 +73,7 @@ func extractViewHints(dql string) map[string]viewHint { if len(call.args) != 2 { continue } - alias := strings.TrimSpace(call.args[0]) + alias := normalizeHintAlias(call.args[0]) ref := unquote(strings.TrimSpace(call.args[1])) if !isIdentifier(alias) || ref == "" { continue @@ -77,11 +81,26 @@ func extractViewHints(dql string) map[string]viewHint { hint := result[alias] hint.CacheRef = ref result[alias] = hint + case "cardinality": + if len(call.args) != 2 { + continue + } + alias := normalizeHintAlias(call.args[0]) + value := strings.ToLower(strings.TrimSpace(unquote(strings.TrimSpace(call.args[1])))) + if !isIdentifier(alias) { + continue + } + if value != "one" && value != "many" { + continue + } + hint := result[alias] + hint.Cardinality = value + result[alias] = hint case "self_ref": if len(call.args) != 4 { continue } - alias := strings.TrimSpace(call.args[0]) + alias := normalizeHintAlias(call.args[0]) holder := unquote(strings.TrimSpace(call.args[1])) child := unquote(strings.TrimSpace(call.args[2])) parent := unquote(strings.TrimSpace(call.args[3])) @@ -91,6 +110,30 @@ func extractViewHints(dql string) map[string]viewHint { hint := result[alias] hint.Self = &plan.SelfReference{Holder: holder, Child: child, Parent: parent} result[alias] = hint + case "dest": + if len(call.args) != 2 { + continue + } + alias := normalizeHintAlias(call.args[0]) + dest := strings.TrimSpace(unquote(strings.TrimSpace(call.args[1]))) + if !isIdentifier(alias) || dest == "" { + continue + } + hint := result[alias] + hint.Dest = dest + result[alias] = hint + case "type": + if len(call.args) != 2 { + continue + } + alias := normalizeHintAlias(call.args[0]) + typeName := strings.TrimSpace(unquote(strings.TrimSpace(call.args[1]))) + if !isIdentifier(alias) || typeName == "" { + continue + } + hint := result[alias] + hint.TypeName = typeName + result[alias] = hint } } return result @@ -102,115 +145,27 @@ type hintCall struct { } func scanHintCalls(input string) []hintCall { - result := make([]hintCall, 0) - for i := 0; i < len(input); { - if !isIdentifierStart(input[i]) { - i++ - continue - } - start := i - i++ - for i < len(input) && isIdentifierPart(input[i]) { - i++ - } - name := strings.ToLower(input[start:i]) - if name != "use_connector" && name != "allow_nulls" && name != "set_limit" && name != "set_cache" && name != "self_ref" { - continue - } - j := skipSpaces(input, i) - if j >= len(input) || input[j] != '(' { - continue - } - body, end, ok := readCallBody(input, j) - if !ok { - continue - } - result = append(result, hintCall{name: name, args: splitCallArgs(body)}) - i = end + 1 - } - return result -} - -func readCallBody(input string, openParen int) (string, int, bool) { - depth := 0 - quote := byte(0) - for i := openParen; i < len(input); i++ { - ch := input[i] - if quote != 0 { - if ch == '\\' && i+1 < len(input) { - i++ - continue - } - if ch == quote { - quote = 0 - } - continue - } - if ch == '\'' || ch == '"' { - quote = ch - continue - } - if ch == '(' { - depth++ - continue - } - if ch == ')' { - depth-- - if depth == 0 { - return input[openParen+1 : i], i, true - } - } - } - return "", -1, false -} - -func splitCallArgs(input string) []string { - args := make([]string, 0) - current := strings.Builder{} - depth := 0 - quote := byte(0) - for i := 0; i < len(input); i++ { - ch := input[i] - if quote != 0 { - current.WriteByte(ch) - if ch == '\\' && i+1 < len(input) { - i++ - current.WriteByte(input[i]) - continue - } - if ch == quote { - quote = 0 - } - continue - } - if ch == '\'' || ch == '"' { - quote = ch - current.WriteByte(ch) - continue - } - if ch == '(' { - depth++ - current.WriteByte(ch) - continue - } - if ch == ')' { - if depth > 0 { - depth-- - } - current.WriteByte(ch) - continue - } - if ch == ',' && depth == 0 { - args = append(args, strings.TrimSpace(current.String())) - current.Reset() - continue - } - current.WriteByte(ch) + names := map[string]bool{ + "use_connector": true, + "allow_nulls": true, + "set_limit": true, + "set_cache": true, + "cardinality": true, + "self_ref": true, + "dest": true, + "type": true, } - if value := strings.TrimSpace(current.String()); value != "" { - args = append(args, value) + parsed, _ := decl.ScanCalls(input, decl.CallScanOptions{ + AllowedNames: names, + RequireDollar: false, + AllowDollar: false, + Strict: false, + }) + result := make([]hintCall, 0, len(parsed)) + for _, call := range parsed { + result = append(result, hintCall{name: call.Name, args: call.Args}) } - return args + return result } func isIdentifierStart(ch byte) bool { @@ -245,22 +200,19 @@ func unquote(value string) string { return value } -func skipSpaces(input string, index int) int { - for index < len(input) { - switch input[index] { - case ' ', '\t', '\n', '\r': - index++ - default: - return index - } - } - return index -} - -func appendRelationViews(result *plan.Result, root *plan.View, hints map[string]viewHint) { +func appendRelationViews(result *plan.Result, root *plan.View, hints map[string]viewHint, rawDQL string) { if result == nil || root == nil || len(root.Relations) == 0 { return } + joinSQLByAlias := map[string]string{} + for _, item := range scanJoinSubqueries(rawDQL) { + alias := strings.TrimSpace(item.alias) + body := strings.TrimSpace(item.body) + if alias == "" || body == "" { + continue + } + joinSQLByAlias[strings.ToLower(alias)] = body + } for _, relation := range root.Relations { if relation == nil { continue @@ -279,6 +231,10 @@ func appendRelationViews(result *plan.Result, root *plan.View, hints map[string] continue } table := strings.TrimSpace(relation.Table) + sqlText := strings.TrimSpace(joinSQLByAlias[strings.ToLower(name)]) + if sqlText == "" { + sqlText = relationSQLText(table) + } if table == "" { table = name } @@ -288,10 +244,14 @@ func appendRelationViews(result *plan.Result, root *plan.View, hints map[string] Holder: name, Name: name, Table: table, + SQL: sqlText, Cardinality: "many", FieldType: reflect.TypeOf([]map[string]interface{}{}), ElementType: reflect.TypeOf(map[string]interface{}{}), } + if len(relation.ColumnsConfig) > 0 { + view.Declaration = &plan.ViewDeclaration{ColumnsConfig: relation.ColumnsConfig} + } result.Views = append(result.Views, view) result.ViewsByName[name] = view } @@ -309,11 +269,7 @@ func applyViewHints(result *plan.Result, hints map[string]viewHint) { continue } for _, key := range []string{item.Name, item.Holder} { - key = strings.TrimSpace(key) - if key == "" { - continue - } - hint, ok := hints[key] + hint, ok := lookupViewHint(hints, key) if !ok { continue } @@ -335,13 +291,40 @@ func applyViewHints(result *plan.Result, hints map[string]viewHint) { if item.CacheRef == "" && hint.CacheRef != "" { item.CacheRef = hint.CacheRef } + if hint.Cardinality != "" { + item.Cardinality = hint.Cardinality + } if item.Self == nil && hint.Self != nil { item.Self = hint.Self } + if hint.Dest != "" || hint.TypeName != "" { + if item.Declaration == nil { + item.Declaration = &plan.ViewDeclaration{} + } + if item.Declaration.Dest == "" && hint.Dest != "" { + item.Declaration.Dest = hint.Dest + } + if item.Declaration.TypeName == "" && hint.TypeName != "" { + item.Declaration.TypeName = hint.TypeName + } + } } } } +func normalizeHintAlias(value string) string { + return strings.ToLower(strings.TrimSpace(value)) +} + +func lookupViewHint(hints map[string]viewHint, key string) (viewHint, bool) { + key = normalizeHintAlias(key) + if key == "" { + return viewHint{}, false + } + hint, ok := hints[key] + return hint, ok +} + func normalizeRelationTable(table string) string { table = strings.TrimSpace(table) if table == "" { @@ -374,3 +357,65 @@ func normalizeRelationTable(table string) string { } return normalized } + +func relationSQLText(table string) string { + trimmed := strings.TrimSpace(table) + if trimmed == "" { + return "" + } + normalized := strings.ToLower(trimmed) + if strings.HasPrefix(normalized, "select ") { + return trimmed + } + if strings.HasPrefix(trimmed, "(") { + unwrapped := unwrapRelationParens(trimmed) + unwrappedLower := strings.ToLower(strings.TrimSpace(unwrapped)) + if strings.HasPrefix(unwrappedLower, "select ") { + return strings.TrimSpace(unwrapped) + } + } + return "" +} + +func unwrapRelationParens(input string) string { + input = strings.TrimSpace(input) + if len(input) < 2 || input[0] != '(' || input[len(input)-1] != ')' { + return input + } + depth := 0 + quote := byte(0) + for i := 0; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + switch ch { + case '(': + depth++ + case ')': + depth-- + if depth == 0 && i != len(input)-1 { + return input + } + } + } + if depth != 0 { + return input + } + inner := strings.TrimSpace(input[1 : len(input)-1]) + if inner == "" { + return input + } + return inner +} diff --git a/repository/shape/compile/hints_strip.go b/repository/shape/compile/hints_strip.go new file mode 100644 index 000000000..5fd1847c5 --- /dev/null +++ b/repository/shape/compile/hints_strip.go @@ -0,0 +1,80 @@ +package compile + +import ( + "strings" + + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/query" +) + +var projectionHintCalls = map[string]bool{ + "useconnector": true, + "allownulls": true, + "setlimit": true, + "setcache": true, + "cardinality": true, + "selfref": true, + "dest": true, + "type": true, +} + +// stripProjectionHintCalls removes hint-only projection functions (e.g. self_ref, dest) +// from executable SQL while preserving metadata extraction from original DQL. +func stripProjectionHintCalls(sqlText string) string { + sqlText = strings.TrimSpace(sqlText) + if sqlText == "" { + return sqlText + } + queryNode, err := sqlparser.ParseQuery(sqlText) + if err != nil || queryNode == nil { + return sqlText + } + if !stripHintCallsFromSelect(queryNode) { + return sqlText + } + return strings.TrimSpace(sqlparser.Stringify(queryNode)) +} + +func stripHintCallsFromSelect(node *query.Select) bool { + if node == nil || len(node.List) == 0 { + return false + } + filtered := make(query.List, 0, len(node.List)) + changed := false + for _, item := range node.List { + if item == nil { + continue + } + if isHintProjectionItem(item) { + changed = true + continue + } + filtered = append(filtered, item) + } + // Keep original list if stripping would produce invalid SELECT list. + if changed && len(filtered) > 0 { + node.List = filtered + return true + } + return false +} + +func isHintProjectionItem(item *query.Item) bool { + if item == nil || item.Expr == nil { + return false + } + call, ok := item.Expr.(*expr.Call) + if !ok || call.X == nil { + return false + } + name := normalizeHintCallName(sqlparser.Stringify(call.X)) + return projectionHintCalls[name] +} + +func normalizeHintCallName(name string) string { + name = strings.ToLower(strings.TrimSpace(name)) + name = strings.Trim(name, "`\"'") + name = strings.ReplaceAll(name, "_", "") + return name +} diff --git a/repository/shape/compile/hints_test.go b/repository/shape/compile/hints_test.go index c40470e0f..6ead9fde7 100644 --- a/repository/shape/compile/hints_test.go +++ b/repository/shape/compile/hints_test.go @@ -1,6 +1,7 @@ package compile import ( + "strings" "testing" "github.com/stretchr/testify/assert" @@ -31,18 +32,41 @@ func TestExtractViewHints_MixedCaseAndUnquotedConnector(t *testing.T) { assert.False(t, *hints["match"].NoLimit) } +func TestExtractViewHints_DestAndType(t *testing.T) { + dql := "SELECT dest(vendor,'vendor.go'), type(vendor,'Vendor'), dest(products,'vendor.go'), type(products,'Products') FROM VENDOR vendor" + hints := extractViewHints(dql) + require.Contains(t, hints, "vendor") + require.Contains(t, hints, "products") + assert.Equal(t, "vendor.go", hints["vendor"].Dest) + assert.Equal(t, "Vendor", hints["vendor"].TypeName) + assert.Equal(t, "vendor.go", hints["products"].Dest) + assert.Equal(t, "Products", hints["products"].TypeName) +} + +func TestExtractViewHints_Cardinality(t *testing.T) { + dql := "SELECT cardinality(products_meta, 'one'), cardinality(products, 'many')" + hints := extractViewHints(dql) + require.Contains(t, hints, "products_meta") + require.Contains(t, hints, "products") + assert.Equal(t, "one", hints["products_meta"].Cardinality) + assert.Equal(t, "many", hints["products"].Cardinality) +} + func TestApplyViewHints_Metadata(t *testing.T) { trueValue := true result := &plan.Result{ Views: []*plan.View{ - {Name: "match", Table: "MATCH"}, + {Name: "match", Table: "MATCH", Cardinality: "many"}, }, } applyViewHints(result, map[string]viewHint{ "match": { - Connector: "ci_ads", - AllowNulls: &trueValue, - NoLimit: &trueValue, + Connector: "ci_ads", + AllowNulls: &trueValue, + NoLimit: &trueValue, + Cardinality: "one", + Dest: "match.go", + TypeName: "Match", }, }) require.Len(t, result.Views, 1) @@ -51,4 +75,201 @@ func TestApplyViewHints_Metadata(t *testing.T) { assert.True(t, *result.Views[0].AllowNulls) require.NotNil(t, result.Views[0].SelectorNoLimit) assert.True(t, *result.Views[0].SelectorNoLimit) + assert.Equal(t, "one", strings.ToLower(result.Views[0].Cardinality)) + require.NotNil(t, result.Views[0].Declaration) + assert.Equal(t, "match.go", result.Views[0].Declaration.Dest) + assert.Equal(t, "Match", result.Views[0].Declaration.TypeName) +} + +func TestApplyViewHints_MetadataCaseInsensitiveAlias(t *testing.T) { + trueValue := true + result := &plan.Result{ + Views: []*plan.View{ + {Name: "User", Holder: "User", Table: "USER"}, + }, + } + applyViewHints(result, map[string]viewHint{ + "user": { + Self: &plan.SelfReference{Holder: "Team", Child: "ID", Parent: "MGR_ID"}, + AllowNulls: &trueValue, + }, + }) + require.Len(t, result.Views, 1) + require.NotNil(t, result.Views[0].Self) + assert.Equal(t, "Team", result.Views[0].Self.Holder) + assert.Equal(t, "ID", result.Views[0].Self.Child) + assert.Equal(t, "MGR_ID", result.Views[0].Self.Parent) +} + +func TestStripProjectionHintCalls_RemovesSelfRefFromSQL(t *testing.T) { + sqlText := "SELECT user.* EXCEPT MGR_ID, self_ref(user, 'Team', 'ID', 'MGR_ID'), cardinality(user, 'one') FROM (SELECT t.* FROM USER t) user" + actual := stripProjectionHintCalls(sqlText) + assert.NotContains(t, strings.ToLower(actual), "self_ref(") + assert.NotContains(t, strings.ToLower(actual), "cardinality(") + assert.Contains(t, strings.ToLower(actual), "user.* except mgr_id") +} + +func TestAppendRelationViews_SQLSelection(t *testing.T) { + testCases := []struct { + name string + rawDQL string + relationTable string + expectContains string + expectNotContain string + }{ + { + name: "prefers raw join subquery SQL when available", + rawDQL: ` +SELECT wrapper.*, + vendor.* +FROM (SELECT ID FROM VENDOR WHERE ID = $vendorID) wrapper +JOIN (SELECT * FROM VENDOR t WHERE t.ID = $criteria.AppendBinding($Unsafe.vendorID)) vendor ON vendor.ID = wrapper.ID`, + relationTable: "(SELECT * FROM VENDOR t WHERE t.ID = 1)", + expectContains: "$criteria.AppendBinding($Unsafe.vendorID)", + expectNotContain: "t.ID = 1", + }, + { + name: "falls back to relation table SQL when raw join SQL missing", + rawDQL: ` +SELECT wrapper.* +FROM (SELECT ID FROM VENDOR WHERE ID = $vendorID) wrapper`, + relationTable: "(SELECT * FROM VENDOR t WHERE t.ID = 1)", + expectContains: "t.ID = 1", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + result := &plan.Result{ + ViewsByName: map[string]*plan.View{}, + } + root := &plan.View{ + Relations: []*plan.Relation{ + { + Name: "vendor", + Ref: "vendor", + Table: testCase.relationTable, + On: []*plan.RelationLink{ + {Expression: "vendor.ID = wrapper.ID"}, + }, + }, + }, + } + + appendRelationViews(result, root, nil, testCase.rawDQL) + require.Len(t, result.Views, 1) + assert.Contains(t, result.Views[0].SQL, testCase.expectContains) + if testCase.expectNotContain != "" { + assert.NotContains(t, result.Views[0].SQL, testCase.expectNotContain) + } + }) + } +} + +func TestAppendRelationViews_ComplexTreeAnyLevel(t *testing.T) { + rawDQL := ` +SELECT wrapper.*, + vendor.*, + products.*, + reviews.* +FROM (SELECT ID FROM VENDOR WHERE ID = $vendorID) wrapper +JOIN (SELECT * FROM VENDOR t WHERE t.ID = $criteria.AppendBinding($Unsafe.vendorID)) vendor ON vendor.ID = wrapper.ID +JOIN (SELECT * FROM PRODUCT p WHERE p.VENDOR_ID = $criteria.AppendBinding($Unsafe.vendorID)) products ON products.VENDOR_ID = vendor.ID +JOIN (SELECT * FROM REVIEW r WHERE r.PRODUCT_ID = products.ID) reviews ON reviews.PRODUCT_ID = products.ID` + + result := &plan.Result{ + ViewsByName: map[string]*plan.View{}, + } + root := &plan.View{ + Relations: []*plan.Relation{ + { + Name: "vendor", + Ref: "vendor", + Parent: "wrapper", + Table: "(SELECT * FROM VENDOR t WHERE t.ID = 1)", + On: []*plan.RelationLink{ + {Expression: "vendor.ID = wrapper.ID"}, + }, + }, + { + Name: "products", + Ref: "products", + Parent: "vendor", + Table: "(SELECT * FROM PRODUCT p WHERE p.VENDOR_ID = 1)", + On: []*plan.RelationLink{ + {Expression: "products.VENDOR_ID = vendor.ID"}, + }, + }, + { + Name: "reviews", + Ref: "reviews", + Parent: "products", + Table: "(SELECT * FROM REVIEW r WHERE r.PRODUCT_ID = products.ID)", + On: []*plan.RelationLink{ + {Expression: "reviews.PRODUCT_ID = products.ID"}, + }, + }, + }, + } + + appendRelationViews(result, root, nil, rawDQL) + require.Len(t, result.Views, 3) + require.Contains(t, result.ViewsByName, "vendor") + require.Contains(t, result.ViewsByName, "products") + require.Contains(t, result.ViewsByName, "reviews") + assert.Contains(t, result.ViewsByName["vendor"].SQL, "$criteria.AppendBinding($Unsafe.vendorID)") + assert.Contains(t, result.ViewsByName["products"].SQL, "$criteria.AppendBinding($Unsafe.vendorID)") + assert.Contains(t, result.ViewsByName["reviews"].SQL, "r.PRODUCT_ID = products.ID") +} + +func TestAppendRelationViews_ComplexTreeCrossLevelJoin(t *testing.T) { + rawDQL := ` +SELECT wrapper.*, + vendor.*, + products.*, + stats.* +FROM (SELECT ID FROM VENDOR WHERE ID = $vendorID) wrapper +JOIN (SELECT * FROM VENDOR t WHERE t.ID = $criteria.AppendBinding($Unsafe.vendorID)) vendor ON vendor.ID = wrapper.ID +JOIN (SELECT * FROM PRODUCT p WHERE p.VENDOR_ID = vendor.ID) products ON products.VENDOR_ID = vendor.ID +JOIN (SELECT COUNT(1) AS CNT, v.ID AS VENDOR_ID FROM VENDOR v WHERE v.ID = wrapper.ID) stats ON stats.VENDOR_ID = wrapper.ID` + + result := &plan.Result{ + ViewsByName: map[string]*plan.View{}, + } + root := &plan.View{ + Relations: []*plan.Relation{ + { + Name: "vendor", + Ref: "vendor", + Parent: "wrapper", + Table: "(SELECT * FROM VENDOR t WHERE t.ID = 1)", + On: []*plan.RelationLink{ + {Expression: "vendor.ID = wrapper.ID"}, + }, + }, + { + Name: "products", + Ref: "products", + Parent: "vendor", + Table: "(SELECT * FROM PRODUCT p WHERE p.VENDOR_ID = vendor.ID)", + On: []*plan.RelationLink{ + {Expression: "products.VENDOR_ID = vendor.ID"}, + }, + }, + { + Name: "stats", + Ref: "stats", + Parent: "products", + Table: "(SELECT COUNT(1) AS CNT, v.ID AS VENDOR_ID FROM VENDOR v WHERE v.ID = wrapper.ID)", + On: []*plan.RelationLink{ + {Expression: "stats.VENDOR_ID = wrapper.ID"}, + }, + }, + }, + } + + appendRelationViews(result, root, nil, rawDQL) + require.Len(t, result.Views, 3) + require.Contains(t, result.ViewsByName, "stats") + assert.Contains(t, result.ViewsByName["stats"].SQL, "v.ID = wrapper.ID") } diff --git a/repository/shape/compile/pipeline/infer.go b/repository/shape/compile/pipeline/infer.go index 7ad212431..3d0e11e15 100644 --- a/repository/shape/compile/pipeline/infer.go +++ b/repository/shape/compile/pipeline/infer.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" "strings" + "unicode" "github.com/viant/sqlparser" "github.com/viant/sqlparser/query" @@ -152,10 +153,14 @@ func InferProjectionType(queryNode *query.Select) (reflect.Type, reflect.Type, s used[fieldName]++ typ := parseColumnType(column.Type) + veltyNames := []string{columnName} + if fieldName != "" && fieldName != columnName { + veltyNames = append(veltyNames, fieldName) + } fields = append(fields, reflect.StructField{ Name: fieldName, Type: typ, - Tag: reflect.StructTag(fmt.Sprintf(`json:"%s,omitempty" sqlx:"name=%s"`, strings.ToLower(fieldName), columnName)), + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s,omitempty" sqlx:"name=%s" velty:"names=%s"`, strings.ToLower(fieldName), columnName, strings.Join(veltyNames, "|"))), }) } element := reflect.StructOf(fields) @@ -182,7 +187,11 @@ func SanitizeName(value string) string { } func ExportedName(value string) string { - value = replaceNonWordWithUnderscore(strings.TrimSpace(value)) + value = strings.TrimSpace(value) + if preserved := preserveMixedCaseIdentifier(value); preserved != "" { + return preserved + } + value = replaceNonWordWithUnderscore(value) value = strings.Trim(value, "_") if value == "" { return "" @@ -204,6 +213,37 @@ func ExportedName(value string) string { return name } +func preserveMixedCaseIdentifier(value string) string { + if value == "" { + return "" + } + hasLower := false + hasUpperAfterFirst := false + for i, r := range value { + if !(unicode.IsLetter(r) || unicode.IsDigit(r)) { + return "" + } + if unicode.IsLower(r) { + hasLower = true + } + if i > 0 && unicode.IsUpper(r) { + hasUpperAfterFirst = true + } + } + if !hasLower || !hasUpperAfterFirst { + return "" + } + runes := []rune(value) + if len(runes) == 0 { + return "" + } + if unicode.IsDigit(runes[0]) { + return "N" + value + } + runes[0] = unicode.ToUpper(runes[0]) + return string(runes) +} + func replaceNonWordWithUnderscore(value string) string { if value == "" { return "" diff --git a/repository/shape/compile/pipeline/infer_test.go b/repository/shape/compile/pipeline/infer_test.go index 748fcded4..7a2073eef 100644 --- a/repository/shape/compile/pipeline/infer_test.go +++ b/repository/shape/compile/pipeline/infer_test.go @@ -1,9 +1,11 @@ package pipeline import ( + "reflect" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/viant/sqlparser" ) @@ -40,3 +42,22 @@ func TestInferTableFromSQL_ResolvesTopLevelFrom(t *testing.T) { sqlText := `SELECT a.*, EXISTS(SELECT 1 FROM CI_ENTITY_WATCHLIST w WHERE w.ENTITY_ID = a.ID) AS watching FROM (SELECT x.* FROM CI_ADVERTISER x) a` assert.Equal(t, "CI_ADVERTISER", InferTableFromSQL(sqlText)) } + +func TestExportedName_PreservesMixedCaseIdentifiers(t *testing.T) { + assert.Equal(t, "UserID", ExportedName("UserID")) + assert.Equal(t, "IsReadOnly", ExportedName("IsReadOnly")) + assert.Equal(t, "VendorName", ExportedName("vendor_name")) +} + +func TestInferProjectionType_AddsVeltyNames(t *testing.T) { + queryNode, err := sqlparser.ParseQuery(`SELECT ID, IS_AUTH FROM PRODUCT`) + require.NoError(t, err) + _, element, _ := InferProjectionType(queryNode) + require.Equal(t, reflect.Struct, element.Kind()) + field, ok := element.FieldByName("IsAuth") + assert.True(t, ok) + assert.Equal(t, `names=IS_AUTH|IsAuth`, field.Tag.Get("velty")) + idField, ok := element.FieldByName("Id") + assert.True(t, ok) + assert.Equal(t, `names=ID|Id`, idField.Tag.Get("velty")) +} diff --git a/repository/shape/compile/pipeline/parse.go b/repository/shape/compile/pipeline/parse.go index c897454a4..9609225cc 100644 --- a/repository/shape/compile/pipeline/parse.go +++ b/repository/shape/compile/pipeline/parse.go @@ -11,7 +11,8 @@ import ( ) func ParseSelectWithDiagnostic(sqlText string) (*query.Select, *dqlshape.Diagnostic, error) { - sqlText = trimLeadingBlockComments(sqlText) + original := sqlText + sqlText, trimPrefix := trimLeadingBlockComments(sqlText) var diagnostic *dqlshape.Diagnostic onError := func(err error, cur *parsly.Cursor, _ interface{}) error { offset := 0 @@ -26,7 +27,7 @@ func ParseSelectWithDiagnostic(sqlText string) (*query.Select, *dqlshape.Diagnos Severity: dqlshape.SeverityError, Message: strings.TrimSpace(err.Error()), Hint: "check SQL syntax near the reported location", - Span: pointSpan(sqlText, offset), + Span: pointSpan(original, offset+trimPrefix), } return err } @@ -38,7 +39,7 @@ func ParseSelectWithDiagnostic(sqlText string) (*query.Select, *dqlshape.Diagnos Severity: dqlshape.SeverityError, Message: strings.TrimSpace(err.Error()), Hint: "check SQL syntax near the reported location", - Span: pointSpan(sqlText, 0), + Span: pointSpan(original, trimPrefix), } } return nil, diagnostic, err @@ -49,14 +50,16 @@ func ParseSelectWithDiagnostic(sqlText string) (*query.Select, *dqlshape.Diagnos return result, nil, nil } -func trimLeadingBlockComments(sqlText string) string { +func trimLeadingBlockComments(sqlText string) (string, int) { remaining := strings.TrimLeft(sqlText, " \t\r\n") + trimPrefix := len(sqlText) - len(remaining) for strings.HasPrefix(remaining, "/*") { end := strings.Index(remaining, "*/") if end == -1 { - return remaining + return remaining, trimPrefix } remaining = strings.TrimLeft(remaining[end+2:], " \t\r\n") + trimPrefix = len(sqlText) - len(remaining) } - return remaining + return remaining, trimPrefix } diff --git a/repository/shape/compile/pipeline/parse_test.go b/repository/shape/compile/pipeline/parse_test.go index 69292fc8a..e0222ff28 100644 --- a/repository/shape/compile/pipeline/parse_test.go +++ b/repository/shape/compile/pipeline/parse_test.go @@ -23,7 +23,7 @@ func TestParseSelectWithDiagnostic_Syntax(t *testing.T) { require.NotNil(t, diag) assert.Equal(t, dqldiag.CodeParseSyntax, diag.Code) assert.Equal(t, 1, diag.Span.Start.Line) - assert.Greater(t, diag.Span.Start.Char, 1) + assert.Equal(t, 29, diag.Span.Start.Char) } func TestParseSelectWithDiagnostic_LeadingBlockComment(t *testing.T) { @@ -33,3 +33,43 @@ func TestParseSelectWithDiagnostic_LeadingBlockComment(t *testing.T) { require.NotNil(t, queryNode) assert.Equal(t, "o", queryNode.From.Alias) } + +func TestParseSelectWithDiagnostic_SyntaxPositionMatrix(t *testing.T) { + testCases := []struct { + name string + sql string + expectedLine int + expectedChar int + }{ + { + name: "plain sql", + sql: "SELECT id FROM orders WHERE (", + expectedLine: 1, + expectedChar: 29, + }, + { + name: "with leading block comment", + sql: "/* {\"URI\":\"/x\"} */\nSELECT id FROM orders WHERE (", + expectedLine: 2, + expectedChar: 29, + }, + { + name: "with multiple leading lines and comments", + sql: "\n\n/*a*/\n/*b*/\nSELECT id FROM orders WHERE (", + expectedLine: 5, + expectedChar: 29, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + queryNode, diag, err := ParseSelectWithDiagnostic(testCase.sql) + require.Error(t, err) + require.Nil(t, queryNode) + require.NotNil(t, diag) + assert.Equal(t, dqldiag.CodeParseSyntax, diag.Code) + assert.Equal(t, testCase.expectedLine, diag.Span.Start.Line) + assert.Equal(t, testCase.expectedChar, diag.Span.Start.Char) + }) + } +} diff --git a/repository/shape/compile/pipeline/read.go b/repository/shape/compile/pipeline/read.go index c665d154b..462d51d2a 100644 --- a/repository/shape/compile/pipeline/read.go +++ b/repository/shape/compile/pipeline/read.go @@ -10,6 +10,8 @@ import ( dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/expr" "github.com/viant/sqlparser/query" ) @@ -17,6 +19,10 @@ import ( // It applies multiple parse strategies and gracefully degrades to a // loose (schema-less) view for template-driven SQL that cannot be fully parsed. func BuildRead(sourceName, sqlText string) (*plan.View, []*dqlshape.Diagnostic, error) { + return BuildReadWithConsts(sourceName, sqlText, nil) +} + +func BuildReadWithConsts(sourceName, sqlText string, consts map[string]string) (*plan.View, []*dqlshape.Diagnostic, error) { queryNode, parseDiag, parserSQL, err := resolveQueryNode(sqlText) // Template-driven SQL may legitimately fail strict parsing; treat as warning. @@ -24,7 +30,9 @@ func BuildRead(sourceName, sqlText string) (*plan.View, []*dqlshape.Diagnostic, if parseDiag != nil { parseDiag.Severity = dqlshape.SeverityWarning } - return buildLooseRead(sourceName, sqlText), collectDiags(parseDiag), nil + view := buildLooseRead(sourceName, sqlText) + applyConstTables(view, consts) + return view, collectDiags(parseDiag), nil } var diags []*dqlshape.Diagnostic @@ -56,21 +64,114 @@ func BuildRead(sourceName, sqlText string) (*plan.View, []*dqlshape.Diagnostic, elementType = reflect.TypeOf(map[string]interface{}{}) cardinality = "many" } + rootSQL := sqlText + if rawRoot := extractRootSQLFromRaw(sqlText); rawRoot != "" { + rootSQL = rawRoot + } else if queryNode.From.X != nil { + fromExpr := strings.TrimSpace(sqlparser.Stringify(queryNode.From.X)) + fromExpr = trimJoinSuffix(fromExpr) + candidate := extractParenthesizedSelect(fromExpr) + if candidate == "" { + candidate = unwrapReadParens(fromExpr) + } + if candidate != "" { + rootSQL = candidate + } + } view := &plan.View{ Path: name, Holder: name, Name: name, Mode: "SQLQuery", Table: table, - SQL: sqlText, + SQL: rootSQL, Cardinality: cardinality, FieldType: fieldType, ElementType: elementType, Relations: relations, } + exceptByAlias := extractExceptColumnsByNamespace(queryNode) + if except := lookupExceptColumns(exceptByAlias, name); len(except) > 0 { + view.Declaration = &plan.ViewDeclaration{ColumnsConfig: except} + } + applyRelationExceptColumns(relations, exceptByAlias) + applyConstTables(view, consts) return view, diags, nil } +func applyConstTables(view *plan.View, consts map[string]string) { + if view == nil || len(consts) == 0 { + return + } + view.Table = resolveConstTable(view.Table, consts) + for _, relation := range view.Relations { + if relation == nil { + continue + } + relation.Table = resolveConstTable(relation.Table, consts) + } +} + +func resolveConstTable(table string, consts map[string]string) string { + trimmed := strings.TrimSpace(strings.Trim(table, "`\"")) + if token := unsafeSelectorToken(trimmed); token != "" { + if resolved := resolveConstValue(token, consts); resolved != "" { + return resolved + } + } + for key, value := range consts { + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key == "" || value == "" { + continue + } + placeholder := "Unsafe_" + key + if strings.EqualFold(trimmed, placeholder) { + return value + } + templatePlaceholder := "${Unsafe." + key + "}" + if strings.EqualFold(trimmed, templatePlaceholder) { + return value + } + selectorPlaceholder := "$Unsafe." + key + if strings.EqualFold(trimmed, selectorPlaceholder) { + return value + } + if strings.Contains(table, placeholder) { + table = strings.ReplaceAll(table, placeholder, value) + } + if strings.Contains(table, templatePlaceholder) { + table = strings.ReplaceAll(table, templatePlaceholder, value) + } + if strings.Contains(table, selectorPlaceholder) { + table = strings.ReplaceAll(table, selectorPlaceholder, value) + } + } + return table +} + +func unsafeSelectorToken(input string) string { + if strings.HasPrefix(input, "${Unsafe.") && strings.HasSuffix(input, "}") { + return strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(input, "${Unsafe."), "}")) + } + if strings.HasPrefix(input, "$Unsafe.") { + return strings.TrimSpace(strings.TrimPrefix(input, "$Unsafe.")) + } + return "" +} + +func resolveConstValue(token string, consts map[string]string) string { + if token == "" || len(consts) == 0 { + return "" + } + for key, value := range consts { + if strings.EqualFold(strings.TrimSpace(key), token) && strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + // resolveQueryNode attempts to parse sqlText into a query AST using up to // three strategies: // 1. Parse the normalised form. @@ -170,6 +271,84 @@ func inferRootFromRelations(relations []*plan.Relation) string { return "" } +func extractRootExceptColumns(queryNode *query.Select, rootName string) map[string]*plan.ViewColumnConfig { + return lookupExceptColumns(extractExceptColumnsByNamespace(queryNode), rootName) +} + +func applyRelationExceptColumns(relations []*plan.Relation, exceptByAlias map[string]map[string]*plan.ViewColumnConfig) { + if len(relations) == 0 || len(exceptByAlias) == 0 { + return + } + for _, relation := range relations { + if relation == nil { + continue + } + if columns := lookupExceptColumns(exceptByAlias, relation.Ref); len(columns) > 0 { + relation.ColumnsConfig = columns + } + } +} + +func lookupExceptColumns(exceptByAlias map[string]map[string]*plan.ViewColumnConfig, alias string) map[string]*plan.ViewColumnConfig { + if len(exceptByAlias) == 0 { + return nil + } + alias = strings.ToLower(strings.TrimSpace(alias)) + if alias == "" { + return nil + } + result := exceptByAlias[alias] + if len(result) == 0 { + return nil + } + ret := make(map[string]*plan.ViewColumnConfig, len(result)) + for key, cfg := range result { + ret[key] = cfg + } + return ret +} + +func extractExceptColumnsByNamespace(queryNode *query.Select) map[string]map[string]*plan.ViewColumnConfig { + if queryNode == nil { + return nil + } + result := map[string]map[string]*plan.ViewColumnConfig{} + for _, item := range queryNode.List { + if item == nil || item.Expr == nil { + continue + } + star, ok := item.Expr.(*expr.Star) + if !ok || len(star.Except) == 0 { + continue + } + selectorNs := "" + if selector, ok := star.X.(*expr.Selector); ok { + selectorNs = strings.ToLower(strings.TrimSpace(selector.Name)) + } + if selectorNs == "" { + continue + } + nsColumns := result[selectorNs] + if nsColumns == nil { + nsColumns = map[string]*plan.ViewColumnConfig{} + result[selectorNs] = nsColumns + } + for _, exceptColumn := range star.Except { + exceptColumn = strings.TrimSpace(exceptColumn) + if exceptColumn == "" { + continue + } + nsColumns[exceptColumn] = &plan.ViewColumnConfig{ + Tag: `internal:"true"`, + } + } + } + if len(result) == 0 { + return nil + } + return result +} + func extractSimpleFromTable(sqlText string) string { lower := strings.ToLower(sqlText) for i := 0; i+4 <= len(lower); i++ { @@ -195,6 +374,227 @@ func extractSimpleFromTable(sqlText string) string { return "" } +func extractRootSQLFromRaw(sqlText string) string { + if strings.TrimSpace(sqlText) == "" { + return "" + } + lower := strings.ToLower(sqlText) + depth := 0 + quote := byte(0) + for i := 0; i < len(sqlText); i++ { + ch := sqlText[i] + if quote != 0 { + if ch == '\\' && i+1 < len(sqlText) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + switch ch { + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + } + if depth != 0 || !hasReadWordAt(lower, i, "from") { + continue + } + start := skipReadSpaces(sqlText, i+4) + if start >= len(sqlText) { + return "" + } + fromExpr := trimJoinSuffix(sqlText[start:]) + fromExpr = strings.TrimSpace(fromExpr) + if fromExpr == "" { + return "" + } + if candidate := extractParenthesizedSelect(fromExpr); candidate != "" { + return candidate + } + fromExpr = unwrapReadParens(fromExpr) + fromExpr = strings.TrimSpace(fromExpr) + if fromExpr == "" { + return "" + } + if strings.HasPrefix(strings.ToLower(fromExpr), "select ") { + return fromExpr + } + return "SELECT * FROM " + fromExpr + } + return "" +} + +func unwrapReadParens(input string) string { + input = strings.TrimSpace(input) + if len(input) < 2 || input[0] != '(' || input[len(input)-1] != ')' { + return input + } + depth := 0 + quote := byte(0) + for i := 0; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + switch ch { + case '(': + depth++ + case ')': + depth-- + if depth == 0 && i != len(input)-1 { + return input + } + } + } + if depth != 0 { + return input + } + inner := strings.TrimSpace(input[1 : len(input)-1]) + if inner == "" { + return input + } + return inner +} + +func trimJoinSuffix(input string) string { + input = strings.TrimSpace(input) + if input == "" { + return "" + } + depth := 0 + quote := byte(0) + for i := 0; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + switch ch { + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + } + if depth != 0 || !isReadWordStart(input[i]) { + continue + } + if hasReadWordAt(strings.ToLower(input), i, "join") { + return strings.TrimSpace(input[:i]) + } + } + return input +} + +func extractParenthesizedSelect(input string) string { + input = strings.TrimSpace(input) + if input == "" || input[0] != '(' { + return "" + } + body, end, ok := readReadParenBody(input, 0) + if !ok { + return "" + } + tail := strings.TrimSpace(input[end+1:]) + if tail != "" && !isReadIdentifierStart(tail[0]) { + return "" + } + body = strings.TrimSpace(body) + if strings.HasPrefix(strings.ToLower(body), "select ") { + return body + } + return "" +} + +func readReadParenBody(input string, openParen int) (string, int, bool) { + depth := 0 + quote := byte(0) + for i := openParen; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + if ch == '(' { + depth++ + continue + } + if ch == ')' { + depth-- + if depth == 0 { + return input[openParen+1 : i], i, true + } + } + } + return "", -1, false +} + +func hasReadWordAt(lower string, pos int, word string) bool { + if pos < 0 || pos+len(word) > len(lower) { + return false + } + if lower[pos:pos+len(word)] != word { + return false + } + if pos > 0 && isReadWordPart(lower[pos-1]) { + return false + } + next := pos + len(word) + if next < len(lower) && isReadWordPart(lower[next]) { + return false + } + return true +} + +func isReadWordStart(ch byte) bool { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' +} + +func isReadWordPart(ch byte) bool { + return isReadWordStart(ch) || (ch >= '0' && ch <= '9') +} + // collectDiags returns a single-element slice for a non-nil diagnostic, // or nil otherwise. Used to avoid repeated nil checks at call sites. func collectDiags(diag *dqlshape.Diagnostic) []*dqlshape.Diagnostic { diff --git a/repository/shape/compile/pipeline/read_normalize.go b/repository/shape/compile/pipeline/read_normalize.go index 6ff42af2f..41b2dd8ce 100644 --- a/repository/shape/compile/pipeline/read_normalize.go +++ b/repository/shape/compile/pipeline/read_normalize.go @@ -154,6 +154,9 @@ func normalizeTemplateExprBody(body string) (string, bool) { if isReadReservedName(trimmed) { return "", true } + if selector := normalizeTemplateSelector(trimmed); selector != "" { + return selector, false + } lower := strings.ToLower(trimmed) if strings.Contains(lower, `build("where")`) || strings.Contains(lower, "build('where')") { return " WHERE 1 ", false @@ -164,6 +167,38 @@ func normalizeTemplateExprBody(body string) (string, bool) { return "1", false } +func normalizeTemplateSelector(input string) string { + if input == "" { + return "" + } + for i := 0; i < len(input); i++ { + ch := input[i] + if !(isReadIdentifierPart(ch) || ch == '.') { + return "" + } + } + parts := strings.Split(input, ".") + builder := strings.Builder{} + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if builder.Len() > 0 { + builder.WriteByte('_') + } + builder.WriteString(part) + } + result := builder.String() + if result == "" { + return "" + } + if !isReadIdentifierStart(result[0]) { + return "" + } + return result +} + func readReadTemplateExpr(input string, openBrace int) (string, int, bool) { if openBrace <= 0 || openBrace >= len(input) || input[openBrace] != '{' || input[openBrace-1] != '$' { return "", -1, false diff --git a/repository/shape/compile/pipeline/read_test.go b/repository/shape/compile/pipeline/read_test.go index 9d414beb8..5fb54e710 100644 --- a/repository/shape/compile/pipeline/read_test.go +++ b/repository/shape/compile/pipeline/read_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape/plan" "github.com/viant/sqlparser/expr" "github.com/viant/sqlparser/query" ) @@ -31,10 +32,26 @@ JOIN (SELECT * FROM session/attributes) attribute ON attribute.user_id = session require.NotNil(t, view) assert.Equal(t, "session", view.Name) assert.Equal(t, "session", view.Table) + assert.Contains(t, view.SQL, "$criteria.AppendBinding($Unsafe.Jwt.UserID)") require.NotEmpty(t, view.Relations) assert.Equal(t, "attribute", view.Relations[0].Ref) } +func TestExtractRootSQLFromRaw_JoinRootTable(t *testing.T) { + sqlText := "SELECT o.id, i.sku FROM orders o JOIN items i ON o.id = i.order_id" + assert.Equal(t, "SELECT * FROM orders o", extractRootSQLFromRaw(sqlText)) +} + +func TestExtractRootSQLFromRaw_PreservesTemplateVariables(t *testing.T) { + sqlText := `SELECT wrapper.* EXCEPT ID, + vendor.* +FROM (SELECT ID FROM VENDOR WHERE ID = $vendorID ) wrapper +JOIN (SELECT * FROM VENDOR t WHERE t.ID = $vendorID ) vendor ON vendor.ID = wrapper.ID` + root := extractRootSQLFromRaw(sqlText) + assert.Contains(t, root, "$vendorID") + assert.NotContains(t, root, " ID = 1 ") +} + func TestNormalizeParserSQL(t *testing.T) { input := "SELECT * FROM session WHERE user_id = $criteria.AppendBinding($Unsafe.Jwt.UserID) AND x = $Jwt.UserID" actual := normalizeParserSQL(input) @@ -50,6 +67,13 @@ func TestNormalizeParserSQL_VeltyBlockExpression(t *testing.T) { assert.Contains(t, actual, "SELECT b.* FROM CI_BROWSER b WHERE 1 AND b.ARCHIVED = 0") } +func TestNormalizeParserSQL_TemplateSelector(t *testing.T) { + input := `SELECT * FROM ${Unsafe.Vendor} t WHERE t.ID = ${Unsafe.VendorID}` + actual := normalizeParserSQL(input) + assert.Contains(t, actual, "FROM Unsafe_Vendor t") + assert.Contains(t, actual, "t.ID = Unsafe_VendorID") +} + func TestNormalizeParserSQL_PrivateShorthand(t *testing.T) { input := `SELECT private(audience.FREQ_CAPPING) AS freq_capping FROM CI_AUDIENCE audience` actual := normalizeParserSQL(input) @@ -71,3 +95,78 @@ func TestBuildRead_FallbackWhenInitialParseFails(t *testing.T) { assert.Equal(t, "CI_BROWSER", view.Table) assert.Empty(t, diags) } + +func TestBuildRead_NoJoin_UsesFromSourceSQL(t *testing.T) { + sqlText := `SELECT user.* EXCEPT MGR_ID, self_ref(user, 'Team', 'ID', 'MGR_ID') FROM (SELECT t.* FROM USER t) user` + view, _, err := BuildRead("user_tree", sqlText) + require.NoError(t, err) + require.NotNil(t, view) + assert.Equal(t, "SELECT t.* FROM USER t", strings.TrimSpace(view.SQL)) +} + +func TestBuildRead_ExceptBecomesInternalColumnConfig(t *testing.T) { + sqlText := `SELECT user.* EXCEPT MGR_ID FROM (SELECT t.* FROM USER t) user` + view, _, err := BuildRead("user_tree", sqlText) + require.NoError(t, err) + require.NotNil(t, view) + require.NotNil(t, view.Declaration) + require.NotNil(t, view.Declaration.ColumnsConfig) + cfg, ok := view.Declaration.ColumnsConfig["MGR_ID"] + require.True(t, ok) + require.NotNil(t, cfg) + assert.Equal(t, `internal:"true"`, cfg.Tag) +} + +func TestBuildRead_ChildExceptBecomesRelationColumnConfig(t *testing.T) { + sqlText := `SELECT wrapper.* EXCEPT ID, + products.* EXCEPT VENDOR_ID, + setting.* EXCEPT ID +FROM (SELECT ID FROM VENDOR WHERE ID = $VendorID) wrapper +JOIN (SELECT * FROM (SELECT (1) AS IS_ACTIVE, (3) AS CHANNEL, CAST($VendorID AS SIGNED) AS ID) t) setting ON setting.ID = wrapper.ID +JOIN (SELECT * FROM PRODUCT t) products ON products.VENDOR_ID = wrapper.ID` + view, _, err := BuildRead("vendor_details", sqlText) + require.NoError(t, err) + require.NotNil(t, view) + require.Len(t, view.Relations, 2) + + var productsCfg, settingCfg map[string]*plan.ViewColumnConfig + for _, rel := range view.Relations { + switch rel.Ref { + case "products": + productsCfg = rel.ColumnsConfig + case "setting": + settingCfg = rel.ColumnsConfig + } + } + require.Contains(t, productsCfg, "VENDOR_ID") + assert.Equal(t, `internal:"true"`, productsCfg["VENDOR_ID"].Tag) + require.Contains(t, settingCfg, "ID") + assert.Equal(t, `internal:"true"`, settingCfg["ID"].Tag) +} + +func TestBuildRead_TemplateTableSelector_PreservesRelations(t *testing.T) { + sqlText := `SELECT vendor.*, products.* +FROM (SELECT * FROM ${Unsafe.Vendor} t WHERE t.ID IN ($criteria.AppendBinding($Unsafe.vendorIDs))) vendor +JOIN (SELECT * FROM ${Unsafe.Product} t) products ON products.VENDOR_ID = vendor.ID` + view, _, err := BuildRead("const", sqlText) + require.NoError(t, err) + require.NotNil(t, view) + assert.Equal(t, "vendor", view.Name) + require.NotEmpty(t, view.Relations) + assert.Equal(t, "products", view.Relations[0].Ref) +} + +func TestBuildReadWithConsts_ResolvesUnsafeTablePlaceholders(t *testing.T) { + sqlText := `SELECT vendor.*, products.* +FROM (SELECT * FROM ${Unsafe.Vendor} t WHERE t.ID IN ($criteria.AppendBinding($Unsafe.vendorIDs))) vendor +JOIN (SELECT * FROM ${Unsafe.Product} t) products ON products.VENDOR_ID = vendor.ID` + view, _, err := BuildReadWithConsts("const", sqlText, map[string]string{ + "Vendor": "VENDOR", + "Product": "PRODUCT", + }) + require.NoError(t, err) + require.NotNil(t, view) + assert.Equal(t, "VENDOR", view.Table) + require.NotEmpty(t, view.Relations) + assert.Contains(t, view.Relations[0].Table, "PRODUCT") +} diff --git a/repository/shape/compile/pipeline/relation.go b/repository/shape/compile/pipeline/relation.go index dc94b8955..d60b56816 100644 --- a/repository/shape/compile/pipeline/relation.go +++ b/repository/shape/compile/pipeline/relation.go @@ -30,6 +30,7 @@ func ExtractJoinRelations(raw string, queryNode *query.Select) ([]*plan.Relation ref, table := relationRef(join, idx+1) relation := &plan.Relation{ Name: ref, + Parent: relationParentAlias(join, rootAlias), Holder: ExportedName(ref), Ref: ref, Table: table, @@ -346,6 +347,31 @@ func rootNamespace(queryNode *query.Select) string { return root } +func relationParentAlias(join *query.Join, rootAlias string) string { + if join == nil || join.On == nil { + return strings.TrimSpace(rootAlias) + } + parent := "" + sqlparser.Traverse(join.On, func(n node.Node) bool { + selector, ok := n.(*expr.Selector) + if !ok { + return true + } + name := strings.TrimSpace(selector.Name) + if name == "" || strings.EqualFold(name, strings.TrimSpace(join.Alias)) { + return true + } + if parent == "" { + parent = name + } + return true + }) + if parent != "" { + return parent + } + return strings.TrimSpace(rootAlias) +} + func relationRef(join *query.Join, ordinal int) (string, string) { if join == nil { return fmt.Sprintf("join_%d", ordinal), "" diff --git a/repository/shape/compile/pipeline/relation_test.go b/repository/shape/compile/pipeline/relation_test.go index 62f5c9d3a..dd358aadb 100644 --- a/repository/shape/compile/pipeline/relation_test.go +++ b/repository/shape/compile/pipeline/relation_test.go @@ -49,6 +49,9 @@ func TestExtractJoinRelations_NonRootParentChain(t *testing.T) { require.NoError(t, err) relations, diags := ExtractJoinRelations(sqlText, queryNode) require.Len(t, relations, 3) + assert.Equal(t, "sl", relations[0].Parent) + assert.Equal(t, "m", relations[1].Parent) + assert.Equal(t, "s", relations[2].Parent) require.Len(t, relations[0].On, 1) assert.Equal(t, "sl", relations[0].On[0].ParentNamespace) @@ -70,6 +73,56 @@ func TestExtractJoinRelations_NonRootParentChain(t *testing.T) { assert.Empty(t, diags) } +func TestExtractJoinRelations_ParentAliasMatrix(t *testing.T) { + testCases := []struct { + name string + sqlText string + expected map[string]string + }{ + { + name: "root parent", + sqlText: "SELECT o.id FROM orders o JOIN order_items i ON o.id = i.order_id", + expected: map[string]string{ + "i": "o", + }, + }, + { + name: "multi level chain", + sqlText: "SELECT sl.id FROM site_list sl JOIN site_list_match m ON m.site_list_id = sl.id JOIN ci_site s ON s.id = m.site_id JOIN ci_publisher p ON p.id = s.publisher_id", + expected: map[string]string{ + "m": "sl", + "s": "m", + "p": "s", + }, + }, + { + name: "left join child of child", + sqlText: "SELECT a.id FROM alpha a LEFT JOIN beta b ON b.a_id = a.id LEFT JOIN gamma g ON g.b_id = b.id", + expected: map[string]string{ + "b": "a", + "g": "b", + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + queryNode, err := sqlparser.ParseQuery(testCase.sqlText) + require.NoError(t, err) + relations, diags := ExtractJoinRelations(testCase.sqlText, queryNode) + assert.Empty(t, diags) + got := map[string]string{} + for _, relation := range relations { + if relation == nil { + continue + } + got[relation.Ref] = relation.Parent + } + assert.Equal(t, testCase.expected, got) + }) + } +} + func TestExtractJoinRelations_DoesNotFallbackForComplexRawPredicate(t *testing.T) { sqlText := "SELECT o.id FROM orders o JOIN order_items i ON COALESCE(o.id, 0) = i.order_id" queryNode, err := sqlparser.ParseQuery(sqlText) diff --git a/repository/shape/compile/statedecl.go b/repository/shape/compile/statedecl.go index bd401b11d..409718e0c 100644 --- a/repository/shape/compile/statedecl.go +++ b/repository/shape/compile/statedecl.go @@ -1,9 +1,13 @@ package compile import ( + "fmt" "strconv" "strings" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/view/extension" st "github.com/viant/datly/view/state" @@ -16,23 +20,29 @@ func appendDeclaredStates(rawDQL string, result *plan.Result) { } seen := map[string]bool{} for _, block := range extractSetBlocks(rawDQL) { - holder, kind, location, tail, ok := parseSetDeclarationBody(block.Body) + holder, kind, location, tail, tailOffset, ok := parseSetDeclarationBody(block.Body) if !ok { continue } - if kind == "view" || kind == "data_view" { - continue - } key := declaredStateKey(holder, kind, location) if seen[key] { continue } + inName := location + if kind == "view" || kind == "data_view" { + if isAttachedSummaryState(result, holder) { + continue + } + // Keep parity with legacy translator: view declarations are addressed + // by declaration holder name (e.g. $Authorization(view/authorization)). + inName = holder + } state := &plan.State{ Parameter: st.Parameter{ Name: holder, In: &st.Location{ Kind: st.Kind(kind), - Name: location, + Name: inName, }, }, } @@ -48,10 +58,85 @@ func appendDeclaredStates(rawDQL string, result *plan.Result) { required := true state.Required = &required } - applyDeclaredStateOptions(state, tail) + applyDeclaredStateOptions(state, tail, rawDQL, block.BodyOffset+tailOffset, &result.Diagnostics) result.States = append(result.States, state) seen[key] = true } + appendInferredPathStates(rawDQL, result, seen) +} + +func appendInferredPathStates(rawDQL string, result *plan.Result, seen map[string]bool) { + if result == nil || strings.TrimSpace(rawDQL) == "" { + return + } + prepared := dqlpre.Prepare(rawDQL) + if prepared.Directives == nil || prepared.Directives.Route == nil { + return + } + for _, name := range extractRoutePathParams(prepared.Directives.Route.URI) { + key := declaredStateKey(name, string(st.KindPath), name) + if seen[key] { + continue + } + result.States = append(result.States, &plan.State{ + Parameter: st.Parameter{ + Name: name, + In: st.NewPathLocation(name), + Schema: &st.Schema{ + DataType: "string", + Cardinality: st.One, + }, + }, + }) + seen[key] = true + } +} + +func extractRoutePathParams(uri string) []string { + uri = strings.TrimSpace(uri) + if uri == "" { + return nil + } + var result []string + seen := map[string]bool{} + for { + start := strings.IndexByte(uri, '{') + if start == -1 { + break + } + uri = uri[start+1:] + end := strings.IndexByte(uri, '}') + if end == -1 { + break + } + name := strings.TrimSpace(uri[:end]) + uri = uri[end+1:] + if name == "" { + continue + } + key := strings.ToLower(name) + if seen[key] { + continue + } + seen[key] = true + result = append(result, name) + } + return result +} + +func isAttachedSummaryState(result *plan.Result, holder string) bool { + if result == nil || strings.TrimSpace(holder) == "" { + return false + } + for _, item := range result.Views { + if item == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(item.SummaryName), strings.TrimSpace(holder)) { + return true + } + } + return false } func declaredStateKey(name, kind, in string) string { @@ -60,77 +145,156 @@ func declaredStateKey(name, kind, in string) string { strings.ToLower(strings.TrimSpace(in)) } -func applyDeclaredStateOptions(state *plan.State, tail string) { +func applyDeclaredStateOptions(state *plan.State, tail, dql string, baseOffset int, diags *[]*dqlshape.Diagnostic) { if state == nil || strings.TrimSpace(tail) == "" { return } cursor := newOptionCursor(tail) for cursor.next() { name, args := cursor.option() + optionOffset := baseOffset + cursor.start switch { case strings.EqualFold(name, "WithURI"): - if len(args) == 1 { - state.URI = trimQuote(args[0]) + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue + } + state.URI = trimQuote(args[0]) + case strings.EqualFold(name, "WithTag"), strings.EqualFold(name, "Tag"): + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue } + state.Tag = trimQuote(args[0]) case strings.EqualFold(name, "Optional"): + if !expectStateArgs(state, name, args, 0, 0, dql, optionOffset, diags) { + continue + } required := false state.Required = &required case strings.EqualFold(name, "Required"): + if !expectStateArgs(state, name, args, 0, 0, dql, optionOffset, diags) { + continue + } required := true state.Required = &required case strings.EqualFold(name, "Cacheable"): - if len(args) == 1 { - if value, err := strconv.ParseBool(strings.TrimSpace(trimQuote(args[0]))); err == nil { - state.Cacheable = &value - } + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue + } + value, err := strconv.ParseBool(strings.TrimSpace(trimQuote(args[0]))) + if err != nil { + appendStateOptionDiagnostic(state, name, fmt.Sprintf("invalid bool cacheable %q", args[0]), dql, optionOffset, diags) + continue } + state.Cacheable = &value case strings.EqualFold(name, "QuerySelector"): - if len(args) == 1 { - state.QuerySelector = trimQuote(args[0]) - if state.Cacheable == nil { - cacheable := false - state.Cacheable = &cacheable - } + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue + } + state.QuerySelector = trimQuote(args[0]) + if state.Cacheable == nil { + cacheable := false + state.Cacheable = &cacheable } case strings.EqualFold(name, "WithPredicate"), strings.EqualFold(name, "Predicate"): + if !expectStateArgs(state, name, args, 1, -1, dql, optionOffset, diags) { + continue + } appendStatePredicate(state, args, false) case strings.EqualFold(name, "EnsurePredicate"): + if !expectStateArgs(state, name, args, 1, -1, dql, optionOffset, diags) { + continue + } appendStatePredicate(state, args, true) case strings.EqualFold(name, "When"): - if len(args) == 1 { - state.When = trimQuote(args[0]) + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue } + state.When = trimQuote(args[0]) case strings.EqualFold(name, "Scope"): - if len(args) == 1 { - state.Scope = trimQuote(args[0]) + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue } + state.Scope = trimQuote(args[0]) case strings.EqualFold(name, "WithType"): - if len(args) == 1 { - ensureStateSchema(state).DataType = trimQuote(args[0]) + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue } + ensureStateSchema(state).DataType = trimQuote(args[0]) case strings.EqualFold(name, "WithCodec"): - if len(args) >= 1 { - state.Output = &st.Codec{ - Name: trimQuote(args[0]), - Args: append([]string{}, trimQuotedArgs(args[1:])...), - } + if !expectStateArgs(state, name, args, 1, -1, dql, optionOffset, diags) { + continue + } + state.Output = &st.Codec{ + Name: trimQuote(args[0]), + Args: append([]string{}, trimQuotedArgs(args[1:])...), } case strings.EqualFold(name, "WithStatusCode"): - if len(args) == 1 { - if value, err := strconv.Atoi(strings.TrimSpace(trimQuote(args[0]))); err == nil { - state.ErrorStatusCode = value - } + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue } + value, err := strconv.Atoi(strings.TrimSpace(trimQuote(args[0]))) + if err != nil { + appendStateOptionDiagnostic(state, name, fmt.Sprintf("invalid status code %q", args[0]), dql, optionOffset, diags) + continue + } + state.ErrorStatusCode = value case strings.EqualFold(name, "WithErrorMessage"): - if len(args) == 1 { - state.ErrorMessage = trimQuote(args[0]) + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue } + state.ErrorMessage = trimQuote(args[0]) case strings.EqualFold(name, "Value"): - if len(args) == 1 { - state.Value = trimQuote(args[0]) + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue + } + state.Value = trimQuote(args[0]) + case strings.EqualFold(name, "Embed"): + if !expectStateArgs(state, name, args, 0, 0, dql, optionOffset, diags) { + continue + } + if !strings.Contains(state.Tag, `anonymous:"true"`) { + if strings.TrimSpace(state.Tag) != "" { + state.Tag += " " + } + state.Tag += `anonymous:"true"` + } + case strings.EqualFold(name, "Cardinality"): + if !expectStateArgs(state, name, args, 1, 1, dql, optionOffset, diags) { + continue + } + card := strings.ToLower(strings.TrimSpace(trimQuote(args[0]))) + switch card { + case "one": + ensureStateSchema(state).Cardinality = st.One + case "many": + ensureStateSchema(state).Cardinality = st.Many + default: + if state != nil && state.In != nil { + kind := strings.ToLower(state.KindString()) + if kind == "view" || kind == "data_view" { + // Declared views already validate cardinality with DQL-VIEW-CARDINALITY. + // Avoid duplicating that diagnostic on the shadow state projection. + continue + } + } + appendStateOptionDiagnostic(state, name, fmt.Sprintf("unsupported cardinality %q", args[0]), dql, optionOffset, diags) } case strings.EqualFold(name, "Async"): + if !expectStateArgs(state, name, args, 0, 0, dql, optionOffset, diags) { + continue + } state.Async = true + default: + if state != nil && state.In != nil { + kind := strings.ToLower(state.KindString()) + if kind == "view" || kind == "data_view" { + // View declarations carry many view-level options (e.g. Cardinality, + // WithURI, WithColumnType). Those are handled by declared-view parsing + // and should not emit state-option diagnostics. + continue + } + } + appendStateOptionDiagnostic(state, name, "unknown option", dql, optionOffset, diags) } } } @@ -222,6 +386,7 @@ func ensureStateSchema(state *plan.State) *st.Schema { type optionCursor struct { raw string cursor int + start int name string args []string } @@ -233,12 +398,14 @@ func newOptionCursor(raw string) *optionCursor { func (o *optionCursor) next() bool { o.name = "" o.args = nil + o.start = 0 for o.cursor < len(o.raw) && (o.raw[o.cursor] == ' ' || o.raw[o.cursor] == '\n' || o.raw[o.cursor] == '\t' || o.raw[o.cursor] == '\r') { o.cursor++ } if o.cursor >= len(o.raw) || o.raw[o.cursor] != '.' { return false } + o.start = o.cursor o.cursor++ start := o.cursor for o.cursor < len(o.raw) { @@ -305,3 +472,29 @@ func (o *optionCursor) next() bool { func (o *optionCursor) option() (string, []string) { return o.name, o.args } + +func expectStateArgs(state *plan.State, option string, args []string, min, max int, dql string, offset int, diags *[]*dqlshape.Diagnostic) bool { + if len(args) < min { + appendStateOptionDiagnostic(state, option, fmt.Sprintf("expected at least %d args, got %d", min, len(args)), dql, offset, diags) + return false + } + if max >= 0 && len(args) > max { + appendStateOptionDiagnostic(state, option, fmt.Sprintf("expected at most %d args, got %d", max, len(args)), dql, offset, diags) + return false + } + return true +} + +func appendStateOptionDiagnostic(state *plan.State, option, detail, dql string, offset int, diags *[]*dqlshape.Diagnostic) { + stateName := "" + if state != nil { + stateName = state.Name + } + *diags = append(*diags, &dqlshape.Diagnostic{ + Code: dqldiag.CodeDeclOptionArgs, + Severity: dqlshape.SeverityWarning, + Message: fmt.Sprintf("invalid %s declaration for state %q: %s", option, stateName, detail), + Hint: "check option name, arity and argument formatting", + Span: relationSpan(dql, offset), + }) +} diff --git a/repository/shape/compile/statedecl_test.go b/repository/shape/compile/statedecl_test.go index 33c9241a4..d8cd344fd 100644 --- a/repository/shape/compile/statedecl_test.go +++ b/repository/shape/compile/statedecl_test.go @@ -1,17 +1,20 @@ package compile import ( + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" "github.com/viant/datly/repository/shape/plan" ) func TestAppendDeclaredStates(t *testing.T) { dql := ` #set($_ = $Jwt(header/Authorization).WithCodec(JwtClaim).WithStatusCode(401)) -#set($_ = $Claims(header/Authorization).WithCodec(JwtClaim)) +#set($_ = $Claims(header/Authorization).WithCodec(JwtClaim).WithTag('json:"claims,omitempty"')) #set($_ = $Name(query/name).WithPredicate(0,'contains','sl','NAME').Optional()) #set($_ = $Fields<[]string>(query/fields).QuerySelector(site_list)) #set($_ = $Meta(output/summary)) @@ -37,6 +40,7 @@ SELECT id FROM SITE_LIST sl` require.NotNil(t, byName["Claims"]) assert.Equal(t, "string", byName["Claims"].Schema.DataType) assert.Equal(t, "*JwtClaims", byName["Claims"].OutputDataType) + assert.Equal(t, `json:"claims,omitempty"`, byName["Claims"].Tag) require.NotNil(t, byName["Name"]) assert.Equal(t, "query", byName["Name"].KindString()) @@ -77,3 +81,121 @@ SELECT id FROM USERS u` require.NotNil(t, result.States[0].Required) assert.True(t, *result.States[0].Required) } + +func TestAppendDeclaredStates_ViewDeclarationBecomesViewInput(t *testing.T) { + dql := ` +#define($_ = $Authorization(view/authorization).Required().WithStatusCode(403) /* SELECT Authorized FROM AUTH */) +SELECT id FROM USERS u` + result := &plan.Result{} + appendDeclaredStates(dql, result) + require.Len(t, result.States, 1) + state := result.States[0] + require.NotNil(t, state) + assert.Equal(t, "Authorization", state.Name) + assert.Equal(t, "view", state.KindString()) + assert.Equal(t, "Authorization", state.In.Name) + require.NotNil(t, state.Required) + assert.True(t, *state.Required) + assert.Equal(t, 403, state.ErrorStatusCode) +} + +func TestAppendDeclaredStates_SkipsSummaryAttachedViewDeclaration(t *testing.T) { + dql := ` +#define($_ = $ProductsMeta(view/products_meta) /* SELECT COUNT(1) CNT FROM ($View.products.SQL) t */) +SELECT vendor.*, products.* +FROM (SELECT * FROM VENDOR t) vendor +JOIN (SELECT * FROM PRODUCT t) products ON products.VENDOR_ID = vendor.ID` + result := &plan.Result{ + Views: []*plan.View{ + {Name: "vendor"}, + {Name: "products", SummaryName: "ProductsMeta"}, + }, + } + + appendDeclaredStates(dql, result) + + require.Empty(t, result.States) +} + +func TestAppendDeclaredStates_EmbedSetsAnonymousTag(t *testing.T) { + dql := ` +#set($_ = $Data(output/view).Embed()) +SELECT id FROM USERS u` + result := &plan.Result{} + appendDeclaredStates(dql, result) + require.Len(t, result.States, 1) + assert.Equal(t, "Data", result.States[0].Name) + assert.Equal(t, "output", result.States[0].KindString()) + assert.Contains(t, result.States[0].Tag, `anonymous:"true"`) +} + +func TestAppendDeclaredStates_OutputViewCardinalityIsParsed(t *testing.T) { + dql := ` +#define($_ = $Data(output/view).Cardinality('One').Embed()) +SELECT id FROM USERS u` + result := &plan.Result{} + + appendDeclaredStates(dql, result) + + require.Len(t, result.States, 1) + require.NotNil(t, result.States[0].Schema) + assert.Equal(t, "output", result.States[0].KindString()) + assert.Equal(t, "One", string(result.States[0].Schema.Cardinality)) + assert.Contains(t, result.States[0].Tag, `anonymous:"true"`) +} + +func TestAppendDeclaredStates_InvalidOption_ReportsExactSpan(t *testing.T) { + dql := ` +#set($_ = $Auth(header/Authorization).Cacheable('x').UnknownFlag()) +SELECT id FROM USERS u` + result := &plan.Result{} + appendDeclaredStates(dql, result) + require.NotEmpty(t, result.Diagnostics) + require.GreaterOrEqual(t, len(result.Diagnostics), 2) + assert.Equal(t, dqldiag.CodeDeclOptionArgs, result.Diagnostics[0].Code) + assert.Equal(t, dqldiag.CodeDeclOptionArgs, result.Diagnostics[1].Code) + + cacheableOffset := strings.Index(dql, ".Cacheable") + require.GreaterOrEqual(t, cacheableOffset, 0) + cacheablePos := dqlpre.PointSpan(dql, cacheableOffset).Start + assert.Equal(t, cacheablePos.Line, result.Diagnostics[0].Span.Start.Line) + assert.Equal(t, cacheablePos.Char, result.Diagnostics[0].Span.Start.Char) + + unknownOffset := strings.Index(dql, ".UnknownFlag") + require.GreaterOrEqual(t, unknownOffset, 0) + unknownPos := dqlpre.PointSpan(dql, unknownOffset).Start + assert.Equal(t, unknownPos.Line, result.Diagnostics[1].Span.Start.Line) + assert.Equal(t, unknownPos.Char, result.Diagnostics[1].Span.Start.Char) +} + +func TestAppendDeclaredStates_InferPathStatesFromRouteDirective(t *testing.T) { + dql := ` +#setting($_ = $route('/v1/api/shape/dev/team/{teamID}', 'DELETE')) +DELETE FROM TEAM WHERE ID = ${teamID}` + result := &plan.Result{} + + appendDeclaredStates(dql, result) + + require.Len(t, result.States, 1) + assert.Equal(t, "teamID", result.States[0].Name) + assert.Equal(t, "path", result.States[0].KindString()) + assert.Equal(t, "teamID", result.States[0].In.Name) + require.NotNil(t, result.States[0].Schema) + assert.Equal(t, "string", result.States[0].Schema.DataType) +} + +func TestAppendDeclaredStates_ExplicitPathStateWinsOverInferredRouteParam(t *testing.T) { + dql := ` +#setting($_ = $route('/v1/api/shape/dev/vendors/{vendorID}', 'GET')) +#define($_ = $VendorID(path/vendorID)) +SELECT * FROM VENDOR WHERE ID = $VendorID` + result := &plan.Result{} + + appendDeclaredStates(dql, result) + + require.Len(t, result.States, 1) + assert.Equal(t, "VendorID", result.States[0].Name) + assert.Equal(t, "path", result.States[0].KindString()) + assert.Equal(t, "vendorID", result.States[0].In.Name) + assert.Equal(t, "int", result.States[0].Schema.DataType) +} diff --git a/repository/shape/compile/type_support.go b/repository/shape/compile/type_support.go index 44a8b9bca..d3faa9329 100644 --- a/repository/shape/compile/type_support.go +++ b/repository/shape/compile/type_support.go @@ -1,13 +1,20 @@ package compile import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" "reflect" "strings" + "time" "github.com/viant/datly/repository/shape" "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/repository/shape/typectx" "github.com/viant/x" + "github.com/viant/xunsafe" ) func applyLinkedTypeSupport(result *plan.Result, source *shape.Source) { @@ -15,9 +22,6 @@ func applyLinkedTypeSupport(result *plan.Result, source *shape.Source) { return } registry := source.EnsureTypeRegistry() - if registry == nil || len(registry.Keys()) == 0 { - return - } resolver := typectx.NewResolver(registry, result.TypeContext) rootTypeKey := resolveRootTypeKey(source, resolver, registry) existing := existingTypesByName(result.Types) @@ -26,11 +30,7 @@ func applyLinkedTypeSupport(result *plan.Result, source *shape.Source) { if item == nil { continue } - resolvedKey := resolveViewTypeKey(item, idx == 0, rootTypeKey, resolver, registry) - if resolvedKey == "" { - continue - } - resolvedType := registry.Lookup(resolvedKey) + resolvedType := resolveViewType(item, idx == 0, rootTypeKey, resolver, registry, result.TypeContext, source) if resolvedType == nil || resolvedType.Type == nil { continue } @@ -38,6 +38,12 @@ func applyLinkedTypeSupport(result *plan.Result, source *shape.Source) { if rType == nil { continue } + item.ElementType = rType + if strings.EqualFold(strings.TrimSpace(item.Cardinality), "many") { + item.FieldType = reflect.SliceOf(rType) + } else { + item.FieldType = rType + } typeExpr, typePkg := schemaTypeExpression(rType, result.TypeContext) if shouldSetSchemaType(item) && typeExpr != "" { item.SchemaType = typeExpr @@ -61,6 +67,22 @@ func applyLinkedTypeSupport(result *plan.Result, source *shape.Source) { } } +func resolveViewType(item *plan.View, root bool, rootTypeKey string, resolver *typectx.Resolver, registry *x.Registry, ctx *typectx.Context, source *shape.Source) *x.Type { + for _, candidate := range viewTypeCandidates(item, root, rootTypeKey) { + if key := resolveTypeKey(candidate, resolver, registry); key != "" { + if registry != nil { + if resolved := registry.Lookup(key); resolved != nil && resolved.Type != nil { + return resolved + } + } + } + if linked := lookupLinkedType(candidate, ctx, source); linked != nil { + return x.NewType(linked) + } + } + return nil +} + func resolveRootTypeKey(source *shape.Source, resolver *typectx.Resolver, registry *x.Registry) string { if source == nil || registry == nil { return "" @@ -75,9 +97,9 @@ func resolveRootTypeKey(source *shape.Source, resolver *typectx.Resolver, regist return resolveTypeKey(x.NewType(rType).Key(), resolver, registry) } -func resolveViewTypeKey(item *plan.View, root bool, rootTypeKey string, resolver *typectx.Resolver, registry *x.Registry) string { - if item == nil || registry == nil { - return "" +func viewTypeCandidates(item *plan.View, root bool, rootTypeKey string) []string { + if item == nil { + return nil } candidates := make([]string, 0, 8) seen := map[string]bool{} @@ -106,12 +128,7 @@ func resolveViewTypeKey(item *plan.View, root bool, rootTypeKey string, resolver appendCandidate(name + "View") appendCandidate(name) } - for _, candidate := range candidates { - if key := resolveTypeKey(candidate, resolver, registry); key != "" { - return key - } - } - return "" + return candidates } func resolveTypeKey(typeExpr string, resolver *typectx.Resolver, registry *x.Registry) string { @@ -236,3 +253,297 @@ func unwrapResolvedType(rType reflect.Type) reflect.Type { } return nil } + +func lookupLinkedType(typeExpr string, ctx *typectx.Context, source *shape.Source) reflect.Type { + base := normalizeTypeLookupKey(typeExpr) + if base == "" { + return nil + } + if pkg, name, ok := splitQualifiedType(base); ok { + if fullPkg := packagePathForAlias(pkg, ctx); fullPkg != "" { + if linked := xunsafe.LookupType(fullPkg + "/" + name); linked != nil { + return linked + } + if linked := lookupASTType(fullPkg, name, ctx, source); linked != nil { + return linked + } + } + return nil + } + if ctx != nil && strings.TrimSpace(ctx.PackagePath) != "" { + if linked := xunsafe.LookupType(strings.TrimSpace(ctx.PackagePath) + "/" + base); linked != nil { + return linked + } + if linked := lookupASTType(strings.TrimSpace(ctx.PackagePath), base, ctx, source); linked != nil { + return linked + } + } + return nil +} + +func splitQualifiedType(value string) (string, string, bool) { + index := strings.Index(value, ".") + if index <= 0 || index+1 >= len(value) { + return "", "", false + } + return strings.TrimSpace(value[:index]), strings.TrimSpace(value[index+1:]), true +} + +func packagePathForAlias(alias string, ctx *typectx.Context) string { + alias = strings.TrimSpace(alias) + if alias == "" || ctx == nil { + return "" + } + for _, item := range ctx.Imports { + if strings.TrimSpace(item.Alias) == alias { + return strings.TrimSpace(item.Package) + } + } + if strings.TrimSpace(ctx.PackageName) == alias { + return strings.TrimSpace(ctx.PackagePath) + } + return "" +} + +func lookupASTType(pkgPath, typeName string, ctx *typectx.Context, source *shape.Source) reflect.Type { + pkgDir := resolveTypePackageDir(pkgPath, ctx, source) + if pkgDir == "" { + return nil + } + return parseNamedStructType(pkgDir, typeName) +} + +func resolveTypePackageDir(pkgPath string, ctx *typectx.Context, source *shape.Source) string { + if ctx == nil { + return "" + } + moduleRoot := nearestModuleRoot(source) + if moduleRoot == "" { + if strings.TrimSpace(ctx.PackagePath) == strings.TrimSpace(pkgPath) { + if dir := strings.TrimSpace(ctx.PackageDir); dir != "" { + if filepath.IsAbs(dir) { + return dir + } + } + } + return "" + } + modulePath := detectModulePath(moduleRoot) + if modulePath != "" { + if rel, ok := packagePathRelative(modulePath, pkgPath); ok { + if rel == "" { + return moduleRoot + } + return filepath.Join(moduleRoot, filepath.FromSlash(rel)) + } + } + if strings.TrimSpace(ctx.PackagePath) == strings.TrimSpace(pkgPath) { + if dir := strings.TrimSpace(ctx.PackageDir); dir != "" { + if filepath.IsAbs(dir) { + return dir + } + return filepath.Join(moduleRoot, filepath.FromSlash(dir)) + } + } + return "" +} + +func packageNameForPath(pkgPath string, ctx *typectx.Context) string { + if ctx != nil && strings.TrimSpace(ctx.PackagePath) == strings.TrimSpace(pkgPath) && strings.TrimSpace(ctx.PackageName) != "" { + return strings.TrimSpace(ctx.PackageName) + } + if index := strings.LastIndex(strings.TrimSpace(pkgPath), "/"); index != -1 { + return strings.TrimSpace(pkgPath[index+1:]) + } + return strings.TrimSpace(pkgPath) +} + +func nearestModuleRoot(source *shape.Source) string { + if source == nil || strings.TrimSpace(source.Path) == "" { + return "" + } + current := filepath.Dir(strings.TrimSpace(source.Path)) + for current != "" && current != string(filepath.Separator) && current != "." { + if _, err := os.Stat(filepath.Join(current, "go.mod")); err == nil { + return current + } + parent := filepath.Dir(current) + if parent == current { + break + } + current = parent + } + return "" +} + +func parseNamedStructType(pkgDir, typeName string) reflect.Type { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, pkgDir, nil, parser.ParseComments) + if err != nil || len(pkgs) == 0 { + return nil + } + specs := map[string]*ast.TypeSpec{} + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.TYPE { + continue + } + for _, spec := range gen.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok || typeSpec.Name == nil { + continue + } + specs[typeSpec.Name.Name] = typeSpec + } + } + } + } + cache := map[string]reflect.Type{} + inProgress := map[string]bool{} + var buildNamed func(name string) reflect.Type + var buildExpr func(expr ast.Expr) reflect.Type + + buildNamed = func(name string) reflect.Type { + if cached, ok := cache[name]; ok { + return cached + } + if inProgress[name] { + return reflect.TypeOf(new(interface{})).Elem() + } + spec := specs[name] + if spec == nil { + return nil + } + inProgress[name] = true + rType := buildExpr(spec.Type) + delete(inProgress, name) + if rType != nil { + cache[name] = rType + } + return rType + } + + buildExpr = func(expr ast.Expr) reflect.Type { + switch actual := expr.(type) { + case *ast.Ident: + switch actual.Name { + case "string": + return reflect.TypeOf("") + case "bool": + return reflect.TypeOf(true) + case "int": + return reflect.TypeOf(int(0)) + case "int8": + return reflect.TypeOf(int8(0)) + case "int16": + return reflect.TypeOf(int16(0)) + case "int32": + return reflect.TypeOf(int32(0)) + case "int64": + return reflect.TypeOf(int64(0)) + case "uint": + return reflect.TypeOf(uint(0)) + case "uint8": + return reflect.TypeOf(uint8(0)) + case "uint16": + return reflect.TypeOf(uint16(0)) + case "uint32": + return reflect.TypeOf(uint32(0)) + case "uint64": + return reflect.TypeOf(uint64(0)) + case "float32": + return reflect.TypeOf(float32(0)) + case "float64": + return reflect.TypeOf(float64(0)) + case "interface{}", "any": + return reflect.TypeOf(new(interface{})).Elem() + default: + return buildNamed(actual.Name) + } + case *ast.StarExpr: + if inner := buildExpr(actual.X); inner != nil { + return reflect.PtrTo(inner) + } + case *ast.ArrayType: + if actual.Len == nil { + if inner := buildExpr(actual.Elt); inner != nil { + return reflect.SliceOf(inner) + } + } + case *ast.MapType: + key := buildExpr(actual.Key) + value := buildExpr(actual.Value) + if key != nil && value != nil { + return reflect.MapOf(key, value) + } + case *ast.InterfaceType: + return reflect.TypeOf(new(interface{})).Elem() + case *ast.SelectorExpr: + if ident, ok := actual.X.(*ast.Ident); ok { + if ident.Name == "time" && actual.Sel != nil && actual.Sel.Name == "Time" { + return reflect.TypeOf(time.Time{}) + } + } + case *ast.StructType: + fields := make([]reflect.StructField, 0, len(actual.Fields.List)) + seen := map[string]bool{} + for _, field := range actual.Fields.List { + if field == nil { + continue + } + fieldType := buildExpr(field.Type) + if fieldType == nil { + continue + } + tag := reflect.StructTag("") + if field.Tag != nil { + tag = reflect.StructTag(strings.Trim(field.Tag.Value, "`")) + } + if len(field.Names) == 0 { + if name := exportedEmbeddedFieldName(field.Type); name != "" { + if seen[name] { + continue + } + seen[name] = true + fields = append(fields, reflect.StructField{Name: name, Type: fieldType, Tag: tag, Anonymous: true}) + } + continue + } + for _, name := range field.Names { + if name == nil || !name.IsExported() { + continue + } + if seen[name.Name] { + continue + } + seen[name.Name] = true + fields = append(fields, reflect.StructField{Name: name.Name, Type: fieldType, Tag: tag}) + } + } + if len(fields) > 0 { + return reflect.StructOf(fields) + } + } + return nil + } + + return buildNamed(typeName) +} + +func exportedEmbeddedFieldName(expr ast.Expr) string { + switch actual := expr.(type) { + case *ast.Ident: + if actual.IsExported() { + return actual.Name + } + case *ast.SelectorExpr: + if actual.Sel != nil && actual.Sel.IsExported() { + return actual.Sel.Name + } + case *ast.StarExpr: + return exportedEmbeddedFieldName(actual.X) + } + return "" +} diff --git a/repository/shape/compile/typectx_defaults.go b/repository/shape/compile/typectx_defaults.go index 5561d36a3..89d4492ec 100644 --- a/repository/shape/compile/typectx_defaults.go +++ b/repository/shape/compile/typectx_defaults.go @@ -5,6 +5,7 @@ import ( "path" "path/filepath" "strings" + "unicode" "github.com/viant/datly/repository/shape" "github.com/viant/datly/repository/shape/typectx" @@ -13,6 +14,7 @@ import ( func applyTypeContextDefaults(ctx *typectx.Context, source *shape.Source, opts *shape.CompileOptions, layout compilePathLayout) *typectx.Context { ret := cloneTypeContext(ctx) + ret = hydrateExplicitTypeContext(ret, source, layout) if shouldInferTypeContext(opts) { ret = mergeTypeContext(ret, inferDatlyGenTypeContext(source, layout)) } @@ -34,6 +36,75 @@ func applyTypeContextDefaults(ctx *typectx.Context, source *shape.Source, opts * return normalizeTypeContext(ret) } +func hydrateExplicitTypeContext(ctx *typectx.Context, source *shape.Source, layout compilePathLayout) *typectx.Context { + if ctx == nil { + return nil + } + if strings.TrimSpace(ctx.PackagePath) == "" && strings.TrimSpace(ctx.DefaultPackage) != "" { + ctx.PackagePath = strings.TrimSpace(ctx.DefaultPackage) + } + if strings.TrimSpace(ctx.PackageName) == "" { + base := path.Base(strings.TrimSpace(ctx.PackagePath)) + if base == "." || base == "/" || base == "" { + base = path.Base(strings.TrimSpace(ctx.DefaultPackage)) + } + ctx.PackageName = sanitizePackageName(base) + } + if strings.TrimSpace(ctx.PackageDir) == "" { + parsed, ok := parseSourceLayout(source, layout) + if ok { + modulePath := detectModulePath(parsed.projectRoot) + pkgPath := strings.TrimSpace(ctx.PackagePath) + if modulePath != "" && pkgPath != "" { + if rel, ok := packagePathRelative(modulePath, pkgPath); ok { + ctx.PackageDir = rel + } + } + } + } + return ctx +} + +func packagePathRelative(modulePath, packagePath string) (string, bool) { + modulePath = strings.Trim(strings.TrimSpace(modulePath), "/") + packagePath = strings.Trim(strings.TrimSpace(packagePath), "/") + if modulePath == "" || packagePath == "" { + return "", false + } + if packagePath == modulePath { + return "", true + } + prefix := modulePath + "/" + if !strings.HasPrefix(packagePath, prefix) { + return "", false + } + return strings.TrimPrefix(packagePath, prefix), true +} + +func sanitizePackageName(name string) string { + name = strings.TrimSpace(strings.ToLower(name)) + if name == "" { + return "" + } + var out strings.Builder + for _, r := range name { + switch { + case r == '_' || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'): + out.WriteRune(r) + case r == '-' || unicode.IsSpace(r): + out.WriteRune('_') + } + } + result := strings.Trim(out.String(), "_") + if result == "" { + return "" + } + if result[0] >= '0' && result[0] <= '9' { + return "p" + result + } + return result +} + func shouldInferTypeContext(opts *shape.CompileOptions) bool { if opts == nil || opts.InferTypeContext == nil { return true diff --git a/repository/shape/compile/typectx_defaults_test.go b/repository/shape/compile/typectx_defaults_test.go index aa3d01d8c..d396fbfc3 100644 --- a/repository/shape/compile/typectx_defaults_test.go +++ b/repository/shape/compile/typectx_defaults_test.go @@ -43,6 +43,17 @@ func TestApplyTypeContextDefaults_Matrix(t *testing.T) { require.Equal(t, "github.com/acme/manual", got.DefaultPackage) }) + t.Run("default package hydrates package path dir and sanitized name", func(t *testing.T) { + input := &typectx.Context{ + DefaultPackage: "github.vianttech.com/viant/platform/pkg/dev/events-one-one", + } + got := applyTypeContextDefaults(input, source, nil, layout) + require.NotNil(t, got) + require.Equal(t, "pkg/dev/events-one-one", got.PackageDir) + require.Equal(t, "events_one_one", got.PackageName) + require.Equal(t, "github.vianttech.com/viant/platform/pkg/dev/events-one-one", got.PackagePath) + }) + t.Run("compile override wins over both", func(t *testing.T) { input := &typectx.Context{ PackageDir: "pkg/manual", diff --git a/repository/shape/compile/viewdecl.go b/repository/shape/compile/viewdecl.go index 359a3712e..44c139a79 100644 --- a/repository/shape/compile/viewdecl.go +++ b/repository/shape/compile/viewdecl.go @@ -12,30 +12,36 @@ import ( ) type declaredView struct { - Name string - SQL string - URI string - Connector string - Cardinality string - Tag string - Codec string - CodecArgs []string - HandlerName string - HandlerArgs []string - StatusCode *int - ErrorMessage string - QuerySelector string - CacheRef string - Limit *int - Cacheable *bool - When string - Scope string - DataType string - Of string - Value string - Async bool - Output bool - Predicates []declaredPredicate + Name string + VirtualSummary bool + SQL string + URI string + Connector string + Cardinality string + Required bool + CardinalitySet bool + Tag string + TypeName string + Dest string + Codec string + CodecArgs []string + HandlerName string + HandlerArgs []string + StatusCode *int + ErrorMessage string + QuerySelector string + CacheRef string + Limit *int + Cacheable *bool + When string + Scope string + DataType string + Of string + Value string + Async bool + Output bool + Predicates []declaredPredicate + ColumnsConfig map[string]*declaredColumnConfig } type declaredPredicate struct { @@ -45,6 +51,11 @@ type declaredPredicate struct { Arguments []string } +type declaredColumnConfig struct { + DataType string + Tag string +} + const ( vdWhitespaceToken = iota vdSetToken @@ -74,11 +85,11 @@ func extractDeclaredViews(dql string) ([]*declaredView, []*dqlshape.Diagnostic) var views []*declaredView var diags []*dqlshape.Diagnostic for _, block := range extractSetBlocks(dql) { - holder, kind, location, tail, ok := parseSetDeclarationBody(block.Body) + holder, kind, location, tail, tailOffset, ok := parseSetDeclarationBody(block.Body) if !ok { continue } - if kind != "view" && kind != "data_view" { + if kind != "view" && kind != "data_view" && !isOutputSummaryDeclaration(kind, location) { continue } sqlText, errorStatusCode := extractDeclarationSQLWithStatus(tail) @@ -92,19 +103,28 @@ func extractDeclaredViews(dql string) ([]*declaredView, []*dqlshape.Diagnostic) }) continue } - name := pipeline.SanitizeName(location) + name := pipeline.SanitizeName(holder) if name == "" { - name = pipeline.SanitizeName(holder) + name = pipeline.SanitizeName(location) } if name == "" { continue } - view := &declaredView{Name: name, SQL: strings.TrimSpace(sqlText)} + view := &declaredView{ + Name: name, + SQL: strings.TrimSpace(sqlText), + VirtualSummary: isOutputSummaryDeclaration(kind, location), + } if errorStatusCode != nil { view.StatusCode = errorStatusCode } - applyDeclaredViewOptions(view, tail, dql, block.Offset, &diags) + applyDeclaredViewOptions(view, tail, dql, block.BodyOffset+tailOffset, &diags) views = append(views, view) } return views, diags } + +func isOutputSummaryDeclaration(kind, location string) bool { + return strings.EqualFold(strings.TrimSpace(kind), "output") && + strings.EqualFold(strings.TrimSpace(location), "summary") +} diff --git a/repository/shape/compile/viewdecl_append.go b/repository/shape/compile/viewdecl_append.go index 6ae2fc663..297f75349 100644 --- a/repository/shape/compile/viewdecl_append.go +++ b/repository/shape/compile/viewdecl_append.go @@ -21,9 +21,17 @@ func appendDeclaredViews(rawDQL string, result *plan.Result) { if item == nil || strings.TrimSpace(item.Name) == "" || strings.TrimSpace(item.SQL) == "" { continue } + if item.VirtualSummary { + if root := lookupRootView(result); root != nil && strings.TrimSpace(root.Summary) == "" { + root.Summary = strings.TrimSpace(item.SQL) + root.SummaryName = strings.TrimSpace(item.Name) + } + continue + } if parent := lookupSummaryParentView(result, item.SQL); parent != nil { if strings.TrimSpace(parent.Summary) == "" { - parent.Summary = strings.TrimSpace(item.SQL) + parent.Summary = normalizeSummarySQLForParent(parent, item.SQL) + parent.SummaryName = strings.TrimSpace(item.Name) } continue } @@ -43,20 +51,23 @@ func appendDeclaredViews(rawDQL string, result *plan.Result) { ElementType: reflect.TypeOf(map[string]interface{}{}), Declaration: buildViewDeclaration(item), } + if item.Required && !item.CardinalitySet { + view.Cardinality = "one" + } if item.Cardinality != "" { view.Cardinality = item.Cardinality } if queryNode, err := sqlparser.ParseQuery(item.SQL); err == nil && queryNode != nil { if inferredName, inferredTable, err := pipeline.InferRoot(queryNode, item.Name); err == nil { - view.Name = inferredName - view.Holder = inferredName - view.Path = inferredName - view.Table = inferredTable + _ = inferredName + if strings.TrimSpace(inferredTable) != "" { + view.Table = inferredTable + } } if fType, eType, card := pipeline.InferProjectionType(queryNode); fType != nil && eType != nil { view.FieldType = fType view.ElementType = eType - if item.Cardinality == "" { + if item.Cardinality == "" && !(item.Required && !item.CardinalitySet) { view.Cardinality = card } } @@ -66,10 +77,48 @@ func appendDeclaredViews(rawDQL string, result *plan.Result) { } } +func normalizeSummarySQLForParent(parent *plan.View, sqlText string) string { + normalized := strings.TrimSpace(sqlText) + if parent == nil || normalized == "" { + return normalized + } + parentName := strings.TrimSpace(parent.Name) + if parentName == "" { + return normalized + } + for _, candidate := range []string{ + "$View." + parentName + ".SQL", + "$view." + strings.ToLower(parentName) + ".sql", + } { + normalized = strings.ReplaceAll(normalized, candidate, "$View.NonWindowSQL") + } + return normalized +} + +func lookupRootView(result *plan.Result) *plan.View { + if result == nil { + return nil + } + if len(result.Views) > 0 && result.Views[0] != nil { + return result.Views[0] + } + for _, item := range result.ViewsByName { + if item != nil { + return item + } + } + return nil +} + func lookupSummaryParentView(result *plan.Result, sqlText string) *plan.View { if result == nil || strings.TrimSpace(sqlText) == "" { return nil } + if hasRootSummaryReference(sqlText) { + if len(result.Views) > 0 && result.Views[0] != nil { + return result.Views[0] + } + } parent, ok := findSummaryParentReference(sqlText) if !ok { return nil @@ -134,6 +183,13 @@ func findSummaryParentReference(input string) (string, bool) { return "", false } +func hasRootSummaryReference(input string) bool { + if strings.TrimSpace(input) == "" { + return false + } + return strings.Contains(strings.ToLower(input), "$view.nonwindowsql") +} + func isCompileIdentifierStart(ch byte) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' } @@ -148,6 +204,8 @@ func buildViewDeclaration(item *declaredView) *plan.ViewDeclaration { } ret := &plan.ViewDeclaration{ Tag: item.Tag, + TypeName: item.TypeName, + Dest: item.Dest, Codec: item.Codec, CodecArgs: append([]string{}, item.CodecArgs...), HandlerName: item.HandlerName, @@ -177,11 +235,27 @@ func buildViewDeclaration(item *declaredView) *plan.ViewDeclaration { }) } } - if ret.Tag == "" && ret.Codec == "" && len(ret.CodecArgs) == 0 && ret.HandlerName == "" && + if len(item.ColumnsConfig) > 0 { + ret.ColumnsConfig = map[string]*plan.ViewColumnConfig{} + for name, cfg := range item.ColumnsConfig { + if strings.TrimSpace(name) == "" || cfg == nil { + continue + } + ret.ColumnsConfig[name] = &plan.ViewColumnConfig{ + DataType: strings.TrimSpace(cfg.DataType), + Tag: strings.TrimSpace(cfg.Tag), + } + } + if len(ret.ColumnsConfig) == 0 { + ret.ColumnsConfig = nil + } + } + if ret.Tag == "" && ret.TypeName == "" && ret.Dest == "" && + ret.Codec == "" && len(ret.CodecArgs) == 0 && ret.HandlerName == "" && len(ret.HandlerArgs) == 0 && ret.StatusCode == nil && ret.ErrorMessage == "" && ret.QuerySelector == "" && ret.CacheRef == "" && ret.Limit == nil && ret.Cacheable == nil && ret.When == "" && ret.Scope == "" && ret.DataType == "" && ret.Of == "" && ret.Value == "" && - !ret.Async && !ret.Output && len(ret.Predicates) == 0 { + !ret.Async && !ret.Output && len(ret.Predicates) == 0 && len(ret.ColumnsConfig) == 0 { return nil } return ret diff --git a/repository/shape/compile/viewdecl_options.go b/repository/shape/compile/viewdecl_options.go index 835b25834..93c3aad29 100644 --- a/repository/shape/compile/viewdecl_options.go +++ b/repository/shape/compile/viewdecl_options.go @@ -49,87 +49,83 @@ func normalizeHintSQLWithStatus(body string) (string, *int) { if body == "" { return "", nil } - var statusCode *int switch body[0] { case '?': body = strings.TrimSpace(body[1:]) case '!': + // Deprecated: legacy `!!NNN` prefix is still supported for backward compatibility. + // Prefer explicit declaration option: .WithStatusCode(NNN). + var statusCode *int body = strings.TrimSpace(body[1:]) if strings.HasPrefix(body, "!") { body = strings.TrimSpace(body[1:]) } if len(body) >= 3 { - var status int - if _, err := fmt.Sscanf(body[:3], "%d", &status); err == nil { - statusCode = &status + var legacyStatus int + if _, err := fmt.Sscanf(body[:3], "%d", &legacyStatus); err == nil { + statusCode = &legacyStatus body = strings.TrimSpace(body[3:]) } } + return strings.TrimSpace(body), statusCode } - return strings.TrimSpace(body), statusCode + return strings.TrimSpace(body), nil } func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, diags *[]*dqlshape.Diagnostic) { if view == nil || strings.TrimSpace(tail) == "" { return } - cursor := parsly.NewCursor("", []byte(tail), 0) - for cursor.Pos < cursor.InputSize { - _ = cursor.MatchOne(vdWhitespaceMatcher) - if cursor.MatchOne(vdDotMatcher).Code != vdDotToken { - cursor.Pos++ - continue - } - _ = cursor.MatchOne(vdWhitespaceMatcher) - name, ok := readIdentifier(cursor) - if !ok { - continue - } - _ = cursor.MatchOne(vdWhitespaceMatcher) - group := cursor.MatchOne(vdExprGroupMatcher) - if group.Code != vdExprGroupToken { - continue - } - content := group.Text(cursor) - if len(content) < 2 { - continue - } - args := splitArgs(content[1 : len(content)-1]) + cursor := newOptionCursor(tail) + for cursor.next() { + name, args := cursor.option() + optionOffset := offset + cursor.start switch { case strings.EqualFold(name, "WithURI"): - if !expectArgs(view, name, args, 1, -1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, -1, dql, optionOffset, diags) { continue } view.URI = trimQuote(args[0]) case strings.EqualFold(name, "WithConnector"), strings.EqualFold(name, "Connector"): - if !expectArgs(view, name, args, 1, -1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, -1, dql, optionOffset, diags) { continue } view.Connector = trimQuote(args[0]) case strings.EqualFold(name, "Cardinality"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } card := strings.ToLower(strings.TrimSpace(trimQuote(args[0]))) switch card { case "one", "many": view.Cardinality = card + view.CardinalitySet = true default: *diags = append(*diags, &dqlshape.Diagnostic{ Code: dqldiag.CodeViewCardinality, Severity: dqlshape.SeverityWarning, Message: fmt.Sprintf("unsupported cardinality %q for declared view %q", args[0], view.Name), Hint: "use Cardinality('one') or Cardinality('many')", - Span: relationSpan(dql, offset), + Span: relationSpan(dql, optionOffset), }) } case strings.EqualFold(name, "WithTag"), strings.EqualFold(name, "Tag"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } view.Tag = trimQuote(args[0]) + case strings.EqualFold(name, "WithTypeName"), strings.EqualFold(name, "TypeName"), strings.EqualFold(name, "Type"): + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { + continue + } + view.TypeName = trimQuote(args[0]) + case strings.EqualFold(name, "WithDest"), strings.EqualFold(name, "Dest"): + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { + continue + } + view.Dest = trimQuote(args[0]) case strings.EqualFold(name, "WithCodec"), strings.EqualFold(name, "Codec"): - if !expectArgs(view, name, args, 1, -1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, -1, dql, optionOffset, diags) { continue } view.Codec = trimQuote(args[0]) @@ -138,7 +134,7 @@ func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, view.CodecArgs = append(view.CodecArgs, strings.TrimSpace(arg)) } case strings.EqualFold(name, "WithHandler"), strings.EqualFold(name, "Handler"): - if !expectArgs(view, name, args, 1, -1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, -1, dql, optionOffset, diags) { continue } view.HandlerName = trimQuote(args[0]) @@ -147,7 +143,7 @@ func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, view.HandlerArgs = append(view.HandlerArgs, strings.TrimSpace(arg)) } case strings.EqualFold(name, "WithStatusCode"), strings.EqualFold(name, "StatusCode"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } statusCode, err := strconv.Atoi(strings.TrimSpace(trimQuote(args[0]))) @@ -157,18 +153,18 @@ func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, Severity: dqlshape.SeverityWarning, Message: fmt.Sprintf("invalid status code %q for declared view %q", args[0], view.Name), Hint: "use numeric status code, e.g. StatusCode(400)", - Span: relationSpan(dql, offset), + Span: relationSpan(dql, optionOffset), }) continue } view.StatusCode = &statusCode case strings.EqualFold(name, "WithErrorMessage"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } view.ErrorMessage = trimQuote(args[0]) case strings.EqualFold(name, "WithPredicate"), strings.EqualFold(name, "Predicate"): - if !expectArgs(view, name, args, 2, -1, dql, offset, diags) { + if !expectArgs(view, name, args, 2, -1, dql, optionOffset, diags) { continue } view.Predicates = append(view.Predicates, declaredPredicate{ @@ -177,7 +173,7 @@ func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, Arguments: append([]string{}, args[2:]...), }) case strings.EqualFold(name, "EnsurePredicate"): - if !expectArgs(view, name, args, 2, -1, dql, offset, diags) { + if !expectArgs(view, name, args, 2, -1, dql, optionOffset, diags) { continue } view.Predicates = append(view.Predicates, declaredPredicate{ @@ -187,7 +183,7 @@ func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, Arguments: append([]string{}, args[2:]...), }) case strings.EqualFold(name, "QuerySelector"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } view.QuerySelector = trimQuote(args[0]) @@ -197,69 +193,105 @@ func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, Severity: dqlshape.SeverityWarning, Message: fmt.Sprintf("query selector %q can only be used with limit, offset, page, fields, orderby", view.QuerySelector), Hint: "use QuerySelector on declarations named limit/offset/page/fields/orderby", - Span: relationSpan(dql, offset), + Span: relationSpan(dql, optionOffset), }) } case strings.EqualFold(name, "WithCache"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } view.CacheRef = trimQuote(args[0]) case strings.EqualFold(name, "WithLimit"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } limit, err := strconv.Atoi(strings.TrimSpace(trimQuote(args[0]))) if err != nil { - appendOptionArgDiagnostic(view, name, fmt.Sprintf("invalid integer limit %q", args[0]), dql, offset, diags) + appendOptionArgDiagnostic(view, name, fmt.Sprintf("invalid integer limit %q", args[0]), dql, optionOffset, diags) continue } view.Limit = &limit case strings.EqualFold(name, "Cacheable"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } value, err := strconv.ParseBool(strings.TrimSpace(trimQuote(args[0]))) if err != nil { - appendOptionArgDiagnostic(view, name, fmt.Sprintf("invalid bool cacheable %q", args[0]), dql, offset, diags) + appendOptionArgDiagnostic(view, name, fmt.Sprintf("invalid bool cacheable %q", args[0]), dql, optionOffset, diags) continue } view.Cacheable = &value case strings.EqualFold(name, "When"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } view.When = trimQuote(args[0]) case strings.EqualFold(name, "Scope"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } view.Scope = trimQuote(args[0]) case strings.EqualFold(name, "WithType"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } view.DataType = trimQuote(args[0]) + case strings.EqualFold(name, "WithColumnType"), strings.EqualFold(name, "ColumnType"): + if !expectArgs(view, name, args, 2, 2, dql, optionOffset, diags) { + continue + } + columnName := strings.TrimSpace(trimQuote(args[0])) + dataType := strings.TrimSpace(trimQuote(args[1])) + if columnName == "" || dataType == "" { + appendOptionArgDiagnostic(view, name, "column name and type must be non-empty", dql, optionOffset, diags) + continue + } + cfg := ensureDeclaredColumnConfig(view, columnName) + cfg.DataType = dataType + case strings.EqualFold(name, "WithColumnTag"), strings.EqualFold(name, "ColumnTag"): + if !expectArgs(view, name, args, 2, 2, dql, optionOffset, diags) { + continue + } + columnName := strings.TrimSpace(trimQuote(args[0])) + tag := strings.TrimSpace(trimQuote(args[1])) + if columnName == "" || tag == "" { + appendOptionArgDiagnostic(view, name, "column name and tag must be non-empty", dql, optionOffset, diags) + continue + } + cfg := ensureDeclaredColumnConfig(view, columnName) + cfg.Tag = tag case strings.EqualFold(name, "Of"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } view.Of = trimQuote(args[0]) case strings.EqualFold(name, "Value"): - if !expectArgs(view, name, args, 1, 1, dql, offset, diags) { + if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue } view.Value = trimQuote(args[0]) case strings.EqualFold(name, "Async"): - if !expectArgs(view, name, args, 0, 0, dql, offset, diags) { + if !expectArgs(view, name, args, 0, 0, dql, optionOffset, diags) { continue } view.Async = true case strings.EqualFold(name, "Output"): - if !expectArgs(view, name, args, 0, 0, dql, offset, diags) { + if !expectArgs(view, name, args, 0, 0, dql, optionOffset, diags) { continue } view.Output = true + case strings.EqualFold(name, "Required"): + if !expectArgs(view, name, args, 0, 0, dql, optionOffset, diags) { + continue + } + view.Required = true + case strings.EqualFold(name, "Optional"): + if !expectArgs(view, name, args, 0, 0, dql, optionOffset, diags) { + continue + } + view.Required = false + default: + appendOptionArgDiagnostic(view, name, "unknown option", dql, optionOffset, diags) } } } @@ -392,3 +424,15 @@ func isAllowedQuerySelector(name string) bool { return false } } + +func ensureDeclaredColumnConfig(view *declaredView, columnName string) *declaredColumnConfig { + if view.ColumnsConfig == nil { + view.ColumnsConfig = map[string]*declaredColumnConfig{} + } + cfg := view.ColumnsConfig[columnName] + if cfg == nil { + cfg = &declaredColumnConfig{} + view.ColumnsConfig[columnName] = cfg + } + return cfg +} diff --git a/repository/shape/compile/viewdecl_parse.go b/repository/shape/compile/viewdecl_parse.go index 51fd45a9d..dd6469df2 100644 --- a/repository/shape/compile/viewdecl_parse.go +++ b/repository/shape/compile/viewdecl_parse.go @@ -8,8 +8,9 @@ import ( ) type setBlock struct { - Offset int - Body string + Offset int + BodyOffset int + Body string } func extractSetBlocks(dql string) []setBlock { @@ -26,26 +27,29 @@ func extractSetBlocks(dql string) []setBlock { if group.Code != vdExprGroupToken { continue } + groupText := group.Text(cursor) + groupStart := cursor.Pos - len(groupText) body := group.Text(cursor) if len(body) < 2 { continue } result = append(result, setBlock{ - Offset: offset, - Body: body[1 : len(body)-1], + Offset: offset, + BodyOffset: groupStart + 1, + Body: body[1 : len(body)-1], }) } return result } -func parseSetDeclarationBody(body string) (holder, kind, location, tail string, ok bool) { +func parseSetDeclarationBody(body string) (holder, kind, location, tail string, tailOffset int, ok bool) { cursor := parsly.NewCursor("", []byte(body), 0) if cursor.MatchAfterOptional(vdWhitespaceMatcher, vdParamDeclMatcher).Code != vdParamDeclToken { - return "", "", "", "", false + return "", "", "", "", 0, false } id, matched := readIdentifier(cursor) if !matched { - return "", "", "", "", false + return "", "", "", "", 0, false } holder = id _ = cursor.MatchOne(vdWhitespaceMatcher) @@ -53,21 +57,22 @@ func parseSetDeclarationBody(body string) (holder, kind, location, tail string, _ = cursor.MatchOne(vdWhitespaceMatcher) kindLoc := cursor.MatchOne(vdExprGroupMatcher) if kindLoc.Code != vdExprGroupToken { - return "", "", "", "", false + return "", "", "", "", 0, false } inGroup := kindLoc.Text(cursor) if len(inGroup) < 2 { - return "", "", "", "", false + return "", "", "", "", 0, false } raw := strings.TrimSpace(inGroup[1 : len(inGroup)-1]) slash := strings.Index(raw, "/") if slash == -1 { - return "", "", "", "", false + return "", "", "", "", 0, false } kind = strings.ToLower(strings.TrimSpace(raw[:slash])) location = strings.TrimSpace(raw[slash+1:]) + tailOffset = cursor.Pos tail = strings.TrimSpace(string(cursor.Input[cursor.Pos:])) - return holder, kind, location, tail, true + return holder, kind, location, tail, tailOffset, true } func readIdentifier(cursor *parsly.Cursor) (string, bool) { diff --git a/repository/shape/compile/viewdecl_test.go b/repository/shape/compile/viewdecl_test.go index 0136c64a1..a4cd80543 100644 --- a/repository/shape/compile/viewdecl_test.go +++ b/repository/shape/compile/viewdecl_test.go @@ -1,11 +1,13 @@ package compile import ( + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" dqldiag "github.com/viant/datly/repository/shape/dql/diag" + dqlpre "github.com/viant/datly/repository/shape/dql/preprocess" dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" ) @@ -21,12 +23,13 @@ func TestViewDecl_ExtractSetBlocks(t *testing.T) { } func TestViewDecl_ParseSetDeclarationBody(t *testing.T) { - holder, kind, location, tail, ok := parseSetDeclarationBody("$_ = $Extra(view/extra_view).WithURI('/x')") + holder, kind, location, tail, tailOffset, ok := parseSetDeclarationBody("$_ = $Extra(view/extra_view).WithURI('/x')") require.True(t, ok) assert.Equal(t, "Extra", holder) assert.Equal(t, "view", kind) assert.Equal(t, "extra_view", location) assert.Contains(t, tail, ".WithURI('/x')") + assert.Greater(t, tailOffset, 0) } func TestViewDecl_ApplyOptions_InvalidCardinality(t *testing.T) { @@ -55,13 +58,21 @@ func TestViewDecl_AppendDeclaredViews(t *testing.T) { assert.True(t, found) } +func TestViewDecl_ExtractDeclarationSQLWithLegacyStatusPrefix(t *testing.T) { + sqlText, status := extractDeclarationSQLWithStatus("/* !!403 SELECT id FROM EXTRA e */") + assert.Equal(t, "SELECT id FROM EXTRA e", sqlText) + require.NotNil(t, status) + assert.Equal(t, 403, *status) +} + func TestViewDecl_ApplyOptions_Extended(t *testing.T) { view := &declaredView{Name: "limit"} var diags []*dqlshape.Diagnostic tail := ".WithTag('json:\"id\"').WithCodec(AsJSON,'x').WithHandler('Build',a,b)." + "WithStatusCode(422).WithErrorMessage('bad req').WithPredicate('ByID','id = ?', 101)." + "EnsurePredicate('Tenant','tenant_id = ?', 7).QuerySelector('qs').WithCache('c1').WithLimit(10)." + - "Cacheable(true).When('x > 1').Scope('team').WithType('[]Order').Of('list').Value('abc').Async().Output()" + "Cacheable(true).When('x > 1').Scope('team').Type('OrderView').Dest('orders.go').WithType('[]Order')." + + "WithColumnType('Authorized','bool').WithColumnTag('Authorized','internal:\"true\"').Of('list').Value('abc').Async().Output()" applyDeclaredViewOptions(view, tail, "SELECT 1", 0, &diags) require.Empty(t, diags) @@ -89,7 +100,13 @@ func TestViewDecl_ApplyOptions_Extended(t *testing.T) { assert.True(t, *view.Cacheable) assert.Equal(t, "x > 1", view.When) assert.Equal(t, "team", view.Scope) + assert.Equal(t, "OrderView", view.TypeName) + assert.Equal(t, "orders.go", view.Dest) assert.Equal(t, "[]Order", view.DataType) + require.NotNil(t, view.ColumnsConfig) + require.Contains(t, view.ColumnsConfig, "Authorized") + assert.Equal(t, "bool", view.ColumnsConfig["Authorized"].DataType) + assert.Equal(t, `internal:"true"`, view.ColumnsConfig["Authorized"].Tag) assert.Equal(t, "list", view.Of) assert.Equal(t, "abc", view.Value) assert.True(t, view.Async) @@ -104,6 +121,27 @@ func TestViewDecl_ApplyOptions_QuerySelectorValidation(t *testing.T) { assert.Equal(t, dqldiag.CodeDeclQuerySelector, diags[0].Code) } +func TestViewDecl_ApplyOptions_ExactSpanAndUnknownOption(t *testing.T) { + dql := "#set($_ = $Extra(view/extra).UnknownOpt('x').WithLimit('x') /* SELECT id FROM EXTRA e */)" + declared, diags := extractDeclaredViews(dql) + require.NotEmpty(t, declared) + require.Len(t, diags, 2) + assert.Equal(t, dqldiag.CodeDeclOptionArgs, diags[0].Code) + assert.Equal(t, dqldiag.CodeDeclOptionArgs, diags[1].Code) + + unknownOffset := strings.Index(dql, ".UnknownOpt") + require.GreaterOrEqual(t, unknownOffset, 0) + unknownPos := dqlpre.PointSpan(dql, unknownOffset).Start + assert.Equal(t, unknownPos.Line, diags[0].Span.Start.Line) + assert.Equal(t, unknownPos.Char, diags[0].Span.Start.Char) + + limitOffset := strings.Index(dql, ".WithLimit") + require.GreaterOrEqual(t, limitOffset, 0) + limitPos := dqlpre.PointSpan(dql, limitOffset).Start + assert.Equal(t, limitPos.Line, diags[1].Span.Start.Line) + assert.Equal(t, limitPos.Char, diags[1].Span.Start.Char) +} + func TestViewDecl_SplitArgs_Nested(t *testing.T) { args := splitArgs(`'a', fn(1,2), {'k': [1,2]}, "x,y"`) require.Len(t, args, 4) @@ -117,7 +155,8 @@ func TestViewDecl_AppendDeclaredViews_ExtendedDeclarationMetadata(t *testing.T) dql := "#set($_ = $limit(view/limit).WithTag('json:\"id\"').WithCodec(AsJSON).WithHandler('Build',a)." + "WithStatusCode(409).WithErrorMessage('conflict').WithPredicate('ByID','id=?',1)." + "EnsurePredicate('Tenant','tenant=?',2).QuerySelector('items').WithCache('c1').WithLimit(5)." + - "Cacheable(false).When('x').Scope('s').WithType('Order').Of('o').Value('v').Async().Output() /* SELECT id FROM EXTRA e */)" + "Cacheable(false).When('x').Scope('s').Type('OrderView').Dest('order.go').WithType('Order')." + + "WithColumnType('Authorized','bool').WithColumnTag('Authorized','internal:\"true\"').Of('o').Value('v').Async().Output() /* SELECT id FROM EXTRA e */)" result := &plan.Result{ ViewsByName: map[string]*plan.View{}, ByPath: map[string]*plan.Field{}, @@ -126,7 +165,7 @@ func TestViewDecl_AppendDeclaredViews_ExtendedDeclarationMetadata(t *testing.T) require.NotEmpty(t, result.Views) var target *plan.View for _, item := range result.Views { - if item != nil && item.Name == "e" { + if item != nil && item.Name == "limit" { target = item break } @@ -147,7 +186,13 @@ func TestViewDecl_AppendDeclaredViews_ExtendedDeclarationMetadata(t *testing.T) assert.False(t, *target.Declaration.Cacheable) assert.Equal(t, "x", target.Declaration.When) assert.Equal(t, "s", target.Declaration.Scope) + assert.Equal(t, "OrderView", target.Declaration.TypeName) + assert.Equal(t, "order.go", target.Declaration.Dest) assert.Equal(t, "Order", target.Declaration.DataType) + require.NotNil(t, target.Declaration.ColumnsConfig) + require.Contains(t, target.Declaration.ColumnsConfig, "Authorized") + assert.Equal(t, "bool", target.Declaration.ColumnsConfig["Authorized"].DataType) + assert.Equal(t, `internal:"true"`, target.Declaration.ColumnsConfig["Authorized"].Tag) assert.Equal(t, "o", target.Declaration.Of) assert.Equal(t, "v", target.Declaration.Value) assert.True(t, target.Declaration.Async) @@ -172,6 +217,99 @@ func TestViewDecl_AppendDeclaredViews_AttachSummaryFromMetaViewSQL(t *testing.T) assert.Contains(t, root.Summary, "$View.browser.SQL") } +func TestViewDecl_AppendDeclaredViews_AttachSummaryFromOutputSummarySQL(t *testing.T) { + root := &plan.View{Name: "Vendor", Path: "Vendor", Holder: "Vendor"} + result := &plan.Result{ + Views: []*plan.View{root}, + ViewsByName: map[string]*plan.View{"Vendor": root}, + ByPath: map[string]*plan.Field{}, + } + dql := "#define($_ = $Meta(output/summary) /* SELECT COUNT(1) CNT FROM ($View.vendor.SQL) t */)" + + appendDeclaredViews(dql, result) + + require.Len(t, result.Views, 1) + require.NotNil(t, root) + assert.Contains(t, root.Summary, "COUNT(1)") + assert.Contains(t, root.Summary, "$View.vendor.SQL") +} + +func TestViewDecl_AppendDeclaredViews_AttachSummaryFromOutputSummaryNonWindowSQL(t *testing.T) { + root := &plan.View{Name: "Vendor", Path: "Vendor", Holder: "Vendor"} + result := &plan.Result{ + Views: []*plan.View{root}, + ViewsByName: map[string]*plan.View{"Vendor": root}, + ByPath: map[string]*plan.Field{}, + } + dql := "#define($_ = $Meta(output/summary) /* SELECT COUNT(1) CNT FROM ($View.NonWindowSQL) t */)" + + appendDeclaredViews(dql, result) + + require.Len(t, result.Views, 1) + require.NotNil(t, root) + assert.Contains(t, root.Summary, "COUNT(1)") + assert.Contains(t, root.Summary, "$View.NonWindowSQL") +} + +func TestViewDecl_AppendDeclaredViews_AttachSummaryToReferencedChildView(t *testing.T) { + root := &plan.View{Name: "Vendor", Path: "Vendor", Holder: "Vendor"} + child := &plan.View{Name: "products", Path: "products", Holder: "Products"} + result := &plan.Result{ + Views: []*plan.View{root, child}, + ViewsByName: map[string]*plan.View{ + "Vendor": root, + "products": child, + }, + ByPath: map[string]*plan.Field{}, + } + dql := "#define($_ = $ProductsMeta(view/products_meta) /* SELECT COUNT(1) CNT FROM ($View.products.SQL) t */)" + + appendDeclaredViews(dql, result) + + require.Len(t, result.Views, 2) + require.NotNil(t, child) + assert.Equal(t, "ProductsMeta", child.SummaryName) + assert.Contains(t, child.Summary, "COUNT(1)") + assert.Contains(t, child.Summary, "$View.NonWindowSQL") + assert.Empty(t, root.Summary) +} + +func TestViewDecl_AppendDeclaredViews_OutputSummaryWithoutRoot_DoesNotCreateView(t *testing.T) { + result := &plan.Result{ + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + } + dql := "#define($_ = $Meta(output/summary) /* SELECT COUNT(1) CNT FROM ($View.NonWindowSQL) t */)" + + appendDeclaredViews(dql, result) + + assert.Empty(t, result.Views) +} + +func TestViewDecl_RequiredImpliesOneCardinalityByDefault(t *testing.T) { + dql := "#define($_ = $Authorization(view/authorization).Required() /* SELECT Authorized FROM AUTH */)" + result := &plan.Result{ + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + } + appendDeclaredViews(dql, result) + + require.Len(t, result.Views, 1) + assert.Equal(t, "one", strings.ToLower(result.Views[0].Cardinality)) +} + +func TestViewDecl_ExplicitCardinalityOverridesRequiredDefault(t *testing.T) { + dql := "#define($_ = $Authorization(view/authorization).Required().Cardinality('many') /* SELECT Authorized FROM AUTH */)" + result := &plan.Result{ + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + } + appendDeclaredViews(dql, result) + + require.Len(t, result.Views, 1) + assert.Equal(t, "many", strings.ToLower(result.Views[0].Cardinality)) +} + func TestViewDecl_AppendDeclaredViews_MetaViewSQL_NoParentFallbackToView(t *testing.T) { result := &plan.Result{ ViewsByName: map[string]*plan.View{}, diff --git a/repository/shape/componenttag/component.go b/repository/shape/componenttag/component.go new file mode 100644 index 000000000..d9c0890b5 --- /dev/null +++ b/repository/shape/componenttag/component.go @@ -0,0 +1,99 @@ +package componenttag + +import ( + "fmt" + "reflect" + "strings" + + tagtags "github.com/viant/tagly/tags" +) + +const TagName = "component" + +type Component struct { + Name string + Path string + Method string + Connector string + Marshaller string + Handler string + Input string + Output string + View string + Source string + Summary string +} + +type Tag struct { + Component *Component +} + +func (c *Component) Tag() *tagtags.Tag { + if c == nil { + return nil + } + builder := &strings.Builder{} + builder.WriteString(c.Name) + appendNonEmpty(builder, "path", c.Path) + appendNonEmpty(builder, "method", c.Method) + appendNonEmpty(builder, "connector", c.Connector) + appendNonEmpty(builder, "marshaller", c.Marshaller) + appendNonEmpty(builder, "handler", c.Handler) + appendNonEmpty(builder, "input", c.Input) + appendNonEmpty(builder, "output", c.Output) + appendNonEmpty(builder, "view", c.View) + appendNonEmpty(builder, "source", c.Source) + appendNonEmpty(builder, "summary", c.Summary) + return &tagtags.Tag{Name: TagName, Values: tagtags.Values(builder.String())} +} + +func Parse(tag reflect.StructTag) (*Tag, error) { + tagValue, ok := tag.Lookup(TagName) + if !ok { + return &Tag{}, nil + } + name, values := tagtags.Values(tagValue).Name() + component := &Component{Name: name} + if err := values.MatchPairs(func(key, value string) error { + switch strings.ToLower(strings.TrimSpace(key)) { + case "name": + component.Name = strings.TrimSpace(value) + case "path": + component.Path = strings.TrimSpace(value) + case "method": + component.Method = strings.TrimSpace(value) + case "connector": + component.Connector = strings.TrimSpace(value) + case "marshaller": + component.Marshaller = strings.TrimSpace(value) + case "handler": + component.Handler = strings.TrimSpace(value) + case "input": + component.Input = strings.TrimSpace(value) + case "output": + component.Output = strings.TrimSpace(value) + case "view": + component.View = strings.TrimSpace(value) + case "source": + component.Source = strings.TrimSpace(value) + case "summary": + component.Summary = strings.TrimSpace(value) + default: + return fmt.Errorf("unsupported component tag option: '%s'", key) + } + return nil + }); err != nil { + return nil, err + } + return &Tag{Component: component}, nil +} + +func appendNonEmpty(builder *strings.Builder, key, value string) { + if value == "" { + return + } + builder.WriteString(",") + builder.WriteString(key) + builder.WriteString("=") + builder.WriteString(value) +} diff --git a/repository/shape/dql/decl/calls.go b/repository/shape/dql/decl/calls.go new file mode 100644 index 000000000..ac8016973 --- /dev/null +++ b/repository/shape/dql/decl/calls.go @@ -0,0 +1,97 @@ +package decl + +import ( + "strings" + + "github.com/viant/parsly" +) + +// Call represents a parsed function call with offsets in the scanned input. +type Call struct { + Name string + Args []string + Offset int + EndOffset int + Dollar bool +} + +// CallParseError represents a malformed call span. +type CallParseError struct { + Name string + Offset int + Message string +} + +// CallScanOptions controls call scanning behavior. +type CallScanOptions struct { + AllowedNames map[string]bool + RequireDollar bool + AllowDollar bool + Strict bool +} + +// ScanCalls parses function calls and returns parsed calls plus malformed-call errors. +func ScanCalls(input string, options CallScanOptions) ([]Call, []CallParseError) { + calls := make([]Call, 0) + parseErrors := make([]CallParseError, 0) + cursor := parsly.NewCursor("", []byte(input), 0) + for cursor.Pos < cursor.InputSize { + matched := cursor.MatchAfterOptional( + whitespaceMatcher, + commentBlockMatcher, + singleQuotedMatcher, + doubleQuotedMatcher, + dollarIdentifierMatcher, + identifierMatcher, + anyMatcher, + ) + switch matched.Code { + case dollarIdentifierToken, identifierToken: + rawName := matched.Text(cursor) + hasDollar := matched.Code == dollarIdentifierToken + name := strings.ToLower(strings.TrimPrefix(rawName, "$")) + if options.AllowedNames != nil && !options.AllowedNames[name] { + continue + } + if options.RequireDollar && !hasDollar { + continue + } + if !options.AllowDollar && hasDollar { + continue + } + nameOffset := matched.Offset + block := cursor.MatchAfterOptional(whitespaceMatcher, parenthesesBlockMatcher) + if block.Code != parenthesesBlockToken { + if options.Strict { + parseErrors = append(parseErrors, CallParseError{ + Name: name, + Offset: nameOffset, + Message: "invalid call syntax, expected (...)", + }) + } + continue + } + blockText := block.Text(cursor) + argsText := "" + if len(blockText) >= 2 { + argsText = blockText[1 : len(blockText)-1] + } + calls = append(calls, Call{ + Name: name, + Args: splitArgs(argsText), + Offset: nameOffset, + EndOffset: block.Offset + len(blockText), + Dollar: hasDollar, + }) + case parsly.Invalid: + if options.Strict { + parseErrors = append(parseErrors, CallParseError{ + Offset: cursor.Pos, + Message: "invalid token while scanning calls", + }) + } + cursor.Pos++ + } + } + return calls, parseErrors +} diff --git a/repository/shape/dql/decl/calls_test.go b/repository/shape/dql/decl/calls_test.go new file mode 100644 index 000000000..c467f977f --- /dev/null +++ b/repository/shape/dql/decl/calls_test.go @@ -0,0 +1,52 @@ +package decl + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScanCalls_DollarStrict(t *testing.T) { + input := "$connector('dev') $dest('a.go')" + calls, errs := ScanCalls(input, CallScanOptions{ + AllowedNames: map[string]bool{"connector": true, "dest": true}, + RequireDollar: true, + AllowDollar: true, + Strict: true, + }) + require.Empty(t, errs) + require.Len(t, calls, 2) + assert.Equal(t, "connector", calls[0].Name) + assert.Equal(t, []string{"'dev'"}, calls[0].Args) + assert.True(t, calls[0].Dollar) + assert.Equal(t, "dest", calls[1].Name) +} + +func TestScanCalls_ReportsMalformedCallOffset(t *testing.T) { + input := "$dest('a.go'" + calls, errs := ScanCalls(input, CallScanOptions{ + AllowedNames: map[string]bool{"dest": true}, + RequireDollar: true, + AllowDollar: true, + Strict: true, + }) + require.Empty(t, calls) + require.Len(t, errs, 1) + assert.Equal(t, "dest", errs[0].Name) + assert.Equal(t, 0, errs[0].Offset) +} + +func TestScanCalls_BareOnly(t *testing.T) { + input := "dest(vendor,'vendor.go'), type(vendor,'Vendor'), $dest('x.go')" + calls, errs := ScanCalls(input, CallScanOptions{ + AllowedNames: map[string]bool{"dest": true, "type": true}, + RequireDollar: false, + AllowDollar: false, + Strict: false, + }) + require.Empty(t, errs) + require.Len(t, calls, 2) + assert.Equal(t, "dest", calls[0].Name) + assert.Equal(t, "type", calls[1].Name) +} diff --git a/repository/shape/dql/decl/lex.go b/repository/shape/dql/decl/lex.go index fbf6270ac..7d9e5ed74 100644 --- a/repository/shape/dql/decl/lex.go +++ b/repository/shape/dql/decl/lex.go @@ -11,6 +11,7 @@ const ( doubleQuotedToken commentBlockToken parenthesesBlockToken + dollarIdentifierToken identifierToken anyToken ) @@ -21,6 +22,7 @@ var doubleQuotedMatcher = parsly.NewToken(doubleQuotedToken, "DoubleQuote", matc var commentBlockMatcher = parsly.NewToken(commentBlockToken, "CommentBlock", matcher.NewSeqBlock("/*", "*/")) var parenthesesBlockMatcher = parsly.NewToken(parenthesesBlockToken, "Parentheses", matcher.NewBlock('(', ')', '\\')) +var dollarIdentifierMatcher = parsly.NewToken(dollarIdentifierToken, "DollarIdentifier", &dollarIdentifierMatch{}) var identifierMatcher = parsly.NewToken(identifierToken, "Identifier", &identifierMatch{}) var anyMatcher = parsly.NewToken(anyToken, "Any", &anyMatch{}) @@ -35,6 +37,29 @@ func (a *anyMatch) Match(cursor *parsly.Cursor) int { type identifierMatch struct{} +type dollarIdentifierMatch struct{} + +func (d *dollarIdentifierMatch) Match(cursor *parsly.Cursor) int { + if cursor.Pos >= cursor.InputSize { + return 0 + } + if cursor.Input[cursor.Pos] != '$' { + return 0 + } + next := cursor.Pos + 1 + if next >= cursor.InputSize { + return 0 + } + if !isIdentifierStart(cursor.Input[next]) { + return 0 + } + pos := next + 1 + for pos < cursor.InputSize && isIdentifierPart(cursor.Input[pos]) { + pos++ + } + return pos - cursor.Pos +} + func (i *identifierMatch) Match(cursor *parsly.Cursor) int { if cursor.Pos >= cursor.InputSize { return 0 diff --git a/repository/shape/dql/diag/codes.go b/repository/shape/dql/diag/codes.go index c4de8559f..70a08506e 100644 --- a/repository/shape/dql/diag/codes.go +++ b/repository/shape/dql/diag/codes.go @@ -18,6 +18,12 @@ const ( CodeDirDateFormat = "DQL-DIR-DATE-FORMAT" CodeDirCaseFormat = "DQL-DIR-CASE-FORMAT" CodeDirConst = "DQL-DIR-CONST" + CodeDirDest = "DQL-DIR-DEST" + CodeDirInputDest = "DQL-DIR-INPUT-DEST" + CodeDirOutputDest = "DQL-DIR-OUTPUT-DEST" + CodeDirRouterDest = "DQL-DIR-ROUTER-DEST" + CodeDirInputType = "DQL-DIR-INPUT-TYPE" + CodeDirOutputType = "DQL-DIR-OUTPUT-TYPE" CodeDirUnsupported = "DQL-DIR-UNSUPPORTED" CodeOptParse = "DQL-OPT-PARSE" diff --git a/repository/shape/dql/preprocess/directive_parser.go b/repository/shape/dql/preprocess/directive_parser.go index 9a13320cc..edc5e98bd 100644 --- a/repository/shape/dql/preprocess/directive_parser.go +++ b/repository/shape/dql/preprocess/directive_parser.go @@ -1,45 +1,54 @@ package preprocess -import "strings" +import ( + "strings" + + "github.com/viant/datly/repository/shape/dql/decl" +) type directiveCall struct { name string args []string start int + end int +} + +type directiveParseError struct { + name string + start int + message string } func scanDollarCalls(input string, names map[string]bool) []directiveCall { + calls, _ := scanDollarCallsStrict(input, names) + return calls +} + +func scanDollarCallsStrict(input string, names map[string]bool) ([]directiveCall, []directiveParseError) { + parsed, parseErrors := decl.ScanCalls(input, decl.CallScanOptions{ + AllowedNames: names, + RequireDollar: true, + AllowDollar: true, + Strict: true, + }) result := make([]directiveCall, 0) - for i := 0; i < len(input); { - if input[i] != '$' || i+1 >= len(input) || !isIdentifierStart(input[i+1]) { - i++ - continue - } - start := i + 1 - i += 2 - for i < len(input) && isIdentifierPart(input[i]) { - i++ - } - name := strings.ToLower(input[start:i]) - if !names[name] { - continue - } - j := skipSpaces(input, i) - if j >= len(input) || input[j] != '(' { - continue - } - body, end, ok := readCallBody(input, j) - if !ok { - continue - } + for _, call := range parsed { result = append(result, directiveCall{ - name: name, - args: splitCallArgs(body), - start: start - 1, + name: call.Name, + args: call.Args, + start: call.Offset, + end: call.EndOffset, + }) + } + errs := make([]directiveParseError, 0, len(parseErrors)) + for _, parseErr := range parseErrors { + errs = append(errs, directiveParseError{ + name: parseErr.Name, + start: parseErr.Offset, + message: parseErr.Message, }) - i = end + 1 } - return result + return result, errs } func readCallBody(input string, openParen int) (string, int, bool) { diff --git a/repository/shape/dql/preprocess/extract.go b/repository/shape/dql/preprocess/extract.go index 67b761a41..e73308041 100644 --- a/repository/shape/dql/preprocess/extract.go +++ b/repository/shape/dql/preprocess/extract.go @@ -18,11 +18,13 @@ func extractSQLAndContext(dql string) (string, *typectx.Context, *dqlshape.Direc blocks := extractSetDirectiveBlocks(dql) for _, block := range blocks { - applyMask(mask, dql, block.start, block.end) + if shouldMaskDirectiveBlock(block) { + applyMask(mask, dql, block.start, block.end) + } if block.kind != directiveSettings { continue } - diagnostics = append(diagnostics, parseSettingsDirectives(block.body, dql, block.start, directives)...) + diagnostics = append(diagnostics, parseSettingsDirectives(block.body, dql, block.bodyStart, directives)...) } lines := strings.SplitAfter(dql, "\n") @@ -43,6 +45,10 @@ func extractSQLAndContext(dql string) (string, *typectx.Context, *dqlshape.Direc } if kind := lineDirectiveKind(trimmed); kind != directiveUnknown { if !hasMasked(mask, lineStart, lineEnd) { + if kind != directiveSettings && !shouldMaskDirectiveLine(kind, trimmed) { + offset += len(line) + continue + } if kind != directiveSettings { applyMask(mask, dql, lineStart, lineEnd) offset += len(line) @@ -72,6 +78,38 @@ func extractSQLAndContext(dql string) (string, *typectx.Context, *dqlshape.Direc return string(masked), ctx, directives, diagnostics } +func shouldMaskDirectiveBlock(block setDirectiveBlock) bool { + switch block.kind { + case directiveSettings, directiveDefine: + return true + case directiveSet: + return isDeclarationDirectiveBody(block.body) + default: + return false + } +} + +func shouldMaskDirectiveLine(kind directiveKind, line string) bool { + switch kind { + case directiveSettings, directiveDefine: + return true + case directiveSet: + start := strings.Index(line, "(") + end := strings.LastIndex(line, ")") + if start == -1 || end <= start { + return false + } + return isDeclarationDirectiveBody(line[start+1 : end]) + default: + return false + } +} + +func isDeclarationDirectiveBody(body string) bool { + text := strings.TrimSpace(body) + return strings.HasPrefix(text, "$_") +} + func applyMask(mask []bool, text string, start, end int) { if start < 0 { start = 0 diff --git a/repository/shape/dql/preprocess/preprocess.go b/repository/shape/dql/preprocess/preprocess.go index a09e1b90a..28d63f70e 100644 --- a/repository/shape/dql/preprocess/preprocess.go +++ b/repository/shape/dql/preprocess/preprocess.go @@ -41,12 +41,30 @@ func Prepare(dql string) *Result { ret.Optimized = optimized sanitized := dqlsanitize.Rewrite(optimized, dqlsanitize.Options{ Declared: dqlsanitize.Declared(optimized), + Foreach: dqlsanitize.ForeachDeclared(optimized), + Consts: constNames(ret.Directives), }) ret.SQL = sanitized.SQL ret.Mapper = newMapper(len(optimized), sanitized.Patches, sanitized.TrimPrefix, dql) return ret } +func constNames(directives *dqlshape.Directives) map[string]bool { + if directives == nil || len(directives.Const) == 0 { + return nil + } + result := make(map[string]bool, len(directives.Const)) + for name := range directives.Const { + if trimmed := strings.TrimSpace(name); trimmed != "" { + result[trimmed] = true + } + } + if len(result) == 0 { + return nil + } + return result +} + func stripDecorators(sql string) string { if strings.TrimSpace(sql) == "" { return sql @@ -72,6 +90,9 @@ func isStandaloneDecoratorLine(line string) bool { if open <= 0 || close <= open { return false } + if strings.TrimSpace(trimmed[close+1:]) != "" { + return false + } name := strings.ToLower(strings.TrimSpace(trimmed[:open])) switch name { case "use_connector", "allow_nulls", "allownulls", "tag", "cast", "required", "cardinality", "set_limit": @@ -115,6 +136,12 @@ func normalizeDirectives(input *dqlshape.Directives) *dqlshape.Directives { ret := &dqlshape.Directives{ Meta: strings.TrimSpace(input.Meta), DefaultConnector: strings.TrimSpace(input.DefaultConnector), + Dest: strings.TrimSpace(input.Dest), + InputDest: strings.TrimSpace(input.InputDest), + OutputDest: strings.TrimSpace(input.OutputDest), + RouterDest: strings.TrimSpace(input.RouterDest), + InputType: strings.TrimSpace(input.InputType), + OutputType: strings.TrimSpace(input.OutputType), JSONMarshalType: strings.TrimSpace(input.JSONMarshalType), JSONUnmarshalType: strings.TrimSpace(input.JSONUnmarshalType), XMLUnmarshalType: strings.TrimSpace(input.XMLUnmarshalType), @@ -124,8 +151,12 @@ func normalizeDirectives(input *dqlshape.Directives) *dqlshape.Directives { } if input.Cache != nil { ret.Cache = &dqlshape.CacheDirective{ - Enabled: input.Cache.Enabled, - TTL: strings.TrimSpace(input.Cache.TTL), + Enabled: input.Cache.Enabled, + TTL: strings.TrimSpace(input.Cache.TTL), + Name: strings.TrimSpace(input.Cache.Name), + Provider: strings.TrimSpace(input.Cache.Provider), + Location: strings.TrimSpace(input.Cache.Location), + TimeToLiveMs: input.Cache.TimeToLiveMs, } } if input.MCP != nil { @@ -153,7 +184,10 @@ func normalizeDirectives(input *dqlshape.Directives) *dqlshape.Directives { ret.Const[k] = v } } - if ret.Meta == "" && ret.DefaultConnector == "" && ret.Cache == nil && ret.MCP == nil && ret.Route == nil && + if ret.Meta == "" && ret.DefaultConnector == "" && + ret.Dest == "" && ret.InputDest == "" && ret.OutputDest == "" && ret.RouterDest == "" && + ret.InputType == "" && ret.OutputType == "" && + ret.Cache == nil && ret.MCP == nil && ret.Route == nil && ret.JSONMarshalType == "" && ret.JSONUnmarshalType == "" && ret.XMLUnmarshalType == "" && ret.Format == "" && ret.DateFormat == "" && ret.CaseFormat == "" && len(ret.Const) == 0 { return nil diff --git a/repository/shape/dql/preprocess/preprocess_test.go b/repository/shape/dql/preprocess/preprocess_test.go index 337b9ca41..8ea4b3480 100644 --- a/repository/shape/dql/preprocess/preprocess_test.go +++ b/repository/shape/dql/preprocess/preprocess_test.go @@ -1,6 +1,7 @@ package preprocess import ( + "strings" "testing" "github.com/stretchr/testify/assert" @@ -53,6 +54,58 @@ FROM t` assert.NotContains(t, pre.DirectSQL, ",\nFROM") } +func TestPrepare_PreservesSQLCastProjection(t *testing.T) { + dql := `SELECT + CAST($Var3 AS SIGNED) AS Key3, + cast(status, 'int') +FROM t` + pre := Prepare(dql) + require.NotNil(t, pre) + assert.Contains(t, pre.DirectSQL, "CAST($Var3 AS SIGNED) AS Key3") + assert.NotContains(t, pre.DirectSQL, "cast(status, 'int')") +} + +func TestPrepare_PreservesExecControlDirectives(t *testing.T) { + dql := "#define($_ = $Ids<[]int>(body/Ids))\n" + + "#foreach($rec in $Unsafe.Records)\n" + + "#if($rec.IS_AUTH == 0)\n" + + "$logger.Fatal('x')\n" + + "#else\n" + + "UPDATE PRODUCT SET STATUS = $Status WHERE ID = $rec.ID;\n" + + "#end\n" + + "#end" + pre := Prepare(dql) + require.NotNil(t, pre) + assert.Contains(t, pre.SQL, "#foreach($rec in $Unsafe.Records)") + assert.Contains(t, pre.SQL, "#if($rec.IS_AUTH == 0)") + assert.Contains(t, pre.SQL, "#else") + assert.Contains(t, pre.SQL, "#end") +} + +func TestPrepare_PreservesLocalSetDirectivesInExecTemplate(t *testing.T) { + dql := "#define($_ = $Ids<[]int>(query/Ids))\n" + + "#set($byID = $Unsafe.Rows.IndexBy(\"ID\"))\n" + + "#foreach($id in $Unsafe.Ids)\n" + + " #set($row = $byID[$id])\n" + + " UPDATE T SET ACTIVE = 0 WHERE ID = $id;\n" + + "#end" + pre := Prepare(dql) + require.NotNil(t, pre) + assert.Contains(t, pre.SQL, "#set($byID = $Unsafe.Rows.IndexBy(\"ID\"))") + assert.Contains(t, pre.SQL, "#set($row = $byID[$id])") + assert.NotContains(t, pre.SQL, "#define($_ = $Ids<[]int>(query/Ids))") +} + +func TestPrepare_ConstDirective_UsesUnsafeSelectors(t *testing.T) { + dql := "#setting($_ = $const('Vendor','VENDOR'))\n" + + "SELECT * FROM ${Vendor} t WHERE t.ID = $id" + pre := Prepare(dql) + require.NotNil(t, pre) + assert.Contains(t, pre.SQL, "FROM ${Unsafe.Vendor} t") + assert.Contains(t, pre.SQL, "$criteria.AppendBinding($Unsafe.id)") + assert.NotContains(t, pre.SQL, "${criteria.AppendBinding($Unsafe.Vendor)}") +} + func TestPrepare_MultilineSetDirective_TypeContext(t *testing.T) { dql := "#package('a/b')\n#import('x','github.com/acme/x')\nSELECT id FROM t" pre := Prepare(dql) @@ -78,6 +131,12 @@ func TestPrepare_InvalidMultilineImportDiagnostic(t *testing.T) { func TestPrepare_SpecialDirectives(t *testing.T) { dql := "#settings($_ = $meta('docs/orders.md'))\n" + "#setting($_ = $connector('analytics'))\n" + + "#setting($_ = $dest('vendor.go'))\n" + + "#setting($_ = $input_dest('vendor_input.go'))\n" + + "#setting($_ = $output_dest('vendor_output.go'))\n" + + "#setting($_ = $router_dest('vendor_router.go'))\n" + + "#setting($_ = $input_type('VendorInput'))\n" + + "#setting($_ = $output_type('VendorOutput'))\n" + "#settings($_ = $cache(true, '5m'))\n" + "#settings($_ = $mcp('orders.search', 'Search orders', 'docs/mcp/orders.md'))\n" + "#settings($_ = $marshal('application/json','pkg.OrderJSON'))\n" + @@ -92,6 +151,12 @@ func TestPrepare_SpecialDirectives(t *testing.T) { require.NotNil(t, pre.Directives) assert.Equal(t, "docs/orders.md", pre.Directives.Meta) assert.Equal(t, "analytics", pre.Directives.DefaultConnector) + assert.Equal(t, "vendor.go", pre.Directives.Dest) + assert.Equal(t, "vendor_input.go", pre.Directives.InputDest) + assert.Equal(t, "vendor_output.go", pre.Directives.OutputDest) + assert.Equal(t, "vendor_router.go", pre.Directives.RouterDest) + assert.Equal(t, "VendorInput", pre.Directives.InputType) + assert.Equal(t, "VendorOutput", pre.Directives.OutputType) require.NotNil(t, pre.Directives.Cache) assert.True(t, pre.Directives.Cache.Enabled) assert.Equal(t, "5m", pre.Directives.Cache.TTL) @@ -107,6 +172,27 @@ func TestPrepare_SpecialDirectives(t *testing.T) { assert.Equal(t, "lc", pre.Directives.CaseFormat) } +func TestPrepare_InvalidDestDirectiveDiagnostic(t *testing.T) { + dql := "SELECT 1\n#settings($_ = $dest())" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirDest, pre.Diagnostics[0].Code) + assert.Equal(t, 2, pre.Diagnostics[0].Span.Start.Line) +} + +func TestPrepare_CacheProviderDirective(t *testing.T) { + dql := "#setting($_ = $cache('aerospike').WithProvider('aerospike://127.0.0.1:3000/test').WithLocation('${view.Name}').WithTimeToLiveMs(3600000))\nSELECT 1" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotNil(t, pre.Directives) + require.NotNil(t, pre.Directives.Cache) + assert.Equal(t, "aerospike", pre.Directives.Cache.Name) + assert.Equal(t, "aerospike://127.0.0.1:3000/test", pre.Directives.Cache.Provider) + assert.Equal(t, "${view.Name}", pre.Directives.Cache.Location) + assert.Equal(t, 3600000, pre.Directives.Cache.TimeToLiveMs) +} + func TestPrepare_InvalidSpecialDirectiveDiagnostic(t *testing.T) { dql := "SELECT 1\n#settings($_ = $mcp())" pre := Prepare(dql) @@ -125,6 +211,28 @@ func TestPrepare_InvalidConnectorDirectiveDiagnostic(t *testing.T) { assert.Equal(t, 2, pre.Diagnostics[0].Span.Start.Line) } +func TestPrepare_InvalidDirective_UsesExactCallSpan(t *testing.T) { + lineText := "#settings($_ = $dest())" + dql := "SELECT 1\n" + lineText + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirDest, pre.Diagnostics[0].Code) + assert.Equal(t, 2, pre.Diagnostics[0].Span.Start.Line) + assert.Equal(t, strings.Index(lineText, "$dest(")+1, pre.Diagnostics[0].Span.Start.Char) +} + +func TestPrepare_MalformedDirective_UsesExactCallSpan(t *testing.T) { + lineText := "#settings($_ = $dest('x'" + dql := "SELECT 1\n" + lineText + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotEmpty(t, pre.Diagnostics) + assert.Equal(t, dqldiag.CodeDirDest, pre.Diagnostics[0].Code) + assert.Equal(t, 2, pre.Diagnostics[0].Span.Start.Line) + assert.Equal(t, strings.Index(lineText, "$dest(")+1, pre.Diagnostics[0].Span.Start.Char) +} + func TestPrepare_RouteDirective(t *testing.T) { dql := "SELECT 1\n#settings($_ = $route('/v1/api/orders', 'GET', 'POST', 'PATCH'))" pre := Prepare(dql) diff --git a/repository/shape/dql/preprocess/scanner.go b/repository/shape/dql/preprocess/scanner.go index 3b7513917..f60031149 100644 --- a/repository/shape/dql/preprocess/scanner.go +++ b/repository/shape/dql/preprocess/scanner.go @@ -16,10 +16,11 @@ var ( ) type setDirectiveBlock struct { - start int - end int - body string - kind directiveKind + start int + end int + bodyStart int + body string + kind directiveKind } type directiveKind int @@ -41,9 +42,6 @@ func isDirectiveLine(line string) bool { if isSetLine(line) { return true } - if strings.HasPrefix(line, "#if(") || strings.HasPrefix(line, "#elseif(") || strings.HasPrefix(line, "#else") || strings.HasPrefix(line, "#end") { - return true - } return false } @@ -76,10 +74,11 @@ func extractSetDirectiveBlocks(dql string) []setDirectiveBlock { } end := cursor.Pos result = append(result, setDirectiveBlock{ - start: start, - end: end, - body: groupText[1 : len(groupText)-1], - kind: kind, + start: start, + end: end, + bodyStart: group.Offset + 1, + body: groupText[1 : len(groupText)-1], + kind: kind, }) } return result diff --git a/repository/shape/dql/preprocess/settings_directives.go b/repository/shape/dql/preprocess/settings_directives.go index 6a140fa67..53e1eb92b 100644 --- a/repository/shape/dql/preprocess/settings_directives.go +++ b/repository/shape/dql/preprocess/settings_directives.go @@ -2,6 +2,8 @@ package preprocess import ( "net/http" + "regexp" + "strconv" "strings" "github.com/viant/datly/repository/content" @@ -22,6 +24,15 @@ var ( formatDirectiveName = map[string]bool{"format": true} dateFormatDirectiveName = map[string]bool{"date_format": true} caseFormatDirectiveName = map[string]bool{"case_format": true} + destDirectiveName = map[string]bool{"dest": true} + inputDestDirectiveName = map[string]bool{"input_dest": true} + outputDestDirectiveName = map[string]bool{"output_dest": true} + routerDestDirectiveName = map[string]bool{"router_dest": true} + inputTypeDirectiveName = map[string]bool{"input_type": true} + outputTypeDirectiveName = map[string]bool{"output_type": true} + cacheProviderExpr = regexp.MustCompile(`(?i)\.withprovider\s*\(\s*['"]([^'"]+)['"]\s*\)`) + cacheLocationExpr = regexp.MustCompile(`(?i)\.withlocation\s*\(\s*['"]([^'"]+)['"]\s*\)`) + cacheTTLMsExpr = regexp.MustCompile(`(?i)\.withtimetolivems\s*\(\s*([0-9]+)\s*\)`) ) func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, directives *dqlshape.Directives) []*dqlshape.Diagnostic { @@ -40,49 +51,73 @@ func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, direct )) } if strings.Contains(lower, "$meta") { - values := parseMetaDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, metaDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirMeta, fullDQL, diagnosticOffset) + values := parseMetaDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirMeta, "invalid $meta directive", "expected: #settings($_ = $meta('relative/or/absolute/path'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirMeta, "invalid $meta directive", "expected: #settings($_ = $meta('relative/or/absolute/path'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { directives.Meta = values[len(values)-1] } } if strings.Contains(lower, "$connector") { - values := parseConnectorDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, connectorDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirConnector, fullDQL, diagnosticOffset) + values := parseConnectorDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirConnector, "invalid $connector directive", "expected: #settings($_ = $connector('connector_name'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirConnector, "invalid $connector directive", "expected: #settings($_ = $connector('connector_name'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { directives.DefaultConnector = values[len(values)-1] } } if strings.Contains(lower, "$cache") { - values := parseCacheDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, cacheDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirCache, fullDQL, diagnosticOffset) + values := parseCacheDirectiveCalls(input, calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirCache, "invalid $cache directive", "expected: #settings($_ = $cache(true, '5m'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirCache, "invalid $cache directive", "expected: #settings($_ = $cache(true, '5m')) or #setting($_ = $cache('name').WithProvider('...').WithLocation('...').WithTimeToLiveMs(1000))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { directives.Cache = values[len(values)-1] } } if strings.Contains(lower, "$mcp") { - values := parseMCPDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, mcpDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirMCP, fullDQL, diagnosticOffset) + values := parseMCPDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirMCP, "invalid $mcp directive", "expected: #settings($_ = $mcp('tool.name','description','docs/path.md'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirMCP, "invalid $mcp directive", "expected: #settings($_ = $mcp('tool.name','description','docs/path.md'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { directives.MCP = values[len(values)-1] } } if strings.Contains(lower, "$route") { - values := parseRouteDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, routeDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirRoute, fullDQL, diagnosticOffset) + values := parseRouteDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirRoute, "invalid $route directive", "expected: #settings($_ = $route('/v1/api/path','GET','POST'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirRoute, "invalid $route directive", "expected: #settings($_ = $route('/v1/api/path','GET','POST'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { directives.Route = values[len(values)-1] } } if strings.Contains(lower, "$const") { - values := parseConstDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, constDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirConst, fullDQL, diagnosticOffset) + values := parseConstDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirConst, "invalid $const directive", "expected: #settings($_ = $const('Name','VALUE'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirConst, "invalid $const directive", "expected: #settings($_ = $const('Name','VALUE'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { if directives.Const == nil { directives.Const = map[string]string{} @@ -93,17 +128,25 @@ func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, direct } } if strings.Contains(lower, "$marshal") { - values := parseMarshalDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, marshalDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirMarshal, fullDQL, diagnosticOffset) + values := parseMarshalDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirMarshal, "invalid $marshal directive", "expected: #settings($_ = $marshal('application/json','pkg.Type'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirMarshal, "invalid $marshal directive", "expected: #settings($_ = $marshal('application/json','pkg.Type'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { directives.JSONMarshalType = values[len(values)-1] } } if strings.Contains(lower, "$unmarshal") { - values := parseUnmarshalDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, unmarshalDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirUnmarshal, fullDQL, diagnosticOffset) + values := parseUnmarshalDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirUnmarshal, "invalid $unmarshal directive", "expected: #settings($_ = $unmarshal('application/json','pkg.Type'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirUnmarshal, "invalid $unmarshal directive", "expected: #settings($_ = $unmarshal('application/json','pkg.Type'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { last := values[len(values)-1] if last.JSONType != "" { @@ -115,34 +158,187 @@ func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, direct } } if strings.Contains(lower, "$format") { - values := parseFormatDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, formatDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirFormat, fullDQL, diagnosticOffset) + values := parseFormatDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirFormat, "invalid $format directive", "expected: #settings($_ = $format('tabular_json'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirFormat, "invalid $format directive", "expected: #settings($_ = $format('tabular_json'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { directives.Format = values[len(values)-1] } } if strings.Contains(lower, "$date_format") { - values := parseDateFormatDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, dateFormatDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirDateFormat, fullDQL, diagnosticOffset) + values := parseDateFormatDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirDateFormat, "invalid $date_format directive", "expected: #settings($_ = $date_format('2006-01-02'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirDateFormat, "invalid $date_format directive", "expected: #settings($_ = $date_format('2006-01-02'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { directives.DateFormat = values[len(values)-1] } } if strings.Contains(lower, "$case_format") { - values := parseCaseFormatDirectives(input) + calls, parseErrors := scanDollarCallsStrict(input, caseFormatDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirCaseFormat, fullDQL, diagnosticOffset) + values := parseCaseFormatDirectiveCalls(calls) if len(values) == 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirCaseFormat, "invalid $case_format directive", "expected: #settings($_ = $case_format('lc'))", fullDQL, diagnosticOffset)) + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirCaseFormat, "invalid $case_format directive", "expected: #settings($_ = $case_format('lc'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } } else { directives.CaseFormat = values[len(values)-1] } } + if strings.Contains(lower, "$dest") { + calls, parseErrors := scanDollarCallsStrict(input, destDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirDest, fullDQL, diagnosticOffset) + values := parseSingleArgQuotedDirectiveCalls(calls) + if len(values) == 0 { + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirDest, "invalid $dest directive", "expected: #settings($_ = $dest('file.go'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } + } else { + directives.Dest = values[len(values)-1] + } + } + if strings.Contains(lower, "$input_dest") { + calls, parseErrors := scanDollarCallsStrict(input, inputDestDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirInputDest, fullDQL, diagnosticOffset) + values := parseSingleArgQuotedDirectiveCalls(calls) + if len(values) == 0 { + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirInputDest, "invalid $input_dest directive", "expected: #settings($_ = $input_dest('input.go'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } + } else { + directives.InputDest = values[len(values)-1] + } + } + if strings.Contains(lower, "$output_dest") { + calls, parseErrors := scanDollarCallsStrict(input, outputDestDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirOutputDest, fullDQL, diagnosticOffset) + values := parseSingleArgQuotedDirectiveCalls(calls) + if len(values) == 0 { + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirOutputDest, "invalid $output_dest directive", "expected: #settings($_ = $output_dest('output.go'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } + } else { + directives.OutputDest = values[len(values)-1] + } + } + if strings.Contains(lower, "$router_dest") { + calls, parseErrors := scanDollarCallsStrict(input, routerDestDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirRouterDest, fullDQL, diagnosticOffset) + values := parseSingleArgQuotedDirectiveCalls(calls) + if len(values) == 0 { + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirRouterDest, "invalid $router_dest directive", "expected: #settings($_ = $router_dest('router.go'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } + } else { + directives.RouterDest = values[len(values)-1] + } + } + if strings.Contains(lower, "$input_type") { + calls, parseErrors := scanDollarCallsStrict(input, inputTypeDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirInputType, fullDQL, diagnosticOffset) + values := parseSingleArgQuotedDirectiveCalls(calls) + if len(values) == 0 { + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirInputType, "invalid $input_type directive", "expected: #settings($_ = $input_type('TypeName'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } + } else { + directives.InputType = values[len(values)-1] + } + } + if strings.Contains(lower, "$output_type") { + calls, parseErrors := scanDollarCallsStrict(input, outputTypeDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirOutputType, fullDQL, diagnosticOffset) + values := parseSingleArgQuotedDirectiveCalls(calls) + if len(values) == 0 { + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirOutputType, "invalid $output_type directive", "expected: #settings($_ = $output_type('TypeName'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } + } else { + directives.OutputType = values[len(values)-1] + } + } + return diagnostics +} + +func appendDirectiveParseErrors(diagnostics []*dqlshape.Diagnostic, parseErrors []directiveParseError, code, fullDQL string, diagnosticOffset int) []*dqlshape.Diagnostic { + for _, parseErr := range parseErrors { + message := "invalid directive syntax" + if parseErr.name != "" { + message = "invalid $" + parseErr.name + " directive" + } + diagnostics = append(diagnostics, directiveDiagnostic(code, message, "fix malformed directive call syntax", fullDQL, diagnosticOffset+parseErr.start)) + } return diagnostics } +func lastDirectiveCallOffset(calls []directiveCall, diagnosticOffset int) int { + if len(calls) == 0 { + return diagnosticOffset + } + return diagnosticOffset + calls[len(calls)-1].start +} + +func parseDestDirectives(input string) []string { + calls := scanDollarCalls(input, destDirectiveName) + return parseSingleArgQuotedDirectiveCalls(calls) +} + +func parseInputDestDirectives(input string) []string { + calls := scanDollarCalls(input, inputDestDirectiveName) + return parseSingleArgQuotedDirectiveCalls(calls) +} + +func parseOutputDestDirectives(input string) []string { + calls := scanDollarCalls(input, outputDestDirectiveName) + return parseSingleArgQuotedDirectiveCalls(calls) +} + +func parseRouterDestDirectives(input string) []string { + calls := scanDollarCalls(input, routerDestDirectiveName) + return parseSingleArgQuotedDirectiveCalls(calls) +} + +func parseInputTypeDirectives(input string) []string { + calls := scanDollarCalls(input, inputTypeDirectiveName) + return parseSingleArgQuotedDirectiveCalls(calls) +} + +func parseOutputTypeDirectives(input string) []string { + calls := scanDollarCalls(input, outputTypeDirectiveName) + return parseSingleArgQuotedDirectiveCalls(calls) +} + +func parseSingleArgQuotedDirectiveCalls(calls []directiveCall) []string { + result := make([]string, 0, len(calls)) + for _, call := range calls { + if len(call.args) != 1 { + continue + } + value, ok := parseQuotedLiteral(call.args[0]) + if !ok { + continue + } + if value = strings.TrimSpace(value); value != "" { + result = append(result, value) + } + } + return result +} + func parseMetaDirectives(input string) []string { calls := scanDollarCalls(input, metaDirectiveName) + return parseMetaDirectiveCalls(calls) +} + +func parseMetaDirectiveCalls(calls []directiveCall) []string { result := make([]string, 0, len(calls)) for _, call := range calls { if len(call.args) != 1 { @@ -161,6 +357,10 @@ func parseMetaDirectives(input string) []string { func parseConnectorDirectives(input string) []string { calls := scanDollarCalls(input, connectorDirectiveName) + return parseConnectorDirectiveCalls(calls) +} + +func parseConnectorDirectiveCalls(calls []directiveCall) []string { result := make([]string, 0, len(calls)) for _, call := range calls { if len(call.args) != 1 { @@ -179,36 +379,81 @@ func parseConnectorDirectives(input string) []string { func parseCacheDirectives(input string) []*dqlshape.CacheDirective { calls := scanDollarCalls(input, cacheDirectiveName) + return parseCacheDirectiveCalls(input, calls) +} + +func parseCacheDirectiveCalls(input string, calls []directiveCall) []*dqlshape.CacheDirective { result := make([]*dqlshape.CacheDirective, 0, len(calls)) for _, call := range calls { if len(call.args) == 0 || len(call.args) > 2 { continue } - enabledRaw := strings.TrimSpace(call.args[0]) - var enabled bool - switch { - case strings.EqualFold(enabledRaw, "true"): - enabled = true - case strings.EqualFold(enabledRaw, "false"): - enabled = false - default: + firstArg := strings.TrimSpace(call.args[0]) + if strings.EqualFold(firstArg, "true") || strings.EqualFold(firstArg, "false") { + ttl := "" + if len(call.args) == 2 { + value, ok := parseQuotedLiteral(call.args[1]) + if !ok { + continue + } + ttl = strings.TrimSpace(value) + } + result = append(result, &dqlshape.CacheDirective{ + Enabled: strings.EqualFold(firstArg, "true"), + TTL: ttl, + }) + continue + } + name, ok := parseQuotedLiteral(firstArg) + if !ok { + continue + } + name = strings.TrimSpace(name) + if name == "" { + continue + } + tail := "" + if call.end > 0 && call.end <= len(input) { + tail = input[call.end:] + } + cacheDirective := &dqlshape.CacheDirective{ + Enabled: true, + Name: name, + } + if match := cacheProviderExpr.FindStringSubmatch(tail); len(match) > 1 { + cacheDirective.Provider = strings.TrimSpace(match[1]) + } + if match := cacheLocationExpr.FindStringSubmatch(tail); len(match) > 1 { + cacheDirective.Location = strings.TrimSpace(match[1]) + } + if match := cacheTTLMsExpr.FindStringSubmatch(tail); len(match) > 1 { + if ttlMs, err := strconv.Atoi(strings.TrimSpace(match[1])); err == nil && ttlMs > 0 { + cacheDirective.TimeToLiveMs = ttlMs + } + } + if cacheDirective.Provider == "" || cacheDirective.Location == "" || cacheDirective.TimeToLiveMs <= 0 { continue } - ttl := "" if len(call.args) == 2 { value, ok := parseQuotedLiteral(call.args[1]) - if !ok { - continue + if ok { + cacheDirective.TTL = strings.TrimSpace(value) } - ttl = strings.TrimSpace(value) } - result = append(result, &dqlshape.CacheDirective{Enabled: enabled, TTL: ttl}) + if cacheDirective.TTL == "" { + cacheDirective.TTL = strconv.Itoa(cacheDirective.TimeToLiveMs) + "ms" + } + result = append(result, cacheDirective) } return result } func parseMCPDirectives(input string) []*dqlshape.MCPDirective { calls := scanDollarCalls(input, mcpDirectiveName) + return parseMCPDirectiveCalls(calls) +} + +func parseMCPDirectiveCalls(calls []directiveCall) []*dqlshape.MCPDirective { result := make([]*dqlshape.MCPDirective, 0, len(calls)) for _, call := range calls { if len(call.args) < 1 || len(call.args) > 3 { @@ -249,6 +494,10 @@ func parseMCPDirectives(input string) []*dqlshape.MCPDirective { func parseRouteDirectives(input string) []*dqlshape.RouteDirective { calls := scanDollarCalls(input, routeDirectiveName) + return parseRouteDirectiveCalls(calls) +} + +func parseRouteDirectiveCalls(calls []directiveCall) []*dqlshape.RouteDirective { result := make([]*dqlshape.RouteDirective, 0, len(calls)) for _, call := range calls { if len(call.args) == 0 { @@ -322,6 +571,10 @@ func normalizeHTTPMethods(input []string) ([]string, bool) { func parseMarshalDirectives(input string) []string { calls := scanDollarCalls(input, marshalDirectiveName) + return parseMarshalDirectiveCalls(calls) +} + +func parseMarshalDirectiveCalls(calls []directiveCall) []string { result := make([]string, 0, len(calls)) for _, call := range calls { if len(call.args) != 2 { @@ -353,6 +606,10 @@ type unmarshalDirectiveValue struct { func parseUnmarshalDirectives(input string) []unmarshalDirectiveValue { calls := scanDollarCalls(input, unmarshalDirectiveName) + return parseUnmarshalDirectiveCalls(calls) +} + +func parseUnmarshalDirectiveCalls(calls []directiveCall) []unmarshalDirectiveValue { result := make([]unmarshalDirectiveValue, 0, len(calls)) for _, call := range calls { if len(call.args) != 2 { @@ -387,6 +644,10 @@ func parseUnmarshalDirectives(input string) []unmarshalDirectiveValue { func parseFormatDirectives(input string) []string { calls := scanDollarCalls(input, formatDirectiveName) + return parseFormatDirectiveCalls(calls) +} + +func parseFormatDirectiveCalls(calls []directiveCall) []string { result := make([]string, 0, len(calls)) for _, call := range calls { if len(call.args) != 1 { @@ -409,6 +670,10 @@ func parseFormatDirectives(input string) []string { func parseDateFormatDirectives(input string) []string { calls := scanDollarCalls(input, dateFormatDirectiveName) + return parseDateFormatDirectiveCalls(calls) +} + +func parseDateFormatDirectiveCalls(calls []directiveCall) []string { result := make([]string, 0, len(calls)) for _, call := range calls { if len(call.args) != 1 { @@ -427,6 +692,10 @@ func parseDateFormatDirectives(input string) []string { func parseCaseFormatDirectives(input string) []string { calls := scanDollarCalls(input, caseFormatDirectiveName) + return parseCaseFormatDirectiveCalls(calls) +} + +func parseCaseFormatDirectiveCalls(calls []directiveCall) []string { result := make([]string, 0, len(calls)) for _, call := range calls { if len(call.args) != 1 { @@ -450,6 +719,10 @@ func parseCaseFormatDirectives(input string) []string { func parseConstDirectives(input string) [][2]string { calls := scanDollarCalls(input, constDirectiveName) + return parseConstDirectiveCalls(calls) +} + +func parseConstDirectiveCalls(calls []directiveCall) [][2]string { var result [][2]string for _, call := range calls { if len(call.args) != 2 { diff --git a/repository/shape/dql/sanitize/context_test.go b/repository/shape/dql/sanitize/context_test.go new file mode 100644 index 000000000..b19424763 --- /dev/null +++ b/repository/shape/dql/sanitize/context_test.go @@ -0,0 +1,118 @@ +package sanitize + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/velty" +) + +type criteriaContextMock struct{} + +type criteriaContextCollector struct { + Args []interface{} +} + +func (c *criteriaContextCollector) AppendBinding(value interface{}) string { + c.Args = append(c.Args, value) + return "?" +} + +type predicateContextMock struct{} + +func (p predicateContextMock) Builder() *predicateBuilderContextMock { + return &predicateBuilderContextMock{} +} + +func (p predicateContextMock) FilterGroup(group int, op string) string { + return fmt.Sprintf("P%d:%s", group, op) +} + +type predicateBuilderContextMock struct { + value string +} + +func (b *predicateBuilderContextMock) CombineOr(group string) *predicateBuilderContextMock { + b.value = group + return b +} + +func (b *predicateBuilderContextMock) Build(kind string) string { + switch kind { + case "AND": + return " AND (" + b.value + ") " + case "WHERE": + return " WHERE (" + b.value + ") " + default: + return "" + } +} + +type sqlContextMock struct{} + +func (s sqlContextMock) Eq(column string, value interface{}) string { + return fmt.Sprintf("%s = %v", column, value) +} + +type unsafeContextMock struct { + VendorID int +} + +func TestRenderVelty_WithShapeContext_DataDriven(t *testing.T) { + testCases := []struct { + name string + template string + expect string + args []interface{} + }{ + { + name: "criteria append binding", + template: "SELECT * FROM VENDOR t WHERE t.ID = $criteria.AppendBinding($Unsafe.VendorID)", + expect: "t.ID = ?", + args: []interface{}{101}, + }, + { + name: "predicate builder chain", + template: "SELECT * FROM PRODUCT t WHERE 1=1 ${predicate.Builder().CombineOr($predicate.FilterGroup(0, \"AND\")).Build(\"AND\")}", + expect: " AND (P0:AND) ", + }, + { + name: "sql helper", + template: "SELECT * FROM VENDOR t WHERE $sql.Eq(\"ID\", $Unsafe.VendorID)", + expect: "ID = 101", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + actual, args := renderVeltyWithShapeContext(t, testCase.template) + assert.Contains(t, actual, testCase.expect) + if len(testCase.args) > 0 { + assert.Equal(t, testCase.args, args) + } + }) + } +} + +func renderVeltyWithShapeContext(t *testing.T, template string) (string, []interface{}) { + t.Helper() + planner := velty.New() + require.NoError(t, planner.DefineVariable("criteria", &criteriaContextCollector{})) + require.NoError(t, planner.DefineVariable("predicate", predicateContextMock{})) + require.NoError(t, planner.DefineVariable("sql", sqlContextMock{})) + require.NoError(t, planner.DefineVariable("Unsafe", unsafeContextMock{})) + + exec, newState, err := planner.Compile([]byte(template)) + require.NoError(t, err) + + state := newState() + criteria := &criteriaContextCollector{} + require.NoError(t, state.SetValue("criteria", criteria)) + require.NoError(t, state.SetValue("predicate", predicateContextMock{})) + require.NoError(t, state.SetValue("sql", sqlContextMock{})) + require.NoError(t, state.SetValue("Unsafe", unsafeContextMock{VendorID: 101})) + require.NoError(t, exec.Exec(state)) + return state.Buffer.String(), criteria.Args +} diff --git a/repository/shape/dql/sanitize/policy.go b/repository/shape/dql/sanitize/policy.go index 4b6d6b5a4..a33b100c0 100644 --- a/repository/shape/dql/sanitize/policy.go +++ b/repository/shape/dql/sanitize/policy.go @@ -1,32 +1,91 @@ package sanitize -import "strings" +import ( + "strings" + + "github.com/viant/datly/view/keywords" + "github.com/viant/velty" +) type rewritePolicy struct { declared map[string]bool + foreach map[string]bool consts map[string]bool } -func newRewritePolicy(declared, consts map[string]bool) *rewritePolicy { +func newRewritePolicy(declared, foreach, consts map[string]bool) *rewritePolicy { return &rewritePolicy{ declared: declared, + foreach: foreach, consts: consts, } } -func (p *rewritePolicy) rewrite(raw string) string { +func (p *rewritePolicy) rewrite(raw string, kind velty.ExprContextKind) string { holder := holderName(raw) if holder == "" { return raw } + if keywords.Has(holder) { + return raw + } if strings.HasPrefix(raw, "$Unsafe.") || strings.HasPrefix(raw, "${Unsafe.") || strings.HasPrefix(raw, "$Has.") || strings.HasPrefix(raw, "${Has.") { return raw } if p.consts != nil && p.consts[holder] { return addUnsafePrefix(raw) } + if isControlOrFuncContext(kind) { + if p.declared != nil && p.declared[holder] { + return raw + } + if hasExplicitPrefix(raw) { + return raw + } + return addUnsafePrefix(raw) + } if p.declared != nil && p.declared[holder] { + if hasExplicitPrefix(raw) { + if p.foreach != nil && p.foreach[holder] { + return asPlaceholder(raw) + } + return asPlaceholder(addUnsafePrefix(raw)) + } return asPlaceholder(raw) } + if hasExplicitPrefix(raw) { + if p.foreach != nil && p.foreach[holder] { + return asPlaceholder(raw) + } + return asPlaceholder(addUnsafePrefix(raw)) + } return asPlaceholder(addUnsafePrefix(raw)) } + +func isControlOrFuncContext(kind velty.ExprContextKind) bool { + switch kind { + case velty.CtxFuncArg, + velty.CtxForEachCond, + velty.CtxIfCond, velty.CtxElseIfCond, + velty.CtxSetRHS, + velty.CtxForLoopInit, velty.CtxForLoopCond, velty.CtxForLoopPost: + return true + default: + return false + } +} + +func hasExplicitPrefix(raw string) bool { + name := strings.TrimSpace(raw) + if strings.HasPrefix(name, "${") && strings.HasSuffix(name, "}") { + name = "$" + name[2:len(name)-1] + } + if !strings.HasPrefix(name, "$") { + return false + } + name = strings.TrimPrefix(name, "$") + if idx := strings.Index(name, "("); idx != -1 { + return false + } + return strings.Index(name, ".") != -1 +} diff --git a/repository/shape/dql/sanitize/policy_test.go b/repository/shape/dql/sanitize/policy_test.go index c3ceb5f23..518a7575c 100644 --- a/repository/shape/dql/sanitize/policy_test.go +++ b/repository/shape/dql/sanitize/policy_test.go @@ -1,13 +1,19 @@ package sanitize -import "testing" +import ( + "testing" + + "github.com/viant/velty" +) func TestRewritePolicy_Rewrite(t *testing.T) { testCases := []struct { name string raw string declared map[string]bool + foreach map[string]bool consts map[string]bool + kind velty.ExprContextKind expect string }{ { @@ -26,6 +32,32 @@ func TestRewritePolicy_Rewrite(t *testing.T) { declared: map[string]bool{"x": true}, expect: "$criteria.AppendBinding($x)", }, + { + name: "declared foreach variable in body uses placeholder", + raw: "$rec.ID", + declared: map[string]bool{"rec": true}, + foreach: map[string]bool{"rec": true}, + kind: velty.CtxForEachBody, + expect: "$criteria.AppendBinding($rec.ID)", + }, + { + name: "declared dotted parameter uses unsafe placeholder in append context", + raw: "$Jwt.UserID", + declared: map[string]bool{"Jwt": true}, + expect: "$criteria.AppendBinding($Unsafe.Jwt.UserID)", + }, + { + name: "function arg gets unsafe prefix", + raw: "$VendorID", + kind: velty.CtxFuncArg, + expect: "$Unsafe.VendorID", + }, + { + name: "prefixed function arg is preserved", + raw: "$sql.Eq", + kind: velty.CtxFuncArg, + expect: "$sql.Eq", + }, { name: "const selector keeps raw unsafe path", raw: "$ConstID", @@ -40,8 +72,8 @@ func TestRewritePolicy_Rewrite(t *testing.T) { } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - policy := newRewritePolicy(testCase.declared, testCase.consts) - if actual := policy.rewrite(testCase.raw); actual != testCase.expect { + policy := newRewritePolicy(testCase.declared, testCase.foreach, testCase.consts) + if actual := policy.rewrite(testCase.raw, testCase.kind); actual != testCase.expect { t.Fatalf("unexpected rewrite: %s", actual) } }) diff --git a/repository/shape/dql/sanitize/sanitizer.go b/repository/shape/dql/sanitize/sanitizer.go index ec414620d..83e94968e 100644 --- a/repository/shape/dql/sanitize/sanitizer.go +++ b/repository/shape/dql/sanitize/sanitizer.go @@ -11,6 +11,7 @@ import ( type Options struct { Declared map[string]bool + Foreach map[string]bool Consts map[string]bool } @@ -29,6 +30,21 @@ func Declared(input string) map[string]bool { ret[name] = true } } + for _, name := range scanForeachDeclaredHolders(input) { + if name != "" { + ret[name] = true + } + } + return ret +} + +func ForeachDeclared(input string) map[string]bool { + ret := map[string]bool{} + for _, name := range scanForeachDeclaredHolders(input) { + if name != "" { + ret[name] = true + } + } return ret } @@ -129,6 +145,56 @@ func isSanitizeIdentifierPart(ch byte) bool { return isSanitizeIdentifierStart(ch) || (ch >= '0' && ch <= '9') } +func scanForeachDeclaredHolders(input string) []string { + result := make([]string, 0) + lower := strings.ToLower(input) + for i := 0; i < len(input); i++ { + if input[i] != '#' { + continue + } + if !strings.HasPrefix(lower[i:], "#foreach") { + continue + } + j := i + len("#foreach") + for j < len(input) && (input[j] == ' ' || input[j] == '\t' || input[j] == '\r' || input[j] == '\n') { + j++ + } + if j >= len(input) || input[j] != '(' { + continue + } + body, end, ok := readSetDirectiveBody(input, j) + if !ok { + continue + } + if name, ok := parseForeachHolder(body); ok { + result = append(result, name) + } + i = end + } + return result +} + +func parseForeachHolder(body string) (string, bool) { + text := strings.TrimSpace(body) + if text == "" || text[0] != '$' || len(text) < 2 { + return "", false + } + text = text[1:] + end := 0 + for end < len(text) && isSanitizeIdentifierPart(text[end]) { + end++ + } + if end == 0 { + return "", false + } + name := text[:end] + rest := strings.TrimSpace(text[end:]) + if !strings.HasPrefix(strings.ToLower(rest), "in ") { + return "", false + } + return name, true +} + func SQL(input string, opts Options) string { return Rewrite(input, opts).SQL } @@ -140,8 +206,9 @@ func Rewrite(input string, opts Options) RewriteResult { adjuster := &bindingAdjuster{ source: []byte(input), declared: opts.Declared, + foreach: opts.Foreach, consts: opts.Consts, - policy: newRewritePolicy(opts.Declared, opts.Consts), + policy: newRewritePolicy(opts.Declared, opts.Foreach, opts.Consts), } out, err := velty.TransformTemplate([]byte(input), adjuster) if err != nil { @@ -158,12 +225,18 @@ func Rewrite(input string, opts Options) RewriteResult { type bindingAdjuster struct { source []byte declared map[string]bool + foreach map[string]bool consts map[string]bool policy *rewritePolicy patches []velty.Patch } func (b *bindingAdjuster) Adjust(node ast.Node, ctx *velty.ParserContext) (velty.Action, error) { + if call, ok := node.(*aexpr.Call); ok { + b.rewriteCallNodeArgs(call, ctx) + return velty.Keep(), nil + } + sel, ok := node.(*aexpr.Select) if !ok { return velty.Keep(), nil @@ -179,7 +252,7 @@ func (b *bindingAdjuster) Adjust(node ast.Node, ctx *velty.ParserContext) (velty return velty.Keep(), nil } raw := string(b.source[span.Start : span.End+1]) - replacement := b.rewrite(raw) + replacement := b.rewrite(raw, ctx.CurrentExprContext().Kind) if replacement == raw { return velty.Keep(), nil } @@ -190,6 +263,50 @@ func (b *bindingAdjuster) Adjust(node ast.Node, ctx *velty.ParserContext) (velty return velty.PatchSpan(span, []byte(replacement)), nil } +func (b *bindingAdjuster) rewriteCallNodeArgs(call *aexpr.Call, ctx *velty.ParserContext) { + selectors := make([]*aexpr.Select, 0, 4) + for _, arg := range call.Args { + b.collectSelectors(arg, &selectors) + } + for _, sel := range selectors { + span, ok := ctx.GetSpan(sel) + if !ok { + continue + } + if b.inSetDirective(span.Start) { + continue + } + raw := string(b.source[span.Start : span.End+1]) + replacement := b.rewrite(raw, velty.CtxFuncArg) + if replacement == raw { + continue + } + b.patches = append(b.patches, velty.Patch{ + Span: span, + Replacement: []byte(replacement), + }) + } +} + +func (b *bindingAdjuster) collectSelectors(expr ast.Expression, selectors *[]*aexpr.Select) { + switch actual := expr.(type) { + case *aexpr.Select: + *selectors = append(*selectors, actual) + case *aexpr.Call: + b.collectSelectors(actual.X, selectors) + for _, arg := range actual.Args { + b.collectSelectors(arg, selectors) + } + case *aexpr.Binary: + b.collectSelectors(actual.X, selectors) + b.collectSelectors(actual.Y, selectors) + case *aexpr.Unary: + b.collectSelectors(actual.X, selectors) + case *aexpr.Parentheses: + b.collectSelectors(actual.P, selectors) + } +} + func (b *bindingAdjuster) inSetDirective(pos int) bool { if pos <= 0 || pos > len(b.source) { return false @@ -206,11 +323,166 @@ func (b *bindingAdjuster) inSetDirective(pos int) bool { return strings.Count(segment, "(") > strings.Count(segment, ")") } -func (b *bindingAdjuster) rewrite(raw string) string { +func (b *bindingAdjuster) rewrite(raw string, kind velty.ExprContextKind) string { if b.policy == nil { - b.policy = newRewritePolicy(b.declared, b.consts) + b.policy = newRewritePolicy(b.declared, b.foreach, b.consts) + } + if rewritten, ok := b.rewriteCallArgsInSelector(raw); ok { + return rewritten + } + return b.policy.rewrite(raw, kind) +} + +func (b *bindingAdjuster) rewriteCallArgsInSelector(raw string) (string, bool) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" || !strings.HasPrefix(trimmed, "$") || !strings.Contains(trimmed, "(") { + return "", false + } + + hasBraces := strings.HasPrefix(trimmed, "${") && strings.HasSuffix(trimmed, "}") + expr := trimmed + if hasBraces { + expr = "$" + trimmed[2:len(trimmed)-1] + } + + open := strings.Index(expr, "(") + if open <= 0 || !strings.HasPrefix(expr, "$") { + return "", false + } + closeIdx, ok := matchingParen(expr, open) + if !ok || closeIdx != len(expr)-1 { + return "", false + } + + args := expr[open+1 : closeIdx] + rewrittenArgs, changed := b.rewriteCallArgs(args) + if !changed { + return "", false + } + rewritten := expr[:open+1] + rewrittenArgs + expr[closeIdx:] + if hasBraces { + rewritten = "${" + rewritten[1:] + "}" + } + return rewritten, true +} + +func (b *bindingAdjuster) rewriteCallArgs(args string) (string, bool) { + parts := splitArgs(args) + if len(parts) == 0 { + return args, false + } + changed := false + for i := range parts { + part := parts[i] + lead, core, tail := trimArgWhitespace(part) + if core == "" { + continue + } + rewrittenCore := core + if strings.HasPrefix(core, "$") { + if nested, ok := b.rewriteCallArgsInSelector(core); ok { + rewrittenCore = nested + } else { + rewrittenCore = b.policy.rewrite(core, velty.CtxFuncArg) + } + } + if rewrittenCore != core { + changed = true + parts[i] = lead + rewrittenCore + tail + } + } + if !changed { + return args, false + } + return strings.Join(parts, ","), true +} + +func splitArgs(input string) []string { + if input == "" { + return nil + } + result := make([]string, 0, 4) + start := 0 + depth := 0 + quote := byte(0) + for i := 0; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + if ch == '(' { + depth++ + continue + } + if ch == ')' { + if depth > 0 { + depth-- + } + continue + } + if ch == ',' && depth == 0 { + result = append(result, input[start:i]) + start = i + 1 + } + } + result = append(result, input[start:]) + return result +} + +func trimArgWhitespace(input string) (string, string, string) { + start := 0 + for start < len(input) && (input[start] == ' ' || input[start] == '\t' || input[start] == '\n' || input[start] == '\r') { + start++ + } + end := len(input) + for end > start && (input[end-1] == ' ' || input[end-1] == '\t' || input[end-1] == '\n' || input[end-1] == '\r') { + end-- + } + return input[:start], input[start:end], input[end:] +} + +func matchingParen(input string, open int) (int, bool) { + depth := 0 + quote := byte(0) + for i := open; i < len(input); i++ { + ch := input[i] + if quote != 0 { + if ch == '\\' && i+1 < len(input) { + i++ + continue + } + if ch == quote { + quote = 0 + } + continue + } + if ch == '\'' || ch == '"' { + quote = ch + continue + } + if ch == '(' { + depth++ + continue + } + if ch == ')' { + depth-- + if depth == 0 { + return i, true + } + } } - return b.policy.rewrite(raw) + return -1, false } func holderName(raw string) string { diff --git a/repository/shape/dql/sanitize/sanitizer_test.go b/repository/shape/dql/sanitize/sanitizer_test.go index 326bd3b1d..308bb5068 100644 --- a/repository/shape/dql/sanitize/sanitizer_test.go +++ b/repository/shape/dql/sanitize/sanitizer_test.go @@ -39,6 +39,20 @@ func TestSQL_ParityWithLegacySanitizer(t *testing.T) { &inference.Parameter{Parameter: vstate.Parameter{Name: "ConstId", In: vstate.NewConstLocation("ConstId")}}, }, }, + { + name: "exec foreach and logger hooks", + sql: "#foreach($rec in $Unsafe.Records)\n" + + "#if($rec.IS_AUTH == 0)\n" + + " $logger.Fatal(\"Unauthorized access to product: %v\", $rec.ID)\n" + + "#end\n" + + "UPDATE PRODUCT SET STATUS = $Status WHERE ID = $rec.ID\n" + + "#end", + }, + { + name: "predicate and sql hooks", + sql: "SELECT * FROM PRODUCT t WHERE 1=1 ${predicate.Builder().CombineOr($predicate.FilterGroup(0, \"AND\")).Build(\"AND\")} " + + "AND $sql.Eq(\"ID\", $VendorID)", + }, } for _, testCase := range testCases { @@ -50,6 +64,7 @@ func TestSQL_ParityWithLegacySanitizer(t *testing.T) { actual := SQL(testCase.sql, Options{ Declared: tpl.Declared, + Foreach: ForeachDeclared(testCase.sql), Consts: constNames(state), }) assert.Equal(t, expected, actual) @@ -82,6 +97,20 @@ func TestSQL_ParityWithLegacySanitizer_RuntimeExpansion(t *testing.T) { &inference.Parameter{Parameter: vstate.Parameter{Name: "ConstId", In: vstate.NewConstLocation("ConstId")}}, }, }, + { + name: "exec foreach and logger hooks", + sql: "#foreach($rec in $Unsafe.Records)\n" + + "#if($rec.IS_AUTH == 0)\n" + + " $logger.Fatal(\"Unauthorized access to product: %v\", $rec.ID)\n" + + "#end\n" + + "UPDATE PRODUCT SET STATUS = $Status WHERE ID = $rec.ID\n" + + "#end", + }, + { + name: "predicate and sql hooks", + sql: "SELECT * FROM PRODUCT t WHERE 1=1 ${predicate.Builder().CombineOr($predicate.FilterGroup(0, \"AND\")).Build(\"AND\")} " + + "AND $sql.Eq(\"ID\", $VendorID)", + }, } for _, testCase := range testCases { @@ -93,6 +122,7 @@ func TestSQL_ParityWithLegacySanitizer_RuntimeExpansion(t *testing.T) { shapeSQL := SQL(testCase.sql, Options{ Declared: tpl.Declared, + Foreach: ForeachDeclared(testCase.sql), Consts: constNames(state), }) require.Equal(t, legacySQL, shapeSQL) @@ -156,6 +186,11 @@ func TestDeclared_ParameterDeclarationStyle(t *testing.T) { assert.True(t, declared["Jwt"]) } +func TestDeclared_ForeachVariable(t *testing.T) { + declared := Declared("#foreach($rec in $Unsafe.Records)\nUPDATE t SET v = $rec.ID\n#end") + assert.True(t, declared["rec"]) +} + func TestDeclaredListener_OnEventBranches(t *testing.T) { declared := map[string]bool{} l := &declaredListener{declared: declared} @@ -218,9 +253,57 @@ func (c criteriaMock) AppendBinding(value interface{}) string { } type unsafeMock struct { - Id int - Name string - ConstId int + Id int + Name string + ConstId int + VendorID int + Status int + Records []recordMock +} + +type recordMock struct { + ID int + IS_AUTH int +} + +type sqlMock struct{} + +func (s sqlMock) Eq(column string, value interface{}) string { + return fmt.Sprintf("%s = %v", column, value) +} + +type predicateMock struct{} + +func (p predicateMock) Builder() *predicateBuilderMock { + return &predicateBuilderMock{} +} + +func (p predicateMock) FilterGroup(group int, op string) string { + return fmt.Sprintf("P%d:%s", group, op) +} + +type predicateBuilderMock struct { + value string +} + +func (b *predicateBuilderMock) CombineOr(group string) *predicateBuilderMock { + b.value = group + return b +} + +func (b *predicateBuilderMock) Build(kind string) string { + switch kind { + case "AND": + return " AND (" + b.value + ") " + default: + return "" + } +} + +type loggerMock struct{} + +func (l loggerMock) Fatal(_ string, _ ...interface{}) string { + return "" } func renderVeltySQL(t *testing.T, template string) string { @@ -228,18 +311,35 @@ func renderVeltySQL(t *testing.T, template string) string { planner := velty.New() require.NoError(t, planner.DefineVariable("criteria", criteriaMock{})) require.NoError(t, planner.DefineVariable("Unsafe", unsafeMock{})) + require.NoError(t, planner.DefineVariable("sql", sqlMock{})) + require.NoError(t, planner.DefineVariable("predicate", predicateMock{})) + require.NoError(t, planner.DefineVariable("logger", loggerMock{})) require.NoError(t, planner.DefineVariable("Id", 0)) require.NoError(t, planner.DefineVariable("Name", "")) require.NoError(t, planner.DefineVariable("ConstId", 0)) + require.NoError(t, planner.DefineVariable("VendorID", 0)) + require.NoError(t, planner.DefineVariable("Status", 0)) exec, newState, err := planner.Compile([]byte(template)) require.NoError(t, err) state := newState() require.NoError(t, state.SetValue("criteria", criteriaMock{})) - require.NoError(t, state.SetValue("Unsafe", unsafeMock{Id: 10, Name: "ann", ConstId: 77})) + require.NoError(t, state.SetValue("Unsafe", unsafeMock{ + Id: 10, + Name: "ann", + ConstId: 77, + VendorID: 101, + Status: 1, + Records: []recordMock{{ID: 10, IS_AUTH: 1}}, + })) + require.NoError(t, state.SetValue("sql", sqlMock{})) + require.NoError(t, state.SetValue("predicate", predicateMock{})) + require.NoError(t, state.SetValue("logger", loggerMock{})) require.NoError(t, state.SetValue("Id", 10)) require.NoError(t, state.SetValue("Name", "ann")) require.NoError(t, state.SetValue("ConstId", 77)) + require.NoError(t, state.SetValue("VendorID", 101)) + require.NoError(t, state.SetValue("Status", 1)) require.NoError(t, exec.Exec(state)) return state.Buffer.String() } diff --git a/repository/shape/dql/shape/model.go b/repository/shape/dql/shape/model.go index 3f6bc3a6d..bbe53463f 100644 --- a/repository/shape/dql/shape/model.go +++ b/repository/shape/dql/shape/model.go @@ -42,6 +42,12 @@ type Diagnostic struct { type Directives struct { Meta string DefaultConnector string + Dest string + InputDest string + OutputDest string + RouterDest string + InputType string + OutputType string Cache *CacheDirective MCP *MCPDirective Route *RouteDirective @@ -55,8 +61,12 @@ type Directives struct { } type CacheDirective struct { - Enabled bool - TTL string + Enabled bool + TTL string + Name string + Provider string + Location string + TimeToLiveMs int } type MCPDirective struct { diff --git a/repository/shape/dql_engine_test.go b/repository/shape/dql_engine_test.go index a7a40fff5..1b52a0e01 100644 --- a/repository/shape/dql_engine_test.go +++ b/repository/shape/dql_engine_test.go @@ -64,3 +64,23 @@ SELECT id FROM ORDERS t` require.NotEmpty(t, component.Predicates["o"]) assert.Equal(t, "ByID", component.Predicates["o"][0].Name) } + +func TestEngine_LoadDQLComponent_PreservesExplicitOutputViewOneCardinality(t *testing.T) { + engine := shape.New( + shape.WithCompiler(shapeCompile.New()), + shape.WithLoader(shapeLoad.New()), + shape.WithName("/v1/api/shape/dev/auth/user-acl"), + ) + dql := ` +#define($_ = $Data(output/view).Cardinality('One').Embed()) +SELECT 1 AS UserID` + artifact, err := engine.LoadDQLComponent(context.Background(), dql) + require.NoError(t, err) + require.NotNil(t, artifact) + + component, ok := shapeLoad.ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.Output, 1) + require.NotNil(t, component.Output[0].Schema) + assert.Equal(t, "One", string(component.Output[0].Schema.Cardinality)) +} diff --git a/repository/shape/gorouter/discover.go b/repository/shape/gorouter/discover.go new file mode 100644 index 000000000..ddab0b597 --- /dev/null +++ b/repository/shape/gorouter/discover.go @@ -0,0 +1,671 @@ +package gorouter + +import ( + "bufio" + "context" + "fmt" + "go/ast" + "go/token" + "io/fs" + "os" + "path" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/componenttag" + "github.com/viant/datly/view/extension" + tagtags "github.com/viant/tagly/tags" + "github.com/viant/x" + "github.com/viant/xreflect" + "golang.org/x/tools/go/packages" +) + +// Discover scans Go packages for router holders and returns one route source per component-tagged field. +func Discover(ctx context.Context, baseDir string, include, exclude []string) ([]*RouteSource, error) { + baseDir = strings.TrimSpace(baseDir) + if baseDir == "" { + return nil, fmt.Errorf("go router discovery: base dir was empty") + } + if len(include) == 0 { + return nil, fmt.Errorf("go router discovery: include package patterns were empty") + } + include, err := expandPackagePatterns(ctx, baseDir, include) + if err != nil { + return nil, err + } + if len(include) == 0 { + return nil, fmt.Errorf("go router discovery: no packages matched include patterns") + } + loadCfg := &packages.Config{ + Context: ctx, + Dir: baseDir, + Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax, + } + pkgs, err := packages.Load(loadCfg, include...) + if err != nil { + return nil, fmt.Errorf("go router discovery: failed to load packages: %w", err) + } + index, err := newPackageIndex(ctx, baseDir) + if err != nil { + return nil, err + } + var result []*RouteSource + for _, pkg := range pkgs { + if pkg == nil || pkg.PkgPath == "" || matchesPackagePatternList(pkg.PkgPath, exclude) { + continue + } + dir := firstPackageDir(pkg) + if dir == "" { + continue + } + for i, file := range pkg.Syntax { + if file == nil || i >= len(pkg.GoFiles) { + continue + } + filePath := pkg.GoFiles[i] + imports := importMap(file) + discovered, err := index.routesInFile(pkg.PkgPath, pkg.Name, dir, filePath, file, imports) + if err != nil { + return nil, err + } + result = append(result, discovered...) + } + } + sort.SliceStable(result, func(i, j int) bool { + if result[i].PackagePath == result[j].PackagePath { + if result[i].FilePath == result[j].FilePath { + return result[i].FieldName < result[j].FieldName + } + return result[i].FilePath < result[j].FilePath + } + return result[i].PackagePath < result[j].PackagePath + }) + return result, nil +} + +func expandPackagePatterns(ctx context.Context, baseDir string, patterns []string) ([]string, error) { + unique := map[string]bool{} + var result []string + for _, pattern := range patterns { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } + expanded, err := expandPackagePattern(ctx, baseDir, pattern) + if err != nil { + return nil, err + } + for _, item := range expanded { + item = strings.TrimSpace(item) + if item == "" || unique[item] { + continue + } + unique[item] = true + result = append(result, item) + } + } + sort.Strings(result) + return result, nil +} + +func expandPackagePattern(ctx context.Context, baseDir, pattern string) ([]string, error) { + if !strings.HasSuffix(pattern, "/...") { + return []string{pattern}, nil + } + moduleDir, modulePath, err := locateModule(baseDir) + if err == nil { + if packages, ok, expandErr := expandModuleWildcardPattern(moduleDir, modulePath, pattern); expandErr != nil { + return nil, expandErr + } else if ok { + return packages, nil + } + } + cfg := &packages.Config{ + Context: ctx, + Dir: baseDir, + Mode: packages.NeedName | packages.NeedFiles, + } + pkgs, err := packages.Load(cfg, pattern) + if err != nil { + return nil, fmt.Errorf("go router discovery: failed to expand package pattern %s: %w", pattern, err) + } + var result []string + unique := map[string]bool{} + for _, pkg := range pkgs { + if pkg == nil || pkg.PkgPath == "" || unique[pkg.PkgPath] { + continue + } + unique[pkg.PkgPath] = true + result = append(result, pkg.PkgPath) + } + sort.Strings(result) + return result, nil +} + +func expandModuleWildcardPattern(moduleDir, modulePath, pattern string) ([]string, bool, error) { + prefix := strings.TrimSuffix(strings.TrimSpace(pattern), "/...") + if prefix == "" || moduleDir == "" || modulePath == "" { + return nil, false, nil + } + if prefix != modulePath && !strings.HasPrefix(prefix, modulePath+"/") { + return nil, false, nil + } + rel := strings.TrimPrefix(prefix, modulePath) + rel = strings.TrimPrefix(rel, "/") + rootDir := moduleDir + if rel != "" { + rootDir = filepath.Join(moduleDir, filepath.FromSlash(rel)) + } + info, err := os.Stat(rootDir) + if err != nil { + if os.IsNotExist(err) { + return nil, true, nil + } + return nil, true, err + } + if !info.IsDir() { + return nil, true, nil + } + unique := map[string]bool{} + var result []string + err = filepath.WalkDir(rootDir, func(current string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !d.IsDir() { + return nil + } + name := d.Name() + if strings.HasPrefix(name, ".") || name == "testdata" { + if current != rootDir { + return filepath.SkipDir + } + return nil + } + hasGo, err := containsPackageGoFiles(current) + if err != nil { + return err + } + if !hasGo { + return nil + } + relDir, err := filepath.Rel(moduleDir, current) + if err != nil { + return err + } + importPath := modulePath + if relDir != "." { + importPath += "/" + filepath.ToSlash(relDir) + } + if !unique[importPath] { + unique[importPath] = true + result = append(result, importPath) + } + return nil + }) + if err != nil { + return nil, true, err + } + sort.Strings(result) + return result, true, nil +} + +func containsPackageGoFiles(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return false, err + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + return true, nil + } + return false, nil +} + +func locateModule(baseDir string) (string, string, error) { + dir := filepath.Clean(baseDir) + for { + goModPath := filepath.Join(dir, "go.mod") + data, err := os.ReadFile(goModPath) + if err == nil { + modulePath := parseModulePath(data) + if modulePath == "" { + return "", "", fmt.Errorf("go router discovery: module path not found in %s", goModPath) + } + return dir, modulePath, nil + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + return "", "", fmt.Errorf("go router discovery: go.mod not found from %s", baseDir) +} + +func parseModulePath(data []byte) string { + scanner := bufio.NewScanner(strings.NewReader(string(data))) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "//") { + continue + } + if strings.HasPrefix(line, "module ") { + return strings.TrimSpace(strings.TrimPrefix(line, "module ")) + } + } + return "" +} + +type packageIndex struct { + ctx context.Context + baseDir string + pkgs map[string]*packageMeta + dirTypes map[string]*xreflect.DirTypes +} + +type packageMeta struct { + importPath string + name string + dir string +} + +func newPackageIndex(ctx context.Context, baseDir string) (*packageIndex, error) { + return &packageIndex{ + ctx: ctx, + baseDir: baseDir, + pkgs: map[string]*packageMeta{}, + dirTypes: map[string]*xreflect.DirTypes{}, + }, nil +} + +func (p *packageIndex) routesInFile(pkgPath, pkgName, pkgDir, filePath string, file *ast.File, imports map[string]string) ([]*RouteSource, error) { + var result []*RouteSource + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.TYPE { + continue + } + for _, spec := range gen.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + structType, ok := typeSpec.Type.(*ast.StructType) + if !ok { + continue + } + for _, field := range structType.Fields.List { + route, err := p.routeFromField(pkgPath, pkgName, pkgDir, filePath, field, imports) + if err != nil { + return nil, err + } + if route != nil { + result = append(result, route) + } + } + } + } + return result, nil +} + +func (p *packageIndex) routeFromField(pkgPath, pkgName, pkgDir, filePath string, field *ast.Field, imports map[string]string) (*RouteSource, error) { + if field == nil || field.Tag == nil || len(field.Names) == 0 { + return nil, nil + } + tagLiteral, err := strconv.Unquote(field.Tag.Value) + if err != nil { + return nil, fmt.Errorf("go router discovery: invalid struct tag in %s: %w", filePath, err) + } + parsed, err := componenttag.Parse(reflect.StructTag(tagLiteral)) + if err != nil { + return nil, fmt.Errorf("go router discovery: invalid component tag in %s: %w", filePath, err) + } + if parsed == nil || parsed.Component == nil { + return nil, nil + } + fieldName := strings.TrimSpace(field.Names[0].Name) + if fieldName == "" { + return nil, nil + } + inputRef := normalizeTypeRef(strings.TrimSpace(parsed.Component.Input), pkgPath) + outputRef := normalizeTypeRef(strings.TrimSpace(parsed.Component.Output), pkgPath) + viewRef := normalizeTypeRef(strings.TrimSpace(parsed.Component.View), pkgPath) + handlerRef := normalizeTypeRef(strings.TrimSpace(parsed.Component.Handler), pkgPath) + sourceURL := strings.TrimSpace(parsed.Component.Source) + summaryURL := strings.TrimSpace(parsed.Component.Summary) + if inputRef == "" || outputRef == "" { + inferredInput, inferredOutput := inferComponentTypeRefs(field.Type, pkgPath, imports) + if inputRef == "" { + inputRef = inferredInput + } + if outputRef == "" { + outputRef = inferredOutput + } + } + if inputRef == "" && outputRef == "" { + if viewRef == "" && sourceURL == "" { + return nil, nil + } + } + if inputRef == "" && outputRef == "" && viewRef == "" { + return nil, nil + } + registry := x.NewRegistry() + tagCopy := *parsed.Component + if inputRef != "" { + rType, err := p.resolveType(inputRef) + if err != nil { + return nil, fmt.Errorf("go router discovery: failed to resolve %s input %s: %w", fieldName, inputRef, err) + } + registerType(registry, inputRef, rType) + tagCopy.Input = inputRef + } + if outputRef != "" { + rType, err := p.resolveType(outputRef) + if err != nil { + return nil, fmt.Errorf("go router discovery: failed to resolve %s output %s: %w", fieldName, outputRef, err) + } + registerType(registry, outputRef, rType) + tagCopy.Output = outputRef + } + if viewRef != "" { + rType, err := p.resolveType(viewRef) + if err != nil { + return nil, fmt.Errorf("go router discovery: failed to resolve %s view %s: %w", fieldName, viewRef, err) + } + registerType(registry, viewRef, rType) + tagCopy.View = viewRef + } + if handlerRef != "" { + rType, err := p.resolveType(handlerRef) + if err != nil { + return nil, fmt.Errorf("go router discovery: failed to resolve %s handler %s: %w", fieldName, handlerRef, err) + } + registerType(registry, handlerRef, rType) + tagCopy.Handler = handlerRef + } + if sourceURL != "" { + tagCopy.Source = sourceURL + } + if summaryURL != "" { + tagCopy.Summary = summaryURL + } + componentTag := tagCopy.Tag() + rootType := reflect.StructOf([]reflect.StructField{{ + Name: exportName(fieldName), + Type: reflect.TypeOf(struct{}{}), + Tag: reflect.StructTag(tagtags.Tags{componentTag}.Stringify()), + }}) + name := strings.TrimSpace(tagCopy.Name) + if name == "" { + name = exportName(fieldName) + } + return &RouteSource{ + Name: name, + FieldName: fieldName, + FilePath: filePath, + PackageName: pkgName, + PackagePath: pkgPath, + PackageDir: pkgDir, + RoutePath: strings.TrimSpace(tagCopy.Path), + Method: strings.TrimSpace(tagCopy.Method), + Connector: strings.TrimSpace(tagCopy.Connector), + InputRef: inputRef, + OutputRef: outputRef, + ViewRef: viewRef, + SourceURL: sourceURL, + SummaryURL: summaryURL, + Source: &shape.Source{ + Name: name, + Path: filePath, + Type: rootType, + TypeRegistry: registry, + }, + }, nil +} + +func (p *packageIndex) resolveType(ref string) (reflect.Type, error) { + pkgPath, typeName := splitTypeRef(ref) + if pkgPath == "" || typeName == "" { + return nil, fmt.Errorf("invalid type reference %q", ref) + } + if extension.Config != nil && extension.Config.Types != nil { + if linked, err := extension.Config.Types.Lookup(typeName, xreflect.WithPackage(pkgPath)); err == nil && linked != nil { + return linked, nil + } + } + meta, err := p.packageMeta(pkgPath) + if err != nil { + return nil, err + } + dirTypes, err := p.dirTypesFor(meta.dir) + if err != nil { + return nil, err + } + rType, err := dirTypes.Type(typeName) + if err != nil { + return nil, err + } + return rType, nil +} + +func (p *packageIndex) packageMeta(importPath string) (*packageMeta, error) { + if meta, ok := p.pkgs[importPath]; ok { + return meta, nil + } + cfg := &packages.Config{ + Context: p.ctx, + Dir: p.baseDir, + Mode: packages.NeedName | packages.NeedFiles, + } + pkgs, err := packages.Load(cfg, importPath) + if err != nil { + return nil, fmt.Errorf("go router discovery: failed to load package %s: %w", importPath, err) + } + for _, pkg := range pkgs { + if pkg == nil || pkg.PkgPath == "" { + continue + } + dir := firstPackageDir(pkg) + if dir == "" { + continue + } + meta := &packageMeta{importPath: pkg.PkgPath, name: pkg.Name, dir: dir} + p.pkgs[pkg.PkgPath] = meta + if pkg.PkgPath == importPath { + return meta, nil + } + } + return nil, fmt.Errorf("go router discovery: package %s not resolved", importPath) +} + +func (p *packageIndex) dirTypesFor(dir string) (*xreflect.DirTypes, error) { + if cached, ok := p.dirTypes[dir]; ok { + return cached, nil + } + options := []xreflect.Option{} + if extension.Config != nil && extension.Config.Types != nil { + options = append(options, xreflect.WithTypeLookup(extension.Config.Types.Lookup)) + } + parsed, err := xreflect.ParseTypes(dir, options...) + if err != nil { + return nil, fmt.Errorf("go router discovery: failed to parse package dir %s: %w", dir, err) + } + p.dirTypes[dir] = parsed + return parsed, nil +} + +func inferComponentTypeRefs(expr ast.Expr, pkgPath string, imports map[string]string) (string, string) { + args := componentTypeArgs(expr) + if len(args) < 2 { + return "", "" + } + return qualifyTypeExpr(args[0], pkgPath, imports), qualifyTypeExpr(args[1], pkgPath, imports) +} + +func componentTypeArgs(expr ast.Expr) []ast.Expr { + switch actual := expr.(type) { + case *ast.IndexListExpr: + if !isComponentSelector(actual.X) { + return nil + } + return actual.Indices + case *ast.IndexExpr: + if !isComponentSelector(actual.X) { + return nil + } + return []ast.Expr{actual.Index} + default: + return nil + } +} + +func isComponentSelector(expr ast.Expr) bool { + switch actual := expr.(type) { + case *ast.SelectorExpr: + return actual.Sel != nil && actual.Sel.Name == "Component" + case *ast.Ident: + return actual.Name == "Component" + default: + return false + } +} + +func qualifyTypeExpr(expr ast.Expr, pkgPath string, imports map[string]string) string { + switch actual := expr.(type) { + case *ast.Ident: + if pkgPath == "" || actual.Name == "" { + return "" + } + return pkgPath + "." + actual.Name + case *ast.SelectorExpr: + ident, ok := actual.X.(*ast.Ident) + if !ok || ident.Name == "" || actual.Sel == nil || actual.Sel.Name == "" { + return "" + } + importPath := imports[ident.Name] + if importPath == "" { + return "" + } + return importPath + "." + actual.Sel.Name + default: + return "" + } +} + +func importMap(file *ast.File) map[string]string { + result := map[string]string{} + if file == nil { + return result + } + for _, item := range file.Imports { + if item == nil || item.Path == nil { + continue + } + importPath, err := strconv.Unquote(item.Path.Value) + if err != nil || importPath == "" { + continue + } + alias := path.Base(importPath) + if item.Name != nil && strings.TrimSpace(item.Name.Name) != "" { + alias = strings.TrimSpace(item.Name.Name) + } + result[alias] = importPath + } + return result +} + +func splitTypeRef(ref string) (string, string) { + ref = strings.TrimSpace(ref) + if ref == "" { + return "", "" + } + index := strings.LastIndex(ref, ".") + if index == -1 || index+1 >= len(ref) { + return "", "" + } + return strings.TrimSpace(ref[:index]), strings.TrimSpace(ref[index+1:]) +} + +func normalizeTypeRef(ref, pkgPath string) string { + ref = strings.TrimSpace(ref) + if ref == "" { + return "" + } + if strings.Contains(ref, ".") { + return ref + } + if pkgPath == "" { + return ref + } + return pkgPath + "." + ref +} + +func registerType(registry *x.Registry, ref string, rType reflect.Type) { + if registry == nil || rType == nil { + return + } + pkgPath, typeName := splitTypeRef(ref) + registry.Register(x.NewType(rType, x.WithPkgPath(pkgPath), x.WithName(typeName))) +} + +func firstPackageDir(pkg *packages.Package) string { + if pkg == nil { + return "" + } + for _, filePath := range pkg.GoFiles { + if filePath == "" { + continue + } + return filepath.Dir(filePath) + } + return "" +} + +func exportName(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "Route" + } + runes := []rune(name) + if len(runes) == 0 { + return "Route" + } + if runes[0] >= 'a' && runes[0] <= 'z' { + runes[0] = runes[0] - 32 + } + return string(runes) +} + +func matchesPackagePatternList(pkg string, patterns []string) bool { + for _, pattern := range patterns { + if matchesPackagePattern(pkg, pattern) { + return true + } + } + return false +} + +func matchesPackagePattern(pkg, pattern string) bool { + pkg = strings.TrimSpace(pkg) + pattern = strings.TrimSpace(pattern) + if pkg == "" || pattern == "" { + return false + } + if strings.HasSuffix(pattern, "/...") { + prefix := strings.TrimSuffix(pattern, "/...") + return pkg == prefix || strings.HasPrefix(pkg, prefix+"/") + } + return pkg == pattern +} diff --git a/repository/shape/gorouter/discover_test.go b/repository/shape/gorouter/discover_test.go new file mode 100644 index 000000000..5a21b2d64 --- /dev/null +++ b/repository/shape/gorouter/discover_test.go @@ -0,0 +1,266 @@ +package gorouter + +import ( + "context" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/viant/datly/view/extension" + "github.com/viant/xreflect" +) + +func TestDiscover_MultiFieldRouters(t *testing.T) { + baseDir := t.TempDir() + writeFile(t, filepath.Join(baseDir, "go.mod"), "module example.com/app\n\ngo 1.24\n") + writeFile(t, filepath.Join(baseDir, "pkg", "routes", "routes.go"), `package routes + +type ReportView struct { + ID int `+"`"+`sqlx:"ID"`+"`"+` +} + +type ReportInput struct { + ID int `+"`"+`parameter:",kind=path,in=id"`+"`"+` +} + +type ReportOutput struct { + Data []*ReportView `+"`"+`parameter:",kind=output,in=view"`+"`"+` +} + +type CreateInput struct { + Name string `+"`"+`parameter:",kind=body,in=name"`+"`"+` +} + +type CreateOutput struct { + Status string `+"`"+`parameter:",kind=output,in=status"`+"`"+` +} + +type Router struct { + Report struct{} `+"`"+`component:",path=/v1/report/{id},method=GET,input=ReportInput,output=ReportOutput"`+"`"+` + Create struct{} `+"`"+`component:",path=/v1/report,method=POST,input=CreateInput,output=CreateOutput"`+"`"+` +} +`) + + routes, err := Discover(context.Background(), baseDir, []string{"example.com/app/pkg/..."}, nil) + if err != nil { + t.Fatalf("discover failed: %v", err) + } + if len(routes) != 2 { + t.Fatalf("unexpected route count: %d", len(routes)) + } + if routes[0].InputRef == "" || routes[0].OutputRef == "" { + t.Fatalf("expected fully-qualified contract refs, but had %#v", routes[0]) + } + if routes[0].Source == nil || routes[0].Source.TypeRegistry == nil { + t.Fatalf("expected synthetic source with registry") + } + if routes[0].Source.Type == nil { + t.Fatalf("expected synthetic root type") + } +} + +func TestDiscover_ExcludePattern(t *testing.T) { + baseDir := t.TempDir() + writeFile(t, filepath.Join(baseDir, "go.mod"), "module example.com/app\n\ngo 1.24\n") + writeFile(t, filepath.Join(baseDir, "pkg", "one", "one.go"), "package one\ntype In struct{}\ntype Out struct{}\ntype Router struct { Route struct{} `component:\",path=/one,method=GET,input=In,output=Out\"` }\n") + writeFile(t, filepath.Join(baseDir, "pkg", "two", "two.go"), "package two\ntype In struct{}\ntype Out struct{}\ntype Router struct { Route struct{} `component:\",path=/two,method=GET,input=In,output=Out\"` }\n") + + routes, err := Discover(context.Background(), baseDir, []string{"example.com/app/pkg/..."}, []string{"example.com/app/pkg/two"}) + if err != nil { + t.Fatalf("discover failed: %v", err) + } + if len(routes) != 1 { + t.Fatalf("unexpected route count: %d", len(routes)) + } + if routes[0].RoutePath != "/one" { + t.Fatalf("unexpected route path: %s", routes[0].RoutePath) + } +} + +func TestDiscover_WildcardIncludesVendorSubtree(t *testing.T) { + baseDir := t.TempDir() + writeFile(t, filepath.Join(baseDir, "go.mod"), "module example.com/app\n\ngo 1.24\n") + writeFile(t, filepath.Join(baseDir, "shape", "dev", "vendor", "list", "vendor.go"), `package list + +type VendorInput struct { + ID int `+"`"+`parameter:",kind=path,in=id"`+"`"+` +} + +type VendorOutput struct { +} + +type VendorRouter struct { + Vendor struct{} `+"`"+`component:",path=/v1/vendors/{id},method=GET,input=VendorInput,output=VendorOutput"`+"`"+` +} +`) + writeFile(t, filepath.Join(baseDir, "shape", "dev", "team", "delete", "team.go"), `package delete + +type TeamInput struct { + ID int `+"`"+`parameter:",kind=path,in=id"`+"`"+` +} + +type TeamOutput struct{} + +type TeamRouter struct { + Team struct{} `+"`"+`component:",path=/v1/team/{id},method=DELETE,input=TeamInput,output=TeamOutput"`+"`"+` +} +`) + + routes, err := Discover(context.Background(), baseDir, []string{"example.com/app/shape/dev/..."}, nil) + if err != nil { + t.Fatalf("discover failed: %v", err) + } + if len(routes) != 2 { + t.Fatalf("unexpected route count: %d", len(routes)) + } + foundVendor := false + for _, route := range routes { + if route != nil && route.PackagePath == "example.com/app/shape/dev/vendor/list" { + foundVendor = true + break + } + } + if !foundVendor { + t.Fatalf("expected vendor subtree package to be discovered, got %#v", routes) + } +} + +func TestDiscover_ResolvesImportedEmbeddedOutputType(t *testing.T) { + extension.InitRegistry() + baseDir := t.TempDir() + writeFile(t, filepath.Join(baseDir, "go.mod"), "module example.com/app\n\ngo 1.24\n") + writeFile(t, filepath.Join(baseDir, "pkg", "routes", "routes.go"), `package routes + +import "github.com/viant/xdatly/handler/response" + +type ReportInput struct { + ID int `+"`"+`parameter:",kind=path,in=id"`+"`"+` +} + +type ReportOutput struct { + response.Status `+"`"+`parameter:",kind=output,in=status"`+"`"+` +} + +type Router struct { + Report struct{} `+"`"+`component:",path=/v1/report/{id},method=GET,input=ReportInput,output=ReportOutput"`+"`"+` +} +`) + + routes, err := Discover(context.Background(), baseDir, []string{"example.com/app/pkg/..."}, nil) + if err != nil { + t.Fatalf("discover failed: %v", err) + } + if len(routes) != 1 { + t.Fatalf("unexpected route count: %d", len(routes)) + } + if routes[0].OutputRef != "example.com/app/pkg/routes.ReportOutput" { + t.Fatalf("unexpected output ref: %s", routes[0].OutputRef) + } +} + +func TestDiscover_NormalizesHandlerType(t *testing.T) { + baseDir := t.TempDir() + writeFile(t, filepath.Join(baseDir, "go.mod"), "module example.com/app\n\ngo 1.24\n") + writeFile(t, filepath.Join(baseDir, "pkg", "routes", "routes.go"), `package routes + +import ( + "context" + xhandler "github.com/viant/xdatly/handler" +) + +type ReportInput struct { + ID int `+"`"+`parameter:",kind=path,in=id"`+"`"+` +} + +type ReportOutput struct { + OK bool `+"`"+`parameter:",kind=output,in=view"`+"`"+` +} + +type Handler struct{} + +func (h *Handler) Exec(ctx context.Context, sess xhandler.Session) (interface{}, error) { + return ReportOutput{OK: true}, nil +} + +type Router struct { + Report struct{} `+"`"+`component:",path=/v1/report/{id},method=GET,input=ReportInput,output=ReportOutput,handler=Handler"`+"`"+` +} +`) + + routes, err := Discover(context.Background(), baseDir, []string{"example.com/app/pkg/..."}, nil) + if err != nil { + t.Fatalf("discover failed: %v", err) + } + if len(routes) != 1 { + t.Fatalf("unexpected route count: %d", len(routes)) + } + tag := routes[0].Source.Type.Field(0).Tag.Get("component") + if tag == "" || filepath.Base(routes[0].PackagePath) == "" { + t.Fatalf("expected route component tag to be present") + } + if got := routes[0].Source.Type.Field(0).Tag.Get("component"); !strings.Contains(got, "handler=example.com/app/pkg/routes.Handler") { + t.Fatalf("expected normalized handler ref in component tag, got %q", got) + } +} + +func TestDiscover_PrefersLinkedNamedType(t *testing.T) { + extension.InitRegistry() + type linkedReportView struct { + ID int `sqlx:"ID"` + Name string `sqlx:"NAME"` + } + if err := extension.Config.Types.Register("ReportView", + xreflect.WithPackage("example.com/app/pkg/routes"), + xreflect.WithReflectType(reflect.TypeOf(linkedReportView{})), + ); err != nil { + t.Fatalf("register linked type: %v", err) + } + + baseDir := t.TempDir() + writeFile(t, filepath.Join(baseDir, "go.mod"), "module example.com/app\n\ngo 1.24\n") + writeFile(t, filepath.Join(baseDir, "pkg", "routes", "routes.go"), `package routes + +type ReportView struct { + Items []*struct { + ID int `+"`"+`sqlx:"ID"`+"`"+` + } `+"`"+`view:",table=ITEM"`+"`"+` +} + +type ReportInput struct{} + +type ReportOutput struct { + Data *ReportView `+"`"+`parameter:",kind=output,in=view"`+"`"+` +} + +type Router struct { + Report struct{} `+"`"+`component:",path=/v1/report,method=GET,input=ReportInput,output=ReportOutput,view=ReportView"`+"`"+` +} +`) + + routes, err := Discover(context.Background(), baseDir, []string{"example.com/app/pkg/..."}, nil) + if err != nil { + t.Fatalf("discover failed: %v", err) + } + if len(routes) != 1 { + t.Fatalf("unexpected route count: %d", len(routes)) + } + lookup := routes[0].Source.TypeRegistry.Lookup("example.com/app/pkg/routes.ReportView") + if lookup == nil || lookup.Type == nil { + t.Fatalf("expected route view to be registered") + } + if got, want := lookup.Type, reflect.TypeOf(linkedReportView{}); got != want { + t.Fatalf("expected linked route view type %v, got %v", want, got) + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir failed: %v", err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write failed: %v", err) + } +} diff --git a/repository/shape/gorouter/model.go b/repository/shape/gorouter/model.go new file mode 100644 index 000000000..d93bd2ea5 --- /dev/null +++ b/repository/shape/gorouter/model.go @@ -0,0 +1,22 @@ +package gorouter + +import "github.com/viant/datly/repository/shape" + +// RouteSource represents one component route field discovered from a Go source package. +type RouteSource struct { + Name string + FieldName string + FilePath string + PackageName string + PackagePath string + PackageDir string + RoutePath string + Method string + Connector string + InputRef string + OutputRef string + ViewRef string + SourceURL string + SummaryURL string + Source *shape.Source +} diff --git a/repository/shape/load/columns.go b/repository/shape/load/columns.go index 147a2a5b4..015e41efd 100644 --- a/repository/shape/load/columns.go +++ b/repository/shape/load/columns.go @@ -32,6 +32,9 @@ func inferColumnsFromType(rType reflect.Type) []*view.Column { if !f.IsExported() { continue } + if shouldSkipInferredField(f) { + continue + } colName := sqlxColumnName(f) if colName == "" { colName = f.Name @@ -44,6 +47,46 @@ func inferColumnsFromType(rType reflect.Type) []*view.Column { return cols } +func shouldSkipInferredField(field reflect.StructField) bool { + if field.Name == "-" { + return true + } + rawTag := string(field.Tag) + if strings.Contains(rawTag, `view:"`) || strings.Contains(rawTag, `on:"`) { + return true + } + if strings.Contains(rawTag, `sqlx:"-"`) { + return true + } + return false +} + +func inferredColumnsArePlaceholders(columns []*view.Column) bool { + if len(columns) == 0 { + return false + } + for _, column := range columns { + if column == nil || !isPlaceholderColumnName(column.Name) { + return false + } + } + return true +} + +func isPlaceholderColumnName(name string) bool { + name = strings.TrimSpace(strings.ToLower(name)) + name = strings.ReplaceAll(name, "_", "") + if !strings.HasPrefix(name, "col") || len(name) == len("col") { + return false + } + for i := len("col"); i < len(name); i++ { + if name[i] < '0' || name[i] > '9' { + return false + } + } + return true +} + // sqlxColumnName reads the sqlx struct tag to get the database column name. func sqlxColumnName(f reflect.StructField) string { tag := f.Tag.Get("sqlx") @@ -52,9 +95,15 @@ func sqlxColumnName(f reflect.StructField) string { } for _, part := range strings.Split(tag, ",") { part = strings.TrimSpace(part) + if part == "" { + continue + } if strings.HasPrefix(part, "name=") { return strings.TrimPrefix(part, "name=") } + if !strings.Contains(part, "=") { + return part + } } return "" } diff --git a/repository/shape/load/columns_test.go b/repository/shape/load/columns_test.go new file mode 100644 index 000000000..96a326745 --- /dev/null +++ b/repository/shape/load/columns_test.go @@ -0,0 +1,24 @@ +package load + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +type sampleInferredColumnsRoot struct { + ID int `sqlx:"ID"` + Products []*sampleInferredRel `view:",table=PRODUCT" on:"Id:ID=VendorId:VENDOR_ID"` + Ignored string `sqlx:"-"` +} + +type sampleInferredRel struct { + VendorID int `sqlx:"VENDOR_ID"` +} + +func TestInferColumnsFromType_SkipsSemanticFields(t *testing.T) { + cols := inferColumnsFromType(reflect.TypeOf(sampleInferredColumnsRoot{})) + require.Len(t, cols, 1) + require.Equal(t, "ID", cols[0].Name) +} diff --git a/repository/shape/load/loader.go b/repository/shape/load/loader.go index 9f38cb8eb..6cd3a585b 100644 --- a/repository/shape/load/loader.go +++ b/repository/shape/load/loader.go @@ -3,19 +3,25 @@ package load import ( "context" "fmt" + "path/filepath" "reflect" + "sort" "strings" - "time" "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/compile/pipeline" dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/repository/shape/typectx" shapevalidate "github.com/viant/datly/repository/shape/validate" "github.com/viant/datly/shared" + "github.com/viant/datly/utils/types" "github.com/viant/datly/view" "github.com/viant/datly/view/extension" "github.com/viant/datly/view/state" + "github.com/viant/datly/view/tags" + "github.com/viant/sqlparser" + "github.com/viant/xdatly/handler/response" ) // Loader materializes runtime view artifacts from normalized shape plan. @@ -27,11 +33,17 @@ func New() *Loader { } // LoadViews implements shape.Loader. -func (l *Loader) LoadViews(ctx context.Context, planned *shape.PlanResult, _ ...shape.LoadOption) (*shape.ViewArtifacts, error) { +func (l *Loader) LoadViews(ctx context.Context, planned *shape.PlanResult, opts ...shape.LoadOption) (*shape.ViewArtifacts, error) { if err := ctx.Err(); err != nil { return nil, err } - pResult, resource, err := l.materialize(planned) + loadOptions := &shape.LoadOptions{} + for _, opt := range opts { + if opt != nil { + opt(loadOptions) + } + } + pResult, resource, err := l.materialize(planned, loadOptions) if err != nil { return nil, err } @@ -42,25 +54,61 @@ func (l *Loader) LoadViews(ctx context.Context, planned *shape.PlanResult, _ ... } // LoadComponent implements shape.Loader. -func (l *Loader) LoadComponent(ctx context.Context, planned *shape.PlanResult, _ ...shape.LoadOption) (*shape.ComponentArtifact, error) { +func (l *Loader) LoadComponent(ctx context.Context, planned *shape.PlanResult, opts ...shape.LoadOption) (*shape.ComponentArtifact, error) { if err := ctx.Err(); err != nil { return nil, err } - pResult, resource, err := l.materialize(planned) + loadOptions := &shape.LoadOptions{} + for _, opt := range opts { + if opt != nil { + opt(loadOptions) + } + } + pResult, resource, err := l.materialize(planned, loadOptions) if err != nil { return nil, err } + if err := validateComponentRoutes(pResult.Components); err != nil { + return nil, err + } if len(pResult.Views) == 0 { - return nil, ErrEmptyViewPlan + if err := materializeComponentRouteView(planned.Source, pResult, resource); err != nil { + return nil, err + } + if len(resource.Views) == 0 && !allowsViewlessComponent(pResult.Components) { + return nil, ErrEmptyViewPlan + } } - component := buildComponent(planned.Source, pResult) + component := buildComponent(planned.Source, pResult, resource, loadOptions) return &shape.ComponentArtifact{ Resource: resource, Component: component, }, nil } -func (l *Loader) materialize(planned *shape.PlanResult) (*plan.Result, *view.Resource, error) { +func validateComponentRoutes(routes []*plan.ComponentRoute) error { + count := 0 + for _, route := range routes { + if route != nil { + count++ + } + } + if count <= 1 { + return nil + } + return fmt.Errorf("shape load: multiple component routes are not supported for a single component artifact") +} + +func allowsViewlessComponent(routes []*plan.ComponentRoute) bool { + for _, route := range routes { + if route != nil { + return true + } + } + return false +} + +func (l *Loader) materialize(planned *shape.PlanResult, loadOptions *shape.LoadOptions) (*plan.Result, *view.Resource, error) { if planned == nil || planned.Source == nil { return nil, nil, shape.ErrNilSource } @@ -77,8 +125,31 @@ func (l *Loader) materialize(planned *shape.PlanResult) (*plan.Result, *view.Res if err != nil { return nil, nil, err } + if loadOptions != nil && loadOptions.UseTypeContextPackages { + inheritViewSchemaPackage(aView, pResult.TypeContext) + } resource.AddViews(aView) } + attachViewRelations(resource, pResult.Views) + if err := enrichRelationHolderTypes(resource, pResult.Views); err != nil { + return nil, nil, err + } + rootView := rootResourceView(resource, pResult.Views) + for _, item := range pResult.States { + if item == nil { + continue + } + param := cloneStateParameter(item) + if param == nil { + continue + } + normalizeDerivedInputSchema(param, resource) + if rootView != nil { + inheritRootOutputSchema(param, rootView) + } + ensureMaterializedOutputSchema(param, rootView) + resource.AddParameters(param) + } if err := shapevalidate.ValidateRelations(resource, resource.Views...); err != nil { return nil, nil, err } @@ -88,47 +159,368 @@ func (l *Loader) materialize(planned *shape.PlanResult) (*plan.Result, *view.Res Name: k, In: state.NewConstLocation(k), Value: v, + Tag: `internal:"true"`, + Schema: &state.Schema{ + Name: "string", + DataType: "string", + Cardinality: state.One, + }, } resource.AddParameters(constParam) } } - // Gap 7: apply global cache TTL directive to root view. + bindTemplateParameters(resource) + // Apply cache directives only as resource-level provider definitions. + // View-level cache binding comes from explicit view metadata such as set_cache(...). if pResult.Directives != nil && pResult.Directives.Cache != nil { - if ttl := strings.TrimSpace(pResult.Directives.Cache.TTL); ttl != "" { - if dur, err := time.ParseDuration(ttl); err == nil && dur > 0 { - ttlMs := int(dur.Milliseconds()) - if rootPlan := pickRootView(pResult.Views); rootPlan != nil { - for _, rv := range resource.Views { - if rv != nil && rv.Name == rootPlan.Name { - if rv.Cache == nil { - rv.Cache = &view.Cache{} - } - rv.Cache.TimeToLiveMs = ttlMs - break - } - } - } + if name := strings.TrimSpace(pResult.Directives.Cache.Name); name != "" { + provider := strings.TrimSpace(pResult.Directives.Cache.Provider) + location := strings.TrimSpace(pResult.Directives.Cache.Location) + ttlMs := pResult.Directives.Cache.TimeToLiveMs + if provider != "" && location != "" && ttlMs > 0 { + resource.CacheProviders = append(resource.CacheProviders, &view.Cache{ + Name: name, + Provider: provider, + Location: location, + TimeToLiveMs: ttlMs, + }) } } } return pResult, resource, nil } -func buildComponent(source *shape.Source, pResult *plan.Result) *Component { +func buildComponent(source *shape.Source, pResult *plan.Result, resource *view.Resource, loadOptions *shape.LoadOptions) *Component { component := &Component{Method: "GET"} if source != nil { component.Name = source.Name component.URI = source.Name } + component.TypeContext = cloneTypeContext(pResult.TypeContext) + applyComponentRoutes(component, pResult.Components) applyViewMeta(component, pResult.Views) - applyStateBuckets(component, pResult.States) + applyStateBuckets(component, pResult.States, resource, loadOptions) + applyStateBuckets(component, synthesizeConstStates(pResult.Const), resource, loadOptions) + applyStateBuckets(component, synthesizeMissingRouteContractStates(component, pResult.Components), resource, loadOptions) component.Input = append(component.Input, synthesizePredicateStates(component.Input, component.Predicates)...) - component.TypeContext = cloneTypeContext(pResult.TypeContext) component.Directives = cloneDirectives(pResult.Directives) component.ColumnsDiscovery = pResult.ColumnsDiscovery + component.TypeSpecs = resolveTypeSpecs(pResult) return component } +func applyComponentRoutes(component *Component, routes []*plan.ComponentRoute) { + if component == nil || len(routes) == 0 { + return + } + component.ComponentRoutes = cloneComponentRoutes(routes) + primary := firstComponentRoute(routes) + if primary == nil { + return + } + if uri := strings.TrimSpace(primary.RoutePath); uri != "" { + component.URI = uri + if strings.TrimSpace(component.Name) == "" { + component.Name = uri + } + } + if method := strings.TrimSpace(primary.Method); method != "" { + component.Method = method + } + if strings.TrimSpace(component.Name) == "" { + component.Name = strings.TrimSpace(primary.Name) + } + if strings.TrimSpace(component.RootView) == "" && strings.TrimSpace(primary.ViewName) != "" { + component.RootView = routeViewAlias(primary) + if component.RootView != "" { + component.Views = append(component.Views, component.RootView) + } + } +} + +func cloneComponentRoutes(routes []*plan.ComponentRoute) []*plan.ComponentRoute { + if len(routes) == 0 { + return nil + } + result := make([]*plan.ComponentRoute, 0, len(routes)) + for _, item := range routes { + if item == nil { + continue + } + cloned := *item + result = append(result, &cloned) + } + if len(result) == 0 { + return nil + } + return result +} + +func firstComponentRoute(routes []*plan.ComponentRoute) *plan.ComponentRoute { + for _, item := range routes { + if item != nil { + return item + } + } + return nil +} + +func materializeComponentRouteView(source *shape.Source, pResult *plan.Result, resource *view.Resource) error { + route := firstComponentRoute(pResult.Components) + if route == nil || resource == nil { + return nil + } + viewType := resolveRouteViewType(source, route) + if viewType == nil { + return nil + } + viewName := routeViewAlias(route) + if viewName == "" { + viewName = "View" + } + opts := []view.Option{ + view.WithSchema(state.NewSchema(viewType)), + view.WithMode(componentRouteMode(route)), + } + if connectorRef := strings.TrimSpace(route.Connector); connectorRef != "" { + opts = append(opts, view.WithConnectorRef(connectorRef)) + } + rootView := view.NewView(viewName, "", opts...) + if sourceURL := absoluteRouteSourceURL(source, route); sourceURL != "" { + tmpl := view.NewTemplate("") + tmpl.SourceURL = sourceURL + rootView.Template = tmpl + } + resource.AddViews(rootView) + return nil +} + +func componentRouteMode(route *plan.ComponentRoute) view.Mode { + if route == nil { + return view.ModeQuery + } + if strings.TrimSpace(route.Handler) != "" { + return view.ModeHandler + } + switch strings.ToUpper(strings.TrimSpace(route.Method)) { + case "", "GET": + return view.ModeQuery + default: + return view.ModeExec + } +} + +func resolveRouteViewType(source *shape.Source, route *plan.ComponentRoute) reflect.Type { + if source == nil || route == nil { + return nil + } + typeName := strings.TrimSpace(route.ViewName) + if typeName == "" { + return nil + } + registry := source.EnsureTypeRegistry() + if registry == nil { + return nil + } + if lookup := registry.Lookup(typeName); lookup != nil && lookup.Type != nil { + return lookup.Type + } + resolver := typectx.NewResolver(registry, nil) + if resolved, err := resolver.Resolve(typeName); err == nil && resolved != "" { + if lookup := registry.Lookup(resolved); lookup != nil && lookup.Type != nil { + return lookup.Type + } + } + return nil +} + +func routeViewAlias(route *plan.ComponentRoute) string { + if route == nil { + return "" + } + if name := strings.TrimSpace(route.Name); name != "" { + return name + } + viewName := strings.TrimSpace(route.ViewName) + if index := strings.LastIndex(viewName, "."); index >= 0 { + viewName = viewName[index+1:] + } + viewName = strings.TrimSuffix(viewName, "View") + return strings.TrimSpace(viewName) +} + +func absoluteRouteSourceURL(source *shape.Source, route *plan.ComponentRoute) string { + if route == nil { + return "" + } + sourceURL := strings.TrimSpace(route.SourceURL) + return absolutizeRouteAssetURL(source, sourceURL) +} + +func absoluteRouteSummaryURL(source *shape.Source, route *plan.ComponentRoute) string { + if route == nil { + return "" + } + return absolutizeRouteAssetURL(source, strings.TrimSpace(route.SummaryURL)) +} + +func absolutizeRouteAssetURL(source *shape.Source, sourceURL string) string { + if sourceURL == "" || strings.Contains(sourceURL, "://") { + return sourceURL + } + if filepath.IsAbs(sourceURL) { + return sourceURL + } + baseDir := "" + if source != nil { + baseDir = source.BaseDir() + } + if baseDir == "" { + return sourceURL + } + return filepath.Join(baseDir, filepath.FromSlash(sourceURL)) +} + +func resolveTypeSpecs(pResult *plan.Result) map[string]*TypeSpec { + if pResult == nil { + return nil + } + specs := map[string]*TypeSpec{} + directives := pResult.Directives + if directives != nil { + if typeName := strings.TrimSpace(directives.InputType); typeName != "" { + specs["input"] = &TypeSpec{Key: "input", Role: TypeRoleInput, TypeName: typeName, Source: "directive"} + } + if typeName := strings.TrimSpace(directives.OutputType); typeName != "" { + specs["output"] = &TypeSpec{Key: "output", Role: TypeRoleOutput, TypeName: typeName, Source: "directive"} + } + if dest := strings.TrimSpace(directives.InputDest); dest != "" { + spec := ensureTypeSpec(specs, "input", TypeRoleInput) + spec.Dest = dest + spec.Source = "directive" + } + if dest := strings.TrimSpace(directives.OutputDest); dest != "" { + spec := ensureTypeSpec(specs, "output", TypeRoleOutput) + spec.Dest = dest + spec.Source = "directive" + } + } + globalDest := "" + if directives != nil { + globalDest = strings.TrimSpace(directives.Dest) + } + for _, aView := range pResult.Views { + if aView == nil || strings.TrimSpace(aView.Name) == "" { + continue + } + key := "view:" + aView.Name + spec := ensureTypeSpec(specs, key, TypeRoleView) + spec.Alias = aView.Name + if globalDest != "" && spec.Dest == "" { + spec.Dest = globalDest + spec.Inherited = true + spec.Source = "directive" + } + if aView.Declaration != nil { + if typeName := strings.TrimSpace(aView.Declaration.TypeName); typeName != "" { + spec.TypeName = typeName + spec.Source = "decl" + } + if dest := strings.TrimSpace(aView.Declaration.Dest); dest != "" { + spec.Dest = dest + spec.Inherited = false + spec.Source = "decl" + } + if tagType, tagDest := parseTypeSpecTag(aView.Declaration.Tag); tagType != "" || tagDest != "" { + if spec.TypeName == "" && tagType != "" { + spec.TypeName = tagType + spec.Source = "annotation" + } + if spec.Dest == "" && tagDest != "" { + spec.Dest = tagDest + spec.Inherited = false + spec.Source = "annotation" + } + } + } + } + if root := pickRootView(pResult.Views); root != nil { + if rootSpec := specs["view:"+root.Name]; rootSpec != nil && strings.TrimSpace(rootSpec.Dest) != "" { + rootDest := strings.TrimSpace(rootSpec.Dest) + for _, aView := range pResult.Views { + if aView == nil || strings.TrimSpace(aView.Name) == "" || aView.Name == root.Name { + continue + } + spec := ensureTypeSpec(specs, "view:"+aView.Name, TypeRoleView) + spec.Alias = aView.Name + if strings.TrimSpace(spec.Dest) == "" || spec.Source == "directive" || spec.Source == "inherit" { + spec.Dest = rootDest + spec.Inherited = true + spec.Source = "inherit" + } + } + } + } + if globalDest != "" { + inputSpec := ensureTypeSpec(specs, "input", TypeRoleInput) + if strings.TrimSpace(inputSpec.Dest) == "" { + inputSpec.Dest = globalDest + inputSpec.Inherited = true + if inputSpec.Source == "" { + inputSpec.Source = "directive" + } + } + outputSpec := ensureTypeSpec(specs, "output", TypeRoleOutput) + if strings.TrimSpace(outputSpec.Dest) == "" { + outputSpec.Dest = globalDest + outputSpec.Inherited = true + if outputSpec.Source == "" { + outputSpec.Source = "directive" + } + } + } + if len(specs) == 0 { + return nil + } + return specs +} + +func ensureTypeSpec(specs map[string]*TypeSpec, key string, role TypeRole) *TypeSpec { + if spec, ok := specs[key]; ok && spec != nil { + return spec + } + spec := &TypeSpec{Key: key, Role: role} + specs[key] = spec + return spec +} + +func parseTypeSpecTag(raw string) (string, string) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", "" + } + var typeName, dest string + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + key, value, ok := strings.Cut(part, "=") + if !ok { + continue + } + key = strings.ToLower(strings.TrimSpace(key)) + value = strings.TrimSpace(strings.Trim(value, `"'`)) + switch key { + case "type": + typeName = value + case "dest": + dest = value + } + } + return strings.TrimSpace(typeName), strings.TrimSpace(dest) +} + // applyViewMeta populates the component with view names, declarations, relations, // query selectors, predicate maps, and root view from the plan view list. func applyViewMeta(component *Component, views []*plan.View) { @@ -138,7 +530,7 @@ func applyViewMeta(component *Component, views []*plan.View) { } component.Views = append(component.Views, aView.Name) if aView.Declaration != nil { - indexViewDeclaration(component, aView.Name, aView.Declaration) + indexViewDeclaration(component, declaredViewIndexName(aView), aView.Declaration) } if len(aView.Relations) > 0 { component.Relations = append(component.Relations, aView.Relations...) @@ -153,6 +545,18 @@ func applyViewMeta(component *Component, views []*plan.View) { } } +func declaredViewIndexName(aView *plan.View) string { + if aView == nil { + return "" + } + if queryNode, err := sqlparser.ParseQuery(strings.TrimSpace(aView.SQL)); err == nil && queryNode != nil { + if inferredName, _, err := pipeline.InferRoot(queryNode, aView.Name); err == nil && strings.TrimSpace(inferredName) != "" { + return inferredName + } + } + return aView.Name +} + // indexViewDeclaration registers the declaration's query selector and predicates // on the component index maps, creating them on demand. func indexViewDeclaration(component *Component, viewName string, decl *plan.ViewDeclaration) { @@ -176,31 +580,201 @@ func indexViewDeclaration(component *Component, viewName string, decl *plan.View // applyStateBuckets sorts plan states into the typed buckets on the component // (Input, Output, Meta, Async, Other) based on the state's location kind. -func applyStateBuckets(component *Component, states []*plan.State) { +func applyStateBuckets(component *Component, states []*plan.State, resource *view.Resource, loadOptions *shape.LoadOptions) { for _, item := range states { if item == nil { continue } + cloned := clonePlanState(item) + if cloned == nil { + continue + } + if loadOptions != nil && loadOptions.UseTypeContextPackages { + inheritTypeContextSchemaPackage(&cloned.Parameter, component) + } + normalizeDerivedInputSchema(&cloned.Parameter, resource) + inheritRootBodySchema(&cloned.Parameter, rootResourceView(resource, nil)) kind := state.Kind(strings.ToLower(item.KindString())) inName := item.InName() if kind == "" && inName == "" { - component.Other = append(component.Other, item) + component.Other = append(component.Other, cloned) continue } switch kind { case state.KindQuery, state.KindPath, state.KindHeader, state.KindRequestBody, + state.KindView, state.KindComponent, state.KindConst, state.KindForm, state.KindCookie, state.KindRequest, "": - component.Input = append(component.Input, item) + component.Input = append(component.Input, cloned) case state.KindOutput: - component.Output = append(component.Output, item) + component.Output = append(component.Output, cloned) case state.KindMeta: - component.Meta = append(component.Meta, item) + component.Meta = append(component.Meta, cloned) case state.KindAsync: - component.Async = append(component.Async, item) + component.Async = append(component.Async, cloned) default: - component.Other = append(component.Other, item) + component.Other = append(component.Other, cloned) + } + } +} + +func inheritTypeContextSchemaPackage(param *state.Parameter, component *Component) { + if param == nil || param.Schema == nil || component == nil || component.TypeContext == nil { + return + } + if param.Schema.Type() != nil { + return + } + if strings.TrimSpace(param.Schema.Package) != "" || strings.TrimSpace(param.Schema.PackagePath) != "" { + return + } + typeName := strings.TrimSpace(shared.FirstNotEmpty(param.Schema.Name, param.Schema.DataType)) + if typeName == "" || strings.Contains(typeName, ".") { + return + } + if _, err := types.LookupType(nil, typeName); err == nil { + return + } + pkgPath := strings.TrimSpace(component.TypeContext.PackagePath) + if pkgPath == "" { + pkgPath = strings.TrimSpace(component.TypeContext.DefaultPackage) + } + if pkgPath == "" { + return + } + param.Schema.Package = pkgPath + param.Schema.PackagePath = pkgPath +} + +func inheritViewSchemaPackage(aView *view.View, ctx *typectx.Context) { + if aView == nil || aView.Schema == nil || ctx == nil { + return + } + if aView.Schema.Type() != nil { + return + } + if strings.TrimSpace(aView.Schema.Package) != "" || strings.TrimSpace(aView.Schema.PackagePath) != "" { + return + } + typeName := strings.TrimSpace(shared.FirstNotEmpty(aView.Schema.Name, aView.Schema.DataType)) + if typeName == "" || strings.Contains(typeName, ".") { + return + } + if _, err := types.LookupType(nil, typeName); err == nil { + return + } + pkgPath := strings.TrimSpace(ctx.PackagePath) + if pkgPath == "" { + pkgPath = strings.TrimSpace(ctx.DefaultPackage) + } + if pkgPath == "" { + return + } + aView.Schema.Package = pkgPath + aView.Schema.PackagePath = pkgPath +} + +func synthesizeMissingRouteContractStates(component *Component, routes []*plan.ComponentRoute) []*plan.State { + if component == nil || len(routes) == 0 { + return nil + } + declared := map[string]bool{} + register := func(items []*plan.State) { + for _, item := range items { + if item == nil || item.In == nil { + continue + } + declared[routeContractStateKey(item.Name, item.In.Kind, item.In.Name)] = true + } + } + register(component.Input) + register(component.Output) + register(component.Meta) + register(component.Async) + register(component.Other) + + var result []*plan.State + for _, route := range routes { + if route == nil { + continue + } + for _, item := range contractStates(route.InputType) { + key := routeContractStateKey(item.Name, item.In.Kind, item.In.Name) + if declared[key] { + continue + } + declared[key] = true + result = append(result, item) + } + for _, item := range contractStates(route.OutputType) { + key := routeContractStateKey(item.Name, item.In.Kind, item.In.Name) + if declared[key] { + continue + } + declared[key] = true + result = append(result, item) } } + return result +} + +func contractStates(rType reflect.Type) []*plan.State { + rType = unwrapContractStateType(rType) + if rType == nil || rType.Kind() != reflect.Struct { + return nil + } + var result []*plan.State + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if field.Anonymous { + result = append(result, contractStates(field.Type)...) + } + parsed, err := tags.ParseStateTags(field.Tag, nil) + if err != nil || parsed == nil || parsed.Parameter == nil { + continue + } + param := parsed.Parameter + name := strings.TrimSpace(param.Name) + if name == "" { + name = strings.TrimSpace(field.Name) + } + locationKind := state.Kind(strings.ToLower(strings.TrimSpace(param.Kind))) + locationName := strings.TrimSpace(param.In) + item := &plan.State{ + Parameter: state.Parameter{ + Name: name, + In: &state.Location{Kind: locationKind, Name: locationName}, + When: param.When, + Scope: param.Scope, + Required: param.Required, + Async: param.Async, + Cacheable: param.Cacheable, + With: param.With, + URI: param.URI, + ErrorStatusCode: param.ErrorCode, + ErrorMessage: param.ErrorMessage, + Tag: string(field.Tag), + Schema: state.NewSchema(field.Type), + }, + } + state.BuildCodec(parsed, &item.Parameter) + state.BuildHandler(parsed, &item.Parameter) + if dataType := strings.TrimSpace(param.DataType); dataType != "" && item.Schema != nil { + item.Schema.DataType = dataType + } + result = append(result, item) + } + return result +} + +func unwrapContractStateType(rType reflect.Type) reflect.Type { + for rType != nil && (rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array) { + rType = rType.Elem() + } + return rType +} + +func routeContractStateKey(name string, kind state.Kind, in string) string { + return strings.ToLower(strings.TrimSpace(name)) + "|" + strings.ToLower(strings.TrimSpace(string(kind))) + "|" + strings.ToLower(strings.TrimSpace(in)) } // synthesizePredicateStates creates query parameters for view-level predicates whose @@ -245,6 +819,37 @@ func synthesizePredicateStates(input []*plan.State, predicates map[string][]*pla return result } +func synthesizeConstStates(constants map[string]string) []*plan.State { + if len(constants) == 0 { + return nil + } + keys := make([]string, 0, len(constants)) + for key := range constants { + key = strings.TrimSpace(key) + if key != "" { + keys = append(keys, key) + } + } + sort.Strings(keys) + result := make([]*plan.State, 0, len(keys)) + for _, key := range keys { + result = append(result, &plan.State{ + Parameter: state.Parameter{ + Name: key, + In: state.NewConstLocation(key), + Value: constants[key], + Tag: `internal:"true"`, + Schema: &state.Schema{ + Name: "string", + DataType: "string", + Cardinality: state.One, + }, + }, + }) + } + return result +} + func cloneTypeContext(input *typectx.Context) *typectx.Context { if input == nil { return nil @@ -282,11 +887,21 @@ func cloneDirectives(input *dqlshape.Directives) *dqlshape.Directives { ret := &dqlshape.Directives{ Meta: strings.TrimSpace(input.Meta), DefaultConnector: strings.TrimSpace(input.DefaultConnector), + Dest: strings.TrimSpace(input.Dest), + InputDest: strings.TrimSpace(input.InputDest), + OutputDest: strings.TrimSpace(input.OutputDest), + RouterDest: strings.TrimSpace(input.RouterDest), + InputType: strings.TrimSpace(input.InputType), + OutputType: strings.TrimSpace(input.OutputType), } if input.Cache != nil { ret.Cache = &dqlshape.CacheDirective{ - Enabled: input.Cache.Enabled, - TTL: strings.TrimSpace(input.Cache.TTL), + Enabled: input.Cache.Enabled, + TTL: strings.TrimSpace(input.Cache.TTL), + Name: strings.TrimSpace(input.Cache.Name), + Provider: strings.TrimSpace(input.Cache.Provider), + Location: strings.TrimSpace(input.Cache.Location), + TimeToLiveMs: input.Cache.TimeToLiveMs, } } if input.MCP != nil { @@ -312,7 +927,10 @@ func cloneDirectives(input *dqlshape.Directives) *dqlshape.Directives { } } } - if ret.Meta == "" && ret.DefaultConnector == "" && ret.Cache == nil && ret.MCP == nil && ret.Route == nil && len(ret.Const) == 0 { + if ret.Meta == "" && ret.DefaultConnector == "" && + ret.Dest == "" && ret.InputDest == "" && ret.OutputDest == "" && ret.RouterDest == "" && + ret.InputType == "" && ret.OutputType == "" && + ret.Cache == nil && ret.MCP == nil && ret.Route == nil && len(ret.Const) == 0 { return nil } return ret @@ -348,11 +966,6 @@ func materializeView(item *plan.View) (*view.View, error) { } schemaType := bestSchemaType(item) - if schemaType == nil { - return nil, fmt.Errorf("shape load: missing schema type for view %q", item.Name) - } - - schema := newSchema(schemaType, item.Cardinality) mode := view.ModeQuery switch strings.TrimSpace(item.Mode) { case string(view.ModeExec): @@ -362,6 +975,14 @@ func materializeView(item *plan.View) (*view.View, error) { case string(view.ModeQuery): mode = view.ModeQuery } + if shouldDeferQuerySchemaType(schemaType, mode) { + schemaType = nil + } + if schemaType == nil && !allowsDeferredSchema(item, mode) { + return nil, fmt.Errorf("shape load: missing schema type for view %q", item.Name) + } + + schema := newSchema(schemaType, item.Cardinality) opts := []view.Option{view.WithSchema(schema), view.WithMode(mode)} if item.Connector != "" { @@ -371,8 +992,12 @@ func materializeView(item *plan.View) (*view.View, error) { tmpl := view.NewTemplate(item.SQL) tmpl.SourceURL = item.SQLURI if strings.TrimSpace(item.Summary) != "" { + name := strings.TrimSpace(item.SummaryName) + if name == "" { + name = "Summary" + } tmpl.Summary = &view.TemplateSummary{ - Name: "Summary", + Name: name, Source: item.Summary, Kind: view.MetaKindRecord, } @@ -400,6 +1025,28 @@ func materializeView(item *plan.View) (*view.View, error) { if item.Declaration != nil && strings.TrimSpace(item.Declaration.Tag) != "" { aView.Tag = strings.TrimSpace(item.Declaration.Tag) } + if item.Declaration != nil && len(item.Declaration.ColumnsConfig) > 0 { + if aView.ColumnsConfig == nil { + aView.ColumnsConfig = map[string]*view.ColumnConfig{} + } + for name, cfg := range item.Declaration.ColumnsConfig { + name = strings.TrimSpace(name) + if name == "" || cfg == nil { + continue + } + columnCfg := aView.ColumnsConfig[name] + if columnCfg == nil { + columnCfg = &view.ColumnConfig{Name: name} + aView.ColumnsConfig[name] = columnCfg + } + if dataType := strings.TrimSpace(cfg.DataType); dataType != "" { + columnCfg.DataType = stringPtr(dataType) + } + if tag := strings.TrimSpace(cfg.Tag); tag != "" { + columnCfg.Tag = stringPtr(tag) + } + } + } if strings.TrimSpace(item.SelectorNamespace) != "" || item.SelectorNoLimit != nil || item.SelectorLimit != nil { if aView.Selector == nil { aView.Selector = &view.Config{} @@ -433,13 +1080,33 @@ func materializeView(item *plan.View) (*view.View, error) { // generate accurate Go struct definitions during bootstrap. Only applied when // the view has no columns yet (avoids overwriting explicit column config). if len(aView.Columns) == 0 { - if cols := inferColumnsFromType(item.ElementType); len(cols) > 0 { + if cols := inferColumnsFromType(bestSchemaType(item)); len(cols) > 0 && !inferredColumnsArePlaceholders(cols) { aView.Columns = cols } } return aView, nil } +func allowsDeferredSchema(item *plan.View, mode view.Mode) bool { + if item == nil { + return false + } + if mode != view.ModeQuery { + return false + } + return strings.TrimSpace(item.Table) != "" || strings.TrimSpace(item.SQL) != "" || strings.TrimSpace(item.SQLURI) != "" +} + +func shouldDeferQuerySchemaType(rType reflect.Type, mode view.Mode) bool { + if rType == nil || mode != view.ModeQuery { + return false + } + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + } + return rType.Kind() == reflect.Map || rType.Kind() == reflect.Interface +} + func bestSchemaType(item *plan.View) reflect.Type { if item.FieldType != nil { return item.FieldType @@ -450,6 +1117,11 @@ func bestSchemaType(item *plan.View) reflect.Type { return nil } +func stringPtr(value string) *string { + ret := value + return &ret +} + func toViewRelations(input []*plan.Relation) []*view.Relation { if len(input) == 0 { return nil @@ -460,9 +1132,11 @@ func toViewRelations(input []*plan.Relation) []*view.Relation { continue } relation := &view.Relation{ - Name: item.Name, - Holder: item.Holder, - On: toViewLinks(item.On, true), + Name: item.Name, + Holder: item.Holder, + Cardinality: state.Many, + IncludeColumn: true, + On: toViewLinks(item.On, true), Of: view.NewReferenceView( toViewLinks(item.On, false), view.NewView(item.Ref, item.Table), @@ -498,8 +1172,632 @@ func toViewLinks(input []*plan.RelationLink, parent bool) view.Links { } func newSchema(rType reflect.Type, cardinality string) *state.Schema { + if rType == nil { + schema := &state.Schema{} + if cardinality == "many" { + schema.Cardinality = state.Many + } else { + schema.Cardinality = state.One + } + return schema + } if cardinality == "many" && rType.Kind() != reflect.Slice { return state.NewSchema(rType, state.WithMany()) } return state.NewSchema(rType) } + +func attachViewRelations(resource *view.Resource, planned []*plan.View) { + if resource == nil || len(planned) == 0 { + return + } + index := resource.Views.Index() + byName := map[string]*plan.View{} + for _, item := range planned { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + byName[strings.ToLower(strings.TrimSpace(item.Name))] = item + } + for _, item := range planned { + if item == nil || len(item.Relations) == 0 { + continue + } + candidates := toViewRelations(item.Relations) + for i, relation := range candidates { + if relation == nil || relation.Of == nil { + continue + } + parentName := relationParentName(item, item.Relations, i) + if parentName == "" { + continue + } + parent, err := index.Lookup(parentName) + if err != nil || parent == nil { + continue + } + if plannedParent, ok := byName[strings.ToLower(parentName)]; ok && plannedParent != nil { + parentName = plannedParent.Name + } + refName := strings.TrimSpace(relation.Of.View.Ref) + if refName == "" { + refName = strings.TrimSpace(relation.Of.View.Name) + } + if refName == "" { + continue + } + ref, err := index.Lookup(refName) + if err != nil || ref == nil { + continue + } + if plannedRef, ok := byName[strings.ToLower(refName)]; ok && plannedRef != nil { + if strings.EqualFold(strings.TrimSpace(plannedRef.Cardinality), string(state.One)) { + relation.Cardinality = state.One + } + } + if inferOneToOneRelation(parent, ref, relation) { + relation.Cardinality = state.One + } + relation.Of.View.Ref = ref.Name + relation.Of.View.Name = "" + relation.Of.View.Columns = ref.Columns + parent.With = append(parent.With, relation) + } + } +} + +func enrichRelationHolderTypes(resource *view.Resource, planned []*plan.View) error { + if resource == nil || len(planned) == 0 { + return nil + } + index := resource.Views.Index() + byName := map[string]*plan.View{} + for _, item := range planned { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + byName[strings.ToLower(strings.TrimSpace(item.Name))] = item + } + for _, item := range planned { + if item == nil || len(item.Relations) == 0 { + continue + } + for i, rel := range item.Relations { + if rel == nil { + continue + } + parentName := relationParentName(item, item.Relations, i) + if parentName == "" { + parentName = item.Name + } + parent, err := index.Lookup(parentName) + if err != nil || parent == nil || parent.Schema == nil { + continue + } + parentType := parent.ComponentType() + if parentType == nil { + continue + } + augmented, changed, err := ensureRelationHolderFields(parentType, &plan.View{Relations: []*plan.Relation{rel}}, byName, index) + if err != nil { + return err + } + if !changed || augmented == nil { + continue + } + if parent.Schema.Cardinality == state.Many { + parent.Schema.SetType(reflect.SliceOf(augmented)) + continue + } + parent.Schema.SetType(augmented) + } + } + return nil +} + +func ensureRelationHolderFields(parentType reflect.Type, item *plan.View, byName map[string]*plan.View, index view.NamedViews) (reflect.Type, bool, error) { + parentType = ensureStructType(parentType) + if parentType == nil || item == nil || len(item.Relations) == 0 { + return parentType, false, nil + } + fields := make([]reflect.StructField, 0, parentType.NumField()+len(item.Relations)) + for i := 0; i < parentType.NumField(); i++ { + fields = append(fields, parentType.Field(i)) + } + changed := false + for _, rel := range item.Relations { + if rel == nil || strings.TrimSpace(rel.Holder) == "" { + continue + } + if _, ok := parentType.FieldByName(rel.Holder); ok { + continue + } + childName := strings.TrimSpace(rel.Ref) + if childName == "" { + continue + } + childView, err := index.Lookup(childName) + if err != nil || childView == nil { + continue + } + childType := childView.ComponentType() + if childType == nil && childView.Schema != nil { + childType = childView.Schema.Type() + } + if childType == nil { + if childPlanned, ok := byName[strings.ToLower(childName)]; ok && childPlanned != nil { + childType = bestSchemaType(childPlanned) + } + } + if childType == nil { + continue + } + fieldType := relationHolderFieldType(childType, childPlannedCardinality(childName, byName)) + if fieldType == nil { + continue + } + fields = append(fields, reflect.StructField{ + Name: rel.Holder, + Type: fieldType, + Tag: reflect.StructTag(buildRelationHolderTag(rel, childView)), + }) + changed = true + } + if !changed { + return parentType, false, nil + } + return reflect.StructOf(fields), true, nil +} + +func childPlannedCardinality(childName string, byName map[string]*plan.View) state.Cardinality { + if childPlanned, ok := byName[strings.ToLower(childName)]; ok && childPlanned != nil { + if strings.EqualFold(strings.TrimSpace(childPlanned.Cardinality), string(state.One)) { + return state.One + } + } + return state.Many +} + +func relationHolderFieldType(childType reflect.Type, cardinality state.Cardinality) reflect.Type { + if childType == nil { + return nil + } + childType = normalizeDeferredHolderType(childType) + if cardinality == state.One { + for childType.Kind() == reflect.Slice || childType.Kind() == reflect.Array { + childType = childType.Elem() + } + if childType.Kind() == reflect.Struct { + return reflect.PtrTo(childType) + } + return childType + } + if childType.Kind() == reflect.Slice || childType.Kind() == reflect.Array { + return childType + } + normalized := childType + if normalized.Kind() == reflect.Struct { + normalized = reflect.PtrTo(normalized) + } + return reflect.SliceOf(normalized) +} + +func normalizeDeferredHolderType(rType reflect.Type) reflect.Type { + if rType == nil { + return nil + } + kind := rType.Kind() + if kind == reflect.Slice || kind == reflect.Array { + elem := rType.Elem() + for elem.Kind() == reflect.Ptr { + elem = elem.Elem() + } + if elem.Kind() == reflect.Map || elem.Kind() == reflect.Interface { + return reflect.SliceOf(reflect.TypeOf(struct{}{})) + } + return rType + } + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + if rType.Kind() == reflect.Map || rType.Kind() == reflect.Interface { + return reflect.TypeOf(struct{}{}) + } + return rType +} + +func ensureStructType(rType reflect.Type) reflect.Type { + if rType == nil { + return nil + } + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return nil + } + return rType +} + +func buildRelationHolderTag(rel *plan.Relation, child *view.View) string { + if rel == nil { + return `json:",omitempty" sqlx:"-"` + } + table := "" + sqlExpr := "" + if child != nil { + table = strings.TrimSpace(child.Table) + if child.Template != nil { + if uri := strings.TrimSpace(child.Template.SourceURL); uri != "" { + sqlExpr = "uri=" + uri + } else if source := strings.TrimSpace(child.Template.Source); source != "" { + sqlExpr = source + } + } + } + tagParts := []string{fmt.Sprintf(`view:",table=%s"`, table)} + if onExpr := buildRelationOnTag(rel); onExpr != "" { + tagParts = append(tagParts, fmt.Sprintf(`on:"%s"`, onExpr)) + } + if sqlExpr != "" { + tagParts = append(tagParts, fmt.Sprintf(`sql:%q`, sqlExpr)) + } + tagParts = append(tagParts, `json:",omitempty"`, `sqlx:"-"`) + return strings.Join(tagParts, " ") +} + +func buildRelationOnTag(rel *plan.Relation) string { + if rel == nil || len(rel.On) == 0 { + return "" + } + parts := make([]string, 0, len(rel.On)) + for _, link := range rel.On { + if link == nil { + continue + } + parentField := firstNonEmpty(strings.TrimSpace(link.ParentField), strings.TrimSpace(link.ParentColumn)) + refField := firstNonEmpty(strings.TrimSpace(link.RefField), strings.TrimSpace(link.RefColumn)) + if parentField == "" || refField == "" { + continue + } + parts = append(parts, fmt.Sprintf("%s:%s=%s:%s", parentField, link.ParentColumn, refField, link.RefColumn)) + } + return strings.Join(parts, ",") +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func bindTemplateParameters(resource *view.Resource) { + if resource == nil || len(resource.Parameters) == 0 { + return + } + params := make([]*state.Parameter, 0, len(resource.Parameters)) + for _, param := range resource.Parameters { + if param == nil || param.In == nil { + continue + } + switch param.In.Kind { + case state.KindOutput, state.KindMeta, state.KindAsync: + continue + } + params = append(params, param) + } + if len(params) == 0 { + return + } + for _, item := range resource.Views { + bindViewTemplateParameters(item, params) + } +} + +func bindViewTemplateParameters(aView *view.View, params []*state.Parameter) { + if aView == nil { + return + } + if aView.Template != nil { + seen := map[string]bool{} + for _, item := range aView.Template.Parameters { + if item != nil { + seen[strings.ToLower(strings.TrimSpace(item.Name))] = true + } + } + for _, param := range params { + if param == nil || strings.TrimSpace(param.Name) == "" { + continue + } + if param.In != nil && param.In.Kind == state.KindView && strings.EqualFold(strings.TrimSpace(param.In.Name), strings.TrimSpace(aView.Name)) { + continue + } + key := strings.ToLower(strings.TrimSpace(param.Name)) + if seen[key] { + continue + } + aView.Template.Parameters = append(aView.Template.Parameters, param) + seen[key] = true + } + } + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + bindViewTemplateParameters(&rel.Of.View, params) + } +} + +func inferOneToOneRelation(parent, ref *view.View, relation *view.Relation) bool { + if parent == nil || ref == nil || relation == nil || relation.Of == nil { + return false + } + parentTable := strings.TrimSpace(parent.Table) + refTable := strings.TrimSpace(ref.Table) + if parentTable == "" || refTable == "" || !strings.EqualFold(parentTable, refTable) { + return false + } + if len(relation.On) == 0 || len(relation.Of.On) == 0 { + return false + } + count := len(relation.On) + if len(relation.Of.On) < count { + count = len(relation.Of.On) + } + if count == 0 { + return false + } + for i := 0; i < count; i++ { + parentCol := normalizeRelationColumn(relation.On[i].Column) + refCol := normalizeRelationColumn(relation.Of.On[i].Column) + if parentCol == "" || refCol == "" || !strings.EqualFold(parentCol, refCol) { + return false + } + } + return true +} + +func normalizeRelationColumn(column string) string { + column = strings.TrimSpace(column) + if column == "" { + return "" + } + if idx := strings.LastIndex(column, "."); idx != -1 && idx+1 < len(column) { + column = column[idx+1:] + } + return strings.TrimSpace(column) +} + +func relationParentName(source *plan.View, relations []*plan.Relation, index int) string { + if index >= 0 && index < len(relations) { + item := relations[index] + if item != nil { + if parent := strings.TrimSpace(item.Parent); parent != "" { + return parent + } + for _, link := range item.On { + if link == nil { + continue + } + if parent := strings.TrimSpace(link.ParentNamespace); parent != "" { + return parent + } + } + } + } + if source == nil { + return "" + } + return strings.TrimSpace(source.Name) +} + +func cloneStateParameter(item *plan.State) *state.Parameter { + if item == nil { + return nil + } + param := item.Parameter + if param.In != nil { + in := *param.In + param.In = &in + } + if param.Schema != nil { + schema := *param.Schema + param.Schema = &schema + } + if len(param.Predicates) > 0 { + preds := make([]*extension.PredicateConfig, 0, len(param.Predicates)) + for _, candidate := range param.Predicates { + if candidate == nil { + continue + } + pred := *candidate + if len(candidate.Args) > 0 { + pred.Args = append([]string{}, candidate.Args...) + } + preds = append(preds, &pred) + } + param.Predicates = preds + } + return ¶m +} + +func clonePlanState(item *plan.State) *plan.State { + if item == nil { + return nil + } + cloned := *item + if param := cloneStateParameter(item); param != nil { + cloned.Parameter = *param + } + return &cloned +} + +func normalizeDerivedInputSchema(param *state.Parameter, resource *view.Resource) { + if param == nil || param.In == nil || resource == nil { + return + } + if param.In.Kind != state.KindView { + return + } + viewName := strings.TrimSpace(param.Name) + if name := strings.TrimSpace(param.In.Name); name != "" { + viewName = name + } + aView, _ := resource.View(viewName) + if aView == nil || aView.Schema == nil { + return + } + required := param.Required != nil && *param.Required + if param.Schema == nil { + param.Schema = aView.Schema.Clone() + if required && param.Schema != nil { + param.Schema.Cardinality = state.One + } + return + } + if strings.TrimSpace(param.Schema.Name) == "" { + param.Schema.Name = strings.TrimSpace(aView.Schema.Name) + } + dataType := strings.TrimSpace(param.Schema.DataType) + if dataType == "" || dataType == "?" || dataType == "interface{}" || dataType == "[]interface{}" || dataType == "*interface{}" || dataType == "string" || dataType == "[]string" { + param.Schema.DataType = strings.TrimSpace(aView.Schema.DataType) + if param.Schema.DataType == "" && param.Schema.Name != "" { + param.Schema.DataType = "*" + param.Schema.Name + } + } + if strings.TrimSpace(param.Schema.Package) == "" { + param.Schema.Package = strings.TrimSpace(aView.Schema.Package) + } + if param.Schema.Type() == nil && aView.Schema.Type() != nil { + param.Schema.SetType(aView.Schema.Type()) + } + if param.Schema.Cardinality == "" { + if required { + param.Schema.Cardinality = state.One + } else if aView.Schema.Cardinality != "" { + param.Schema.Cardinality = aView.Schema.Cardinality + } + } +} + +func rootResourceView(resource *view.Resource, planned []*plan.View) *view.View { + if resource == nil { + return nil + } + rootPlan := pickRootView(planned) + if rootPlan == nil || strings.TrimSpace(rootPlan.Name) == "" { + if len(resource.Views) > 0 { + return resource.Views[0] + } + return nil + } + index := resource.Views.Index() + root, _ := index.Lookup(rootPlan.Name) + return root +} + +func inheritRootOutputSchema(param *state.Parameter, root *view.View) { + if param == nil || param.In == nil || root == nil || root.Schema == nil { + return + } + if param.In.Kind != state.KindOutput || !strings.EqualFold(strings.TrimSpace(param.In.Name), "view") { + return + } + dataType := "" + if param.Schema != nil { + dataType = strings.TrimSpace(param.Schema.DataType) + } + if dataType != "" && dataType != "?" { + return + } + if param.Schema == nil { + param.Schema = &state.Schema{} + } + explicit := *param.Schema + schema := *root.Schema + if strings.TrimSpace(explicit.Name) != "" { + schema.Name = strings.TrimSpace(explicit.Name) + } + if dataType := strings.TrimSpace(explicit.DataType); dataType != "" && dataType != "?" { + schema.DataType = dataType + } + if pkg := strings.TrimSpace(explicit.Package); pkg != "" { + schema.Package = pkg + } + if pkgPath := strings.TrimSpace(explicit.PackagePath); pkgPath != "" { + schema.PackagePath = pkgPath + } + if modulePath := strings.TrimSpace(explicit.ModulePath); modulePath != "" { + schema.ModulePath = modulePath + } + if explicit.Cardinality != "" { + schema.Cardinality = explicit.Cardinality + } + param.Schema = &schema +} + +func inheritRootBodySchema(param *state.Parameter, root *view.View) { + if param == nil || param.In == nil || root == nil || root.Schema == nil { + return + } + if param.In.Kind != state.KindRequestBody { + return + } + if !param.IsAnonymous() { + return + } + dataType := "" + if param.Schema != nil { + dataType = strings.TrimSpace(param.Schema.DataType) + } + if dataType != "" && dataType != "?" { + return + } + if param.Schema == nil { + param.Schema = &state.Schema{} + } + explicit := *param.Schema + schema := *root.Schema + if strings.TrimSpace(explicit.Name) != "" { + schema.Name = strings.TrimSpace(explicit.Name) + } + if dataType := strings.TrimSpace(explicit.DataType); dataType != "" && dataType != "?" { + schema.DataType = dataType + } + if pkg := strings.TrimSpace(explicit.Package); pkg != "" { + schema.Package = pkg + } + if pkgPath := strings.TrimSpace(explicit.PackagePath); pkgPath != "" { + schema.PackagePath = pkgPath + } + if modulePath := strings.TrimSpace(explicit.ModulePath); modulePath != "" { + schema.ModulePath = modulePath + } + if explicit.Cardinality != "" { + schema.Cardinality = explicit.Cardinality + } + param.Schema = &schema +} + +func ensureMaterializedOutputSchema(param *state.Parameter, root *view.View) { + if param == nil || param.In == nil { + return + } + if param.In.Kind != state.KindOutput { + return + } + if param.Schema != nil && (param.Schema.Type() != nil || strings.TrimSpace(param.Schema.DataType) != "") { + return + } + switch strings.ToLower(strings.TrimSpace(param.In.Name)) { + case "status": + param.Schema = state.NewSchema(reflect.TypeOf(response.Status{})) + case "summary": + if root != nil && root.Template != nil && root.Template.Summary != nil && root.Template.Summary.Schema != nil { + param.Schema = root.Template.Summary.Schema.Clone() + } + } +} diff --git a/repository/shape/load/loader_contract_state_test.go b/repository/shape/load/loader_contract_state_test.go new file mode 100644 index 000000000..009cd3e10 --- /dev/null +++ b/repository/shape/load/loader_contract_state_test.go @@ -0,0 +1,23 @@ +package load + +import ( + "reflect" + "testing" +) + +func TestContractStates_PreservesCodecAndHandler(t *testing.T) { + type input struct { + Jwt string `parameter:",kind=header,in=Authorization,errorCode=401" codec:"JwtClaim"` + Run string `parameter:",kind=body,in=run" handler:"Exec"` + } + states := contractStates(reflect.TypeOf(input{})) + if got, want := len(states), 2; got != want { + t.Fatalf("expected %d states, got %d", want, got) + } + if states[0].Output == nil || states[0].Output.Name != "JwtClaim" { + t.Fatalf("expected codec to be preserved, got %#v", states[0].Output) + } + if states[1].Handler == nil || states[1].Handler.Name != "Exec" { + t.Fatalf("expected handler to be preserved, got %#v", states[1].Handler) + } +} diff --git a/repository/shape/load/loader_test.go b/repository/shape/load/loader_test.go index 20117b4e9..e5d5a3913 100644 --- a/repository/shape/load/loader_test.go +++ b/repository/shape/load/loader_test.go @@ -3,6 +3,7 @@ package load import ( "context" "embed" + "path/filepath" "reflect" "testing" @@ -13,6 +14,11 @@ import ( "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/repository/shape/scan" "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/x" + "github.com/viant/xdatly" + "github.com/viant/xdatly/handler/response" ) //go:embed testdata/*.sql @@ -29,6 +35,41 @@ type reportRow struct { Name string } +type relationTreeRoot struct { + ID int `sqlx:"name=ID"` +} + +type relationTreeChild struct { + ID int `sqlx:"name=ID"` + RootID int `sqlx:"name=RootID"` +} + +type relationTreeGrandChild struct { + ID int `sqlx:"name=ID"` + ChildID int `sqlx:"name=ChildID"` +} + +type cityRow struct { + ID int `sqlx:"name=ID"` + DistrictID int `sqlx:"name=DISTRICT_ID"` +} + +type vendorProductRow struct { + ID int `sqlx:"name=ID"` + VendorID int `sqlx:"name=VENDOR_ID"` +} + +type fieldOnlyUserACLRow struct { + UserID int `sqlx:"name=UserID"` + IsReadOnly int `sqlx:"name=IsReadOnly"` + Feature1 int `sqlx:"name=Feature1"` +} + +type placeholderDistrictRow struct { + Col1 string `sqlx:"name=col_1"` + Col2 string `sqlx:"name=col_2"` +} + type reportSource struct { embeddedFS Rows []reportRow `view:"rows,table=REPORT,connector=dev,cache=c1" sql:"uri=testdata/report.sql"` @@ -36,6 +77,65 @@ type reportSource struct { Status any `parameter:"status,kind=output,in=status"` Job any `parameter:"job,kind=async,in=job"` Meta any `parameter:"meta,kind=meta,in=view.name"` + Route struct{} `component:",path=/v1/api/dev/report,method=GET,connector=dev"` +} + +type typedRouteInput struct { + ID int +} + +type typedRouteOutput struct { + Data []reportRow +} + +type typedTeamRouteInput struct { + TeamID string `parameter:",kind=path,in=teamID"` +} + +type typedTeamRouteOutput struct{} + +type typedRouteSource struct { + embeddedFS + Rows []reportRow `view:"rows,table=REPORT" sql:"uri=testdata/report.sql"` + Route xdatly.Component[typedRouteInput, typedRouteOutput] `component:",path=/v1/api/dev/report,method=GET"` +} + +type dynamicRouteInput struct { + Name string +} + +type dynamicRouteOutput struct { + Count int +} + +type namedDynamicRouteInput struct { + Name string `parameter:"name,kind=query,in=name"` +} + +type namedDynamicRouteOutput struct { + response.Status `parameter:",kind=output,in=status" json:",omitempty"` + Data []*reportRow `parameter:",kind=output,in=view" view:"rows,table=REPORT" sql:"uri=testdata/report.sql" anonymous:"true"` +} + +type dynamicRouteSource struct { + embeddedFS + Rows []reportRow `view:"rows,table=REPORT" sql:"uri=testdata/report.sql"` + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET"` +} + +type routerOnlyInput struct { + ID int `parameter:"id,kind=query,in=id"` +} + +func (*routerOnlyInput) EmbedFS() *embed.FS { return &testFS } + +type routerOnlyOutput struct { + response.Status `parameter:",kind=output,in=status" json:",omitempty"` + Data []*reportRow `parameter:",kind=output,in=view" view:"rows,table=REPORT" sql:"uri=testdata/report.sql" anonymous:"true"` +} + +type routerOnlySource struct { + Route xdatly.Component[routerOnlyInput, routerOnlyOutput] `component:",path=/v1/api/dev/router-only,method=GET"` } func TestLoader_LoadViews(t *testing.T) { @@ -101,6 +201,14 @@ func TestLoader_LoadViews_Metadata(t *testing.T) { FieldType: reflect.TypeOf([]map[string]interface{}{}), ElementType: reflect.TypeOf(map[string]interface{}{}), SQL: "SELECT * FROM ITEMS", + Declaration: &plan.ViewDeclaration{ + ColumnsConfig: map[string]*plan.ViewColumnConfig{ + "AUTHORIZED": { + DataType: "bool", + Tag: `internal:"true"`, + }, + }, + }, }, }, ViewsByName: map[string]*plan.View{}, @@ -121,6 +229,116 @@ func TestLoader_LoadViews_Metadata(t *testing.T) { assert.True(t, actual.Selector.NoLimit) require.NotNil(t, actual.Schema) assert.Equal(t, "*ItemView", actual.Schema.DataType) + require.NotNil(t, actual.ColumnsConfig) + require.Contains(t, actual.ColumnsConfig, "AUTHORIZED") + require.NotNil(t, actual.ColumnsConfig["AUTHORIZED"].DataType) + assert.Equal(t, "bool", *actual.ColumnsConfig["AUTHORIZED"].DataType) + require.NotNil(t, actual.ColumnsConfig["AUTHORIZED"].Tag) + assert.Equal(t, `internal:"true"`, *actual.ColumnsConfig["AUTHORIZED"].Tag) +} + +func TestLoader_LoadViews_InfersColumnsFromBestSchemaType(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "user_acl"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "user_acl", + SchemaType: "*UserAclView", + FieldType: reflect.TypeOf([]fieldOnlyUserACLRow{}), + ElementType: nil, + SQL: "SELECT 1", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + loader := New() + artifacts, err := loader.LoadViews(context.Background(), planned) + require.NoError(t, err) + require.Len(t, artifacts.Views, 1) + require.Len(t, artifacts.Views[0].Columns, 3) + assert.Equal(t, "UserID", artifacts.Views[0].Columns[0].Name) + assert.Equal(t, "IsReadOnly", artifacts.Views[0].Columns[1].Name) + assert.Equal(t, "Feature1", artifacts.Views[0].Columns[2].Name) +} + +func TestBindTemplateParameters_SkipsSelfViewParameter(t *testing.T) { + resource := &view.Resource{ + Parameters: state.Parameters{ + {Name: "Jwt", In: state.NewHeaderLocation("Authorization")}, + {Name: "VendorID", In: state.NewPathLocation("vendorID")}, + {Name: "Authorization", In: state.NewViewLocation("authorization")}, + {Name: "Auth", In: state.NewComponent("GET:/auth")}, + }, + Views: []*view.View{ + { + Name: "authorization", + Template: view.NewTemplate("SELECT Authorized", view.WithTemplateParameters( + &state.Parameter{Name: "Jwt", In: state.NewHeaderLocation("Authorization")}, + )), + }, + }, + } + + bindTemplateParameters(resource) + + require.Len(t, resource.Views, 1) + require.NotNil(t, resource.Views[0].Template) + var names []string + for _, param := range resource.Views[0].Template.Parameters { + names = append(names, param.Name) + } + assert.ElementsMatch(t, []string{"Jwt", "VendorID", "Auth"}, names) +} + +func TestLoader_LoadComponent_NormalizesViewInputSchemaFromResourceView(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/teams"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "user_team", + Table: "TEAM", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "UPDATE TEAM SET ACTIVE = false", + }, + { + Name: "TeamStats", + Table: "TEAM", + Cardinality: "many", + SchemaType: "*TeamStatsView", + FieldType: reflect.TypeOf([]struct { + ID int `sqlx:"name=ID"` + TeamMembers int `sqlx:"name=TEAM_MEMBERS"` + Name string `sqlx:"name=NAME"` + }{}), + SQL: "SELECT ID, 0 AS TEAM_MEMBERS, NAME FROM TEAM", + }, + }, + States: []*plan.State{ + {Parameter: state.Parameter{Name: "TeamIDs", In: state.NewQueryLocation("TeamIDs"), Schema: &state.Schema{DataType: "[]int"}}}, + {Parameter: state.Parameter{Name: "TeamStats", In: state.NewViewLocation("TeamStats")}}, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + param := component.InputParameters().Lookup("TeamStats") + require.NotNil(t, param) + require.NotNil(t, param.Schema) + assert.Equal(t, "TeamStatsView", param.Schema.Name) + assert.Equal(t, "*TeamStatsView", param.Schema.DataType) + assert.Equal(t, state.Many, param.Schema.Cardinality) } func TestLoader_LoadComponent(t *testing.T) { @@ -143,9 +361,19 @@ func TestLoader_LoadComponent(t *testing.T) { actualPlan.Directives = &dqlshape.Directives{ Meta: "docs/report.md", DefaultConnector: "analytics", + Dest: "all.go", + InputDest: "input.go", + OutputDest: "output.go", + RouterDest: "router.go", + InputType: "CustomInput", + OutputType: "CustomOutput", Cache: &dqlshape.CacheDirective{ - Enabled: true, - TTL: "5m", + Enabled: true, + TTL: "5m", + Name: "aerospike", + Provider: "aerospike://127.0.0.1:3000/test", + Location: "${view.Name}", + TimeToLiveMs: 3600000, }, MCP: &dqlshape.MCPDirective{ Name: "report.list", @@ -155,7 +383,7 @@ func TestLoader_LoadComponent(t *testing.T) { } loader := New() - artifact, err := loader.LoadComponent(context.Background(), planned) + artifact, err := loader.LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) require.NoError(t, err) require.NotNil(t, artifact) require.NotNil(t, artifact.Resource) @@ -164,8 +392,12 @@ func TestLoader_LoadComponent(t *testing.T) { component, ok := ComponentFrom(artifact) require.True(t, ok) assert.Equal(t, "/v1/api/report", component.Name) - assert.Equal(t, "/v1/api/report", component.URI) + assert.Equal(t, "/v1/api/dev/report", component.URI) assert.Equal(t, "GET", component.Method) + require.Len(t, component.ComponentRoutes, 1) + assert.Equal(t, "Route", component.ComponentRoutes[0].FieldName) + assert.Equal(t, "/v1/api/dev/report", component.ComponentRoutes[0].RoutePath) + assert.Equal(t, "dev", component.ComponentRoutes[0].Connector) assert.Equal(t, "rows", component.RootView) assert.Equal(t, []string{"rows"}, component.Views) assert.Len(t, component.Input, 1) @@ -182,45 +414,87 @@ func TestLoader_LoadComponent(t *testing.T) { require.NotNil(t, component.Directives.Cache) assert.True(t, component.Directives.Cache.Enabled) assert.Equal(t, "5m", component.Directives.Cache.TTL) + assert.Equal(t, "aerospike", component.Directives.Cache.Name) require.NotNil(t, component.Directives.MCP) assert.Equal(t, "report.list", component.Directives.MCP.Name) + require.NotEmpty(t, artifact.Resource.CacheProviders) + assert.Equal(t, "aerospike", artifact.Resource.CacheProviders[0].Name) + assert.Equal(t, "aerospike://127.0.0.1:3000/test", artifact.Resource.CacheProviders[0].Provider) + assert.Equal(t, "${view.Name}", artifact.Resource.CacheProviders[0].Location) + assert.Equal(t, 3600000, artifact.Resource.CacheProviders[0].TimeToLiveMs) assert.True(t, component.ColumnsDiscovery) + require.NotNil(t, component.TypeSpecs) + require.NotNil(t, component.TypeSpecs["input"]) + assert.Equal(t, "CustomInput", component.TypeSpecs["input"].TypeName) + assert.Equal(t, "input.go", component.TypeSpecs["input"].Dest) + require.NotNil(t, component.TypeSpecs["output"]) + assert.Equal(t, "CustomOutput", component.TypeSpecs["output"].TypeName) + assert.Equal(t, "output.go", component.TypeSpecs["output"].Dest) + assert.Equal(t, "router.go", component.Directives.RouterDest) } -func TestLoader_LoadComponent_RelationFieldsPreserved(t *testing.T) { +func TestLoader_LoadComponent_UsesComponentRouteWhenSourceNameMissing(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportSource{}}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.ComponentRoutes, 1) + assert.Equal(t, "/v1/api/dev/report", component.URI) + assert.Equal(t, "/v1/api/dev/report", component.Name) + assert.Equal(t, "GET", component.Method) + assert.Equal(t, "Route", component.ComponentRoutes[0].FieldName) +} + +func TestLoader_LoadComponent_InheritsTypeContextPackageForNamedStateSchemas(t *testing.T) { planned := &shape.PlanResult{ - Source: &shape.Source{Name: "/v1/api/report"}, + Source: &shape.Source{Name: "patch_basic_one"}, Plan: &plan.Result{ + TypeContext: &typectx.Context{ + DefaultPackage: "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", + PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", + }, Views: []*plan.View{ { - Path: "Rows", - Name: "rows", - Table: "REPORT", - Cardinality: "many", + Name: "foos", + Holder: "Foos", FieldType: reflect.TypeOf([]reportRow{}), ElementType: reflect.TypeOf(reportRow{}), - Relations: []*plan.Relation{ - { - Name: "detail", - Holder: "Detail", - Ref: "detail", - Table: "REPORT_DETAIL", - On: []*plan.RelationLink{ - { - ParentField: "ReportID", - ParentNamespace: "rows", - ParentColumn: "REPORT_ID", - RefField: "ID", - RefNamespace: "detail", - RefColumn: "ID", - }, - }, + Cardinality: string(state.Many), + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: &state.Location{Kind: state.KindRequestBody, Name: ""}, + Schema: &state.Schema{ + Name: "Foos", + DataType: "Foos", + }, + }, + }, + { + Parameter: state.Parameter{ + Name: "Foos", + In: &state.Location{Kind: state.KindOutput, Name: "view"}, + Tag: `anonymous:"true"`, + Schema: &state.Schema{ + Name: "Foos", + DataType: "Foos", }, }, }, }, - ViewsByName: map[string]*plan.View{}, - ByPath: map[string]*plan.Field{}, }, } @@ -229,16 +503,1219 @@ func TestLoader_LoadComponent_RelationFieldsPreserved(t *testing.T) { require.NoError(t, err) component, ok := ComponentFrom(artifact) require.True(t, ok) - require.Len(t, component.ViewRelations, 1) - require.Len(t, component.ViewRelations[0].On, 1) - require.Len(t, component.ViewRelations[0].Of.On, 1) + require.Len(t, component.Input, 1) + require.Len(t, component.Output, 1) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", component.Input[0].Schema.Package) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", component.Input[0].Schema.PackagePath) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", component.Output[0].Schema.Package) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", component.Output[0].Schema.PackagePath) +} - parent := component.ViewRelations[0].On[0] - ref := component.ViewRelations[0].Of.On[0] - assert.Equal(t, "ReportID", parent.Field) - assert.Equal(t, "rows", parent.Namespace) - assert.Equal(t, "REPORT_ID", parent.Column) - assert.Equal(t, "ID", ref.Field) - assert.Equal(t, "detail", ref.Namespace) - assert.Equal(t, "ID", ref.Column) +func TestLoader_LoadComponent_InheritsTypeContextPackageForNamedViewSchemas(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "patch_basic_one"}, + Plan: &plan.Result{ + TypeContext: &typectx.Context{ + DefaultPackage: "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", + PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", + }, + Views: []*plan.View{ + { + Name: "foos", + Holder: "Foos", + SchemaType: "*FoosView", + Cardinality: string(state.Many), + FieldType: nil, + ElementType: nil, + SQL: "SELECT * FROM FOOS", + }, + }, + }, + } + + artifact, err := New().LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + root, err := artifact.Resource.Views.Index().Lookup("foos") + require.NoError(t, err) + require.NotNil(t, root) + require.NotNil(t, root.Schema) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", root.Schema.Package) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", root.Schema.PackagePath) +} + +func TestLoader_LoadComponent_PreservesComponentHolderTypes(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &typedRouteSource{}}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.ComponentRoutes, 1) + assert.Equal(t, reflect.TypeOf(typedRouteInput{}), component.ComponentRoutes[0].InputType) + assert.Equal(t, reflect.TypeOf(typedRouteOutput{}), component.ComponentRoutes[0].OutputType) + assert.Empty(t, component.ComponentRoutes[0].InputName) + assert.Empty(t, component.ComponentRoutes[0].OutputName) +} + +func TestLoader_LoadComponent_PreservesDynamicComponentHolderTypes(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &dynamicRouteSource{ + Route: xdatly.Component[any, any]{ + Inout: dynamicRouteInput{}, + Output: dynamicRouteOutput{}, + }, + }}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.ComponentRoutes, 1) + assert.Equal(t, reflect.TypeOf(dynamicRouteInput{}), component.ComponentRoutes[0].InputType) + assert.Equal(t, reflect.TypeOf(dynamicRouteOutput{}), component.ComponentRoutes[0].OutputType) +} + +func TestLoader_LoadComponent_PreservesDynamicComponentHolderExplicitNames(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { + embeddedFS + Rows []reportRow `view:"rows,table=REPORT" sql:"uri=testdata/report.sql"` + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` + }{}}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.ComponentRoutes, 1) + assert.Nil(t, component.ComponentRoutes[0].InputType) + assert.Nil(t, component.ComponentRoutes[0].OutputType) + assert.Equal(t, "ReportInput", component.ComponentRoutes[0].InputName) + assert.Equal(t, "ReportOutput", component.ComponentRoutes[0].OutputName) +} + +func TestLoader_LoadComponent_PreservesDynamicComponentHolderExplicitNamesFromRegistry(t *testing.T) { + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(namedDynamicRouteInput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/load"), x.WithName("ReportInput"))) + registry.Register(x.NewType(reflect.TypeOf(namedDynamicRouteOutput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/load"), x.WithName("ReportOutput"))) + + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{ + Struct: &struct { + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` + }{}, + TypeRegistry: registry, + }) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.ComponentRoutes, 1) + assert.Equal(t, reflect.TypeOf(namedDynamicRouteInput{}), component.ComponentRoutes[0].InputType) + assert.Equal(t, reflect.TypeOf(namedDynamicRouteOutput{}), component.ComponentRoutes[0].OutputType) + assert.Equal(t, "ReportInput", component.ComponentRoutes[0].InputName) + assert.Equal(t, "ReportOutput", component.ComponentRoutes[0].OutputName) + require.Len(t, component.Input, 1) + assert.Equal(t, "name", component.Input[0].Name) + require.Len(t, component.Output, 2) + require.Len(t, artifact.Resource.Views, 1) + assert.Equal(t, "rows", artifact.Resource.Views[0].Name) +} + +func TestLoader_LoadComponent_ErrorsOnMultipleComponentRoutes(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { + embeddedFS + Rows []reportRow `view:"rows,table=REPORT" sql:"uri=testdata/report.sql"` + RouteA struct{} `component:",path=/v1/api/dev/report-a,method=GET"` + RouteB struct{} `component:",path=/v1/api/dev/report-b,method=POST"` + }{}}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + _, err = loader.LoadComponent(context.Background(), planned) + require.Error(t, err) + assert.Contains(t, err.Error(), "multiple component routes are not supported") +} + +func TestLoader_LoadComponent_RouterOnlySourceSynthesizesStatesAndViews(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &routerOnlySource{}}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + assert.Equal(t, "/v1/api/dev/router-only", component.URI) + assert.Equal(t, "GET", component.Method) + require.Len(t, component.Input, 1) + assert.Equal(t, "id", component.Input[0].Name) + require.Len(t, component.Output, 2) + require.Len(t, artifact.Resource.Views, 1) + assert.Equal(t, "rows", artifact.Resource.Views[0].Name) +} + +func TestLoader_LoadComponent_SynthesizesStatesFromRouteContractsWhenPlanStatesAreEmpty(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "team"}, + Plan: &plan.Result{ + Components: []*plan.ComponentRoute{ + { + FieldName: "Team", + Name: "Team", + RoutePath: "/v1/api/dev/team/{teamID}", + Method: "DELETE", + InputType: reflect.TypeOf(typedTeamRouteInput{}), + OutputType: reflect.TypeOf(typedTeamRouteOutput{}), + ViewName: "Team", + }, + }, + }, + } + + artifact, err := New().LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.Input, 1) + assert.Equal(t, "TeamID", component.Input[0].Name) + require.NotNil(t, component.Input[0].In) + assert.Equal(t, state.KindPath, component.Input[0].In.Kind) + assert.Equal(t, "teamID", component.Input[0].In.Name) +} + +func TestLoader_LoadComponent_CacheProviderDoesNotBindRootView(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/shape/dev/vendors/"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "vendor", + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]any{}), + ElementType: reflect.TypeOf(map[string]any{}), + SQL: "SELECT * FROM VENDOR", + }, + }, + Directives: &dqlshape.Directives{ + Cache: &dqlshape.CacheDirective{ + Enabled: true, + Name: "aerospike", + Provider: "aerospike://127.0.0.1:3000/test", + Location: "${view.Name}", + TimeToLiveMs: 3600000, + }, + }, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + require.Len(t, artifact.Resource.Views, 1) + require.NotEmpty(t, artifact.Resource.CacheProviders) + + root := artifact.Resource.Views[0] + assert.Nil(t, root.Cache) +} + +func TestLoader_LoadViews_DoesNotSeedPlaceholderColumnsFromLinkedType(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "districts"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "districts", + Table: "DISTRICT", + Cardinality: "many", + SchemaType: "*DistrictsView", + FieldType: reflect.TypeOf([]*placeholderDistrictRow{}), + ElementType: reflect.TypeOf(placeholderDistrictRow{}), + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifact, err := New().LoadViews(context.Background(), planned) + require.NoError(t, err) + require.Len(t, artifact.Views, 1) + assert.Empty(t, artifact.Views[0].Columns) +} + +func TestLoader_LoadViews_DefersMapBackedQuerySchemaType(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "cities"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "cities", + Table: "CITY", + Mode: string(view.ModeQuery), + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM CITY", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifact, err := New().LoadViews(context.Background(), planned) + require.NoError(t, err) + require.Len(t, artifact.Views, 1) + assert.Nil(t, artifact.Views[0].Schema.Type()) +} + +func TestLoader_LoadViews_PreservesChildSummaryGraph(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "meta"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM VENDOR", + Relations: []*plan.Relation{ + { + Name: "products", + Parent: "vendor", + Holder: "Products", + Ref: "products", + Table: "PRODUCT", + On: []*plan.RelationLink{ + {ParentColumn: "ID", RefColumn: "VENDOR_ID"}, + }, + }, + }, + }, + { + Name: "products", + Table: "PRODUCT", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM PRODUCT", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) PROD_META GROUP BY VENDOR_ID", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + loader := New() + artifacts, err := loader.LoadViews(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifacts) + require.Len(t, artifacts.Views, 2) + + index := artifacts.Resource.Views.Index() + products, err := index.Lookup("products") + require.NoError(t, err) + require.NotNil(t, products) + require.NotNil(t, products.Template) + require.NotNil(t, products.Template.Summary) + assert.Contains(t, products.Template.Summary.Source, "TOTAL_PRODUCTS") + assert.Contains(t, products.Template.Summary.Source, "$View.products.SQL") +} + +func TestLoader_LoadComponent_ConstDirectiveCreatesInternalConstParameter(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/shape/dev/vendors-env/"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "vendor", + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]any{}), + ElementType: reflect.TypeOf(map[string]any{}), + SQL: "SELECT * FROM VENDOR", + }, + }, + Directives: &dqlshape.Directives{ + Const: map[string]string{ + "Vendor": "VENDOR", + }, + }, + Const: map[string]string{ + "Vendor": "VENDOR", + }, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + require.NotEmpty(t, artifact.Resource.Parameters) + require.NotEmpty(t, artifact.Component) + + var constParam *state.Parameter + for _, item := range artifact.Resource.Parameters { + if item != nil && item.Name == "Vendor" && item.In != nil && item.In.Kind == state.KindConst { + constParam = item + break + } + } + require.NotNil(t, constParam) + assert.Equal(t, "VENDOR", constParam.Value) + assert.Equal(t, `internal:"true"`, constParam.Tag) + require.NotNil(t, constParam.Schema) + assert.Equal(t, "string", constParam.Schema.DataType) + assert.Equal(t, state.One, constParam.Schema.Cardinality) + + loaded, ok := ComponentFrom(artifact) + require.True(t, ok) + var constInput *plan.State + for _, item := range loaded.Input { + if item != nil && item.Name == "Vendor" && item.In != nil && item.In.Kind == state.KindConst { + constInput = item + break + } + } + require.NotNil(t, constInput) + assert.Equal(t, "VENDOR", constInput.Value) + assert.Equal(t, `internal:"true"`, constInput.Tag) + require.NotNil(t, constInput.Schema) + assert.Equal(t, "string", constInput.Schema.DataType) + assert.Equal(t, state.One, constInput.Schema.Cardinality) +} + +func TestLoader_LoadComponent_ViewKindStateIsInput(t *testing.T) { + required := true + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/auth/vendors/{vendorID}"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "vendor", + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]any{}), + ElementType: reflect.TypeOf(map[string]any{}), + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Jwt", + In: state.NewHeaderLocation("Authorization"), + Required: &required, + ErrorStatusCode: 401, + Schema: &state.Schema{DataType: "string"}, + }, + }, + { + Parameter: state.Parameter{ + Name: "Authorization", + In: state.NewViewLocation("Authorization"), + Required: &required, + ErrorStatusCode: 403, + Schema: &state.Schema{Cardinality: state.Many}, + }, + }, + { + Parameter: state.Parameter{ + Name: "VendorID", + In: state.NewPathLocation("vendorID"), + Required: &required, + Schema: &state.Schema{DataType: "int"}, + }, + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.NotNil(t, component) + + require.Len(t, component.Input, 3) + var hasViewInput bool + for _, input := range component.Input { + if input != nil && input.In != nil && input.In.Kind == state.KindView && input.Name == "Authorization" { + hasViewInput = true + assert.Equal(t, 403, input.ErrorStatusCode) + break + } + } + assert.True(t, hasViewInput) +} + +func TestResolveTypeSpecs_ViewOverridesAndInheritance(t *testing.T) { + result := &plan.Result{ + Directives: &dqlshape.Directives{Dest: "all.go"}, + Views: []*plan.View{ + { + Name: "vendor", + Path: "vendor", + Declaration: &plan.ViewDeclaration{ + Dest: "vendor.go", + TypeName: "Vendor", + }, + }, + {Name: "products", Path: "vendor.products"}, + }, + } + specs := resolveTypeSpecs(result) + require.NotNil(t, specs) + require.NotNil(t, specs["view:vendor"]) + assert.Equal(t, "Vendor", specs["view:vendor"].TypeName) + assert.Equal(t, "vendor.go", specs["view:vendor"].Dest) + require.NotNil(t, specs["view:products"]) + assert.Equal(t, "vendor.go", specs["view:products"].Dest) + assert.True(t, specs["view:products"].Inherited) +} + +func TestLoader_LoadComponent_RelationFieldsPreserved(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/report"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "Rows", + Name: "rows", + Table: "REPORT", + Cardinality: "many", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + Relations: []*plan.Relation{ + { + Name: "detail", + Holder: "Detail", + Ref: "detail", + Table: "REPORT_DETAIL", + On: []*plan.RelationLink{ + { + ParentField: "ReportID", + ParentNamespace: "rows", + ParentColumn: "REPORT_ID", + RefField: "ID", + RefNamespace: "detail", + RefColumn: "ID", + }, + }, + }, + }, + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.ViewRelations, 1) + require.Len(t, component.ViewRelations[0].On, 1) + require.Len(t, component.ViewRelations[0].Of.On, 1) + + parent := component.ViewRelations[0].On[0] + ref := component.ViewRelations[0].Of.On[0] + assert.Equal(t, "ReportID", parent.Field) + assert.Equal(t, "rows", parent.Namespace) + assert.Equal(t, "REPORT_ID", parent.Column) + assert.Equal(t, "ID", ref.Field) + assert.Equal(t, "detail", ref.Namespace) + assert.Equal(t, "ID", ref.Column) +} + +func TestLoader_LoadComponent_AttachesRelationTreeByParent(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/tree"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "root", + Name: "root", + Table: "ROOT", + Cardinality: "many", + FieldType: reflect.TypeOf([]relationTreeRoot{}), + ElementType: reflect.TypeOf(relationTreeRoot{}), + Relations: []*plan.Relation{ + { + Name: "child", + Parent: "root", + Holder: "Child", + Ref: "child", + Table: "CHILD", + On: []*plan.RelationLink{ + { + ParentNamespace: "root", + ParentColumn: "ID", + RefNamespace: "child", + RefField: "RootID", + RefColumn: "RootID", + }, + }, + }, + { + Name: "grand_child", + Parent: "child", + Holder: "GrandChild", + Ref: "grand_child", + Table: "GRAND_CHILD", + On: []*plan.RelationLink{ + { + ParentNamespace: "child", + ParentColumn: "ID", + RefNamespace: "grand_child", + RefField: "ChildID", + RefColumn: "ChildID", + }, + }, + }, + }, + }, + { + Path: "child", + Name: "child", + Table: "CHILD", + Cardinality: "many", + FieldType: reflect.TypeOf([]relationTreeChild{}), + ElementType: reflect.TypeOf(relationTreeChild{}), + }, + { + Path: "grand_child", + Name: "grand_child", + Table: "GRAND_CHILD", + Cardinality: "many", + FieldType: reflect.TypeOf([]relationTreeGrandChild{}), + ElementType: reflect.TypeOf(relationTreeGrandChild{}), + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + + index := artifact.Resource.Views.Index() + root, err := index.Lookup("root") + require.NoError(t, err) + require.NotNil(t, root) + require.Len(t, root.With, 1) + assert.Equal(t, "child", root.With[0].Of.View.Ref) + + child, err := index.Lookup("child") + require.NoError(t, err) + require.NotNil(t, child) + require.Len(t, child.With, 1) + assert.Equal(t, "grand_child", child.With[0].Of.View.Ref) +} + +func TestLoader_LoadComponent_AugmentsRelationHolderField(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/districts"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "districts", + Name: "districts", + Table: "DISTRICT", + Cardinality: "many", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + Relations: []*plan.Relation{ + { + Name: "cities", + Parent: "districts", + Holder: "Cities", + Ref: "cities", + Table: "CITY", + On: []*plan.RelationLink{ + { + ParentField: "ID", + ParentColumn: "ID", + RefField: "DistrictID", + RefColumn: "DistrictID", + }, + }, + }, + }, + }, + { + Path: "cities", + Name: "cities", + Table: "CITY", + Cardinality: "many", + FieldType: reflect.TypeOf([]cityRow{}), + ElementType: reflect.TypeOf(cityRow{}), + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + + index := artifact.Resource.Views.Index() + root, err := index.Lookup("districts") + require.NoError(t, err) + require.NotNil(t, root) + compType := root.ComponentType() + require.NotNil(t, compType) + field, ok := compType.FieldByName("Cities") + require.True(t, ok) + assert.Equal(t, reflect.Slice, field.Type.Kind()) + assert.Equal(t, reflect.Ptr, field.Type.Elem().Kind()) + assert.Equal(t, "cityRow", field.Type.Elem().Elem().Name()) + assert.Contains(t, string(field.Tag), `view:",table=CITY"`) + assert.Contains(t, string(field.Tag), `on:"ID:ID=DistrictID:DistrictID"`) +} + +func TestLoader_LoadComponent_AugmentsRelationHolderField_ForMapBackedChildView(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/districts"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "districts", + Name: "districts", + Table: "DISTRICT", + Cardinality: "many", + FieldType: reflect.TypeOf([]placeholderDistrictRow{}), + ElementType: reflect.TypeOf(placeholderDistrictRow{}), + Relations: []*plan.Relation{ + { + Name: "cities", + Parent: "districts", + Holder: "Cities", + Ref: "cities", + Table: "CITY", + On: []*plan.RelationLink{ + { + ParentField: "ID", + ParentColumn: "ID", + RefField: "DistrictID", + RefColumn: "DISTRICT_ID", + }, + }, + }, + }, + }, + { + Path: "cities", + Name: "cities", + Table: "CITY", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifact, err := New().LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + require.NotNil(t, artifact) + + root, err := artifact.Resource.Views.Index().Lookup("districts") + require.NoError(t, err) + require.NotNil(t, root) + + compType := root.ComponentType() + require.NotNil(t, compType) + + field, ok := compType.FieldByName("Cities") + require.True(t, ok) + assert.Equal(t, reflect.Slice, field.Type.Kind()) + assert.Equal(t, reflect.Struct, field.Type.Elem().Kind()) + assert.Contains(t, string(field.Tag), `view:",table=CITY"`) +} + +func TestLoader_LoadComponent_AugmentsResolvedParentRelationHolderField(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/vendor-details"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "wrapper", + Name: "wrapper", + Table: "WRAPPER", + Cardinality: "many", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + Relations: []*plan.Relation{ + { + Name: "products", + Holder: "Products", + Ref: "products", + Table: "PRODUCT", + On: []*plan.RelationLink{ + { + ParentNamespace: "vendor", + ParentField: "ID", + ParentColumn: "ID", + RefNamespace: "products", + RefField: "VendorID", + RefColumn: "VENDOR_ID", + }, + }, + }, + }, + }, + { + Path: "vendor", + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + }, + { + Path: "products", + Name: "products", + Table: "PRODUCT", + Cardinality: "many", + FieldType: reflect.TypeOf([]vendorProductRow{}), + ElementType: reflect.TypeOf(vendorProductRow{}), + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + index := artifact.Resource.Views.Index() + vendor, err := index.Lookup("vendor") + require.NoError(t, err) + require.NotNil(t, vendor) + compType := vendor.ComponentType() + require.NotNil(t, compType) + field, ok := compType.FieldByName("Products") + require.True(t, ok) + assert.Equal(t, reflect.Slice, field.Type.Kind()) + assert.Equal(t, reflect.Ptr, field.Type.Elem().Kind()) + assert.Equal(t, "vendorProductRow", field.Type.Elem().Elem().Name()) +} + +func TestLoader_LoadComponent_InfersOneToOneOnSameTableJoin(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/vendor-details"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "wrapper", + Name: "wrapper", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + Relations: []*plan.Relation{ + { + Name: "vendor", + Parent: "wrapper", + Holder: "Vendor", + Ref: "vendor", + Table: "VENDOR", + On: []*plan.RelationLink{ + { + ParentNamespace: "wrapper", + ParentColumn: "ID", + RefNamespace: "vendor", + RefColumn: "ID", + }, + }, + }, + { + Name: "setting", + Parent: "wrapper", + Holder: "Setting", + Ref: "setting", + Table: "T", + On: []*plan.RelationLink{ + { + ParentNamespace: "wrapper", + ParentColumn: "ID", + RefNamespace: "setting", + RefColumn: "ID", + }, + }, + }, + }, + }, + { + Path: "vendor", + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + }, + { + Path: "setting", + Name: "setting", + Table: "T", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + + index := artifact.Resource.Views.Index() + root, err := index.Lookup("wrapper") + require.NoError(t, err) + require.NotNil(t, root) + require.Len(t, root.With, 2) + + assert.Equal(t, state.One, root.With[0].Cardinality) + assert.Equal(t, state.Many, root.With[1].Cardinality) +} + +func TestLoader_LoadComponent_IncludesComponentStateInInput(t *testing.T) { + required := true + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/vendor"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "vendor", + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Auth", + In: state.NewComponent("GET:/v1/api/dev/auth"), + Required: &required, + Schema: &state.Schema{DataType: "*Output", Package: "auth"}, + }, + }, + }, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.NotNil(t, component) + require.Len(t, component.Input, 1) + assert.Equal(t, state.KindComponent, component.Input[0].In.Kind) + assert.Equal(t, "Auth", component.Input[0].Name) +} + +func TestLoader_LoadComponent_MaterializesOutputStatusSchema(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/user"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "user", + Name: "user", + Table: "USER", + Cardinality: "many", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Status", + In: state.NewOutputLocation("status"), + Tag: `anonymous:"true"`, + }, + }, + }, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + + param, err := artifact.Resource.LookupParameter("Status") + require.NoError(t, err) + require.NotNil(t, param) + require.NotNil(t, param.Schema) + require.NotNil(t, param.Schema.Type()) + assert.Equal(t, "Status", param.Schema.Type().Name()) +} + +func TestLoader_LoadComponent_PreservesExplicitOutputViewCardinality(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/auth/user-acl"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "user_acl", + Name: "user_acl", + Table: "USER_ACL", + Cardinality: "many", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Data", + In: state.NewOutputLocation("view"), + Schema: &state.Schema{ + Cardinality: state.One, + }, + }, + }, + }, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.NotNil(t, component) + require.Len(t, component.Output, 1) + require.NotNil(t, component.Output[0].Schema) + assert.Equal(t, state.One, component.Output[0].Schema.Cardinality) +} + +func TestLoader_LoadComponent_RequiredViewInputDefaultsToOneCardinality(t *testing.T) { + required := true + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/auth/vendor"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "Authorization", + Name: "Authorization", + Table: "AUTH", + Cardinality: "many", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Authorization", + In: state.NewViewLocation("Authorization"), + Required: &required, + }, + }, + }, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.NotNil(t, component) + require.Len(t, component.Input, 1) + require.NotNil(t, component.Input[0].Schema) + assert.Equal(t, state.One, component.Input[0].Schema.Cardinality) +} + +func TestLoader_LoadComponent_UsesPlannedRefCardinality(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/vendor-meta"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "vendor", + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + Relations: []*plan.Relation{ + { + Name: "products_meta", + Parent: "vendor", + Holder: "ProductsMeta", + Ref: "products_meta", + Table: "PRODUCT", + On: []*plan.RelationLink{ + { + ParentNamespace: "vendor", + ParentColumn: "ID", + RefNamespace: "products_meta", + RefColumn: "VENDOR_ID", + }, + }, + }, + }, + }, + { + Path: "products_meta", + Name: "products_meta", + Table: "PRODUCT", + Cardinality: "one", + FieldType: reflect.TypeOf(map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + + index := artifact.Resource.Views.Index() + root, err := index.Lookup("vendor") + require.NoError(t, err) + require.NotNil(t, root) + require.Len(t, root.With, 1) + assert.Equal(t, state.One, root.With[0].Cardinality) +} + +func TestLoader_LoadComponent_SynthesizesRootViewFromComponentRoute(t *testing.T) { + baseDir := t.TempDir() + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(reportRow{}), x.WithPkgPath("example.com/routes"), x.WithName("ReportView"))) + + artifact, err := New().LoadComponent(context.Background(), &shape.PlanResult{ + Source: &shape.Source{ + Path: filepath.Join(baseDir, "router.go"), + TypeRegistry: registry, + }, + Plan: &plan.Result{ + Components: []*plan.ComponentRoute{{ + Name: "Report", + RoutePath: "/v1/api/report", + Method: "DELETE", + Connector: "dev", + ViewName: "example.com/routes.ReportView", + SourceURL: "report/report.sql", + }}, + }, + }) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Equal(t, "Report", component.RootView) + require.Len(t, artifact.Resource.Views, 1) + require.Equal(t, "Report", artifact.Resource.Views[0].Name) + require.NotNil(t, artifact.Resource.Views[0].Template) + require.Equal(t, filepath.Join(baseDir, "report", "report.sql"), artifact.Resource.Views[0].Template.SourceURL) +} + +func TestLoader_LoadComponent_AllowsViewlessComponentRoute(t *testing.T) { + artifact, err := New().LoadComponent(context.Background(), &shape.PlanResult{ + Source: &shape.Source{Name: "delete_team"}, + Plan: &plan.Result{ + Components: []*plan.ComponentRoute{ + { + Name: "Team", + Method: "DELETE", + RoutePath: "/v1/api/dev/team/{teamID}", + Connector: "dev", + }, + }, + }, + }) + require.NoError(t, err) + require.NotNil(t, artifact) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + assert.Equal(t, "DELETE", component.Method) + assert.Equal(t, "/v1/api/dev/team/{teamID}", component.URI) + assert.Empty(t, artifact.Resource.Views) } diff --git a/repository/shape/load/model.go b/repository/shape/load/model.go index b94a00cea..9666bdffa 100644 --- a/repository/shape/load/model.go +++ b/repository/shape/load/model.go @@ -18,6 +18,7 @@ type Component struct { Name string URI string Method string + ComponentRoutes []*plan.ComponentRoute RootView string Views []string Relations []*plan.Relation @@ -28,6 +29,7 @@ type Component struct { TypeContext *typectx.Context Directives *dqlshape.Directives ColumnsDiscovery bool + TypeSpecs map[string]*TypeSpec Input []*plan.State Output []*plan.State @@ -36,6 +38,24 @@ type Component struct { Other []*plan.State } +type TypeRole string + +const ( + TypeRoleInput TypeRole = "input" + TypeRoleOutput TypeRole = "output" + TypeRoleView TypeRole = "view" +) + +type TypeSpec struct { + Key string + Role TypeRole + Alias string + TypeName string + Dest string + Inherited bool + Source string +} + // ShapeSpecKind implements shape.ComponentSpec. func (c *Component) ShapeSpecKind() string { return "component" } diff --git a/repository/shape/load/model_test.go b/repository/shape/load/model_test.go new file mode 100644 index 000000000..a1c13bb5a --- /dev/null +++ b/repository/shape/load/model_test.go @@ -0,0 +1,94 @@ +package load + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view/state" + "github.com/viant/xreflect" +) + +type packageScopedPatchFoos struct { + ID int + Name *string + Quantity *int +} + +func TestComponent_InputReflectType_UsesComponentPackageForPatchHelpers(t *testing.T) { + const pkgPath = "github.com/viant/datly/e2e/v1/shape/dev/generate_patch_basic_one" + + types := xreflect.NewTypes() + require.NoError(t, types.Register("Foos", + xreflect.WithPackage(pkgPath), + xreflect.WithReflectType(reflect.TypeOf(packageScopedPatchFoos{})), + )) + + component := &Component{ + RootView: "Foos", + TypeContext: &typectx.Context{ + PackagePath: pkgPath, + PackageName: "generate_patch_basic_one", + }, + Input: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewBodyLocation(""), + Tag: `anonymous:"true"`, + Schema: &state.Schema{Name: "Foos"}, + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoosId", + In: state.NewParameterLocation("Foos"), + Tag: `codec:"structql,uri=foos/cur_foos_id.sql"`, + Schema: state.NewSchema(reflect.TypeOf(&struct { + Values []int + }{})), + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoos", + In: state.NewViewLocation("CurFoos"), + Tag: `view:"CurFoos" sql:"uri=foos/cur_foos.sql"`, + Schema: &state.Schema{Name: "Foos", Cardinality: state.Many}, + }, + }, + }, + } + + rType, err := component.InputReflectType(pkgPath, types.Lookup, state.WithSetMarker(), state.WithTypeName("Input")) + require.NoError(t, err) + require.NotNil(t, rType) + + foosField, ok := rType.FieldByName("Foos") + require.True(t, ok) + assert.Equal(t, "packageScopedPatchFoos", namedType(foosField.Type).Name()) + + curFoosField, ok := rType.FieldByName("CurFoos") + require.True(t, ok) + assert.Equal(t, reflect.Slice, curFoosField.Type.Kind()) + assert.Equal(t, "packageScopedPatchFoos", namedType(curFoosField.Type.Elem()).Name()) + + curFoosIDField, ok := rType.FieldByName("CurFoosId") + require.True(t, ok) + assert.Equal(t, reflect.Ptr, curFoosIDField.Type.Kind()) + assert.Equal(t, reflect.Struct, curFoosIDField.Type.Elem().Kind()) + valuesField, ok := curFoosIDField.Type.Elem().FieldByName("Values") + require.True(t, ok) + assert.Equal(t, reflect.Slice, valuesField.Type.Kind()) + assert.Equal(t, reflect.Int, valuesField.Type.Elem().Kind()) +} + +func namedType(rType reflect.Type) reflect.Type { + for rType != nil && (rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice) { + rType = rType.Elem() + } + return rType +} diff --git a/repository/shape/model.go b/repository/shape/model.go index 88c8da537..d48c848ec 100644 --- a/repository/shape/model.go +++ b/repository/shape/model.go @@ -1,6 +1,8 @@ package shape import ( + "os" + "path/filepath" "reflect" "github.com/viant/datly/view" @@ -28,6 +30,26 @@ type Source struct { DQL string } +func (s *Source) BaseDir() string { + if s == nil { + return "" + } + location := filepath.Clean(s.Path) + if location == "" || location == "." { + return "" + } + if info, err := os.Stat(location); err == nil { + if info.IsDir() { + return location + } + return filepath.Dir(location) + } + if ext := filepath.Ext(location); ext != "" { + return filepath.Dir(location) + } + return location +} + // ScanSpec is implemented by every scan-pipeline descriptor result. // The sole production implementation is *scan.Result. type ScanSpec interface { diff --git a/repository/shape/options.go b/repository/shape/options.go index 27b970fae..dbd0529d8 100644 --- a/repository/shape/options.go +++ b/repository/shape/options.go @@ -238,3 +238,13 @@ func WithInferTypeContextDefaults(enabled bool) CompileOption { o.InferTypeContext = &enabled } } + +// WithLinkedTypes enables/disables linked Go type support during compile. +func WithLinkedTypes(enabled bool) CompileOption { + return func(o *CompileOptions) { + if o == nil { + return + } + o.UseLinkedTypes = &enabled + } +} diff --git a/repository/shape/plan/model.go b/repository/shape/plan/model.go index e537907a1..ae3b3879c 100644 --- a/repository/shape/plan/model.go +++ b/repository/shape/plan/model.go @@ -20,6 +20,7 @@ type Result struct { Views []*View ViewsByName map[string]*View States []*State + Components []*ComponentRoute Types []*Type ColumnsDiscovery bool Const map[string]string @@ -28,6 +29,25 @@ type Result struct { Diagnostics []*dqlshape.Diagnostic } +type ComponentRoute struct { + Path string + FieldName string + Type reflect.Type + InputType reflect.Type + OutputType reflect.Type + InputName string + OutputName string + ViewName string + SourceURL string + SummaryURL string + Name string + RoutePath string + Method string + Connector string + Marshaller string + Handler string +} + // Type is normalized type metadata collected during compile. type Type struct { Name string @@ -62,6 +82,7 @@ type View struct { SQL string SQLURI string Summary string + SummaryName string Relations []*Relation Holder string @@ -82,6 +103,8 @@ type View struct { // ViewDeclaration captures declaration options used to derive a view from DQL directives. type ViewDeclaration struct { Tag string + TypeName string + Dest string Codec string CodecArgs []string HandlerName string @@ -100,6 +123,7 @@ type ViewDeclaration struct { Async bool Output bool Predicates []*ViewPredicate + ColumnsConfig map[string]*ViewColumnConfig } // ViewPredicate captures WithPredicate / EnsurePredicate metadata. @@ -110,16 +134,24 @@ type ViewPredicate struct { Arguments []string } +// ViewColumnConfig captures declaration-level per-column overrides. +type ViewColumnConfig struct { + DataType string + Tag string +} + // Relation is normalized relation metadata extracted from DQL joins. type Relation struct { - Name string - Holder string - Ref string - Table string - Kind string - Raw string - On []*RelationLink - Warnings []string + Name string + Parent string + Holder string + Ref string + Table string + Kind string + Raw string + ColumnsConfig map[string]*ViewColumnConfig + On []*RelationLink + Warnings []string } // RelationLink represents one parent/ref join predicate. diff --git a/repository/shape/plan/planner.go b/repository/shape/plan/planner.go index f78b4bcc1..f97139ac5 100644 --- a/repository/shape/plan/planner.go +++ b/repository/shape/plan/planner.go @@ -61,11 +61,16 @@ func (p *Planner) Plan(ctx context.Context, scanned *shape.ScanResult, _ ...shap result.ViewsByName[v.Name] = v } } + assignNestedRelationParents(result.Views) for _, item := range scanResult.StateFields { result.States = append(result.States, normalizeState(item)) } + for _, item := range scanResult.ComponentFields { + result.Components = append(result.Components, normalizeComponent(item)) + } + return &shape.PlanResult{Source: scanned.Source, Plan: result}, nil } @@ -85,6 +90,13 @@ func normalizeView(field *scan.Field) *View { result.Partitioner = tag.View.PartitionerType result.PartitionedConcurrency = tag.View.PartitionedConcurrency result.RelationalConcurrency = tag.View.RelationalConcurrency + if strings.TrimSpace(tag.View.CustomTag) != "" || strings.TrimSpace(field.ViewTypeName) != "" || strings.TrimSpace(field.ViewDest) != "" { + result.Declaration = &ViewDeclaration{ + Tag: strings.TrimSpace(tag.View.CustomTag), + TypeName: strings.TrimSpace(field.ViewTypeName), + Dest: strings.TrimSpace(field.ViewDest), + } + } } result.SQL = tag.SQL.SQL result.SQLURI = tag.SQL.URI @@ -195,12 +207,120 @@ func normalizeState(field *scan.Field) *State { result.ErrorMessage = pTag.ErrorMessage result.Schema = state.NewSchema(resolveStateType(result, field.Type)) + if typeName := strings.TrimSpace(field.StateTag.TypeName); typeName != "" { + applyStateTypeName(result.Schema, typeName) + } + state.BuildCodec(field.StateTag, &result.Parameter) + state.BuildHandler(field.StateTag, &result.Parameter) + if value, err := field.StateTag.GetValue(result.Schema.Type()); err == nil && value != nil { + result.Value = normalizeStateValue(value) + } if dataType := strings.TrimSpace(pTag.DataType); dataType != "" { result.Schema.DataType = dataType } return result } +func normalizeStateValue(value interface{}) interface{} { + switch actual := value.(type) { + case *string: + if actual == nil { + return nil + } + return *actual + } + return value +} + +func applyStateTypeName(schema *state.Schema, typeName string) { + if schema == nil { + return + } + typeName = strings.TrimSpace(strings.TrimPrefix(typeName, "*")) + if typeName == "" { + return + } + if idx := strings.LastIndex(typeName, "."); idx != -1 { + schema.Package = strings.TrimSpace(typeName[:idx]) + schema.PackagePath = schema.Package + schema.Name = strings.TrimSpace(typeName[idx+1:]) + return + } + schema.Name = typeName +} + +func normalizeComponent(field *scan.Field) *ComponentRoute { + result := &ComponentRoute{ + Path: field.Path, + FieldName: field.Name, + Type: field.Type, + InputType: field.ComponentInputType, + OutputType: field.ComponentOutputType, + InputName: field.ComponentInputName, + OutputName: field.ComponentOutputName, + Name: field.Name, + } + if field.ComponentTag != nil && field.ComponentTag.Component != nil { + tag := field.ComponentTag.Component + if strings.TrimSpace(tag.Name) != "" { + result.Name = strings.TrimSpace(tag.Name) + } + result.RoutePath = strings.TrimSpace(tag.Path) + result.Method = strings.TrimSpace(tag.Method) + result.Connector = strings.TrimSpace(tag.Connector) + result.Marshaller = strings.TrimSpace(tag.Marshaller) + result.Handler = strings.TrimSpace(tag.Handler) + result.ViewName = strings.TrimSpace(tag.View) + result.SourceURL = strings.TrimSpace(tag.Source) + result.SummaryURL = strings.TrimSpace(tag.Summary) + } + return result +} + +func assignNestedRelationParents(views []*View) { + if len(views) == 0 { + return + } + byPath := map[string]*View{} + for _, item := range views { + if item == nil || strings.TrimSpace(item.Path) == "" { + continue + } + byPath[strings.TrimSpace(item.Path)] = item + } + for _, item := range views { + if item == nil || len(item.Relations) == 0 { + continue + } + parentPath := parentViewPath(item.Path) + if parentPath == "" { + continue + } + parent := byPath[parentPath] + if parent == nil || strings.TrimSpace(parent.Name) == "" { + continue + } + for _, rel := range item.Relations { + if rel == nil || strings.TrimSpace(rel.Parent) != "" { + continue + } + rel.Parent = parent.Name + } + } +} + +func parentViewPath(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return "" + } + index := strings.LastIndex(path, ".") + if index == -1 { + return "" + } + return strings.TrimSpace(path[:index]) +} + func resolveStateType(item *State, fallback reflect.Type) reflect.Type { if item.In == nil { return fallback diff --git a/repository/shape/plan/planner_test.go b/repository/shape/plan/planner_test.go index dc0416eb8..25fb8ee2c 100644 --- a/repository/shape/plan/planner_test.go +++ b/repository/shape/plan/planner_test.go @@ -3,6 +3,7 @@ package plan import ( "context" "embed" + "reflect" "strings" "testing" @@ -13,6 +14,9 @@ import ( outputkeys "github.com/viant/datly/repository/locator/output/keys" "github.com/viant/datly/repository/shape" "github.com/viant/datly/repository/shape/scan" + "github.com/viant/datly/view/tags" + "github.com/viant/x" + "github.com/viant/xdatly" ) //go:embed testdata/*.sql @@ -35,6 +39,7 @@ type reportSource struct { Job interface{} `parameter:"job,kind=async,in=job"` VName interface{} `parameter:"viewName,kind=meta,in=view.name"` ID int `parameter:"id,kind=query,in=id"` + Route struct{} `component:",path=/v1/api/dev/report,method=GET,connector=dev"` } type relationRow struct { @@ -49,6 +54,51 @@ type relationSourceWithFields struct { Rows []relationRow `view:"rows,table=REPORT" on:"ReportID:rows.report_id=ID:report.id"` } +type viewTypeDestSource struct { + Rows []relationRow `view:"rows,table=REPORT,type=ReportRow,dest=rows.go"` +} + +type typedRouteInput struct { + ID int +} + +type typedRouteOutput struct { + Data []reportRow +} + +type typedRouteSource struct { + Route xdatly.Component[typedRouteInput, typedRouteOutput] `component:",path=/v1/api/dev/report,method=GET"` +} + +type dynamicRouteInput struct { + Name string +} + +type dynamicRouteOutput struct { + Count int +} + +type namedDynamicRouteInput struct { + Name string `parameter:"name,kind=query,in=name"` +} + +type namedDynamicRouteOutput struct { + Count int `parameter:"count,kind=output,in=view"` +} + +type taggedComponentStateSource struct { + Auth interface{} `parameter:",kind=component,in=GET:/v1/api/dev/auth" typeName:"github.com/acme/auth.UserAclOutput"` +} + +type constStateSource struct { + Product string `parameter:",kind=const,in=Product" value:"PRODUCT" internal:"true"` +} + +type codecStateSource struct { + Jwt string `parameter:",kind=header,in=Authorization,errorCode=401" codec:"JwtClaim"` + Run string `parameter:",kind=body,in=run" handler:"Exec"` +} + func TestPlanner_Plan(t *testing.T) { scanner := scan.New() scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportSource{}}) @@ -88,6 +138,12 @@ func TestPlanner_Plan(t *testing.T) { require.NotNil(t, stateByPath["id"]) assert.Equal(t, "query", stateByPath["id"].KindString()) assert.Equal(t, "id", stateByPath["id"].InName()) + + require.Len(t, result.Components, 1) + assert.Equal(t, "Route", result.Components[0].FieldName) + assert.Equal(t, "/v1/api/dev/report", result.Components[0].RoutePath) + assert.Equal(t, "GET", result.Components[0].Method) + assert.Equal(t, "dev", result.Components[0].Connector) } func TestPlanner_Plan_LinkOnProducesStructuredRelations(t *testing.T) { @@ -113,6 +169,190 @@ func TestPlanner_Plan_LinkOnProducesStructuredRelations(t *testing.T) { assert.Equal(t, "id", relation.On[0].RefColumn) } +func TestPlanner_Plan_ComponentHolderTypes(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &typedRouteSource{}}) + require.NoError(t, err) + + planner := New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.Components, 1) + assert.Equal(t, reflect.TypeOf(typedRouteInput{}), result.Components[0].InputType) + assert.Equal(t, reflect.TypeOf(typedRouteOutput{}), result.Components[0].OutputType) + assert.Empty(t, result.Components[0].InputName) + assert.Empty(t, result.Components[0].OutputName) +} + +func TestPlanner_Plan_DynamicComponentHolderTypes(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET"` + }{ + Route: xdatly.Component[any, any]{ + Inout: dynamicRouteInput{}, + Output: dynamicRouteOutput{}, + }, + }}) + require.NoError(t, err) + + planner := New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.Components, 1) + assert.Equal(t, reflect.TypeOf(dynamicRouteInput{}), result.Components[0].InputType) + assert.Equal(t, reflect.TypeOf(dynamicRouteOutput{}), result.Components[0].OutputType) +} + +func TestPlanner_Plan_DynamicComponentHolderExplicitNames(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` + }{}}) + require.NoError(t, err) + + planner := New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.Components, 1) + assert.Nil(t, result.Components[0].InputType) + assert.Nil(t, result.Components[0].OutputType) + assert.Equal(t, "ReportInput", result.Components[0].InputName) + assert.Equal(t, "ReportOutput", result.Components[0].OutputName) +} + +func TestPlanner_Plan_PreservesConstValueTag(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &constStateSource{}}) + require.NoError(t, err) + + planned, err := New().Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.States, 1) + require.NotNil(t, result.States[0].In) + assert.Equal(t, "const", result.States[0].KindString()) + assert.Equal(t, "Product", result.States[0].InName()) + assert.Equal(t, "PRODUCT", result.States[0].Value) +} + +func TestPlanner_Plan_PreservesCodecAndHandlerTags(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &codecStateSource{}}) + require.NoError(t, err) + + planned, err := New().Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.States, 2) + stateByName := map[string]*State{} + for _, item := range result.States { + stateByName[item.Name] = item + } + require.NotNil(t, stateByName["Jwt"]) + require.NotNil(t, stateByName["Run"]) + if stateByName["Jwt"].Output == nil || stateByName["Jwt"].Output.Name != "JwtClaim" { + t.Fatalf("expected Jwt codec to be preserved, got %#v", stateByName["Jwt"].Output) + } + if stateByName["Run"].Handler == nil || stateByName["Run"].Handler.Name != "Exec" { + t.Fatalf("expected Run handler to be preserved, got %#v", stateByName["Run"].Handler) + } +} + +func TestPlanner_Plan_DynamicComponentHolderExplicitNamesFromRegistry(t *testing.T) { + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(namedDynamicRouteInput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/plan"), x.WithName("ReportInput"))) + registry.Register(x.NewType(reflect.TypeOf(namedDynamicRouteOutput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/plan"), x.WithName("ReportOutput"))) + + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{ + Struct: &struct { + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` + }{}, + TypeRegistry: registry, + }) + require.NoError(t, err) + + planner := New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.Components, 1) + assert.Equal(t, reflect.TypeOf(namedDynamicRouteInput{}), result.Components[0].InputType) + assert.Equal(t, reflect.TypeOf(namedDynamicRouteOutput{}), result.Components[0].OutputType) + assert.Equal(t, "ReportInput", result.Components[0].InputName) + assert.Equal(t, "ReportOutput", result.Components[0].OutputName) +} + +func TestPlanner_Plan_StateTypeNameOverridesInterfaceType(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &taggedComponentStateSource{}}) + require.NoError(t, err) + + planner := New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.States, 1) + assert.Equal(t, "github.com/acme/auth", result.States[0].Schema.Package) + assert.Equal(t, "github.com/acme/auth", result.States[0].Schema.PackagePath) + assert.Equal(t, "UserAclOutput", result.States[0].Schema.Name) +} + +func TestPlanner_Plan_AssignsNestedRelationParent(t *testing.T) { + scanned := &shape.ScanResult{ + Source: &shape.Source{Name: "nested"}, + Descriptors: &scan.Result{ + RootType: reflect.TypeOf(struct{}{}), + ViewFields: []*scan.Field{ + { + Path: "Route.Output.Data", + Name: "Data", + Type: reflect.TypeOf([]struct{}{}), + ViewTag: &tags.Tag{ + View: &tags.View{Name: "vendor"}, + SQL: tags.NewViewSQL("", "vendor.sql"), + }, + }, + { + Path: "Route.Output.Data.Products", + Name: "Products", + Type: reflect.TypeOf([]struct{}{}), + ViewTag: &tags.Tag{ + View: &tags.View{Table: "PRODUCT"}, + SQL: tags.NewViewSQL("", "products.sql"), + LinkOn: []string{"Id:ID=VendorId:VENDOR_ID"}, + }, + }, + }, + }, + } + result, err := New().Plan(context.Background(), scanned) + require.NoError(t, err) + planned, ok := result.Plan.(*Result) + require.True(t, ok) + require.Len(t, planned.Views, 2) + require.Len(t, planned.Views[1].Relations, 1) + require.Equal(t, "vendor", planned.Views[1].Relations[0].Parent) +} + func TestPlanner_Plan_LinkOnPreservesFieldSelectors(t *testing.T) { scanner := scan.New() scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &relationSourceWithFields{}}) @@ -138,6 +378,24 @@ func TestPlanner_Plan_LinkOnPreservesFieldSelectors(t *testing.T) { assert.Equal(t, "id", relation.On[0].RefColumn) } +func TestPlanner_Plan_ViewTypeDestDeclaration(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &viewTypeDestSource{}}) + require.NoError(t, err) + + planner := New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.Views, 1) + viewPlan := result.Views[0] + require.NotNil(t, viewPlan.Declaration) + assert.Equal(t, "ReportRow", viewPlan.Declaration.TypeName) + assert.Equal(t, "rows.go", viewPlan.Declaration.Dest) +} + // stubScanSpec is a non-scan-Result implementation of shape.ScanSpec used to // verify that Plan() returns an error when given an unexpected descriptor type. type stubScanSpec struct{} diff --git a/repository/shape/scan/component_contract.go b/repository/shape/scan/component_contract.go new file mode 100644 index 000000000..c26cc3a78 --- /dev/null +++ b/repository/shape/scan/component_contract.go @@ -0,0 +1,229 @@ +package scan + +import ( + "fmt" + "path" + "reflect" + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/componenttag" + "github.com/viant/datly/repository/shape/typectx" +) + +const ( + xdatlyComponentPkg = "github.com/viant/xdatly" + xdatlyComponentName = "Component[" +) + +type componentContract struct { + InputType reflect.Type + OutputType reflect.Type + InputName string + OutputName string + UsesDynamic bool +} + +func resolveComponentContract(source *shape.Source, fieldType reflect.Type, value reflect.Value, tag *componenttag.Tag) (*componentContract, error) { + contract := &componentContract{} + if tag != nil && tag.Component != nil { + contract.InputName = strings.TrimSpace(tag.Component.Input) + contract.OutputName = strings.TrimSpace(tag.Component.Output) + } + if provider := componentContractProvider(value); provider != nil { + contract.InputType = provider.ComponentInputType() + contract.OutputType = provider.ComponentOutputType() + } + typedInput, typedOutput, dynamic := inspectXDatlyComponent(fieldType, value) + isStructuredHolder := providerDefined(contract) || isXDatlyComponentType(fieldType) + if contract.InputType == nil { + contract.InputType = typedInput + } + if contract.OutputType == nil { + contract.OutputType = typedOutput + } + if contract.InputType == nil && contract.InputName != "" { + contract.InputType = resolveNamedComponentContractType(source, contract.InputName) + } + if contract.OutputType == nil && contract.OutputName != "" { + contract.OutputType = resolveNamedComponentContractType(source, contract.OutputName) + } + contract.UsesDynamic = dynamic + if !isStructuredHolder && contract.InputName == "" && contract.OutputName == "" { + return contract, nil + } + if contract.UsesDynamic && + contract.InputType == nil && + contract.OutputType == nil && + contract.InputName == "" && + contract.OutputName == "" { + return nil, fmt.Errorf("dynamic component holder requires explicit input/output tag names or initialized Inout/Output values") + } + if contract.InputType == nil && contract.InputName == "" { + return nil, fmt.Errorf("component input contract type is unresolved") + } + if contract.OutputType == nil && contract.OutputName == "" { + return nil, fmt.Errorf("component output contract type is unresolved") + } + return contract, nil +} + +func resolveNamedComponentContractType(source *shape.Source, typeName string) reflect.Type { + typeName = strings.TrimSpace(typeName) + if source == nil || typeName == "" { + return nil + } + registry := source.EnsureTypeRegistry() + if registry == nil { + return nil + } + resolver := typectx.NewResolver(registry, componentTypeContext(source)) + resolved, err := resolver.Resolve(typeName) + if err != nil || resolved == "" { + return nil + } + lookup := registry.Lookup(resolved) + if lookup == nil || lookup.Type == nil { + return nil + } + return unwrapComponentType(lookup.Type) +} + +func componentTypeContext(source *shape.Source) *typectx.Context { + if source == nil { + return nil + } + rootType, err := source.ResolveRootType() + if err != nil || rootType == nil { + return nil + } + pkgPath := strings.TrimSpace(rootType.PkgPath()) + if pkgPath == "" { + return nil + } + return &typectx.Context{ + DefaultPackage: pkgPath, + PackagePath: pkgPath, + PackageName: path.Base(pkgPath), + } +} + +func providerDefined(contract *componentContract) bool { + return contract != nil && (contract.InputType != nil || contract.OutputType != nil) +} + +type typedComponentContract interface { + ComponentInputType() reflect.Type + ComponentOutputType() reflect.Type +} + +func componentContractProvider(value reflect.Value) typedComponentContract { + if !value.IsValid() { + return nil + } + if value.CanInterface() { + if provider, ok := value.Interface().(typedComponentContract); ok { + return provider + } + } + for value.IsValid() && value.Kind() == reflect.Interface { + if value.IsNil() { + return nil + } + value = value.Elem() + } + if value.CanInterface() { + if provider, ok := value.Interface().(typedComponentContract); ok { + return provider + } + } + for value.IsValid() && value.Kind() == reflect.Ptr { + if value.IsNil() { + return nil + } + value = value.Elem() + if !value.CanInterface() { + continue + } + if provider, ok := value.Interface().(typedComponentContract); ok { + return provider + } + } + return nil +} + +func componentFieldValue(holder reflect.Value, fieldName string) reflect.Value { + if !holder.IsValid() { + return reflect.Value{} + } + for holder.IsValid() && holder.Kind() == reflect.Ptr { + if holder.IsNil() { + return reflect.Value{} + } + holder = holder.Elem() + } + if !holder.IsValid() || holder.Kind() != reflect.Struct { + return reflect.Value{} + } + field := holder.FieldByName(fieldName) + if !field.IsValid() { + return reflect.Value{} + } + return field +} + +func concreteComponentFieldType(fallback reflect.Type, holder reflect.Value, fieldName string) (reflect.Type, bool) { + if fallback != nil && !(fallback.Kind() == reflect.Interface && fallback.NumMethod() == 0) { + return fallback, false + } + field := componentFieldValue(holder, fieldName) + if !field.IsValid() { + return nil, true + } + for field.IsValid() && field.Kind() == reflect.Interface { + if field.IsNil() { + return nil, true + } + field = field.Elem() + } + for field.IsValid() && field.Kind() == reflect.Ptr { + if field.IsNil() { + return nil, true + } + field = field.Elem() + } + if !field.IsValid() { + return nil, true + } + return field.Type(), true +} + +func inspectXDatlyComponent(rType reflect.Type, value reflect.Value) (reflect.Type, reflect.Type, bool) { + rType = unwrapComponentType(rType) + if !isXDatlyComponentType(rType) { + return nil, nil, false + } + inoutField, ok := rType.FieldByName("Inout") + if !ok { + return nil, nil, false + } + outputField, ok := rType.FieldByName("Output") + if !ok { + return nil, nil, false + } + inputType, inputDynamic := concreteComponentFieldType(inoutField.Type, value, "Inout") + outputType, outputDynamic := concreteComponentFieldType(outputField.Type, value, "Output") + return inputType, outputType, inputDynamic || outputDynamic +} + +func isXDatlyComponentType(rType reflect.Type) bool { + rType = unwrapComponentType(rType) + return rType != nil && rType.Kind() == reflect.Struct && rType.PkgPath() == xdatlyComponentPkg && strings.HasPrefix(rType.Name(), xdatlyComponentName) +} + +func unwrapComponentType(rType reflect.Type) reflect.Type { + for rType != nil && rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + return rType +} diff --git a/repository/shape/scan/model.go b/repository/shape/scan/model.go index 357299250..c750fd672 100644 --- a/repository/shape/scan/model.go +++ b/repository/shape/scan/model.go @@ -4,30 +4,40 @@ import ( "embed" "reflect" + "github.com/viant/datly/repository/shape/componenttag" "github.com/viant/datly/view/tags" ) // Result holds scan output produced from a struct source. type Result struct { - RootType reflect.Type - EmbedFS *embed.FS - Fields []*Field - ByPath map[string]*Field - ViewFields []*Field - StateFields []*Field + RootType reflect.Type + EmbedFS *embed.FS + Fields []*Field + ByPath map[string]*Field + ViewFields []*Field + StateFields []*Field + ComponentFields []*Field } // Field describes one scanned struct field. type Field struct { - Path string - Name string - Index []int - Type reflect.Type - Tag reflect.StructTag - Anonymous bool + Path string + Name string + Index []int + Type reflect.Type + ComponentInputType reflect.Type + ComponentOutputType reflect.Type + ComponentInputName string + ComponentOutputName string + Tag reflect.StructTag + Anonymous bool + ViewTypeName string + ViewDest string - HasViewTag bool - HasStateTag bool - ViewTag *tags.Tag - StateTag *tags.Tag + HasViewTag bool + HasStateTag bool + HasComponentTag bool + ViewTag *tags.Tag + StateTag *tags.Tag + ComponentTag *componenttag.Tag } diff --git a/repository/shape/scan/scanner.go b/repository/shape/scan/scanner.go index 255f9cd4b..f798f27a3 100644 --- a/repository/shape/scan/scanner.go +++ b/repository/shape/scan/scanner.go @@ -2,13 +2,19 @@ package scan import ( "context" + "embed" "fmt" + "os" + "path/filepath" "reflect" "strings" + afsembed "github.com/viant/afs/embed" "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/componenttag" "github.com/viant/datly/view/state" "github.com/viant/datly/view/tags" + taglytags "github.com/viant/tagly/tags" ) // StructScanner scans arbitrary struct types and extracts Datly-relevant tags. @@ -35,13 +41,15 @@ func (s *StructScanner) Scan(ctx context.Context, source *shape.Source, _ ...sha } embedder := resolveEmbedder(source) + baseDir := source.BaseDir() + rootValue := resolveRootValue(source) result := &Result{ RootType: root, EmbedFS: embedder.EmbedFS(), ByPath: map[string]*Field{}, } - if err = s.scanStruct(root, "", nil, embedder, result, map[reflect.Type]bool{}); err != nil { + if err = s.scanStruct(source, root, rootValue, "", nil, embedder, baseDir, result, map[reflect.Type]bool{}); err != nil { return nil, err } @@ -85,11 +93,31 @@ func resolveEmbedder(source *shape.Source) *state.FSEmbedder { return embedder } +func resolveRootValue(source *shape.Source) reflect.Value { + if source == nil || source.Struct == nil { + return reflect.Value{} + } + value := reflect.ValueOf(source.Struct) + for value.IsValid() && value.Kind() == reflect.Ptr { + if value.IsNil() { + return reflect.Value{} + } + value = value.Elem() + } + if !value.IsValid() || value.Kind() != reflect.Struct { + return reflect.Value{} + } + return value +} + func (s *StructScanner) scanStruct( + source *shape.Source, rType reflect.Type, + rootValue reflect.Value, prefix string, indexPrefix []int, embedder *state.FSEmbedder, + baseDir string, result *Result, visited map[reflect.Type]bool, ) error { @@ -116,8 +144,10 @@ func (s *StructScanner) scanStruct( Anonymous: field.Anonymous, } + fieldFS := parseFS(field.Tag, embedder.EmbedFS(), baseDir) if hasAny(field.Tag, tags.ViewTag, tags.SQLTag, tags.SQLSummaryTag, tags.LinkOnTag) { - parsed, err := tags.ParseViewTags(field.Tag, embedder.EmbedFS()) + descriptor.ViewTypeName, descriptor.ViewDest = parseShapeViewHints(field.Tag) + parsed, err := tags.ParseViewTags(field.Tag, fieldFS) if err != nil { return fmt.Errorf("shape scan: failed to parse view tags on %s: %w", path, err) } @@ -126,8 +156,8 @@ func (s *StructScanner) scanStruct( result.ViewFields = append(result.ViewFields, descriptor) } - if hasAny(field.Tag, tags.ParameterTag, tags.SQLTag, tags.PredicateTag, tags.CodecTag, tags.HandlerTag) { - parsed, err := tags.ParseStateTags(field.Tag, embedder.EmbedFS()) + if hasAny(field.Tag, tags.ParameterTag, tags.PredicateTag, tags.CodecTag, tags.HandlerTag) { + parsed, err := tags.ParseStateTags(field.Tag, fieldFS) if err != nil { return fmt.Errorf("shape scan: failed to parse state tags on %s: %w", path, err) } @@ -136,15 +166,36 @@ func (s *StructScanner) scanStruct( result.StateFields = append(result.StateFields, descriptor) } + if hasAny(field.Tag, componenttag.TagName) { + parsed, err := componenttag.Parse(field.Tag) + if err != nil { + return fmt.Errorf("shape scan: failed to parse component tags on %s: %w", path, err) + } + descriptor.HasComponentTag = true + descriptor.ComponentTag = parsed + fieldValue := fieldValueByIndex(rootValue, combinedIndex) + contract, err := resolveComponentContract(source, field.Type, fieldValue, parsed) + if err != nil { + return fmt.Errorf("shape scan: failed to resolve component contract on %s: %w", path, err) + } + if contract != nil { + descriptor.ComponentInputType = contract.InputType + descriptor.ComponentOutputType = contract.OutputType + descriptor.ComponentInputName = contract.InputName + descriptor.ComponentOutputName = contract.OutputName + if err := s.scanComponentContracts(source, path, fieldValue, contract, baseDir, result, visited); err != nil { + return err + } + } + result.ComponentFields = append(result.ComponentFields, descriptor) + } + result.Fields = append(result.Fields, descriptor) result.ByPath[path] = descriptor - nextType := field.Type - for nextType.Kind() == reflect.Ptr { - nextType = nextType.Elem() - } - if field.Anonymous && nextType.Kind() == reflect.Struct && !isStdlib(nextType.PkgPath()) { - if err := s.scanStruct(nextType, path, combinedIndex, embedder, result, visited); err != nil { + nextType := nestedStructType(field.Type) + if nextType != nil && shouldRecurseIntoField(field, descriptor, nextType) { + if err := s.scanStruct(source, nextType, rootValue, path, combinedIndex, embedder, baseDir, result, visited); err != nil { return err } } @@ -152,6 +203,92 @@ func (s *StructScanner) scanStruct( return nil } +func nestedStructType(rType reflect.Type) reflect.Type { + for rType != nil { + switch rType.Kind() { + case reflect.Ptr, reflect.Slice, reflect.Array: + rType = rType.Elem() + default: + if rType.Kind() == reflect.Struct { + return rType + } + return nil + } + } + return nil +} + +func shouldRecurseIntoField(field reflect.StructField, descriptor *Field, nextType reflect.Type) bool { + if nextType == nil || nextType.Kind() != reflect.Struct { + return false + } + if field.Anonymous { + return !isStdlib(nextType.PkgPath()) + } + if descriptor != nil && descriptor.HasViewTag { + // Source-reconstructed and StructOf-based semantic view structs often have no package path. + // They still need recursive scanning so nested relation views are preserved. + return true + } + return false +} + +func (s *StructScanner) scanComponentContracts( + source *shape.Source, + prefix string, + fieldValue reflect.Value, + contract *componentContract, + baseDir string, + result *Result, + visited map[reflect.Type]bool, +) error { + if contract == nil { + return nil + } + if contract.InputType != nil { + embedder := state.NewFSEmbedder(nil) + embedder.SetType(contract.InputType) + if err := s.scanStruct(source, contractInputRoot(contract.InputType), componentFieldValue(fieldValue, "Inout"), prefix+".Inout", nil, embedder, baseDir, result, visited); err != nil { + return err + } + } + if contract.OutputType != nil { + embedder := state.NewFSEmbedder(nil) + embedder.SetType(contract.OutputType) + if err := s.scanStruct(source, contractInputRoot(contract.OutputType), componentFieldValue(fieldValue, "Output"), prefix+".Output", nil, embedder, baseDir, result, visited); err != nil { + return err + } + } + return nil +} + +func contractInputRoot(rType reflect.Type) reflect.Type { + for rType != nil && rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + return rType +} + +func fieldValueByIndex(rootValue reflect.Value, index []int) reflect.Value { + if !rootValue.IsValid() || len(index) == 0 { + return reflect.Value{} + } + current := rootValue + for _, idx := range index { + for current.IsValid() && current.Kind() == reflect.Ptr { + if current.IsNil() { + return reflect.Value{} + } + current = current.Elem() + } + if !current.IsValid() || current.Kind() != reflect.Struct || idx < 0 || idx >= current.NumField() { + return reflect.Value{} + } + current = current.Field(idx) + } + return current +} + func hasAny(tag reflect.StructTag, names ...string) bool { for _, name := range names { if _, ok := tag.Lookup(name); ok { @@ -161,9 +298,77 @@ func hasAny(tag reflect.StructTag, names ...string) bool { return false } +func parseFS(tag reflect.StructTag, existing *embed.FS, baseDir string) *embed.FS { + baseDir = strings.TrimSpace(baseDir) + if baseDir == "" { + return existing + } + uris := sqlURIs(tag) + if len(uris) == 0 { + return existing + } + holder := afsembed.NewHolder() + if existing != nil { + holder.AddFs(existing, ".") + } + added := 0 + for _, URI := range uris { + if URI == "" || filepath.IsAbs(URI) || strings.Contains(URI, "://") { + continue + } + absPath := filepath.Join(baseDir, filepath.FromSlash(URI)) + data, err := os.ReadFile(absPath) + if err != nil { + continue + } + holder.Add(filepath.ToSlash(URI), string(data)) + added++ + } + if added == 0 { + return existing + } + return holder.EmbedFs() +} + +func sqlURIs(tag reflect.StructTag) []string { + var result []string + appendURI := func(tagName string) { + value := strings.TrimSpace(tag.Get(tagName)) + if !strings.HasPrefix(value, "uri=") { + return + } + URI := strings.TrimSpace(value[4:]) + if URI != "" { + result = append(result, URI) + } + } + appendURI(tags.SQLTag) + appendURI(tags.SQLSummaryTag) + return result +} + func isStdlib(pkg string) bool { if pkg == "" { return true } return !strings.Contains(pkg, ".") } + +func parseShapeViewHints(tag reflect.StructTag) (string, string) { + raw, ok := tag.Lookup(tags.ViewTag) + if !ok { + return "", "" + } + _, values := taglytags.Values(raw).Name() + var typeName, dest string + _ = values.MatchPairs(func(key, value string) error { + switch strings.ToLower(strings.TrimSpace(key)) { + case "typename": + typeName = strings.TrimSpace(value) + case "dest": + dest = strings.TrimSpace(value) + } + return nil + }) + return typeName, dest +} diff --git a/repository/shape/scan/scanner_test.go b/repository/shape/scan/scanner_test.go index bf57d5cec..42a53750e 100644 --- a/repository/shape/scan/scanner_test.go +++ b/repository/shape/scan/scanner_test.go @@ -3,6 +3,8 @@ package scan import ( "context" "embed" + "os" + "path/filepath" "reflect" "testing" @@ -10,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/viant/datly/repository/shape" "github.com/viant/x" + "github.com/viant/xdatly" ) //go:embed testdata/*.sql @@ -28,8 +31,37 @@ type reportRow struct { type reportSource struct { embeddedFS - Rows []reportRow `view:"rows,table=REPORT,connector=dev" sql:"uri=testdata/report.sql"` - ID int `parameter:"id,kind=query,in=id"` + Rows []reportRow `view:"rows,table=REPORT,connector=dev,type=ReportRow,dest=rows.go" sql:"uri=testdata/report.sql"` + ID int `parameter:"id,kind=query,in=id"` + Route struct{} `component:",path=/v1/api/dev/report,method=GET,connector=dev"` +} + +type reportInput struct { + ID int +} + +type reportOutput struct { + Data []reportRow +} + +type typedComponentSource struct { + Route xdatly.Component[reportInput, reportOutput] `component:",path=/v1/api/dev/report,method=GET"` +} + +type dynamicReportInput struct { + Name string +} + +type dynamicReportOutput struct { + Count int +} + +type namedReportInput struct { + Name string `parameter:"name,kind=query,in=name"` +} + +type namedReportOutput struct { + Data []reportRow `parameter:"data,kind=output,in=view"` } func TestStructScanner_Scan(t *testing.T) { @@ -49,6 +81,8 @@ func TestStructScanner_Scan(t *testing.T) { require.True(t, rows.HasViewTag) require.NotNil(t, rows.ViewTag) assert.Equal(t, "rows", rows.ViewTag.View.Name) + assert.Equal(t, "ReportRow", rows.ViewTag.View.TypeName) + assert.Equal(t, "rows.go", rows.ViewTag.View.Dest) assert.Contains(t, rows.ViewTag.SQL.SQL, "SELECT ID, NAME FROM REPORT") idField := descriptors.ByPath["ID"] @@ -59,6 +93,15 @@ func TestStructScanner_Scan(t *testing.T) { assert.Equal(t, "id", idField.StateTag.Parameter.Name) assert.Equal(t, "query", idField.StateTag.Parameter.Kind) assert.Equal(t, "id", idField.StateTag.Parameter.In) + + route := descriptors.ByPath["Route"] + require.NotNil(t, route) + require.True(t, route.HasComponentTag) + require.NotNil(t, route.ComponentTag) + require.NotNil(t, route.ComponentTag.Component) + assert.Equal(t, "/v1/api/dev/report", route.ComponentTag.Component.Path) + assert.Equal(t, "GET", route.ComponentTag.Component.Method) + assert.Equal(t, "dev", route.ComponentTag.Component.Connector) } func TestStructScanner_Scan_InvalidSource(t *testing.T) { @@ -68,6 +111,94 @@ func TestStructScanner_Scan_InvalidSource(t *testing.T) { assert.Contains(t, err.Error(), "expected struct") } +func TestStructScanner_Scan_ComponentHolderTypes(t *testing.T) { + scanner := New() + result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &typedComponentSource{}}) + require.NoError(t, err) + + descriptors, ok := DescriptorsFrom(result) + require.True(t, ok) + require.Len(t, descriptors.ComponentFields, 1) + route := descriptors.ComponentFields[0] + require.NotNil(t, route) + assert.Equal(t, reflect.TypeOf(reportInput{}), route.ComponentInputType) + assert.Equal(t, reflect.TypeOf(reportOutput{}), route.ComponentOutputType) + assert.Empty(t, route.ComponentInputName) + assert.Empty(t, route.ComponentOutputName) +} + +func TestStructScanner_Scan_DynamicComponentHolderTypes(t *testing.T) { + scanner := New() + result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET"` + }{ + Route: xdatly.Component[any, any]{ + Inout: dynamicReportInput{}, + Output: dynamicReportOutput{}, + }, + }}) + require.NoError(t, err) + + descriptors, ok := DescriptorsFrom(result) + require.True(t, ok) + require.Len(t, descriptors.ComponentFields, 1) + route := descriptors.ComponentFields[0] + require.NotNil(t, route) + assert.Equal(t, reflect.TypeOf(dynamicReportInput{}), route.ComponentInputType) + assert.Equal(t, reflect.TypeOf(dynamicReportOutput{}), route.ComponentOutputType) +} + +func TestStructScanner_Scan_DynamicComponentHolderTypesWithExplicitNames(t *testing.T) { + scanner := New() + result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` + }{}}) + require.NoError(t, err) + + descriptors, ok := DescriptorsFrom(result) + require.True(t, ok) + require.Len(t, descriptors.ComponentFields, 1) + route := descriptors.ComponentFields[0] + require.NotNil(t, route) + assert.Nil(t, route.ComponentInputType) + assert.Nil(t, route.ComponentOutputType) + assert.Equal(t, "ReportInput", route.ComponentInputName) + assert.Equal(t, "ReportOutput", route.ComponentOutputName) +} + +func TestStructScanner_Scan_DynamicComponentHolderTypesWithExplicitNamesFromRegistry(t *testing.T) { + scanner := New() + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(namedReportInput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/scan"), x.WithName("ReportInput"))) + registry.Register(x.NewType(reflect.TypeOf(namedReportOutput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/scan"), x.WithName("ReportOutput"))) + result, err := scanner.Scan(context.Background(), &shape.Source{ + Struct: &struct { + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` + }{}, + TypeRegistry: registry, + }) + require.NoError(t, err) + + descriptors, ok := DescriptorsFrom(result) + require.True(t, ok) + require.Len(t, descriptors.ComponentFields, 1) + route := descriptors.ComponentFields[0] + require.NotNil(t, route) + assert.Equal(t, reflect.TypeOf(namedReportInput{}), route.ComponentInputType) + assert.Equal(t, reflect.TypeOf(namedReportOutput{}), route.ComponentOutputType) + assert.Equal(t, "ReportInput", route.ComponentInputName) + assert.Equal(t, "ReportOutput", route.ComponentOutputName) +} + +func TestStructScanner_Scan_DynamicComponentHolderTypesRequireContract(t *testing.T) { + scanner := New() + _, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET"` + }{}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "dynamic component holder requires explicit input/output tag names or initialized Inout/Output values") +} + func TestStructScanner_Scan_WithRegistryType(t *testing.T) { scanner := New() registry := x.NewRegistry() @@ -81,3 +212,65 @@ func TestStructScanner_Scan_WithRegistryType(t *testing.T) { require.True(t, ok) assert.Equal(t, reflect.TypeOf(reportSource{}), descriptors.RootType) } + +func TestStructScanner_Scan_UsesSourceBaseDirForRelativeSQL(t *testing.T) { + scanner := New() + baseDir := t.TempDir() + sqlPath := filepath.Join(baseDir, "routes", "report.sql") + require.NoError(t, os.MkdirAll(filepath.Dir(sqlPath), 0o755)) + require.NoError(t, os.WriteFile(sqlPath, []byte("SELECT ID FROM REPORT"), 0o644)) + + type reportView struct { + Data []reportRow `view:"rows" sql:"uri=routes/report.sql"` + } + + result, err := scanner.Scan(context.Background(), &shape.Source{ + Type: reflect.TypeOf(reportView{}), + Path: filepath.Join(baseDir, "router.go"), + }) + require.NoError(t, err) + + descriptors, ok := DescriptorsFrom(result) + require.True(t, ok) + viewField := descriptors.ByPath["Data"] + require.NotNil(t, viewField) + require.NotNil(t, viewField.ViewTag) + assert.Equal(t, "SELECT ID FROM REPORT", string(viewField.ViewTag.SQL.SQL)) + assert.Equal(t, "routes/report.sql", string(viewField.ViewTag.SQL.URI)) +} + +func TestStructScanner_Scan_RecursesIntoViewTaggedStructFields(t *testing.T) { + type vendorProduct struct { + ID int `sqlx:"ID"` + VendorID int `sqlx:"VENDOR_ID"` + } + type vendorRow struct { + ID int `sqlx:"ID"` + Products []*vendorProduct `view:",table=PRODUCT" on:"Id:ID=VendorId:VENDOR_ID" sql:"uri=testdata/report.sql"` + } + type nestedViewOutput struct { + Data []*vendorRow `parameter:",kind=output,in=view" view:"vendor" sql:"uri=testdata/report.sql" anonymous:"true"` + } + type nestedViewRouteSource struct { + Route xdatly.Component[reportInput, nestedViewOutput] `component:",path=/v1/api/dev/vendors,method=GET"` + } + + scanner := New() + result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &nestedViewRouteSource{}}) + require.NoError(t, err) + + descriptors, ok := DescriptorsFrom(result) + require.True(t, ok) + + rootView := descriptors.ByPath["Route.Output.Data"] + require.NotNil(t, rootView) + require.True(t, rootView.HasViewTag) + + childView := descriptors.ByPath["Route.Output.Data.Products"] + require.NotNil(t, childView) + require.True(t, childView.HasViewTag) + require.NotNil(t, childView.ViewTag) + assert.Equal(t, "PRODUCT", childView.ViewTag.View.Table) + assert.Len(t, descriptors.ViewFields, 2) + assert.Nil(t, descriptors.ByPath["Route.Output.Data.Products"].StateTag) +} diff --git a/repository/shape/shape.go b/repository/shape/shape.go index 5f7f766d8..94e6f63c8 100644 --- a/repository/shape/shape.go +++ b/repository/shape/shape.go @@ -35,9 +35,11 @@ type ( RegisterComponent(ctx context.Context, artifacts *ComponentArtifact) error } - ScanOptions struct{} - PlanOptions struct{} - LoadOptions struct{} + ScanOptions struct{} + PlanOptions struct{} + LoadOptions struct { + UseTypeContextPackages bool + } CompileOptions struct { Strict bool Profile CompileProfile @@ -50,6 +52,7 @@ type ( TypePackageName string TypePackagePath string InferTypeContext *bool + UseLinkedTypes *bool } ScanOption func(*ScanOptions) @@ -58,6 +61,15 @@ type ( CompileOption func(*CompileOptions) ) +func WithLoadTypeContextPackages(enabled bool) LoadOption { + return func(o *LoadOptions) { + if o == nil { + return + } + o.UseTypeContextPackages = enabled + } +} + const ( CompileMixedModeExecWins CompileMixedMode = "exec_wins" CompileMixedModeReadWins CompileMixedMode = "read_wins" diff --git a/repository/shape/validate/relation.go b/repository/shape/validate/relation.go index 31aee9357..c0774330c 100644 --- a/repository/shape/validate/relation.go +++ b/repository/shape/validate/relation.go @@ -36,6 +36,11 @@ func ValidateRelations(resource *view.Resource, targets ...*view.View) error { } } refIndex := view.Columns(ref.Columns).Index(ref.CaseFormat) + // Shape load runs before DB-backed column discovery in transcribe. + // When either side has no columns yet, defer strict relation checks. + if len(parent.Columns) == 0 || len(ref.Columns) == 0 { + continue + } pairCount := len(rel.On) if len(rel.Of.On) > pairCount { pairCount = len(rel.Of.On) diff --git a/repository/shape/velty/ast/assign.go b/repository/shape/velty/ast/assign.go new file mode 100644 index 000000000..f058ff248 --- /dev/null +++ b/repository/shape/velty/ast/assign.go @@ -0,0 +1,110 @@ +package ast + +import "fmt" + +func (s *Assign) Generate(builder *Builder) (err error) { + if builder.AssignNotifier != nil { + newExpr, err := builder.AssignNotifier(s) + if err != nil { + return err + } + + if newExpr != nil && newExpr != s { + return newExpr.Generate(builder) + } + } + + switch builder.Lang { + case LangVelty: + if err = builder.WriteIndentedString("\n#set("); err != nil { + return err + } + if err = s.Holder.Generate(builder); err != nil { + return err + } + if err = builder.WriteString(" = "); err != nil { + return err + } + if err = s.Expression.Generate(builder); err != nil { + return err + } + if err = builder.WriteString(")"); err != nil { + return err + } + return nil + + case LangGO: + + callExpr, ok := s.Expression.(*CallExpr) + if ok && callExpr.Name == "IndexBy" && builder.Options.Lang == LangGO { + if builder.IndexByCode == nil { + builder.IndexByCode = builder.NewBuilder() + } + indexBuilder := builder.IndexByCode + asIdent, _ := s.Holder.(*Ident) + if holder := asIdent.Holder; holder != "" { + indexBuilder.WriteString(holder + ".") + } + indexBuilder.WriteString(asIdent.Name) + indexBuilder.WriteString(" = ") + if err = s.Expression.Generate(indexBuilder); err != nil { + return err + } + indexBuilder.WriteString("\n") + return nil + } + + if err = builder.WriteIndentedString("\n"); err != nil { + return err + } + asIdent, ok := s.Holder.(*Ident) + wasDeclared := true + if ok { + wasDeclared = builder.State.IsDeclared(asIdent.Name) + } + + if err = s.Holder.Generate(builder); err != nil { + return err + } + + for _, holder := range s.ExtraHolders { + if err = builder.WriteString(", "); err != nil { + return err + } + + if err = holder.Generate(builder); err != nil { + return err + } + } + + if err = s.appendGoAssignToken(builder, wasDeclared); err != nil { + return err + } + + if err = s.Expression.Generate(builder); err != nil { + return err + } + if !wasDeclared { + builder.State.DeclareVariable(asIdent.Name) + } + return nil + } + return fmt.Errorf("unsupported option %T %v\n", s, builder.Lang) + +} + +func (s *Assign) appendGoAssignToken(builder *Builder, isDeclared bool) error { + + if isDeclared { + return builder.WriteString(" = ") + } + + return builder.WriteString(" := ") +} + +func NewAssign(holder Expression, expr Expression) *Assign { + return &Assign{ + Holder: holder, + Expression: expr, + } +} diff --git a/repository/shape/velty/ast/ast.go b/repository/shape/velty/ast/ast.go new file mode 100644 index 000000000..98290ecc7 --- /dev/null +++ b/repository/shape/velty/ast/ast.go @@ -0,0 +1,163 @@ +package ast + +import ( + "fmt" + "github.com/viant/tagly/format/text" +) + +type ( + Node interface { + Generate(builder *Builder) error + } + Statement interface { + Node + } + + Expression interface { + Node + } //can be BinaryExpr or CallExpr or QuerySelector Expr + + Block []Statement + Ident struct { + Holder string + Name string + } + + Foreach struct { + Value *Ident + Set *Ident + Body Block + } + + Assign struct { + Holder Expression + ExtraHolders []Expression + Expression Expression + } + + CallExpr struct { + Terminator bool + Receiver Expression + Name string + Args []Expression + } + + MapExpr struct { + Map Expression + Key Expression + } + + StatementExpression struct { + Expression + } + + TerminatorExpression struct { + X Expression + } + + SelectorExpr struct { + Ident + X *SelectorExpr + } + + BinaryExpr struct { + X Expression + Op string + Y Expression + } + + LiteralExpr struct { + Literal string + } + + ReturnExpr struct { + X Expression + } +) + +func NewReturnExpr(expr Expression) *ReturnExpr { + return &ReturnExpr{X: expr} +} +func (r *ReturnExpr) Generate(builder *Builder) error { + switch builder.Lang { + case LangGO: + if err := builder.WriteIndentedString("\nreturn "); err != nil { + return err + } + + return r.X.Generate(builder) + } + + return fmt.Errorf("unsupported %T with lang %v", r, builder.Lang) +} + +func (m *MapExpr) Generate(builder *Builder) error { + if err := m.Map.Generate(builder); err != nil { + return err + } + + if err := builder.WriteString("["); err != nil { + return err + } + + if err := m.Key.Generate(builder); err != nil { + return err + } + + if err := builder.WriteString("]"); err != nil { + return err + } + + return nil +} + +func (b *Block) Append(statement Statement) { + *b = append(*b, statement) +} + +func (b *Block) AppendEmptyLine() { + b.Append(NewStatementExpression(NewLiteral(""))) +} + +func (b Block) Generate(builder *Builder) error { + if builder.WithoutBusinessLogic { + return nil + } + for _, stmt := range b { + if err := stmt.Generate(builder); err != nil { + return err + } + } + return nil +} + +func (e Ident) Generate(builder *Builder) (err error) { + identName := e.Name + if builder.WithLowerCaseIdent && e.Holder == "" { + upperCamel := text.CaseFormatUpperCamel + identName = upperCamel.Format(identName, text.CaseFormatLowerCamel) + } + + if e.Holder != "" { + identName = e.Holder + "." + identName + } + builder.State.DeclareVariable(identName) + if builder.Lang == LangVelty { + return builder.WriteString("$" + identName) + } + return builder.WriteString(identName) +} + +func (b TerminatorExpression) Generate(builder *Builder) error { + if err := b.X.Generate(builder); err != nil { + return err + } + if builder.Lang == LangVelty { + return builder.WriteByte(';') + } + return nil +} + +func NewTerminatorExpression(x Expression) *TerminatorExpression { + return &TerminatorExpression{X: x} +} diff --git a/repository/shape/velty/ast/ast_test.go b/repository/shape/velty/ast/ast_test.go new file mode 100644 index 000000000..6d226b931 --- /dev/null +++ b/repository/shape/velty/ast/ast_test.go @@ -0,0 +1,124 @@ +package ast + +import ( + "github.com/stretchr/testify/assert" + "strings" + "testing" +) + +func TestBlock_Stringify(t *testing.T) { + + var testCases = []struct { + description string + block Block + options Options + expect string + }{ + { + description: "assign", + options: Options{Lang: LangVelty}, + block: Block{ + &Assign{Holder: &Ident{Name: "inited"}, Expression: &CallExpr{Receiver: Ident{Name: "Campaign"}, Name: "init", Args: []Expression{ + Ident{Name: "CurCampaign"}, + }}}, + }, + expect: `#set($inited = $Campaign.init($CurCampaign))`, + }, + { + description: "for each ", + options: Options{Lang: LangVelty}, + block: Block{ + &Foreach{Set: &Ident{Name: "Sets"}, + Value: &Ident{Name: "Item"}, + Body: Block{ + &Assign{Holder: &Ident{Name: "tested"}, Expression: &CallExpr{Receiver: Ident{Name: "Campaign"}, Name: "Test", Args: []Expression{ + Ident{Name: "Item"}, + }}}}}}, + expect: `#foreach($Item in $Sets) + #set($tested = $Campaign.Test($Item)) +#end`, + }, + { + description: "if condition", + options: Options{Lang: LangVelty}, + block: Block{ + &Condition{ + If: &BinaryExpr{X: &Ident{Name: "Campaign.Id"}, Op: ">", Y: &LiteralExpr{Literal: "1"}}, + IFBlock: Block{ + &Assign{Holder: &Ident{Name: "inited"}, Expression: &CallExpr{Receiver: Ident{Name: "Campaign"}, Name: "init", Args: []Expression{ + Ident{Name: "CurCampaign"}, + }}}, + }, + ElseIfBlocks: []*ConditionalBlock{{ + If: &BinaryExpr{X: &Ident{Name: "Campaign.Name"}, Op: "==", Y: &LiteralExpr{Literal: `"Foo"`}}, + Block: Block{ + &Assign{Holder: &Ident{Name: "fooed"}, Expression: &CallExpr{Receiver: Ident{Name: "Campaign"}, Name: "Foo", Args: []Expression{ + Ident{Name: "CurCampaign"}, + }}}, + }, + }, + }, + }, + }, + expect: `#if($Campaign.Id > 1) + #set($inited = $Campaign.init($CurCampaign)) +#elseif($Campaign.Name == "Foo") + #set($fooed = $Campaign.Foo($CurCampaign)) +#end`, + }, + { + description: "assign condition | go", + options: Options{Lang: LangGO}, + block: Block{ + &Assign{Holder: &Ident{Name: "foo"}, Expression: &LiteralExpr{Literal: "10"}}, + }, + expect: `foo := 10`, + }, + { + description: "if stmt | go", + options: Options{Lang: LangGO}, + block: Block{ + &Condition{ + If: &BinaryExpr{X: &LiteralExpr{"0"}, Y: &Ident{Name: "foo"}, Op: ">"}, + IFBlock: Block{ + &Assign{Holder: &Ident{Name: "foo"}, Expression: &BinaryExpr{X: &Ident{Name: "foo"}, Op: "*", Y: &LiteralExpr{Literal: "-1"}}}, + }, + }, + }, + expect: `if 0 > foo { + foo = foo * -1 +}`, + }, + { + description: "foreach", + options: Options{Lang: LangGO}, + block: Block{ + &Foreach{ + Value: &Ident{Name: "foo"}, + Set: &Ident{Name: "foos"}, + Body: Block{ + &CallExpr{ + Receiver: &Ident{Name: "fmt"}, + Name: "Printf", + Args: []Expression{&Ident{Name: "foo"}}, + }, + }, + }, + }, + expect: `for _, foo := range foos { + fmt.Printf(foo) +}`, + }, + } + + //for _, testCase := range testCases[len(testCases)-1:] { + for _, testCase := range testCases { + builder := NewBuilder(testCase.options) + err := testCase.block.Generate(builder) + if !assert.Nil(t, err, testCase.description) { + continue + } + actual := builder.String() + assert.EqualValues(t, testCase.expect, strings.TrimSpace(actual)) + } +} diff --git a/repository/shape/velty/ast/binary.go b/repository/shape/velty/ast/binary.go new file mode 100644 index 000000000..cb4231e96 --- /dev/null +++ b/repository/shape/velty/ast/binary.go @@ -0,0 +1,22 @@ +package ast + +func NewBinary(x Expression, op string, y Expression) *BinaryExpr { + return &BinaryExpr{X: x, Op: op, Y: y} +} + +func (s *BinaryExpr) Generate(builder *Builder) (err error) { + if err := s.X.Generate(builder); err != nil { + return err + } + if err = builder.WriteString(" "); err != nil { + return err + } + + if err = builder.WriteString(s.Op); err != nil { + return err + } + if err = builder.WriteString(" "); err != nil { + return err + } + return s.Y.Generate(builder) +} diff --git a/repository/shape/velty/ast/builder.go b/repository/shape/velty/ast/builder.go new file mode 100644 index 000000000..53fd18d25 --- /dev/null +++ b/repository/shape/velty/ast/builder.go @@ -0,0 +1,52 @@ +package ast + +import "strings" + +const ( + LangVelty = "velty" + LangGO = "go" +) + +type ( + Builder struct { + *strings.Builder + Options + Indent string + State *Scope + declarations map[string]string + IndexByCode *Builder + } +) + +func (b *Builder) NewBuilder() *Builder { + r := *b + r.Builder = &strings.Builder{} + return &r +} + +func (b *Builder) WriteIndentedString(s string) error { + fragment := strings.ReplaceAll(s, "\n", "\n"+b.Indent) + _, err := b.Builder.WriteString(fragment) + return err +} + +func (b *Builder) IncIndent(indent string) *Builder { + newBuilder := *b + newBuilder.Indent += indent + newBuilder.State = newBuilder.State.NextScope() + return &newBuilder +} + +func (b *Builder) WriteString(s string) error { + _, err := b.Builder.WriteString(s) + return err +} + +func NewBuilder(option Options, declaredVariables ...string) *Builder { + return &Builder{ + Builder: &strings.Builder{}, + Options: option, + declarations: map[string]string{}, + State: NewScope(declaredVariables...), + } +} diff --git a/repository/shape/velty/ast/condition.go b/repository/shape/velty/ast/condition.go new file mode 100644 index 000000000..60b88b3a5 --- /dev/null +++ b/repository/shape/velty/ast/condition.go @@ -0,0 +1,142 @@ +package ast + +import ( + "fmt" +) + +type ( + Condition struct { + If Expression + IFBlock Block + ElseIfBlocks []*ConditionalBlock + ElseBlock Block + } + + ConditionalBlock struct { + If Expression + Block Block + } +) + +func (s *Condition) Generate(builder *Builder) (err error) { + if builder.OnIfNotifier != nil { + if expr, err := builder.OnIfNotifier(s); err != nil { + return err + } else if expr != nil && expr != s { + return expr.Generate(builder) + } + } + + switch builder.Lang { + case LangVelty: + if err = builder.WriteIndentedString("\n#if("); err != nil { + return err + } + if err = s.If.Generate(builder); err != nil { + return err + } + if err = builder.WriteString(")"); err != nil { + return err + } + bodyBuilder := builder.IncIndent(" ") + if err = s.IFBlock.Generate(bodyBuilder); err != nil { + return err + } + for _, item := range s.ElseIfBlocks { + if err = builder.WriteIndentedString("\n#elseif("); err != nil { + return err + } + if err = item.If.Generate(builder); err != nil { + return err + } + if err = builder.WriteString(")"); err != nil { + return err + } + if err = item.Block.Generate(bodyBuilder); err != nil { + return err + } + } + if s.ElseBlock != nil { + if err = builder.WriteIndentedString("\n#else"); err != nil { + return err + } + if err = s.ElseBlock.Generate(bodyBuilder); err != nil { + return err + } + } + if err = builder.WriteIndentedString("\n#end"); err != nil { + return err + } + return nil + + case LangGO: + if err = builder.WriteIndentedString("\nif "); err != nil { + return err + } + + if err = s.If.Generate(builder); err != nil { + return err + } + + if err = builder.WriteString(" {"); err != nil { + return err + } + + bodyBlockBuilder := builder.IncIndent(" ") + if err = s.IFBlock.Generate(bodyBlockBuilder); err != nil { + return err + } + + if err = builder.WriteIndentedString("\n}"); err != nil { + return err + } + + for _, block := range s.ElseIfBlocks { + if err = builder.WriteString(" else if "); err != nil { + return err + } + + if err = block.If.Generate(builder); err != nil { + return err + } + + if err = builder.WriteString(" { "); err != nil { + return err + } + + if err = block.Block.Generate(bodyBlockBuilder); err != nil { + return err + } + + if err = builder.WriteIndentedString("\n} "); err != nil { + return err + } + } + + if len(s.ElseBlock) > 0 { + if err = builder.WriteString(" else "); err != nil { + return err + } + + if err = builder.WriteString(" { "); err != nil { + return err + } + + if err = s.ElseBlock.Generate(bodyBlockBuilder); err != nil { + return err + } + + if err = builder.WriteIndentedString("\n} "); err != nil { + return err + } + } + + return nil + } + + return fmt.Errorf("unsupported option %T %v\n", s, builder.Lang) +} + +func NewCondition(ifExpr Expression, ifBlock, elseBlock Block) *Condition { + return &Condition{If: ifExpr, IFBlock: ifBlock, ElseBlock: elseBlock} +} diff --git a/repository/shape/velty/ast/dml.go b/repository/shape/velty/ast/dml.go new file mode 100644 index 000000000..e6c333d15 --- /dev/null +++ b/repository/shape/velty/ast/dml.go @@ -0,0 +1,97 @@ +package ast + +import ( + "fmt" + "strings" +) + +type Insert struct { + Table string + Columns []string + Fields []string +} + +func (s *Insert) Generate(builder *Builder) (err error) { + switch builder.Lang { + case LangVelty: + builder.WriteString("INSERT INTO ") + builder.WriteString(s.Table) + builder.WriteString("(") + builder.WriteString(strings.Join(s.Columns, ",")) + builder.WriteString(") Fields(") + builder.WriteString(strings.Join(s.Fields, ",")) + builder.WriteString(");") + case LangGO: + return fmt.Errorf("DML not yet supported for golang") + } + return nil +} + +type Update struct { + Table string + Columns []string + Fields []string + PkColumns []string + PkFields []string +} + +func (s *Update) Generate(builder *Builder) (err error) { + switch builder.Lang { + case LangVelty: + if err = builder.WriteString("UPDATE "); err != nil { + return err + } + if err = builder.WriteString(s.Table); err != nil { + return err + } + if err = builder.WriteString("SET "); err != nil { + return err + } + for i, column := range s.PkColumns { + if i > 0 { + if err = builder.WriteString(","); err != nil { + return err + } + } + if err = builder.WriteString(column); err != nil { + return err + } + if err = builder.WriteString(" = "); err != nil { + return err + } + if err = builder.WriteString(s.PkFields[i]); err != nil { + return err + } + } + for i, column := range s.Columns { + if err = builder.WriteString("\t#if("); err == nil { + if err = builder.WriteString(getHasField(s.Fields[i])); err == nil { + if err = builder.WriteString(")"); err == nil { + if err = builder.WriteString(","); err == nil { + if err = builder.WriteString(column); err == nil { + if err = builder.WriteString(" = "); err == nil { + if err = builder.WriteString(s.Fields[i]); err == nil { + err = builder.WriteString("\t#end") + } + } + } + } + } + } + } + } + return err + case LangGO: + return fmt.Errorf("DML not yet supported for golang") + } + return nil +} + +func getHasField(field string) string { + if index := strings.LastIndex(field, "."); index != -1 { + leaf := field[index+1:] + field = field[:index] + return field + "." + "Has." + leaf + } + return field +} diff --git a/repository/shape/velty/ast/errcheck.go b/repository/shape/velty/ast/errcheck.go new file mode 100644 index 000000000..d225faa2d --- /dev/null +++ b/repository/shape/velty/ast/errcheck.go @@ -0,0 +1,29 @@ +package ast + +type ErrorCheck struct { + X Expression +} + +func (e *ErrorCheck) Generate(builder *Builder) error { + switch builder.Options.Lang { + case LangGO: + if err := builder.WriteString("if err ="); err != nil { + return err + } + if err := e.X.Generate(builder); err != nil { + return err + } + return builder.WriteString(";err != nil {\nreturn err\n}") + case LangVelty: + if err := builder.WriteString("\n"); err != nil { + return err + } + return e.X.Generate(builder) + } + + return unsupportedOptionUse(builder, e) +} + +func NewErrorCheck(expr Expression) *ErrorCheck { + return &ErrorCheck{X: expr} +} diff --git a/repository/shape/velty/ast/expression.go b/repository/shape/velty/ast/expression.go new file mode 100644 index 000000000..6c1f8889c --- /dev/null +++ b/repository/shape/velty/ast/expression.go @@ -0,0 +1,95 @@ +package ast + +import "fmt" + +func NewCallExpr(holder Expression, name string, args ...Expression) *CallExpr { + return &CallExpr{ + Receiver: holder, + Name: name, + Args: args, + } +} + +func (s *StatementExpression) Generate(builder *Builder) (err error) { + if err = builder.WriteIndentedString("\n"); err != nil { + return err + } + return s.Expression.Generate(builder) +} + +// NewStatementExpression return new statement expr +func NewStatementExpression(expr Expression) *StatementExpression { + return &StatementExpression{Expression: expr} +} +func (e *CallExpr) Generate(builder *Builder) (err error) { + expr, err := e.actualExpr(builder) + if err != nil { + return err + } + if expr != e { + return expr.Generate(builder) + } + + if e.Receiver != nil { + + if err = e.Receiver.Generate(builder); err != nil { + return err + } + + if err = builder.WriteString("."); err != nil { + return err + } + } + if err = builder.WriteString(e.Name); err != nil { + return err + } + + if err = builder.WriteString("("); err != nil { + return err + } + for i, arg := range e.Args { + if i > 0 { + if err = builder.WriteString(", "); err != nil { + return err + } + } + if err = arg.Generate(builder); err != nil { + return err + } + } + if err = builder.WriteString(")"); err != nil { + return err + } + + return nil +} + +func (e *CallExpr) actualExpr(builder *Builder) (Expression, error) { + if builder.CallNotifier == nil { + return e, nil + } + + notifier, err := builder.CallNotifier(e) + if err != nil || notifier != nil { + return notifier, err + } + + return e, nil +} + +func (s *SelectorExpr) Generate(builder *Builder) error { + return unsupportedOptionUse(builder, s) +} + +func unsupportedOptionUse(builder *Builder, s Expression) error { + return fmt.Errorf("unsupported option %T %v\n", s, builder.Lang) +} + +func NewIdent(name string) *Ident { + return &Ident{Name: name} +} + +func NewHolderIndent(holder, name string) *Ident { + ret := &Ident{Name: name, Holder: holder} + return ret +} diff --git a/repository/shape/velty/ast/foreach.go b/repository/shape/velty/ast/foreach.go new file mode 100644 index 000000000..2e096cde7 --- /dev/null +++ b/repository/shape/velty/ast/foreach.go @@ -0,0 +1,86 @@ +package ast + +import "fmt" + +func (s *Foreach) Generate(builder *Builder) (err error) { + if builder.SliceItemNotifier != nil { + if err = builder.SliceItemNotifier(s.Value, s.Set); err != nil { + return err + } + } + + switch builder.Lang { + case LangVelty: + if err = builder.WriteIndentedString("\n#foreach("); err != nil { + return err + } + if err = s.Value.Generate(builder); err != nil { + return err + } + if err = builder.WriteString(" in "); err != nil { + return err + } + if err = s.Set.Generate(builder); err != nil { + return err + } + if err = builder.WriteString(")"); err != nil { + return err + } + + bodyBuilder := builder.IncIndent(" ") + if err = s.Body.Generate(bodyBuilder); err != nil { + return err + } + if err = builder.WriteIndentedString("\n#end"); err != nil { + return err + } + return nil + + case LangGO: + + if err = builder.WriteIndentedString("\nfor _, "); err != nil { + return err + } + + if err = s.Value.Generate(builder); err != nil { + return err + } + + if err = builder.WriteString(" := range "); err != nil { + return err + } + + if err = s.Set.Generate(builder); err != nil { + return err + } + + if err = builder.WriteString(" { "); err != nil { + return err + } + + bodyBuilder := builder.IncIndent(" ") + if err = bodyBuilder.WriteIndentedString("\n"); err != nil { + return err + } + + if err = s.Body.Generate(bodyBuilder); err != nil { + return err + } + + if err = builder.WriteString("\n}"); err != nil { + return err + } + + return nil + } + + return fmt.Errorf("unsupported option %T %v\n", s, builder.Lang) +} + +func NewForEach(value, set *Ident, body Block) *Foreach { + return &Foreach{ + Value: value, + Set: set, + Body: body, + } +} diff --git a/repository/shape/velty/ast/func.go b/repository/shape/velty/ast/func.go new file mode 100644 index 000000000..28307ee58 --- /dev/null +++ b/repository/shape/velty/ast/func.go @@ -0,0 +1,162 @@ +package ast + +type ( + Function struct { + Receiver *Receiver + Name string + ArgsIn []*FuncArg + ArgsOut []string + Body Block + Return *ReturnExpr + } + + Receiver struct { + Name string + Ident *Ident + } + + FuncArg struct { + Name string + Ident *Ident + } +) + +func (a *FuncArg) Generate(builder *Builder) error { + switch builder.Lang { + case LangGO: + if err := builder.WriteString(a.Name); err != nil { + return err + } + + if err := builder.WriteString(" "); err != nil { + return err + } + + return a.Ident.Generate(builder) + } + + return unsupportedOptionUse(builder, a) +} + +func (r *Receiver) Generate(builder *Builder) error { + switch builder.Lang { + case LangGO: + if err := builder.WriteString(r.Name); err != nil { + return err + } + + if err := builder.WriteString(" "); err != nil { + return err + } + + return r.Ident.Generate(builder) + } + + return unsupportedOptionUse(builder, r) +} + +func (f *Function) Generate(builder *Builder) error { + switch builder.Lang { + case LangGO: + if err := builder.WriteIndentedString("\nfunc "); err != nil { + return err + } + + if f.Receiver != nil { + if err := builder.WriteString("( "); err != nil { + return err + } + + if err := f.Receiver.Generate(builder); err != nil { + return err + } + + if err := builder.WriteString(" ) "); err != nil { + return err + } + } + + if err := builder.WriteString(f.Name); err != nil { + return err + } + + if err := builder.WriteString("("); err != nil { + return err + } + + for i, arg := range f.ArgsIn { + if i != 0 { + if err := builder.WriteString(", "); err != nil { + return err + } + } + + if err := arg.Generate(builder); err != nil { + return err + } + } + + if err := builder.WriteString(") "); err != nil { + return err + } + + switch len(f.ArgsOut) { + case 0: + //Exec nothing + case 1: + if err := builder.WriteString(f.ArgsOut[0]); err != nil { + return err + } + + default: + for i, argType := range f.ArgsOut { + if err := builder.WriteString("("); err != nil { + return err + } + + if i != 0 { + if err := builder.WriteString(", "); err != nil { + return err + } + + } + + if err := builder.WriteString(argType); err != nil { + return err + } + + if err := builder.WriteString(")"); err != nil { + return err + } + } + } + + if err := builder.WriteString(" {"); err != nil { + return err + } + + blockBuilder := builder.IncIndent(" ") + if err := blockBuilder.WriteIndentedString("\n"); err != nil { + return err + } + + if err := f.Body.Generate(blockBuilder); err != nil { + return err + } + + if f.Return != nil { + if err := f.Return.Generate(builder); err != nil { + return err + } + } + + if err := builder.WriteIndentedString("\n}"); err != nil { + return err + } + + return nil + + default: + return unsupportedOptionUse(builder, f) + } +} diff --git a/repository/shape/velty/ast/literal.go b/repository/shape/velty/ast/literal.go new file mode 100644 index 000000000..04b2a7fe5 --- /dev/null +++ b/repository/shape/velty/ast/literal.go @@ -0,0 +1,21 @@ +package ast + +import ( + "strconv" + "strings" +) + +func (s *LiteralExpr) Generate(builder *Builder) error { + return builder.WriteString(s.Literal) +} + +func NewQuotedLiteral(text string) *LiteralExpr { + if !strings.HasPrefix(text, "\"") { + text = strconv.Quote(text) + } + return &LiteralExpr{text} +} + +func NewLiteral(text string) *LiteralExpr { + return &LiteralExpr{text} +} diff --git a/repository/shape/velty/ast/options.go b/repository/shape/velty/ast/options.go new file mode 100644 index 000000000..e03146fd6 --- /dev/null +++ b/repository/shape/velty/ast/options.go @@ -0,0 +1,12 @@ +package ast + +type Options struct { + Lang string + StateName string + CallNotifier func(callExpr *CallExpr) (Expression, error) + AssignNotifier func(assign *Assign) (Expression, error) + SliceItemNotifier func(value, set *Ident) error + WithoutBusinessLogic bool + OnIfNotifier func(value *Condition) (Expression, error) + WithLowerCaseIdent bool +} diff --git a/repository/shape/velty/ast/scope.go b/repository/shape/velty/ast/scope.go new file mode 100644 index 000000000..82cb06fdd --- /dev/null +++ b/repository/shape/velty/ast/scope.go @@ -0,0 +1,63 @@ +package ast + +import "strings" + +type ( + Scope struct { + Variables map[string]*Variable + Parent *Scope + } + + Variable struct { + Name string + } +) + +func NewScope(declaredVariables ...string) *Scope { + variables := map[string]*Variable{} + for _, variable := range declaredVariables { + variables[variable] = &Variable{Name: variable} + } + return &Scope{ + Variables: variables, + } +} + +func (s *Scope) NextScope() *Scope { + scope := NewScope() + scope.Parent = s + return scope +} + +func (s *Scope) DeclareVariable(variable string) { + split := strings.Split(variable, ".") + if len(split) > 0 { + variable = split[0] + } + + if s.Variables[variable] != nil { + return + } + + s.Variables[variable] = &Variable{ + Name: variable, + } +} + +func (s *Scope) IsDeclared(variable string) bool { + dotIndex := strings.Index(variable, ".") + if dotIndex >= 0 { + return true + } + + tmp := s + for tmp != nil { + if _, ok := tmp.Variables[variable]; ok { + return true + } + + tmp = tmp.Parent + } + + return false +} diff --git a/repository/shape/velty/ast/star.go b/repository/shape/velty/ast/star.go new file mode 100644 index 000000000..769b03374 --- /dev/null +++ b/repository/shape/velty/ast/star.go @@ -0,0 +1,51 @@ +package ast + +type ( + DerefExpression struct { + X Expression + } + + RefExpression struct { + X Expression + } +) + +func NewRefExpression(x Expression) *RefExpression { + return &RefExpression{ + X: x, + } +} + +func NewDerefExpression(x Expression) *DerefExpression { + return &DerefExpression{X: x} +} + +func (s *DerefExpression) Generate(builder *Builder) error { + switch builder.Options.Lang { + case LangGO: + if err := builder.WriteString("*"); err != nil { + return err + } + + fallthrough + case LangVelty: + return s.X.Generate(builder) + } + + return unsupportedOptionUse(builder, s) +} + +func (s *RefExpression) Generate(builder *Builder) error { + switch builder.Options.Lang { + case LangGO: + if err := builder.WriteString("&"); err != nil { + return err + } + + fallthrough + case LangVelty: + return s.X.Generate(builder) + } + + return unsupportedOptionUse(builder, s) +} diff --git a/repository/shape/xgen/codegen.go b/repository/shape/xgen/codegen.go index c6e4b522a..c0dd4fc77 100644 --- a/repository/shape/xgen/codegen.go +++ b/repository/shape/xgen/codegen.go @@ -1,18 +1,32 @@ package xgen import ( + "bytes" "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" "os" + "path" "path/filepath" + "sort" "strings" + "time" + "unicode" + "unicode/utf8" "github.com/viant/datly/repository/shape/dql/shape" shapeload "github.com/viant/datly/repository/shape/load" "github.com/viant/datly/repository/shape/typectx" + utypes "github.com/viant/datly/utils/types" "github.com/viant/datly/view" + "github.com/viant/datly/view/extension" "github.com/viant/datly/view/state" + viewtags "github.com/viant/datly/view/tags" "github.com/viant/tagly/format/text" "github.com/viant/xreflect" + "github.com/viant/xunsafe" "reflect" ) @@ -37,11 +51,18 @@ type ComponentCodegen struct { // ComponentCodegenResult captures generation outputs. type ComponentCodegenResult struct { - FilePath string - PackagePath string - PackageName string - Types []string - Embeds map[string]string // SQL file name → SQL content + FilePath string + PackageDir string + PackagePath string + PackageName string + Types []string + GeneratedFiles []string + InputFilePath string + OutputFilePath string + ViewFilePath string + RouterFilePath string + VeltyFilePath string + Embeds map[string]string // SQL file name → SQL content } // Generate produces the component Go source file. @@ -83,40 +104,45 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { } componentName := g.componentName() + inputTypeName := g.inputTypeName(componentName) + outputTypeName := g.outputTypeName(componentName) + rootViewTypeName := g.rootViewTypeName(componentName) embedURI := text.CaseFormatUpperCamel.Format(componentName, text.CaseFormatLowerUnderscore) + explicitOutputParams := cloneCodegenParameters(g.Component.OutputParameters()) + hasExplicitOutput := len(explicitOutputParams) > 0 - fileName := g.FileName - if fileName == "" { - fileName = embedURI + ".go" + defaultFileName := g.FileName + if defaultFileName == "" { + defaultFileName = embedURI + ".go" } + outputFileName := g.resolveOutputDestFileName(defaultFileName) + inputFileName := g.resolveInputDestFileName(outputFileName) + viewFileName := g.resolveViewDestFileName(outputFileName) + routerFileName := g.resolveRouterDestFileName("") - // First generate view shapes via xgen (for entity structs like VendorView, ProductsView) - shapeDoc := resourceToCodegenDoc(g.Resource, g.TypeContext) - shapeCfg := &Config{ - ProjectDir: projectDir, - PackageDir: packageDir, - PackageName: packageName, - PackagePath: packagePath, - FileName: "shapes_gen.go", + shapeFragment, err := g.generateShapeFragment(projectDir, packageDir, packageName, packagePath) + if err != nil { + return nil, err } - shapeResult, _ := GenerateFromDQLShape(shapeDoc, shapeCfg) // Build Input/Output types using state.Parameters.ReflectType - lookupType := func(name string, opts ...xreflect.Option) (reflect.Type, error) { - return nil, fmt.Errorf("type %s not found", name) - } + lookupType := g.componentLookupType(packagePath) var inputType, outputType reflect.Type - if params := g.Component.InputParameters(); len(params) > 0 { - rt, err := params.ReflectType(packagePath, lookupType, state.WithSetMarker(), state.WithTypeName(componentName+"Input")) + if params := g.Component.InputParameters(); len(params) > 0 || strings.TrimSpace(g.Component.URI) != "" { + normalized := normalizeInputParametersForCodegen(params, g.Resource, g.Component.URI) + inputOpts := []state.ReflectOption{state.WithSetMarker(), state.WithTypeName(inputTypeName)} + if g.componentUsesVelty() { + inputOpts = append(inputOpts, state.WithVelty(true)) + } + rt, err := normalized.ReflectType(packagePath, lookupType, inputOpts...) if err == nil && rt != nil { inputType = rt } } // Build output parameters — use explicit ones or synthesize defaults for readers - outputParams := g.Component.OutputParameters() - hasExplicitOutput := len(outputParams) > 0 + outputParams := cloneCodegenParameters(explicitOutputParams) if !hasExplicitOutput { outputParams = g.defaultOutputParameters(componentName) } @@ -129,89 +155,1933 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { } } - // Build the Go source - var builder strings.Builder - builder.WriteString("package " + packageName + "\n\n") - - // Imports - imports := g.buildImports() - if len(imports) > 0 { - builder.WriteString("import (\n") - for _, imp := range imports { - if strings.Contains(imp, " ") { - // aliased import - builder.WriteString("\t" + imp + "\n") - } else { - builder.WriteString("\t\"" + imp + "\"\n") + shapeTypeNames := map[string]bool{} + if shapeFragment != nil { + for _, typeName := range shapeFragment.Types { + typeName = strings.TrimSpace(typeName) + if typeName != "" { + shapeTypeNames[typeName] = true } } - builder.WriteString(")\n\n") } - // Code generated header - builder.WriteString("// Code generated by datly transcribe. DO NOT EDIT.\n\n") + inputHelpers := collectNamedHelperTypes(inputType, packagePath, shapeTypeNames) + outputHelpers := collectNamedHelperTypes(outputType, packagePath, shapeTypeNames) + mutableSupport := g.mutableSupport(inputType) + emitResponseImport := g.outputUsesResponse(outputParams) || mutableSupport != nil + mutableOutputImports := []string{} + if mutableSupport != nil { + mutableOutputImports = append(mutableOutputImports, "github.com/viant/xdatly/handler/validator") + } + var initBuilder strings.Builder // init() registration - builder.WriteString("func init() {\n") + initBuilder.WriteString("func init() {\n") if g.withRegister() { + registryPackage := strings.TrimSpace(packagePath) + if registryPackage == "" { + registryPackage = packageName + } + registered := map[string]bool{} + if inputType != nil { + initBuilder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%s{}), checksum.GeneratedTime)\n", + registryPackage, inputTypeName, inputTypeName)) + registered[inputTypeName] = true + } + initBuilder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%s{}), checksum.GeneratedTime)\n", + registryPackage, outputTypeName, outputTypeName)) + registered[outputTypeName] = true + if shapeFragment != nil { + for _, typeName := range shapeFragment.Types { + typeName = strings.TrimSpace(typeName) + if typeName == "" || registered[typeName] { + continue + } + initBuilder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%s{}), checksum.GeneratedTime)\n", + registryPackage, typeName, typeName)) + registered[typeName] = true + } + } + for _, helper := range inputHelpers { + if helper.TypeName == "" || registered[helper.TypeName] { + continue + } + initBuilder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%s{}), checksum.GeneratedTime)\n", + registryPackage, helper.TypeName, helper.TypeName)) + registered[helper.TypeName] = true + } + for _, helper := range outputHelpers { + if helper.TypeName == "" || registered[helper.TypeName] { + continue + } + initBuilder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%s{}), checksum.GeneratedTime)\n", + registryPackage, helper.TypeName, helper.TypeName)) + registered[helper.TypeName] = true + } + } + initBuilder.WriteString("}\n\n") + + var inputBuilder strings.Builder + if inputType != nil || g.WithContract { + inputBuilder.WriteString(fmt.Sprintf("type %s struct {\n", inputTypeName)) if inputType != nil { - builder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%sInput{}), checksum.GeneratedTime)\n", - packageName, componentName+"Input", componentName)) + inputBuilder.WriteString(structFieldsSource(inputType)) } - builder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%sOutput{}), checksum.GeneratedTime)\n", - packageName, componentName+"Output", componentName)) + if mutableSupport != nil { + mutableSupport.renderInputFields(&inputBuilder) + } + inputBuilder.WriteString("}\n\n") + } + for _, helper := range inputHelpers { + inputBuilder.WriteString(helper.Decl) + } + if g.WithEmbed && inputType != nil { + inputBuilder.WriteString(fmt.Sprintf("func (i *%s) EmbedFS() *embed.FS {\n", inputTypeName)) + inputBuilder.WriteString(fmt.Sprintf("\treturn &%sFS\n", componentName)) + inputBuilder.WriteString("}\n\n") } - builder.WriteString("}\n\n") - // //go:embed + var outputBuilder strings.Builder + var routerBuilder strings.Builder if g.WithEmbed { - builder.WriteString(fmt.Sprintf("//go:embed %s/*.sql\n", embedURI)) - builder.WriteString(fmt.Sprintf("var %sFS embed.FS\n\n", componentName)) + outputBuilder.WriteString(fmt.Sprintf("//go:embed %s/*.sql\n", embedURI)) + outputBuilder.WriteString(fmt.Sprintf("var %sFS embed.FS\n\n", componentName)) } - - // Input struct - if inputType != nil { - builder.WriteString(fmt.Sprintf("type %sInput struct {\n", componentName)) - builder.WriteString(structFieldsSource(inputType)) - builder.WriteString("}\n\n") + outputRenderParams := cloneCodegenParameters(explicitOutputParams) + if !hasExplicitOutput { + outputRenderParams = g.defaultOutputParameters(componentName) + } + g.resolveOutputWildcardTypes(outputRenderParams, componentName) + g.renderOutputStruct(&outputBuilder, outputTypeName, rootViewTypeName, embedURI, outputRenderParams, outputType, mutableSupport) + for _, helper := range outputHelpers { + outputBuilder.WriteString(helper.Decl) + } + if g.WithContract { + g.renderComponentHolder(&routerBuilder, componentName, inputTypeName, outputTypeName) + g.renderDefineComponent(&outputBuilder, componentName, inputTypeName, outputTypeName) } - // Output struct — always render directly (view types may not be registered yet) - g.renderOutputStruct(&builder, componentName, embedURI, outputParams, outputType) + viewDecls := "" + viewImports := []string{} + if shapeFragment != nil { + viewDecls = strings.TrimSpace(shapeFragment.TypeDecls) + viewImports = shapeFragment.Imports + } - // EmbedFS method - if g.WithEmbed && inputType != nil { - builder.WriteString(fmt.Sprintf("func (i *%sInput) EmbedFS() *embed.FS {\n", componentName)) - builder.WriteString(fmt.Sprintf("\treturn &%sFS\n", componentName)) - builder.WriteString("}\n\n") + outputParts := []string{initBuilder.String(), outputBuilder.String()} + routerParts := []string{} + if strings.TrimSpace(routerBuilder.String()) != "" { + if routerFileName == "" || routerFileName == outputFileName { + outputParts = append(outputParts, routerBuilder.String()) + } else { + routerParts = append(routerParts, routerBuilder.String()) + } + } + inputParts := []string{inputBuilder.String()} + viewParts := []string{} + if viewDecls != "" { + viewParts = append(viewParts, viewDecls+"\n") } - // Write file if err := os.MkdirAll(packageDir, 0o755); err != nil { return nil, err } - dest := filepath.Join(packageDir, fileName) - if err := writeAtomic(dest, []byte(builder.String()), 0o644); err != nil { - return nil, err + outputDest := filepath.Join(packageDir, outputFileName) + inputDest := outputDest + viewDest := outputDest + routerDest := "" + inputInitDest := "" + inputValidateDest := "" + if inputFileName != "" { + inputDest = filepath.Join(packageDir, inputFileName) + } + if viewFileName != "" { + viewDest = filepath.Join(packageDir, viewFileName) + } + if routerFileName != "" { + routerDest = filepath.Join(packageDir, routerFileName) + } + if mutableSupport != nil && inputDest != "" { + base := "input" + if inputFileName != "" && inputFileName != outputFileName { + base = strings.TrimSuffix(filepath.Base(inputDest), filepath.Ext(inputDest)) + } else if g.Component != nil && g.Component.Directives != nil { + if dest := strings.TrimSpace(g.Component.Directives.InputDest); dest != "" { + base = strings.TrimSuffix(filepath.Base(dest), filepath.Ext(dest)) + } + } + if strings.TrimSpace(base) == "" { + base = "input" + } + inputInitDest = filepath.Join(packageDir, base+"_init.go") + inputValidateDest = filepath.Join(packageDir, base+"_validate.go") + } + var generatedFiles []string + appendGenerated := func(dest string) { + dest = strings.TrimSpace(dest) + if dest == "" { + return + } + for _, candidate := range generatedFiles { + if candidate == dest { + return + } + } + generatedFiles = append(generatedFiles, dest) + } + split := outputFileName != inputFileName || outputFileName != viewFileName || (routerFileName != "" && routerFileName != outputFileName) + var writeErr error + if !split { + imports := mergeImportPaths( + g.buildImports(g.WithContract && (routerFileName == "" || routerFileName == outputFileName), emitResponseImport), + viewImports, + collectTypeImports(inputType, packagePath), + collectTypeImports(outputType, packagePath), + helperImports(inputHelpers), + helperImports(outputHelpers), + mutableOutputImports, + ) + combined := append(append(outputParts, inputParts...), viewParts...) + writeErr = g.writeSectionFile(outputDest, packageName, imports, combined...) + if writeErr == nil { + appendGenerated(outputDest) + outputFileName = outputDest + } + } else { + outputImports := mergeImportPaths( + g.buildImports(g.WithContract && (routerFileName == "" || routerFileName == outputFileName), emitResponseImport), + collectTypeImports(outputType, packagePath), + helperImports(outputHelpers), + mutableOutputImports, + ) + if viewFileName == outputFileName { + outputImports = mergeImportPaths(outputImports, viewImports) + } + if writeErr = g.writeSectionFile(outputDest, packageName, outputImports, outputParts...); writeErr != nil { + return nil, writeErr + } + appendGenerated(outputDest) + if inputFileName == outputFileName { + if writeErr = g.appendSectionFile(outputDest, inputParts...); writeErr != nil { + return nil, writeErr + } + } else if strings.TrimSpace(strings.Join(inputParts, "")) != "" { + inputImports := []string{} + if g.WithEmbed && inputType != nil { + inputImports = append(inputImports, "embed") + } + inputImports = mergeImportPaths(inputImports, collectTypeImports(inputType, packagePath), helperImports(inputHelpers)) + if viewFileName == inputFileName { + inputImports = mergeImportPaths(inputImports, viewImports) + } + if writeErr = g.writeSectionFile(inputDest, packageName, inputImports, inputParts...); writeErr != nil { + return nil, writeErr + } + appendGenerated(inputDest) + } + if len(viewParts) > 0 { + if viewFileName == outputFileName { + if writeErr = g.appendSectionFile(outputDest, viewParts...); writeErr != nil { + return nil, writeErr + } + } else if viewFileName == inputFileName { + if _, statErr := os.Stat(inputDest); statErr == nil { + if writeErr = g.appendSectionFile(inputDest, viewParts...); writeErr != nil { + return nil, writeErr + } + } else { + if writeErr = g.writeSectionFile(inputDest, packageName, viewImports, viewParts...); writeErr != nil { + return nil, writeErr + } + appendGenerated(inputDest) + } + } else { + if writeErr = g.writeSectionFile(viewDest, packageName, viewImports, viewParts...); writeErr != nil { + return nil, writeErr + } + appendGenerated(viewDest) + } + } + if len(routerParts) > 0 { + if writeErr = g.writeSectionFile(routerDest, packageName, g.buildRouterImports(), routerParts...); writeErr != nil { + return nil, writeErr + } + appendGenerated(routerDest) + } + outputFileName = outputDest + } + if writeErr != nil { + return nil, writeErr + } + if mutableSupport != nil { + if writeErr = g.writeSectionFile(inputInitDest, packageName, []string{"context", "github.com/viant/xdatly/handler"}, mutableSupport.renderInputInit(inputTypeName, outputTypeName)); writeErr != nil { + return nil, writeErr + } + appendGenerated(inputInitDest) + if writeErr = g.writeSectionFile(inputValidateDest, packageName, []string{"context", "github.com/viant/xdatly/handler", "github.com/viant/xdatly/handler/validator"}, mutableSupport.renderInputValidate(inputTypeName, outputTypeName)); writeErr != nil { + return nil, writeErr + } + appendGenerated(inputValidateDest) + } + veltyDest := "" + if mutableSupport != nil && !g.componentUsesHandler() { + var veltyBody string + var ok bool + veltyBody, ok, writeErr = g.renderMutableDSQL(inputType) + if writeErr != nil { + return nil, writeErr + } + if ok { + veltyDest = filepath.Join(packageDir, text.CaseFormatUpperCamel.Format(componentName, text.CaseFormatLowerUnderscore), "patch.sql") + if writeErr = os.MkdirAll(filepath.Dir(veltyDest), 0o755); writeErr != nil { + return nil, writeErr + } + if writeErr = os.WriteFile(veltyDest, []byte(veltyBody), 0o644); writeErr != nil { + return nil, writeErr + } + appendGenerated(veltyDest) + for _, helperFile := range g.mutableHelperSQLFiles(mutableSupport) { + if strings.TrimSpace(helperFile.Path) == "" || strings.TrimSpace(helperFile.Content) == "" { + continue + } + if writeErr = os.MkdirAll(filepath.Dir(helperFile.Path), 0o755); writeErr != nil { + return nil, writeErr + } + if writeErr = os.WriteFile(helperFile.Path, []byte(helperFile.Content), 0o644); writeErr != nil { + return nil, writeErr + } + appendGenerated(helperFile.Path) + } + } } var typeNames []string if inputType != nil { - typeNames = append(typeNames, componentName+"Input") + typeNames = append(typeNames, inputTypeName) } if outputType != nil { - typeNames = append(typeNames, componentName+"Output") + typeNames = append(typeNames, outputTypeName) + } + if shapeFragment != nil { + typeNames = append(typeNames, shapeFragment.Types...) + } + + return &ComponentCodegenResult{ + FilePath: outputFileName, + PackageDir: packageDir, + PackagePath: packagePath, + PackageName: packageName, + Types: typeNames, + GeneratedFiles: generatedFiles, + InputFilePath: inputDest, + OutputFilePath: outputDest, + ViewFilePath: viewDest, + RouterFilePath: routerDest, + VeltyFilePath: veltyDest, + }, nil +} + +func normalizeInputParametersForCodegen(params state.Parameters, resource *view.Resource, uri string) state.Parameters { + result := make(state.Parameters, 0, len(params)+4) + seenPath := map[string]bool{} + var stateResource state.Resource + if resource != nil { + stateResource = view.NewResources(resource, &view.View{}) + } + for _, item := range params { + if item == nil { + continue + } + cloned := *item + schema := normalizeInputSchemaForCodegen(item.Name, item.In, item.Required != nil && *item.Required, item.Schema, resource) + cloned.Schema = schema + if cloned.Schema != nil && stateResource != nil { + _ = cloned.Schema.Init(stateResource) + } + if cloned.Output != nil { + output := *cloned.Output + if cloned.Output.Schema != nil { + output.Schema = cloned.Output.Schema.Clone() + } + cloned.Output = &output + if stateResource != nil && cloned.Schema != nil && cloned.Schema.Type() != nil { + _ = cloned.Output.Init(stateResource, cloned.Schema.Type()) + } + } + if in := item.In; in != nil && in.Kind == state.KindView { + viewName := strings.TrimSpace(item.Name) + if name := strings.TrimSpace(in.Name); name != "" { + viewName = name + } + if v := lookupInputView(resource, viewName); v != nil { + cloned.Tag = mergeViewSQLTag(cloned.Tag, v) + } + cloned.Tag = removeTagKeys(cloned.Tag, "typeName") + } + if in := cloned.In; in != nil && in.Kind == state.KindPath { + key := strings.ToLower(strings.TrimSpace(in.Name)) + if key == "" { + key = strings.ToLower(strings.TrimSpace(cloned.Name)) + } + if key != "" { + seenPath[key] = true + } + } + result = append(result, &cloned) + } + for _, name := range extractCodegenRoutePathParams(uri) { + key := strings.ToLower(strings.TrimSpace(name)) + if key == "" || seenPath[key] { + continue + } + fieldName := name + result = append(result, &state.Parameter{ + Name: fieldName, + In: state.NewPathLocation(name), + Schema: &state.Schema{ + DataType: "string", + Cardinality: state.One, + }, + }) + seenPath[key] = true + } + return result +} + +func exportedCodegenParamName(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "" + } + return strings.ToUpper(name[:1]) + name[1:] +} + +func extractCodegenRoutePathParams(uri string) []string { + uri = strings.TrimSpace(uri) + if uri == "" { + return nil + } + var result []string + seen := map[string]bool{} + for { + start := strings.IndexByte(uri, '{') + if start == -1 { + break + } + uri = uri[start+1:] + end := strings.IndexByte(uri, '}') + if end == -1 { + break + } + name := strings.TrimSpace(uri[:end]) + uri = uri[end+1:] + if name == "" { + continue + } + key := strings.ToLower(name) + if seen[key] { + continue + } + seen[key] = true + result = append(result, name) + } + return result +} + +func cloneCodegenParameters(params state.Parameters) state.Parameters { + if len(params) == 0 { + return nil + } + result := make(state.Parameters, 0, len(params)) + for _, item := range params { + if item == nil { + continue + } + cloned := *item + if item.Schema != nil { + cloned.Schema = item.Schema.Clone() + } + if item.Output != nil { + output := *item.Output + if item.Output.Schema != nil { + output.Schema = item.Output.Schema.Clone() + } + cloned.Output = &output + } + result = append(result, &cloned) + } + return result +} + +func normalizeInputSchemaForCodegen(paramName string, in *state.Location, required bool, schema *state.Schema, resource *view.Resource) *state.Schema { + var cloned state.Schema + if schema != nil { + cloned = exportedSchemaCopy(schema) + } + kind := state.Kind("") + if in != nil { + kind = in.Kind + } + if kind == state.KindView { + if viewSchema := lookupViewSchemaForInput(resource, in, paramName); viewSchema != nil { + base := exportedSchemaCopy(viewSchema) + if explicit := strings.TrimSpace(cloned.Name); explicit != "" && strings.TrimSpace(base.Name) == "" { + base.Name = explicit + } + if explicit := strings.TrimSpace(cloned.DataType); explicit != "" && !isDynamicTypeName(explicit) && strings.TrimSpace(base.DataType) == "" { + base.DataType = explicit + } + if explicit := strings.TrimSpace(cloned.Package); explicit != "" && strings.TrimSpace(base.Package) == "" { + base.Package = explicit + } + if explicit := strings.TrimSpace(cloned.PackagePath); explicit != "" && strings.TrimSpace(base.PackagePath) == "" { + base.PackagePath = explicit + } + if explicit := strings.TrimSpace(cloned.ModulePath); explicit != "" && strings.TrimSpace(base.ModulePath) == "" { + base.ModulePath = explicit + } + if explicit := cloned.Cardinality; explicit != "" { + base.Cardinality = explicit + } + cloned = base + } + } + if cloned.Cardinality == "" { + if kind == state.KindView { + if required { + cloned.Cardinality = state.One + } else { + cloned.Cardinality = state.Many + } + } else { + cloned.Cardinality = state.One + } + } + if kind != state.KindView && strings.TrimSpace(cloned.DataType) == "" { + cloned.DataType = "string" + } + return &cloned +} + +func exportedSchemaCopy(schema *state.Schema) state.Schema { + if schema == nil { + return state.Schema{} + } + return state.Schema{ + Package: schema.Package, + PackagePath: schema.PackagePath, + ModulePath: schema.ModulePath, + Name: schema.Name, + DataType: schema.DataType, + Cardinality: schema.Cardinality, + Methods: append([]reflect.Method(nil), schema.Methods...), + } +} + +func lookupViewSchemaForInput(resource *view.Resource, in *state.Location, paramName string) *state.Schema { + if v := lookupInputView(resource, strings.TrimSpace(paramName)); v != nil && v.Schema != nil { + return v.Schema + } + if in != nil { + if v := lookupInputView(resource, strings.TrimSpace(in.Name)); v != nil && v.Schema != nil { + return v.Schema + } + } + return nil +} + +func lookupInputView(resource *view.Resource, name string) *view.View { + if resource == nil { + return nil + } + name = normalizeViewLookupName(name) + if name == "" { + return nil + } + for _, item := range resource.Views { + if item == nil { + continue + } + candidates := []string{ + item.Name, + item.Reference.Ref, + } + if item.Schema != nil { + candidates = append(candidates, item.Schema.Name) + } + for _, candidate := range candidates { + if normalizeViewLookupName(candidate) == name { + return item + } + } + } + return nil +} + +func normalizeViewLookupName(value string) string { + value = strings.TrimSpace(strings.ToLower(value)) + if value == "" { + return "" + } + var ret strings.Builder + ret.Grow(len(value)) + for _, r := range value { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + ret.WriteRune(r) + } + } + return ret.String() +} + +func mergeViewSQLTag(existing string, aView *view.View) string { + if aView == nil || aView.Template == nil { + return existing + } + viewName := strings.TrimSpace(aView.Name) + sourceURL := strings.TrimSpace(aView.Template.SourceURL) + if viewName == "" && sourceURL == "" { + return existing + } + updated := strings.TrimSpace(existing) + if viewName != "" && !strings.Contains(updated, `view:"`) { + if updated != "" { + updated += " " + } + updated += fmt.Sprintf(`view:"%s"`, viewName) + } + if sourceURL != "" && !strings.Contains(updated, `sql:"`) { + if updated != "" { + updated += " " + } + updated += fmt.Sprintf(`sql:"uri=%s"`, sourceURL) + } + return updated +} + +func removeTagKeys(tag string, keys ...string) string { + tag = strings.TrimSpace(tag) + if tag == "" { + return "" + } + for _, key := range keys { + var updated string + updated, _ = xreflect.RemoveTag(tag, key) + tag = strings.TrimSpace(updated) + } + return tag +} + +func isDynamicTypeName(name string) bool { + n := strings.TrimSpace(strings.ToLower(name)) + n = strings.ReplaceAll(n, " ", "") + switch n { + case "", "interface{}", "any", "*interface{}", "[]interface{}", "[]any": + return true + } + return false +} + +func (g *ComponentCodegen) componentLookupType(packagePath string) xreflect.LookupType { + localTypes := map[string]reflect.Type{} + if g != nil && g.Resource != nil { + for _, aView := range g.Resource.Views { + if aView == nil { + continue + } + typeName := "" + if aView.Schema != nil { + typeName = strings.TrimSpace(aView.Schema.Name) + } + if typeName == "" { + typeName = toUpperCamel(strings.TrimSpace(aView.Name)) + "View" + } + if typeName == "" { + continue + } + var rType reflect.Type + if aView.Schema != nil && aView.Schema.Type() != nil { + rType = aView.Schema.Type() + } + if rType == nil && len(aView.Columns) > 0 { + rType = buildStructType(columnsFromView(aView), g.viewUsesVelty(aView)) + } + if rType == nil { + continue + } + key := strings.ToLower(typeName) + localTypes[key] = rType + } + } + return func(name string, opts ...xreflect.Option) (reflect.Type, error) { + base := normalizeLookupTypeName(name) + if base != "" { + if rType, ok := localTypes[strings.ToLower(base)]; ok { + return rType, nil + } + if packagePath != "" { + if linked := xunsafe.LookupType(packagePath + "/" + base); linked != nil { + return linked, nil + } + } + } + if builtin, ok := builtinTypeByName(name); ok { + return builtin, nil + } + if builtin, ok := builtinTypeByName(base); ok { + return builtin, nil + } + return nil, fmt.Errorf("type %s not found", name) + } +} + +func builtinTypeByName(name string) (reflect.Type, bool) { + name = strings.TrimSpace(name) + if name == "" { + return nil, false + } + if strings.HasPrefix(name, "[]") { + if elem, ok := builtinTypeByName(strings.TrimPrefix(name, "[]")); ok { + return reflect.SliceOf(elem), true + } + } + if strings.HasPrefix(name, "*") { + if elem, ok := builtinTypeByName(strings.TrimPrefix(name, "*")); ok { + return reflect.PtrTo(elem), true + } + } + switch name { + case "string": + return reflect.TypeOf(""), true + case "bool": + return reflect.TypeOf(true), true + case "int": + return reflect.TypeOf(int(0)), true + case "int8": + return reflect.TypeOf(int8(0)), true + case "int16": + return reflect.TypeOf(int16(0)), true + case "int32": + return reflect.TypeOf(int32(0)), true + case "int64": + return reflect.TypeOf(int64(0)), true + case "uint": + return reflect.TypeOf(uint(0)), true + case "uint8": + return reflect.TypeOf(uint8(0)), true + case "uint16": + return reflect.TypeOf(uint16(0)), true + case "uint32": + return reflect.TypeOf(uint32(0)), true + case "uint64": + return reflect.TypeOf(uint64(0)), true + case "float32": + return reflect.TypeOf(float32(0)), true + case "float64": + return reflect.TypeOf(float64(0)), true + case "time.Time": + return reflect.TypeOf(time.Time{}), true + } + return nil, false +} + +func normalizeLookupTypeName(name string) string { + name = strings.TrimSpace(name) + for strings.HasPrefix(name, "*") || strings.HasPrefix(name, "[]") { + if strings.HasPrefix(name, "*") { + name = strings.TrimPrefix(name, "*") + continue + } + name = strings.TrimPrefix(name, "[]") + } + if idx := strings.LastIndex(name, "."); idx != -1 { + name = name[idx+1:] + } + return strings.TrimSpace(name) +} + +func columnsFromView(aView *view.View) []columnDescriptor { + result := make([]columnDescriptor, 0, len(aView.Columns)) + for _, col := range aView.Columns { + if col == nil { + continue + } + result = append(result, columnDescriptor{ + name: strings.TrimSpace(col.Name), + dataType: strings.TrimSpace(col.DataType), + nullable: col.Nullable, + }) + } + return result +} + +func toUpperCamel(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "" + } + var b strings.Builder + capNext := true + for _, r := range s { + if r == '_' || r == '-' || r == ' ' || r == '.' || r == '/' { + capNext = true + continue + } + if capNext { + b.WriteRune(unicode.ToUpper(r)) + capNext = false + continue + } + b.WriteRune(r) } - if shapeResult != nil { - typeNames = append(typeNames, shapeResult.Types...) + return b.String() +} + +type shapeFragment struct { + Types []string + Imports []string + TypeDecls string +} + +func (g *ComponentCodegen) generateShapeFragment(projectDir, packageDir, packageName, packagePath string) (*shapeFragment, error) { + if g == nil || g.Resource == nil || len(g.Resource.Views) == 0 { + return &shapeFragment{}, nil + } + shapeDoc := resourceToCodegenDoc(g.Resource, g.TypeContext) + applyShapeDocViewTypeOverrides(shapeDoc.Root, g.Component) + shapeCfg := &Config{ + ProjectDir: projectDir, + PackageDir: packageDir, + PackageName: packageName, + PackagePath: packagePath, + } + if overrides := collectViewTypeOverrides(g.Component); len(overrides) > 0 { + shapeCfg.ViewTypeNamer = func(ctx ViewTypeContext) string { + if value := strings.TrimSpace(overrides[strings.ToLower(strings.TrimSpace(ctx.ViewName))]); value != "" { + return value + } + return "" + } + } + hydrateConfigFromTypeContext(shapeDoc, shapeCfg) + applyDefaults(shapeCfg) + return g.renderSemanticShapeFragment(shapeCfg, packagePath) +} + +func (g *ComponentCodegen) renderSemanticShapeFragment(shapeCfg *Config, packagePath string) (*shapeFragment, error) { + viewDescriptorsByName := map[string]viewDescriptor{} + shapeDoc := resourceToCodegenDoc(g.Resource, g.TypeContext) + for _, item := range extractViews(shapeDoc.Root) { + viewDescriptorsByName[strings.ToLower(strings.TrimSpace(asString(item.name)))] = item + } + typeNames := make([]string, 0, len(g.Resource.Views)) + registered := map[string]bool{} + imports := map[string]bool{} + var decls strings.Builder + for _, aView := range g.Resource.Views { + if aView == nil { + continue + } + typeName := g.resourceViewTypeName(shapeCfg, aView) + if typeName == "" || registered[typeName] { + continue + } + viewDecl, viewImports, err := g.renderSemanticViewDecl(shapeCfg, aView, packagePath) + if err != nil { + return nil, err + } + if strings.TrimSpace(viewDecl) == "" { + continue + } + registered[typeName] = true + typeNames = append(typeNames, typeName) + decls.WriteString(viewDecl) + decls.WriteString("\n") + for _, imp := range viewImports { + imports[imp] = true + } + + if descriptor, ok := viewDescriptorsByName[strings.ToLower(strings.TrimSpace(aView.Name))]; ok && descriptor.mutable { + structType := buildHasType(columnsFromView(aView)) + if structType != nil { + hasTypeName := typeName + "Has" + if !registered[hasTypeName] { + registered[hasTypeName] = true + typeNames = append(typeNames, hasTypeName) + decls.WriteString(fmt.Sprintf("type %s struct {\n", hasTypeName)) + decls.WriteString(structFieldsSource(structType)) + decls.WriteString("}\n\n") + } + } + } + } + mergedImports := make([]string, 0, len(imports)) + for imp := range imports { + mergedImports = append(mergedImports, imp) + } + sort.Strings(mergedImports) + return &shapeFragment{ + Types: typeNames, + Imports: mergeImportPaths(mergedImports), + TypeDecls: strings.TrimSpace(decls.String()), + }, nil +} + +func (g *ComponentCodegen) resourceViewTypeName(shapeCfg *Config, aView *view.View) string { + if aView == nil { + return "" + } + descriptor := viewDescriptor{ + name: aView.Name, + schemaName: "", + columns: columnsFromView(aView), + } + if aView.Schema != nil { + descriptor.schemaName = aView.Schema.Name + } + return viewTypeName(shapeCfg, descriptor) +} + +func (g *ComponentCodegen) renderSemanticViewDecl(shapeCfg *Config, aView *view.View, currentPackage string) (string, []string, error) { + aView = g.semanticView(aView) + typeName := g.resourceViewTypeName(shapeCfg, aView) + if typeName == "" { + return "", nil, nil + } + var builder strings.Builder + builder.WriteString(fmt.Sprintf("type %s struct {\n", typeName)) + imports := map[string]bool{} + emittedFields := map[string]bool{} + appendField := func(fieldName, fieldSrc string, fieldImports []string) { + if strings.TrimSpace(fieldName) == "" { + fieldName = renderedFieldName(fieldSrc) + } + fieldName = strings.TrimSpace(fieldName) + if fieldName == "" || emittedFields[fieldName] || strings.TrimSpace(fieldSrc) == "" { + return + } + emittedFields[fieldName] = true + builder.WriteString(fieldSrc) + for _, imp := range fieldImports { + imports[imp] = true + } + } + renderedScalar := false + for _, column := range aView.Columns { + fieldSrc, fieldImports := g.renderColumnField(aView, column, currentPackage) + if fieldSrc == "" { + continue + } + renderedScalar = true + appendField("", fieldSrc, fieldImports) + } + if !renderedScalar { + for _, field := range g.renderScalarFallbackFields(aView, currentPackage) { + fieldName := strings.TrimSpace(strings.Split(strings.TrimSpace(field.src), " ")[0]) + appendField(fieldName, field.src, field.imports) + } + } + for _, rel := range aView.With { + fieldSrc, fieldImports := g.renderRelationField(shapeCfg, aView, rel, currentPackage) + if fieldSrc == "" { + continue + } + appendField(strings.TrimSpace(rel.Holder), fieldSrc, fieldImports) + if metaSrc, metaImports := g.renderRelationSummaryField(shapeCfg, rel, currentPackage); metaSrc != "" { + fieldName := "" + if rel.Of.Template != nil && rel.Of.Template.Summary != nil { + fieldName = state.StructFieldName(text.CaseFormatUpperCamel, rel.Of.Template.Summary.Name) + } + appendField(fieldName, metaSrc, metaImports) + } + } + if aView.SelfReference != nil { + if holder := strings.TrimSpace(aView.SelfReference.Holder); holder != "" { + builder.WriteString(fmt.Sprintf("\t%s []interface{} `sqlx:\"-\"`\n", holder)) + } + } + builder.WriteString("}\n\n") + resultImports := make([]string, 0, len(imports)) + for imp := range imports { + resultImports = append(resultImports, imp) + } + sort.Strings(resultImports) + return builder.String(), resultImports, nil +} + +func (g *ComponentCodegen) renderColumnField(aView *view.View, column *view.Column, currentPackage string) (string, []string) { + if aView == nil || column == nil { + return "", nil + } + fieldName := column.FieldName() + if strings.TrimSpace(fieldName) == "" { + caseFormat := aView.CaseFormat + if !caseFormat.IsDefined() { + caseFormat = text.CaseFormatLowerUnderscore + } + fieldName = state.StructFieldName(caseFormat, column.Name) + } + rType := column.ColumnType() + if rType == nil { + if builtin, ok := builtinTypeByName(column.DataType); ok { + rType = builtin + } else if g != nil && g.Resource != nil { + if lookup := g.Resource.LookupType(); lookup != nil { + if resolved, err := utypes.LookupType(lookup, column.DataType); err == nil && resolved != nil { + rType = resolved + } + } + if rType == nil && extension.Config != nil && extension.Config.Types != nil { + if resolved, err := utypes.LookupType(extension.Config.Types.Lookup, column.DataType); err == nil && resolved != nil { + rType = resolved + } + } + } + } + if rType == nil { + rType = reflect.TypeOf((*interface{})(nil)).Elem() + } + rType = g.normalizeColumnType(column, rType) + tag := g.columnFieldTag(aView, column) + return fmt.Sprintf("\t%s %s `%s`\n", fieldName, goTypeString(rType), tag), collectTypeImports(rType, currentPackage) +} + +func (g *ComponentCodegen) renderRelationField(shapeCfg *Config, parent *view.View, rel *view.Relation, currentPackage string) (string, []string) { + if rel == nil { + return "", nil + } + holder := strings.TrimSpace(rel.Holder) + if holder == "" { + return "", nil + } + childTypeName := g.relationTypeName(shapeCfg, rel) + if childTypeName == "" { + return "", nil + } + typeExpr := "*" + childTypeName + if rel.Cardinality == state.Many { + typeExpr = "[]*" + childTypeName + } + tag := g.relationFieldTag(parent, rel) + return fmt.Sprintf("\t%s %s `%s`\n", holder, typeExpr, tag), nil +} + +func (g *ComponentCodegen) renderRelationSummaryField(shapeCfg *Config, rel *view.Relation, currentPackage string) (string, []string) { + if rel == nil || rel.Of.Template == nil || rel.Of.Template.Summary == nil || rel.Of.Template.Summary.Schema == nil { + return "", nil + } + meta := rel.Of.Template.Summary + fieldName := state.StructFieldName(text.CaseFormatUpperCamel, meta.Name) + if strings.TrimSpace(fieldName) == "" { + return "", nil + } + typeName := strings.TrimSpace(meta.Schema.Name) + if typeName == "" { + return "", nil + } + typeExpr := "*" + typeName + tag := fmt.Sprintf(`json:",omitempty" yaml:",omitempty" sqlx:"-" typeName:"%s"`, typeName) + return fmt.Sprintf("\t%s %s `%s`\n", fieldName, typeExpr, tag), collectTypeImports(meta.Schema.Type(), currentPackage) +} + +func (g *ComponentCodegen) relationTypeName(shapeCfg *Config, rel *view.Relation) string { + if rel == nil { + return "" + } + if rel.Of.Schema != nil && strings.TrimSpace(rel.Of.Schema.Name) != "" { + return strings.TrimSpace(rel.Of.Schema.Name) + } + refNames := []string{ + strings.TrimSpace(rel.Of.View.Name), + strings.TrimSpace(rel.Of.View.Reference.Ref), + strings.TrimSpace(rel.Name), + } + for _, refName := range refNames { + if refName == "" { + continue + } + for _, candidate := range g.Resource.Views { + if candidate == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(candidate.Name), refName) || strings.EqualFold(strings.TrimSpace(candidate.Reference.Ref), refName) { + return g.resourceViewTypeName(shapeCfg, candidate) + } + } + } + return "" +} + +func (g *ComponentCodegen) columnFieldTag(aView *view.View, column *view.Column) string { + tag := strings.TrimSpace(column.Tag) + if aView != nil && aView.ColumnsConfig != nil { + if cfg := aView.ColumnsConfig[column.Name]; cfg != nil && cfg.Tag != nil { + configTag := strings.TrimSpace(strings.Trim(*cfg.Tag, ` `)) + if configTag != "" && !strings.Contains(tag, configTag) { + if tag != "" { + tag += " " + } + tag += configTag + } + } + } + if aView != nil && containsFold(aView.Exclude, column.Name) && !strings.Contains(tag, `internal:"true"`) { + if tag != "" { + tag += " " + } + tag += `internal:"true"` + } + sqlxValue := strings.TrimSpace(column.Name) + if column.Codec != nil && strings.TrimSpace(column.DataType) != "" { + sqlxValue += ",type=" + strings.TrimSpace(column.DataType) + } + if sqlxValue != "" && !strings.Contains(tag, `sqlx:"`) { + if tag != "" { + tag += " " + } + tag += fmt.Sprintf(`sqlx:"%s"`, sqlxValue) + } + if g.resourceViewUsesVelty(aView) && !strings.Contains(tag, `velty:"`) { + caseFormat := aView.CaseFormat + if !caseFormat.IsDefined() { + caseFormat = text.CaseFormatLowerUnderscore + } + if tag != "" { + tag += " " + } + tag += fmt.Sprintf(`velty:"%s"`, generateVeltyTagValue(column.Name, caseFormat)) + } + return normalizeGeneratedTagOrder(strings.TrimSpace(tag)) +} + +func (g *ComponentCodegen) viewUsesVelty(aView *view.View) bool { + if aView == nil { + return false + } + switch aView.Mode { + case view.ModeExec: + return true + default: + return false + } +} + +func (g *ComponentCodegen) resourceViewUsesVelty(aView *view.View) bool { + if g.viewUsesVelty(aView) { + return true + } + if g == nil || aView == nil || !g.componentUsesVelty() || g.Component == nil { + return false + } + target := strings.TrimSpace(aView.Name) + if target == "" { + return false + } + for _, input := range g.Component.Input { + if input == nil || input.In == nil || input.In.Kind != state.KindView { + continue + } + if strings.EqualFold(strings.TrimSpace(input.In.Name), target) { + return true + } + } + return false +} + +func (g *ComponentCodegen) componentUsesVelty() bool { + if g == nil { + return false + } + if g.componentUsesHandler() { + return false + } + if g.Resource != nil && g.Component != nil { + rootViewName := strings.TrimSpace(g.Component.RootView) + if rootViewName != "" { + if rootView, _ := g.Resource.View(rootViewName); rootView != nil { + return g.viewUsesVelty(rootView) + } + } + } + if g.Component == nil { + return false + } + method := strings.ToUpper(strings.TrimSpace(g.Component.Method)) + return method != "" && method != "GET" +} + +func (g *ComponentCodegen) componentUsesHandler() bool { + if g == nil || g.Component == nil { + return false + } + for _, route := range g.Component.ComponentRoutes { + if route != nil && strings.TrimSpace(route.Handler) != "" { + return true + } + } + if g.Resource != nil { + if rootViewName := strings.TrimSpace(g.Component.RootView); rootViewName != "" { + if rootView, _ := g.Resource.View(rootViewName); rootView != nil && rootView.Mode == view.ModeHandler { + return true + } + } + } + return false +} + +func normalizeGeneratedTagOrder(tag string) string { + tag = strings.TrimSpace(tag) + if tag == "" { + return tag + } + ordered := make([]string, 0, 4) + for _, key := range []string{"sqlx", "internal", "velty", "json"} { + value := reflect.StructTag(tag).Get(key) + if value == "" { + continue + } + ordered = append(ordered, fmt.Sprintf(`%s:%q`, key, value)) + var updated string + updated, _ = xreflect.RemoveTag(tag, key) + tag = strings.TrimSpace(updated) + } + if tag != "" { + ordered = append(ordered, tag) + } + return strings.Join(ordered, " ") +} + +func (g *ComponentCodegen) relationFieldTag(parent *view.View, rel *view.Relation) string { + child := g.semanticView(g.resolveRelationView(rel)) + tag := &viewtags.Tag{} + if table := strings.TrimSpace(child.Table); isStableTableName(table) { + tag.View = &viewtags.View{Table: table} + } + if relTag := strings.TrimSpace(child.Tag); relTag != "" { + if tag.View == nil { + tag.View = &viewtags.View{} + } + tag.View.CustomTag = relTag + } + if child.Batch != nil && child.Batch.Size > 0 && child.Batch.Size != 10000 { + if tag.View == nil { + tag.View = &viewtags.View{} + } + tag.View.Batch = child.Batch.Size + } + if child.RelationalConcurrency != nil && child.RelationalConcurrency.Number > 0 && child.RelationalConcurrency.Number != 1 { + if tag.View == nil { + tag.View = &viewtags.View{} + } + tag.View.RelationalConcurrency = child.RelationalConcurrency.Number + } + if child.PublishParent { + if tag.View == nil { + tag.View = &viewtags.View{} + } + tag.View.PublishParent = true + } + if child.Partitioned != nil { + if tag.View == nil { + tag.View = &viewtags.View{} + } + tag.View.PartitionerType = child.Partitioned.DataType + tag.View.PartitionedConcurrency = child.Partitioned.Concurrency + } + if child.MatchStrategy != "" && child.MatchStrategy != view.ReadMatched { + if tag.View == nil { + tag.View = &viewtags.View{} + } + tag.View.Match = string(child.MatchStrategy) + } + if parent != nil && parent.Cache != nil { + if tag.View == nil { + tag.View = &viewtags.View{} + } + tag.View.Cache = parent.Cache.Ref + } + if parent != nil && parent.Connector != nil && child.Connector != nil && child.Connector.Ref != parent.Connector.Ref { + if tag.View == nil { + tag.View = &viewtags.View{} + } + tag.View.Connector = child.Connector.Ref + } + tag.LinkOn = g.relationLinkTag(parent, child, rel) + if child.Template != nil { + tag.SQL = viewtags.NewViewSQL("", strings.TrimSpace(child.Template.SourceURL)) + } + return string(tag.UpdateTag(``)) +} + +func (g *ComponentCodegen) normalizeColumnType(column *view.Column, rType reflect.Type) reflect.Type { + if column == nil || rType == nil { + return rType + } + for rType.Kind() == reflect.Ptr { + if column.Nullable { + return rType + } + rType = rType.Elem() + } + if column.Nullable && rType.Kind() != reflect.Interface && rType.Kind() != reflect.Slice && rType.Kind() != reflect.Map { + return reflect.PtrTo(rType) + } + return rType +} + +type renderedField struct { + src string + imports []string +} + +func renderedFieldName(src string) string { + src = strings.TrimSpace(src) + if src == "" { + return "" + } + parts := strings.Fields(src) + if len(parts) == 0 { + return "" + } + return strings.TrimSpace(parts[0]) +} + +func (g *ComponentCodegen) renderScalarFallbackFields(aView *view.View, currentPackage string) []renderedField { + aView = g.semanticView(aView) + rType := g.resourceViewStructType(aView.Name) + rType = ensureCodegenStructType(rType) + if rType == nil && aView != nil { + rType = ensureCodegenStructType(aView.ComponentType()) + if rType == nil && aView.Schema != nil { + rType = ensureCodegenStructType(aView.Schema.Type()) + } + } + if rType == nil { + return nil + } + var result []renderedField + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if !field.IsExported() { + continue + } + if strings.TrimSpace(field.Tag.Get("view")) != "" || strings.TrimSpace(field.Tag.Get("on")) != "" { + continue + } + result = append(result, renderedField{ + src: fmt.Sprintf("\t%s %s `%s`\n", field.Name, goTypeString(field.Type), string(field.Tag)), + imports: collectTypeImports(field.Type, currentPackage), + }) + } + return result +} + +func (g *ComponentCodegen) resolveRelationView(rel *view.Relation) *view.View { + if rel == nil { + return nil + } + names := []string{ + strings.TrimSpace(rel.Of.View.Reference.Ref), + strings.TrimSpace(rel.Of.View.Name), + strings.TrimSpace(rel.Name), + } + for _, name := range names { + if name == "" || g == nil || g.Resource == nil { + continue + } + for _, candidate := range g.Resource.Views { + if candidate == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(candidate.Name), name) || strings.EqualFold(strings.TrimSpace(candidate.Reference.Ref), name) { + return candidate + } + } + } + return &rel.Of.View +} + +func (g *ComponentCodegen) semanticView(aView *view.View) *view.View { + if aView == nil { + return nil + } + merged := *aView + if merged.ColumnsConfig == nil { + merged.ColumnsConfig = map[string]*view.ColumnConfig{} + } + if g == nil || g.Resource == nil { + return &merged + } + for _, parent := range g.Resource.Views { + if parent == nil { + continue + } + for _, rel := range parent.With { + if rel == nil { + continue + } + if !g.matchesViewRef(&merged, rel) { + continue + } + g.mergeViewSemantics(&merged, &rel.Of.View) + } + } + return &merged +} + +func (g *ComponentCodegen) matchesViewRef(target *view.View, rel *view.Relation) bool { + if target == nil || rel == nil { + return false + } + candidates := []string{ + strings.TrimSpace(target.Name), + strings.TrimSpace(target.Reference.Ref), + } + refs := []string{ + strings.TrimSpace(rel.Of.View.Name), + strings.TrimSpace(rel.Of.View.Reference.Ref), + strings.TrimSpace(rel.Name), + } + for _, candidate := range candidates { + if candidate == "" { + continue + } + for _, ref := range refs { + if ref != "" && strings.EqualFold(candidate, ref) { + return true + } + } + } + return false +} + +func (g *ComponentCodegen) mergeViewSemantics(dst, src *view.View) { + if dst == nil || src == nil { + return + } + if len(dst.Columns) == 0 && len(src.Columns) > 0 { + dst.Columns = src.Columns + } + if len(dst.Exclude) == 0 && len(src.Exclude) > 0 { + dst.Exclude = append(dst.Exclude, src.Exclude...) + } + if dst.ColumnsConfig == nil { + dst.ColumnsConfig = map[string]*view.ColumnConfig{} + } + for key, cfg := range src.ColumnsConfig { + if _, ok := dst.ColumnsConfig[key]; !ok { + dst.ColumnsConfig[key] = cfg + } + } + if (dst.Template == nil || strings.TrimSpace(dst.Template.SourceURL) == "") && src.Template != nil { + dst.Template = src.Template + } + if !isStableTableName(dst.Table) && isStableTableName(src.Table) { + dst.Table = src.Table + } + if dst.Schema == nil && src.Schema != nil { + dst.Schema = src.Schema + return + } + if dst.Schema != nil && src.Schema != nil { + if dst.Schema.Type() == nil && src.Schema.Type() != nil { + dst.Schema.SetType(src.Schema.Type()) + } + if strings.TrimSpace(dst.Schema.Name) == "" { + dst.Schema.Name = src.Schema.Name + } + } +} + +func (g *ComponentCodegen) relationLinkTag(parent, child *view.View, rel *view.Relation) viewtags.LinkOn { + if rel == nil { + return nil + } + result := make([]string, 0, len(rel.On)) + for i, parentLink := range rel.On { + if parentLink == nil { + continue + } + var childLink *view.Link + if i < len(rel.Of.On) { + childLink = rel.Of.On[i] + } + left := g.encodeRelationEndpoint(parent, parentLink) + right := g.encodeRelationEndpoint(child, childLink) + if left != "" && right != "" { + result = append(result, left+"="+right) + } + } + return result +} + +func (g *ComponentCodegen) encodeRelationEndpoint(owner *view.View, link *view.Link) string { + if link == nil { + return "" + } + column := stripNamespace(link.Column) + field := strings.TrimSpace(link.Field) + if field == "" { + caseFormat := text.CaseFormatLowerUnderscore + if owner != nil && owner.CaseFormat.IsDefined() { + caseFormat = owner.CaseFormat + } + field = state.StructFieldName(caseFormat, column) + } + if field == "" { + return column + } + if column == "" { + return field + } + return field + ":" + column +} + +func stripNamespace(value string) string { + value = strings.TrimSpace(value) + if idx := strings.LastIndex(value, "."); idx != -1 { + return strings.TrimSpace(value[idx+1:]) + } + return value +} + +func looksLikeSQL(value string) bool { + value = strings.TrimSpace(strings.ToUpper(value)) + return strings.Contains(value, "SELECT ") || strings.Contains(value, "\n") || strings.Contains(value, "(") +} + +func isStableTableName(value string) bool { + value = strings.TrimSpace(value) + if value == "" || looksLikeSQL(value) { + return false + } + for _, r := range value { + switch { + case r >= 'A' && r <= 'Z': + case r >= '0' && r <= '9': + case r == '_' || r == '.': + default: + return false + } + } + return true +} + +func containsFold(items []string, candidate string) bool { + candidate = strings.TrimSpace(candidate) + for _, item := range items { + if strings.EqualFold(strings.TrimSpace(item), candidate) { + return true + } + } + return false +} + +func generateVeltyTagValue(columnName string, caseFormat text.CaseFormat) string { + names := columnName + if fieldName := state.StructFieldName(caseFormat, columnName); fieldName != names { + names += "|" + fieldName + } + return "names=" + names +} + +func goTypeString(rType reflect.Type) string { + if rType == nil { + return "interface{}" + } + switch rType.Kind() { + case reflect.Ptr: + return "*" + goTypeString(rType.Elem()) + case reflect.Slice: + return "[]" + goTypeString(rType.Elem()) + case reflect.Array: + return fmt.Sprintf("[%d]%s", rType.Len(), goTypeString(rType.Elem())) + case reflect.Map: + return fmt.Sprintf("map[%s]%s", goTypeString(rType.Key()), goTypeString(rType.Elem())) + } + if rType.Name() != "" { + if pkg := strings.TrimSpace(rType.PkgPath()); pkg != "" { + prefix := filepath.Base(pkg) + if prefix != "" && prefix != "." { + return prefix + "." + rType.Name() + } + } + return rType.Name() + } + return rType.String() +} + +func (g *ComponentCodegen) resourceViewStructType(name any) reflect.Type { + if g == nil || g.Resource == nil { + return nil + } + viewName := strings.TrimSpace(asString(name)) + if viewName == "" { + return nil + } + for _, aView := range g.Resource.Views { + if aView == nil || !strings.EqualFold(strings.TrimSpace(aView.Name), viewName) { + continue + } + rType := aView.ComponentType() + if rType == nil && aView.Schema != nil { + rType = aView.Schema.Type() + } + rType = ensureCodegenStructType(rType) + if rebuilt := rebuildResourceViewStructType(rType, columnsFromView(aView), g.resourceViewUsesVelty(aView)); rebuilt != nil { + rType = rebuilt + } + if augmented := g.augmentResourceViewStructType(aView, rType); augmented != nil { + return augmented + } + return rType + } + return nil +} + +func ensureCodegenStructType(rType reflect.Type) reflect.Type { + if rType == nil { + return nil + } + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return nil + } + return rType +} + +func shouldRenderResourceViewType(rType reflect.Type, columns []columnDescriptor) bool { + rType = ensureCodegenStructType(rType) + if rType == nil { + return false + } + if resourceViewNeedsRebuild(rType, columns, false) { + return true + } + return rType.NumField() > len(columns) +} + +func rebuildResourceViewStructType(rType reflect.Type, columns []columnDescriptor, includeVelty bool) reflect.Type { + rType = ensureCodegenStructType(rType) + if rType == nil { + if len(columns) == 0 { + return nil + } + return reflect.StructOf(buildStructFields(columns, includeVelty)) + } + if !resourceViewNeedsRebuild(rType, columns, includeVelty) { + return rType + } + fields := buildStructFields(columns, includeVelty) + if len(fields) == 0 { + return rType + } + used := map[string]bool{} + for _, field := range fields { + used[field.Name] = true + } + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if isPlaceholderProjectionField(field) { + continue + } + if used[field.Name] { + continue + } + fields = append(fields, field) + used[field.Name] = true + } + return reflect.StructOf(fields) +} + +func (g *ComponentCodegen) augmentResourceViewStructType(aView *view.View, rType reflect.Type) reflect.Type { + rType = ensureCodegenStructType(rType) + if aView == nil || rType == nil { + return rType + } + fields := make([]reflect.StructField, 0, rType.NumField()+len(aView.With)+1) + used := map[string]bool{} + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + fields = append(fields, field) + used[field.Name] = true + } + if aView.SelfReference != nil { + if holder := strings.TrimSpace(aView.SelfReference.Holder); holder != "" && !used[holder] { + fields = append(fields, reflect.StructField{ + Name: holder, + Type: reflect.TypeOf([]interface{}{}), + Tag: `sqlx:"-"`, + }) + used[holder] = true + } + } + for _, rel := range aView.With { + if rel == nil { + continue + } + holder := strings.TrimSpace(rel.Holder) + if holder == "" || used[holder] { + continue + } + fieldType := g.relationHolderType(rel) + if fieldType == nil { + continue + } + tagParts := []string{} + if table := strings.TrimSpace(rel.Of.View.Table); table != "" { + tagParts = append(tagParts, fmt.Sprintf(`view:",table=%s"`, table)) + } else { + tagParts = append(tagParts, `view:""`) + } + tagParts = append(tagParts, `sqlx:"-"`) + fields = append(fields, reflect.StructField{ + Name: holder, + Type: fieldType, + Tag: reflect.StructTag(strings.Join(tagParts, " ")), + }) + used[holder] = true + } + if len(fields) == rType.NumField() { + return rType + } + return reflect.StructOf(fields) +} + +func (g *ComponentCodegen) relationHolderType(rel *view.Relation) reflect.Type { + if rel == nil { + return nil + } + childType := ensureCodegenStructType(rel.Of.View.ComponentType()) + if childType == nil && rel.Of.Schema != nil { + childType = ensureCodegenStructType(rel.Of.Schema.Type()) + } + if childType == nil && g != nil && g.Resource != nil { + refNames := []string{ + strings.TrimSpace(rel.Of.View.Name), + strings.TrimSpace(rel.Of.View.Reference.Ref), + strings.TrimSpace(rel.Name), + } + for _, refName := range refNames { + if refName == "" { + continue + } + childType = g.resourceViewStructType(refName) + if childType != nil { + break + } + } + } + if childType == nil { + return nil + } + childPtr := childType + if childPtr.Kind() != reflect.Ptr { + childPtr = reflect.PtrTo(childType) + } + if rel.Cardinality == state.One { + return childPtr + } + return reflect.SliceOf(childPtr) +} + +func resourceViewNeedsRebuild(rType reflect.Type, columns []columnDescriptor, includeVelty bool) bool { + rType = ensureCodegenStructType(rType) + if rType == nil { + return false + } + if len(columns) == 0 { + return false + } + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if isPlaceholderProjectionField(field) { + return true + } + if includeVelty && field.Tag.Get("sqlx") != "" && field.Tag.Get("sqlx") != "-" && field.Tag.Get("velty") == "" { + return true + } + } + return false +} + +func isPlaceholderProjectionField(field reflect.StructField) bool { + if tag := strings.TrimSpace(field.Tag.Get("view")); tag != "" { + return false + } + if tag := strings.TrimSpace(field.Tag.Get("sql")); tag != "" { + return false + } + sqlxTag := field.Tag.Get("sqlx") + sqlxName := sqlxTagName(sqlxTag) + if sqlxName == "" || sqlxName == "-" { + return false + } + name := strings.TrimSpace(field.Name) + if strings.HasPrefix(strings.ToLower(name), "col") && strings.HasPrefix(strings.ToLower(sqlxName), "col_") { + return true + } + return false +} + +func sqlxTagName(tag string) string { + tag = strings.TrimSpace(tag) + if tag == "" { + return "" + } + if strings.HasPrefix(tag, "name=") { + tag = strings.TrimPrefix(tag, "name=") + } + if idx := strings.Index(tag, ","); idx != -1 { + tag = tag[:idx] + } + return strings.TrimSpace(tag) +} + +func collectTypeImports(rType reflect.Type, currentPackage string) []string { + seen := map[string]bool{} + var result []string + var visit func(reflect.Type) + visit = func(t reflect.Type) { + if t == nil { + return + } + for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice || t.Kind() == reflect.Array || t.Kind() == reflect.Map { + if t.Kind() == reflect.Map { + visit(t.Key()) + } + t = t.Elem() + if t == nil { + return + } + } + if pkg := strings.TrimSpace(t.PkgPath()); pkg != "" && pkg != currentPackage { + if !seen[pkg] { + seen[pkg] = true + result = append(result, pkg) + } + if t.Name() != "" { + return + } + } + if t.Kind() != reflect.Struct { + return + } + for i := 0; i < t.NumField(); i++ { + visit(t.Field(i).Type) + } + } + visit(rType) + sort.Strings(result) + return result +} + +func applyShapeDocViewTypeOverrides(root map[string]any, component *shapeload.Component) { + if root == nil || component == nil || len(component.TypeSpecs) == 0 { + return + } + resourceMap, _ := root["Resource"].(map[string]any) + if resourceMap == nil { + return + } + views, _ := resourceMap["Views"].([]any) + if len(views) == 0 { + return + } + for _, raw := range views { + viewMap, _ := raw.(map[string]any) + if viewMap == nil { + continue + } + name, _ := viewMap["Name"].(string) + name = strings.TrimSpace(name) + if name == "" { + continue + } + spec, ok := component.TypeSpecs["view:"+name] + if !ok || spec == nil || strings.TrimSpace(spec.TypeName) == "" { + continue + } + schemaMap, _ := viewMap["Schema"].(map[string]any) + if schemaMap == nil { + schemaMap = map[string]any{} + } + schemaMap["Name"] = strings.TrimSpace(spec.TypeName) + viewMap["Schema"] = schemaMap + } +} + +func collectViewTypeOverrides(component *shapeload.Component) map[string]string { + if component == nil || len(component.TypeSpecs) == 0 { + return nil + } + ret := map[string]string{} + for key, spec := range component.TypeSpecs { + if spec == nil || spec.Role != shapeload.TypeRoleView { + continue + } + typeName := strings.TrimSpace(spec.TypeName) + if typeName == "" { + continue + } + alias := strings.TrimSpace(spec.Alias) + if alias == "" && strings.HasPrefix(key, "view:") { + alias = strings.TrimPrefix(key, "view:") + } + if alias == "" { + continue + } + ret[strings.ToLower(alias)] = typeName + } + if len(ret) == 0 { + return nil + } + return ret +} + +func mergeImportPaths(groups ...[]string) []string { + var result []string + seen := map[string]bool{} + for _, group := range groups { + for _, item := range group { + item = strings.TrimSpace(item) + if item == "" || seen[item] { + continue + } + seen[item] = true + result = append(result, item) + } + } + return result +} + +func extractTypeDeclsAndImports(source string) ([]string, string, error) { + fset := token.NewFileSet() + fileNode, err := parser.ParseFile(fset, "", source, parser.ParseComments) + if err != nil { + return nil, "", err + } + var imports []string + for _, spec := range fileNode.Imports { + pathValue := strings.Trim(spec.Path.Value, `"`) + if spec.Name != nil && spec.Name.Name != "" && spec.Name.Name != "." && spec.Name.Name != "_" { + imports = append(imports, spec.Name.Name+` "`+pathValue+`"`) + continue + } + imports = append(imports, pathValue) } - return &ComponentCodegenResult{ - FilePath: dest, - PackagePath: packagePath, - PackageName: packageName, - Types: typeNames, - }, nil + var body bytes.Buffer + for _, decl := range fileNode.Decls { + if typeDecl, ok := decl.(*ast.GenDecl); ok && typeDecl.Tok == token.TYPE { + if err := format.Node(&body, fset, typeDecl); err != nil { + return nil, "", err + } + body.WriteString("\n\n") + } + } + return imports, strings.TrimSpace(body.String()), nil } // renderOutputStruct writes the output struct definition. @@ -221,20 +2091,26 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { // response.Status `parameter:",kind=output,in=status" json:",omitempty"` // Data []*XxxView `parameter:",kind=output,in=view" view:"xxx" sql:"uri=xxx/xxx.sql"` // } -func (g *ComponentCodegen) renderOutputStruct(builder *strings.Builder, componentName, embedURI string, outputParams state.Parameters, outputType reflect.Type) { +func (g *ComponentCodegen) renderOutputStruct(builder *strings.Builder, outputTypeName, viewTypeName, embedURI string, outputParams state.Parameters, outputType reflect.Type, mutableSupport *mutableComponentSupport) { rootView := g.Component.RootView - viewType := componentName + "View" - builder.WriteString(fmt.Sprintf("type %sOutput struct {\n", componentName)) + builder.WriteString(fmt.Sprintf("type %s struct {\n", outputTypeName)) // Check if there's an explicit status parameter hasStatus := false + hasViolations := false for _, p := range outputParams { - if p != nil && p.In != nil && p.In.Name == "status" { + if p == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(p.Name), "Violations") { + hasViolations = true + } + if p.In != nil && p.In.Name == "status" { hasStatus = true } } - if !hasStatus { + if !hasStatus && (len(outputParams) > 0 || g.shouldDefaultReaderOutput() || mutableSupport != nil) { builder.WriteString("\tresponse.Status `parameter:\",kind=output,in=status\" json:\",omitempty\"`\n") } @@ -258,7 +2134,7 @@ func (g *ComponentCodegen) renderOutputStruct(builder *strings.Builder, componen if p.Tag != "" && strings.Contains(p.Tag, "anonymous") { tag += ` anonymous:"true"` } - builder.WriteString(fmt.Sprintf("\t%s %s%s `%s`\n", fieldName, typePrefix, viewType, tag)) + builder.WriteString(fmt.Sprintf("\t%s %s%s `%s`\n", fieldName, typePrefix, viewTypeName, tag)) case "status": builder.WriteString(fmt.Sprintf("\tresponse.Status `parameter:\",kind=output,in=status\" json:\",omitempty\"`\n")) default: @@ -270,8 +2146,17 @@ func (g *ComponentCodegen) renderOutputStruct(builder *strings.Builder, componen builder.WriteString(fmt.Sprintf("\t%s %s `parameter:\",kind=output,in=%s\"`\n", p.Name, typeName, p.In.Name)) } } + if mutableSupport != nil && !hasViolations { + builder.WriteString("\tViolations validator.Violations `json:\",omitempty\"`\n") + } builder.WriteString("}\n\n") + if mutableSupport != nil { + builder.WriteString(fmt.Sprintf("func (o *%s) setError(err error) {\n", outputTypeName)) + builder.WriteString("\to.Status.Message = err.Error()\n") + builder.WriteString("\to.Status.Status = \"error\"\n") + builder.WriteString("}\n\n") + } } // resolveOutputWildcardTypes resolves output parameters with wildcard type `?` or empty @@ -309,6 +2194,9 @@ func (g *ComponentCodegen) resolveOutputWildcardTypes(params state.Parameters, c // - Status: response status (anonymous, kind=output, in=status) // This mirrors internal/translator output.go ensureOutputParameters. func (g *ComponentCodegen) defaultOutputParameters(componentName string) state.Parameters { + if !g.shouldDefaultReaderOutput() { + return nil + } rootView := g.Component.RootView viewType := componentName + "View" @@ -335,6 +2223,26 @@ func (g *ComponentCodegen) defaultOutputParameters(componentName string) state.P return state.Parameters{dataParam, statusParam} } +func (g *ComponentCodegen) shouldDefaultReaderOutput() bool { + if g == nil || g.Resource == nil || g.Component == nil { + return true + } + rootViewName := strings.TrimSpace(g.Component.RootView) + if rootViewName == "" { + return true + } + rootView, _ := g.Resource.View(rootViewName) + if rootView == nil { + return true + } + switch rootView.Mode { + case view.ModeExec, view.ModeHandler: + return false + default: + return true + } +} + func (g *ComponentCodegen) withRegister() bool { if g.WithRegister == nil { return true // default enabled @@ -342,6 +2250,260 @@ func (g *ComponentCodegen) withRegister() bool { return *g.WithRegister } +func (g *ComponentCodegen) typeSpec(key string) *shapeload.TypeSpec { + if g == nil || g.Component == nil || g.Component.TypeSpecs == nil { + return nil + } + return g.Component.TypeSpecs[key] +} + +func (g *ComponentCodegen) inputTypeName(componentName string) string { + if spec := g.typeSpec("input"); spec != nil && strings.TrimSpace(spec.TypeName) != "" { + return strings.TrimSpace(spec.TypeName) + } + return componentName + "Input" +} + +func (g *ComponentCodegen) outputTypeName(componentName string) string { + if spec := g.typeSpec("output"); spec != nil && strings.TrimSpace(spec.TypeName) != "" { + return strings.TrimSpace(spec.TypeName) + } + return componentName + "Output" +} + +func (g *ComponentCodegen) rootViewTypeName(componentName string) string { + rootView := strings.TrimSpace(componentName) + if g.Component != nil && strings.TrimSpace(g.Component.RootView) != "" { + rootView = strings.TrimSpace(g.Component.RootView) + } + if spec := g.typeSpec("view:" + rootView); spec != nil && strings.TrimSpace(spec.TypeName) != "" { + return strings.TrimSpace(spec.TypeName) + } + return componentName + "View" +} + +func (g *ComponentCodegen) rootViewSourceURL() string { + if g == nil || g.Resource == nil { + return "" + } + rootView := "" + if g.Component != nil { + rootView = strings.TrimSpace(g.Component.RootView) + } + if rootView != "" { + if aView, _ := g.Resource.View(rootView); aView != nil && aView.Template != nil { + return strings.TrimSpace(aView.Template.SourceURL) + } + } + if len(g.Resource.Views) == 0 || g.Resource.Views[0] == nil || g.Resource.Views[0].Template == nil { + return "" + } + return strings.TrimSpace(g.Resource.Views[0].Template.SourceURL) +} + +func (g *ComponentCodegen) rootSummarySourceURL() string { + if g == nil || g.Resource == nil { + return "" + } + rootView := "" + var candidate *view.View + if g.Component != nil { + rootView = strings.TrimSpace(g.Component.RootView) + } + if rootView != "" { + if aView, _ := g.Resource.View(rootView); aView != nil { + candidate = aView + } + } + if candidate == nil { + if len(g.Resource.Views) == 0 || g.Resource.Views[0] == nil { + return "" + } + candidate = g.Resource.Views[0] + } + if candidate.Template == nil || candidate.Template.Summary == nil { + return "" + } + if sourceURL := strings.TrimSpace(candidate.Template.Summary.SourceURL); sourceURL != "" { + return sourceURL + } + return path.Join(text.CaseFormatUpperCamel.Format(g.componentName(), text.CaseFormatLowerUnderscore), strings.ToLower(candidate.Name)+"_summary.sql") +} + +func (g *ComponentCodegen) resolveOutputDestFileName(defaultName string) string { + if spec := g.typeSpec("output"); spec != nil { + if dest := strings.TrimSpace(spec.Dest); dest != "" { + return dest + } + } + if g.Component != nil && g.Component.Directives != nil { + if dest := strings.TrimSpace(g.Component.Directives.OutputDest); dest != "" { + return dest + } + } + return defaultName +} + +func (g *ComponentCodegen) resolveInputDestFileName(defaultName string) string { + if spec := g.typeSpec("input"); spec != nil { + if dest := strings.TrimSpace(spec.Dest); dest != "" { + return dest + } + } + if g.Component != nil && g.Component.Directives != nil { + if dest := strings.TrimSpace(g.Component.Directives.InputDest); dest != "" { + return dest + } + } + return defaultName +} + +func (g *ComponentCodegen) resolveViewDestFileName(defaultName string) string { + if root := strings.TrimSpace(g.Component.RootView); root != "" { + if spec := g.typeSpec("view:" + root); spec != nil { + if dest := strings.TrimSpace(spec.Dest); dest != "" { + return dest + } + } + } + if g.Component != nil && g.Component.TypeSpecs != nil { + for _, spec := range g.Component.TypeSpecs { + if spec == nil || spec.Role != shapeload.TypeRoleView { + continue + } + if dest := strings.TrimSpace(spec.Dest); dest != "" { + return dest + } + } + } + if g.Component != nil && g.Component.Directives != nil { + if dest := strings.TrimSpace(g.Component.Directives.Dest); dest != "" { + return dest + } + } + return defaultName +} + +func (g *ComponentCodegen) resolveRouterDestFileName(defaultName string) string { + if g.Component != nil && g.Component.Directives != nil { + if dest := strings.TrimSpace(g.Component.Directives.RouterDest); dest != "" { + return dest + } + } + return defaultName +} + +func (g *ComponentCodegen) writeSectionFile(dest, packageName string, imports []string, sections ...string) error { + var builder strings.Builder + builder.WriteString("package " + packageName + "\n\n") + if len(imports) > 0 { + builder.WriteString("import (\n") + for _, imp := range imports { + imp = strings.TrimSpace(imp) + if imp == "" { + continue + } + if strings.Contains(imp, " ") { + builder.WriteString("\t" + imp + "\n") + } else { + builder.WriteString("\t\"" + imp + "\"\n") + } + } + builder.WriteString(")\n\n") + } + builder.WriteString("// Code generated by datly transcribe. DO NOT EDIT.\n\n") + for _, section := range sections { + if strings.TrimSpace(section) == "" { + continue + } + builder.WriteString(section) + if !strings.HasSuffix(section, "\n\n") { + builder.WriteString("\n") + } + } + return writeAtomic(dest, []byte(dedupeGeneratedStructFields(builder.String())), 0o644) +} + +func (g *ComponentCodegen) appendSectionFile(dest string, sections ...string) error { + data, err := os.ReadFile(dest) + if err != nil { + return err + } + var builder strings.Builder + builder.Write(data) + if len(data) > 0 && !strings.HasSuffix(string(data), "\n") { + builder.WriteString("\n") + } + for _, section := range sections { + if strings.TrimSpace(section) == "" { + continue + } + builder.WriteString("\n") + builder.WriteString(section) + if !strings.HasSuffix(section, "\n") { + builder.WriteString("\n") + } + } + return writeAtomic(dest, []byte(dedupeGeneratedStructFields(builder.String())), 0o644) +} + +func dedupeGeneratedStructFields(source string) string { + lines := strings.Split(source, "\n") + var result []string + inStruct := false + fieldNames := map[string]bool{} + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "type ") && strings.HasSuffix(trimmed, "struct {") { + inStruct = true + fieldNames = map[string]bool{} + result = append(result, line) + continue + } + if inStruct { + if trimmed == "}" { + inStruct = false + fieldNames = nil + result = append(result, line) + continue + } + if name := generatedFieldName(line); name != "" { + if fieldNames[name] { + continue + } + fieldNames[name] = true + } + } + result = append(result, line) + } + return strings.Join(result, "\n") +} + +func generatedFieldName(line string) string { + trimmed := strings.TrimLeft(line, "\t ") + if trimmed == "" { + return "" + } + r, size := utf8.DecodeRuneInString(trimmed) + if r == utf8.RuneError || !unicode.IsUpper(r) { + return "" + } + rest := trimmed[size:] + var b strings.Builder + b.WriteRune(r) + for _, rr := range rest { + if rr == ' ' || rr == '\t' { + break + } + if unicode.IsLetter(rr) || unicode.IsDigit(rr) || rr == '_' { + b.WriteRune(rr) + continue + } + return "" + } + return b.String() +} + func (g *ComponentCodegen) componentName() string { name := "" if g.Component != nil { @@ -356,33 +2518,310 @@ func (g *ComponentCodegen) componentName() string { return state.SanitizeTypeName(name) } -func (g *ComponentCodegen) buildImports() []string { +type namedHelperType struct { + TypeName string + Decl string + Imports []string +} + +func collectNamedHelperTypes(rType reflect.Type, currentPackage string, skip map[string]bool) []namedHelperType { + if rType == nil { + return nil + } + skip = cloneTypeNameSet(skip) + seen := map[string]bool{} + importSet := map[string]bool{} + var result []namedHelperType + var visitType func(reflect.Type) + var visitField func(reflect.StructField) + + visitField = func(field reflect.StructField) { + typeName := strings.TrimSpace(field.Tag.Get("typeName")) + baseType := unwrapAnonymousStructType(field.Type) + if typeName != "" && baseType != nil && baseType.Name() == "" && !skip[typeName] && !seen[typeName] { + seen[typeName] = true + skip[typeName] = true + imports := map[string]bool{} + for _, imp := range collectTypeImports(baseType, currentPackage) { + imports[imp] = true + importSet[imp] = true + } + result = append(result, namedHelperType{ + TypeName: typeName, + Decl: fmt.Sprintf("type %s struct {\n%s}\n\n", typeName, structFieldsSource(baseType)), + Imports: sortedImportSet(imports), + }) + } + visitType(field.Type) + } + + visitType = func(t reflect.Type) { + if t == nil { + return + } + for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice || t.Kind() == reflect.Array { + t = t.Elem() + if t == nil { + return + } + } + if t.Kind() == reflect.Map { + visitType(t.Key()) + visitType(t.Elem()) + return + } + if t.Kind() != reflect.Struct { + return + } + for i := 0; i < t.NumField(); i++ { + visitField(t.Field(i)) + } + } + + visitType(rType) + return result +} + +func helperImports(items []namedHelperType) []string { + imports := map[string]bool{} + for _, item := range items { + for _, imp := range item.Imports { + if strings.TrimSpace(imp) != "" { + imports[imp] = true + } + } + } + return sortedImportSet(imports) +} + +func cloneTypeNameSet(src map[string]bool) map[string]bool { + if len(src) == 0 { + return map[string]bool{} + } + ret := make(map[string]bool, len(src)) + for key, value := range src { + ret[key] = value + } + return ret +} + +func sortedImportSet(src map[string]bool) []string { + if len(src) == 0 { + return nil + } + ret := make([]string, 0, len(src)) + for key := range src { + ret = append(ret, key) + } + sort.Strings(ret) + return ret +} + +func unwrapAnonymousStructType(rType reflect.Type) reflect.Type { + if rType == nil { + return nil + } + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + if rType == nil { + return nil + } + } + if rType.Kind() != reflect.Struct { + return nil + } + return rType +} + +func (g *ComponentCodegen) outputUsesResponse(outputParams state.Parameters) bool { + for _, p := range outputParams { + if p == nil || p.In == nil { + continue + } + if p.In.Name == "status" { + return true + } + } + return len(outputParams) > 0 || g.shouldDefaultReaderOutput() +} + +func (g *ComponentCodegen) buildImports(includeRouter bool, includeResponse bool) []string { var imports []string - if g.withRegister() { + needsReflect := g.withRegister() || g.WithContract + if needsReflect { imports = append(imports, "reflect", - "github.com/viant/xdatly/types/core", ) + } + if g.withRegister() { + imports = append(imports, "github.com/viant/xdatly/types/core") checksumPkg := "github.com/viant/xdatly/types/custom/checksum" if g.PackagePath != "" { if idx := strings.LastIndex(g.PackagePath, "/pkg/"); idx != -1 { - candidate := g.PackagePath[:idx] + "/pkg/checksum" - parent, _ := filepath.Split(candidate) - if !strings.HasSuffix(parent, "dependency/") { - candidate = filepath.Join(parent, "dependency", "checksum") - } - checksumPkg = filepath.ToSlash(candidate) + checksumPkg = g.PackagePath[:idx] + "/pkg/checksum" } } + checksumParent, _ := path.Split(checksumPkg) + if !strings.HasSuffix(strings.TrimSuffix(checksumParent, "/"), "dependency") { + checksumPkg = path.Join(checksumParent, "dependency", "checksum") + } imports = append(imports, checksumPkg) } - imports = append(imports, "github.com/viant/xdatly/handler/response") + if includeResponse { + imports = append(imports, "github.com/viant/xdatly/handler/response") + } if g.WithEmbed { imports = append(imports, "embed") } + if g.WithContract { + imports = append(imports, + "fmt", + "context", + "github.com/viant/datly/view", + "github.com/viant/datly/repository", + "github.com/viant/datly/repository/contract", + "github.com/viant/datly", + ) + } + if includeRouter { + imports = append(imports, "github.com/viant/xdatly") + } return imports } +func (g *ComponentCodegen) buildRouterImports() []string { + return []string{"github.com/viant/xdatly"} +} + +func (g *ComponentCodegen) renderComponentHolder(builder *strings.Builder, componentName, inputTypeName, outputTypeName string) { + method := strings.TrimSpace(g.Component.Method) + if method == "" { + method = "GET" + } + uri := strings.TrimSpace(g.Component.URI) + if uri == "" { + uri = "/" + } + tag := fmt.Sprintf(`component:",path=%s,method=%s`, uri, method) + if connectorRef := strings.TrimSpace(g.rootConnectorRef()); connectorRef != "" { + tag += fmt.Sprintf(`,connector=%s`, connectorRef) + } + if marshaller := strings.TrimSpace(g.rootMarshaller()); marshaller != "" { + tag += fmt.Sprintf(`,marshaller=%s`, marshaller) + } + if handlerRef := strings.TrimSpace(g.rootHandlerRef()); handlerRef != "" { + tag += fmt.Sprintf(`,handler=%s`, handlerRef) + } + if viewTypeName := strings.TrimSpace(g.rootViewTypeName(componentName)); viewTypeName != "" { + tag += fmt.Sprintf(`,view=%s`, viewTypeName) + } + if sourceURL := strings.TrimSpace(g.rootViewSourceURL()); sourceURL != "" { + tag += fmt.Sprintf(`,source=%s`, sourceURL) + } + if summaryURL := strings.TrimSpace(g.rootSummarySourceURL()); summaryURL != "" { + tag += fmt.Sprintf(`,summary=%s`, summaryURL) + } + tag += `"` + builder.WriteString(fmt.Sprintf("type %sRouter struct {\n", componentName)) + builder.WriteString(fmt.Sprintf("\t%s xdatly.Component[%s, %s] `%s`\n", componentName, inputTypeName, outputTypeName, tag)) + builder.WriteString("}\n\n") +} + +func (g *ComponentCodegen) renderDefineComponent(builder *strings.Builder, componentName, inputTypeName, outputTypeName string) { + method := strings.TrimSpace(g.Component.Method) + if method == "" { + method = "GET" + } + uri := strings.TrimSpace(g.Component.URI) + if uri == "" { + uri = "/" + } + connectorRef := strings.TrimSpace(g.rootConnectorRef()) + pathVar := componentName + "PathURI" + builder.WriteString(fmt.Sprintf("var %s = %q\n\n", pathVar, uri)) + builder.WriteString(fmt.Sprintf("func Define%sComponent(ctx context.Context, srv *datly.Service) error {\n", componentName)) + builder.WriteString("\taComponent, err := repository.NewComponent(\n") + builder.WriteString(fmt.Sprintf("\t\tcontract.NewPath(%q, %s),\n", method, pathVar)) + builder.WriteString("\t\trepository.WithResource(srv.Resource()),\n") + builder.WriteString("\t\trepository.WithContract(\n") + if g.WithEmbed { + builder.WriteString(fmt.Sprintf("\t\t\treflect.TypeOf(%s{}),\n", inputTypeName)) + builder.WriteString(fmt.Sprintf("\t\t\treflect.TypeOf(%s{}), &%sFS", outputTypeName, componentName)) + } else { + builder.WriteString(fmt.Sprintf("\t\t\treflect.TypeOf(%s{}),\n", inputTypeName)) + builder.WriteString(fmt.Sprintf("\t\t\treflect.TypeOf(%s{}), nil", outputTypeName)) + } + if connectorRef != "" { + builder.WriteString(fmt.Sprintf(`, view.WithConnectorRef(%q)`, connectorRef)) + } + builder.WriteString("))\n\n") + builder.WriteString("\tif err != nil {\n") + builder.WriteString(fmt.Sprintf("\t\treturn fmt.Errorf(\"failed to create %s component: %%w\", err)\n", componentName)) + builder.WriteString("\t}\n") + builder.WriteString("\tif err := srv.AddComponent(ctx, aComponent); err != nil {\n") + builder.WriteString(fmt.Sprintf("\t\treturn fmt.Errorf(\"failed to add %s component: %%w\", err)\n", componentName)) + builder.WriteString("\t}\n") + builder.WriteString("\treturn nil\n") + builder.WriteString("}\n\n") +} + +func (g *ComponentCodegen) rootConnectorRef() string { + if g.Resource == nil { + return "" + } + root := strings.TrimSpace(g.Component.RootView) + for _, aView := range g.Resource.Views { + if aView == nil || aView.Connector == nil { + continue + } + if root != "" && aView.Name == root { + if aView.Connector.Ref != "" { + return aView.Connector.Ref + } + return aView.Connector.Name + } + } + for _, aView := range g.Resource.Views { + if aView != nil && aView.Connector != nil { + if aView.Connector.Ref != "" { + return aView.Connector.Ref + } + return aView.Connector.Name + } + } + return "" +} + +func (g *ComponentCodegen) rootMarshaller() string { + if g == nil || g.Component == nil { + return "" + } + for _, route := range g.Component.ComponentRoutes { + if route == nil { + continue + } + if marshaller := strings.TrimSpace(route.Marshaller); marshaller != "" { + return marshaller + } + } + return "" +} + +func (g *ComponentCodegen) rootHandlerRef() string { + if g == nil || g.Component == nil { + return "" + } + for _, route := range g.Component.ComponentRoutes { + if route == nil { + continue + } + if handler := strings.TrimSpace(route.Handler); handler != "" { + return handler + } + } + return "" +} + func structFieldsSource(rType reflect.Type) string { if rType == nil { return "" @@ -399,7 +2838,8 @@ func structFieldsSource(rType reflect.Type) string { if !f.IsExported() { continue } - b.WriteString("\t" + f.Name + " " + f.Type.String()) + typeExpr := sourceFieldTypeExpr(f) + b.WriteString("\t" + f.Name + " " + typeExpr) if f.Tag != "" { b.WriteString(" `" + string(f.Tag) + "`") } @@ -408,6 +2848,35 @@ func structFieldsSource(rType reflect.Type) string { return b.String() } +func sourceFieldTypeExpr(field reflect.StructField) string { + typeName := strings.TrimSpace(field.Tag.Get("typeName")) + if typeName == "" { + return field.Type.String() + } + return rewriteFieldTypeExpr(field.Type, typeName) +} + +func rewriteFieldTypeExpr(rType reflect.Type, typeName string) string { + if rType == nil { + return typeName + } + switch rType.Kind() { + case reflect.Ptr: + return "*" + rewriteFieldTypeExpr(rType.Elem(), typeName) + case reflect.Slice: + return "[]" + rewriteFieldTypeExpr(rType.Elem(), typeName) + case reflect.Array: + return fmt.Sprintf("[%d]%s", rType.Len(), rewriteFieldTypeExpr(rType.Elem(), typeName)) + case reflect.Map: + return "map[" + rType.Key().String() + "]" + rewriteFieldTypeExpr(rType.Elem(), typeName) + case reflect.Struct: + if rType.Name() == "" { + return typeName + } + } + return rType.String() +} + func resourceToCodegenDoc(resource *view.Resource, typeCtx *typectx.Context) *shape.Document { root := map[string]any{} var views []any diff --git a/repository/shape/xgen/codegen_contract_parity_test.go b/repository/shape/xgen/codegen_contract_parity_test.go new file mode 100644 index 000000000..7beec9585 --- /dev/null +++ b/repository/shape/xgen/codegen_contract_parity_test.go @@ -0,0 +1,134 @@ +package xgen + +import ( + "os" + "path/filepath" + "strings" + "testing" + + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + shapeload "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" +) + +func TestComponentCodegen_GeneratesDefineComponentParitySnippet(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "pkg", "dev", "wrapper") + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/dev/vendors/{vendorID}", + RootView: "Wrapper", + } + resource := &view.Resource{ + Views: []*view.View{ + { + Name: "Wrapper", + Connector: &view.Connector{Connection: view.Connection{DBConfig: view.DBConfig{Name: "dev"}}}, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + }, + }, + }, + } + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "wrapper", + PackagePath: "github.com/acme/project/pkg/dev/wrapper", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + expectContains(t, generated, `var WrapperPathURI = "/v1/api/dev/vendors/{vendorID}"`) + expectContains(t, generated, `type WrapperRouter struct {`) + expectContains(t, generated, `Wrapper xdatly.Component[WrapperInput, WrapperOutput] `+"`"+`component:",path=/v1/api/dev/vendors/{vendorID},method=GET,connector=dev,view=WrapperView"`+"`") + expectContains(t, generated, `func DefineWrapperComponent(ctx context.Context, srv *datly.Service) error {`) + expectContains(t, generated, `contract.NewPath("GET", WrapperPathURI)`) + expectContains(t, generated, `repository.WithResource(srv.Resource())`) + expectContains(t, generated, `repository.WithContract(`) + expectContains(t, generated, `reflect.TypeOf(WrapperInput{})`) + expectContains(t, generated, `reflect.TypeOf(WrapperOutput{}), &WrapperFS, view.WithConnectorRef("dev"))`) +} + +func expectContains(t *testing.T, actual string, fragment string) { + t.Helper() + if !strings.Contains(actual, fragment) { + t.Fatalf("expected generated source to contain %q\nsource:\n%s", fragment, actual) + } +} + +func TestComponentCodegen_GeneratesSeparateRouterFile(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "pkg", "dev", "vendor") + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/dev/vendors", + RootView: "Vendor", + Directives: &dqlshape.Directives{ + RouterDest: "vendor_router.go", + }, + } + resource := &view.Resource{ + Views: []*view.View{ + { + Name: "Vendor", + Connector: &view.Connector{Connection: view.Connection{DBConfig: view.DBConfig{Name: "dev"}}}, + Columns: []*view.Column{{Name: "ID", DataType: "int"}}, + }, + }, + } + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "vendor", + PackagePath: "github.com/acme/project/pkg/dev/vendor", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + if filepath.Base(result.RouterFilePath) != "vendor_router.go" { + t.Fatalf("expected router file vendor_router.go, got %s", result.RouterFilePath) + } + if len(result.GeneratedFiles) != 2 { + t.Fatalf("expected 2 generated files, got %v", result.GeneratedFiles) + } + routerData, err := os.ReadFile(result.RouterFilePath) + if err != nil { + t.Fatalf("read router file: %v", err) + } + routerSource := string(routerData) + expectContains(t, routerSource, `type VendorRouter struct {`) + expectContains(t, routerSource, `Vendor xdatly.Component[VendorInput, VendorOutput] `+"`"+`component:",path=/v1/api/dev/vendors,method=GET,connector=dev,view=VendorView"`+"`") + outputData, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read primary file: %v", err) + } + if strings.Contains(string(outputData), "type VendorRouter struct") { + t.Fatalf("expected router declaration to be split out of primary output file:\n%s", string(outputData)) + } +} diff --git a/repository/shape/xgen/codegen_imports_test.go b/repository/shape/xgen/codegen_imports_test.go new file mode 100644 index 000000000..648e0dc8e --- /dev/null +++ b/repository/shape/xgen/codegen_imports_test.go @@ -0,0 +1,20 @@ +package xgen + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestComponentCodegen_buildImports_ChecksumPathDefault(t *testing.T) { + g := &ComponentCodegen{} + imports := g.buildImports(false, false) + assert.Contains(t, imports, "github.com/viant/xdatly/types/custom/dependency/checksum") + assert.NotContains(t, imports, "github.com/viant/xdatly/types/custom/checksum") +} + +func TestComponentCodegen_buildImports_ChecksumPathFromPackagePath(t *testing.T) { + g := &ComponentCodegen{PackagePath: "github.com/acme/project/pkg/dev/vendor"} + imports := g.buildImports(false, false) + assert.Contains(t, imports, "github.com/acme/project/pkg/dependency/checksum") +} diff --git a/repository/shape/xgen/codegen_input_view_test.go b/repository/shape/xgen/codegen_input_view_test.go new file mode 100644 index 000000000..fd1fe30f4 --- /dev/null +++ b/repository/shape/xgen/codegen_input_view_test.go @@ -0,0 +1,768 @@ +package xgen + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + shapeload "github.com/viant/datly/repository/shape/load" + shapeplan "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/sqlx/types" +) + +func TestComponentCodegen_ViewInput_UsesResolvedViewType(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "update") + + component := &shapeload.Component{ + Method: "POST", + URI: "/v1/api/shape/dev/auth/products/", + RootView: "ProductUpdate", + Input: []*shapeplan.State{ + {Parameter: state.Parameter{Name: "Jwt", In: state.NewHeaderLocation("Authorization"), Schema: &state.Schema{DataType: "string"}}}, + {Parameter: state.Parameter{Name: "Ids", In: state.NewBodyLocation("Ids"), Schema: &state.Schema{DataType: "[]int"}}}, + {Parameter: state.Parameter{Name: "Records", In: state.NewViewLocation("Records"), Schema: nil}}, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{ + Name: "ProductUpdate", + Mode: view.ModeExec, + }, + &view.View{ + Name: "Records", + Schema: &state.Schema{ + Name: "RecordsView", + DataType: "*RecordsView", + Cardinality: state.Many, + }, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "STATUS", DataType: "int", Nullable: true}, + }, + }, + ) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "update", + PackagePath: "github.com/acme/project/shape/dev/vendor/update", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + if strings.Contains(generated, "Records []interface {}") { + t.Fatalf("expected typed Records input field, got interface slice:\n%s", generated) + } + if !strings.Contains(generated, "Records []") || !strings.Contains(generated, "RecordsView") { + t.Fatalf("expected Records view input field to reference RecordsView:\n%s", generated) + } + if !strings.Contains(generated, `Status *int `+"`"+`sqlx:"STATUS" velty:"names=STATUS|Status"`+"`") { + t.Fatalf("expected exec view input helper type to retain velty aliases:\n%s", generated) + } +} + +func TestComponentCodegen_InputSynthesizesRoutePathParams(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "team", "delete") + + component := &shapeload.Component{ + Method: "DELETE", + URI: "/v1/api/shape/dev/team/{teamID}", + RootView: "Team", + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "Team", + Mode: view.ModeExec, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + }, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "delete", + PackagePath: "github.com/acme/project/shape/dev/team/delete", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + if !strings.Contains(generated, "TeamID string") { + t.Fatalf("expected implicit route path parameter in generated input:\n%s", generated) + } + if !strings.Contains(generated, `parameter:"teamID,kind=path,in=teamID"`) { + t.Fatalf("expected TeamID path parameter tag in generated input:\n%s", generated) + } + if !strings.Contains(generated, `velty:"names=TeamID|teamID"`) { + t.Fatalf("expected TeamID path parameter velty aliases in generated input:\n%s", generated) + } +} + +func TestComponentCodegen_ExportsLowercaseInputFieldNames(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "env") + + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/vendors-env/", + RootView: "Vendor", + Input: []*shapeplan.State{ + {Parameter: state.Parameter{Name: "vendorIDs", In: state.NewQueryLocation("vendorIDs"), Schema: &state.Schema{DataType: "[]int", Cardinality: state.Many}}}, + {Parameter: state.Parameter{Name: "Vendor", In: state.NewConstLocation("Vendor"), Value: "VENDOR", Tag: `internal:"true"`, Schema: &state.Schema{DataType: "string", Cardinality: state.One}}}, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "Vendor", + Mode: view.ModeQuery, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + }, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "env", + PackagePath: "github.com/acme/project/shape/dev/vendor/env", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + if !strings.Contains(generated, "VendorIDs ") { + t.Fatalf("expected exported generated field for lowercase query input:\n%s", generated) + } + if !strings.Contains(generated, `parameter:"vendorIDs,kind=query,in=vendorIDs"`) { + t.Fatalf("expected original query parameter name to be preserved in tag:\n%s", generated) + } +} + +func TestComponentCodegen_ReadView_OmitsVeltyTags(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "list") + + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/vendors/", + RootView: "Vendor", + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "Vendor", + Mode: view.ModeQuery, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "NAME", DataType: "string", Nullable: true}, + }, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "list", + PackagePath: "github.com/acme/project/shape/dev/vendor/list", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + if strings.Contains(generated, `velty:"names=ID|Id"`) || strings.Contains(generated, `velty:"names=NAME|Name"`) { + t.Fatalf("expected read view fields to omit velty tags:\n%s", generated) + } +} + +func TestComponentCodegen_ReadInput_OmitsVeltyTags(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "list") + + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/vendors/", + RootView: "Vendor", + Input: []*shapeplan.State{ + {Parameter: state.Parameter{Name: "VendorName", In: state.NewFormLocation("name"), Schema: &state.Schema{DataType: "string"}}}, + {Parameter: state.Parameter{Name: "Fields", In: state.NewQueryLocation("fields"), Schema: &state.Schema{DataType: "[]string", Cardinality: state.Many}}}, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "Vendor", + Mode: view.ModeQuery, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + }, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "list", + PackagePath: "github.com/acme/project/shape/dev/vendor/list", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + if strings.Contains(generated, `velty:"names=`) { + t.Fatalf("expected read input fields to omit velty tags:\n%s", generated) + } +} + +func TestComponentCodegen_HandlerExec_OmitsVeltyTags(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "auth") + + component := &shapeload.Component{ + Method: "POST", + URI: "/v1/api/shape/dev/auth/vendor", + RootView: "Auth", + ComponentRoutes: []*shapeplan.ComponentRoute{{ + Name: "Auth", + RoutePath: "/v1/api/shape/dev/auth/vendor", + Method: "POST", + Handler: "github.com/acme/project/shape/dev/vendor/auth.Handler", + }}, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "Auth", + Mode: view.ModeHandler, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "NAME", DataType: "string", Nullable: true}, + }, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "auth", + PackagePath: "github.com/acme/project/shape/dev/vendor/auth", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + if strings.Contains(generated, `velty:"names=ID|Id"`) || strings.Contains(generated, `velty:"names=NAME|Name"`) { + t.Fatalf("expected handler-generated fields to omit velty tags:\n%s", generated) + } + if !strings.Contains(generated, `handler=github.com/acme/project/shape/dev/vendor/auth.Handler`) { + t.Fatalf("expected generated router tag to include handler reference:\n%s", generated) + } +} + +func TestComponentCodegen_CodecBackedInput_UsesCodecResultType(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "user_acl") + + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/auth/user-acl", + RootView: "UserAcl", + Input: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Jwt", + In: state.NewHeaderLocation("Authorization"), + Schema: &state.Schema{DataType: "string", Cardinality: state.One}, + Output: &state.Codec{Name: "JwtClaim"}, + }, + }, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "UserAcl", + Mode: view.ModeQuery, + Columns: []*view.Column{ + {Name: "UserID", DataType: "int"}, + }, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "user_acl", + PackagePath: "github.com/acme/project/shape/dev/vendor/user_acl", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + if !strings.Contains(generated, `"github.com/viant/scy/auth/jwt"`) { + t.Fatalf("expected generated input to import jwt claims package:\n%s", generated) + } + if strings.Contains(generated, `"github.com/golang-jwt/jwt/v5"`) { + t.Fatalf("expected generated input to avoid nested jwt dependency import drift:\n%s", generated) + } + if !strings.Contains(generated, "Jwt *jwt.Claims") { + t.Fatalf("expected codec-backed Jwt field to use codec result type:\n%s", generated) + } + if !strings.Contains(generated, `dataType=string`) || + !strings.Contains(generated, `codec:"JwtClaim"`) { + t.Fatalf("expected Jwt field tag to preserve raw datatype and codec metadata:\n%s", generated) + } +} + +func TestComponentCodegen_ViewInput_OverridesStaleInlineSchemaType(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "team", "user_team") + + staleFieldType := reflect.StructOf([]reflect.StructField{ + {Name: "Id", Type: reflect.TypeOf(""), Tag: `json:"id,omitempty"`}, + {Name: "TeamMembers", Type: reflect.TypeOf(""), Tag: `json:"teamMembers,omitempty"`}, + {Name: "Name", Type: reflect.TypeOf(""), Tag: `json:"name,omitempty"`}, + }) + + component := &shapeload.Component{ + Method: "PUT", + URI: "/v1/api/shape/dev/teams", + RootView: "UserTeam", + Input: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "TeamStats", + In: state.NewViewLocation("TeamStats"), + Schema: state.NewSchema(reflect.SliceOf(staleFieldType)), + }, + }, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{ + Name: "UserTeam", + Mode: view.ModeExec, + }, + &view.View{ + Name: "TeamStats", + Schema: &state.Schema{ + Name: "TeamStatsView", + DataType: "*TeamStatsView", + Cardinality: state.Many, + }, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "TEAM_MEMBERS", DataType: "int"}, + {Name: "NAME", DataType: "string", Nullable: true}, + }, + }, + ) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "user_team", + PackagePath: "github.com/acme/project/shape/dev/team/user_team", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + if strings.Contains(generated, "TeamStats []struct") { + t.Fatalf("expected TeamStats view input field to use resolved TeamStatsView, got anonymous struct:\n%s", generated) + } + if !strings.Contains(generated, "TeamStats []") || !strings.Contains(generated, "TeamStatsView") { + t.Fatalf("expected TeamStats view input field to reference TeamStatsView:\n%s", generated) + } +} + +func TestComponentCodegen_ViewInput_ResolvesSnakeCaseViewName(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "team", "user_team") + + staleFieldType := reflect.StructOf([]reflect.StructField{ + {Name: "Id", Type: reflect.TypeOf(""), Tag: `json:"id,omitempty"`}, + }) + + component := &shapeload.Component{ + Method: "PUT", + URI: "/v1/api/shape/dev/teams", + RootView: "UserTeam", + Input: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "TeamStats", + In: state.NewViewLocation("TeamStats"), + Schema: state.NewSchema(reflect.SliceOf(staleFieldType)), + }, + }, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{Name: "UserTeam", Mode: view.ModeExec}, + &view.View{ + Name: "team_stats", + Schema: &state.Schema{ + Name: "TeamStatsView", + DataType: "*TeamStatsView", + Cardinality: state.Many, + }, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + }, + }, + ) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "user_team", + PackagePath: "github.com/acme/project/shape/dev/team/user_team", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + if strings.Contains(generated, "UserTeamTeamStatsView") { + t.Fatalf("expected snake_case resource view to resolve to TeamStatsView, got stale nested helper:\n%s", generated) + } + if !strings.Contains(generated, "TeamStats []*TeamStatsView") { + t.Fatalf("expected TeamStats view input field to reference TeamStatsView:\n%s", generated) + } +} + +func TestComponentCodegen_ExecWithoutExplicitOutput_DoesNotSynthesizeReaderData(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "update") + + component := &shapeload.Component{ + Method: "POST", + URI: "/v1/api/shape/dev/auth/products/", + RootView: "ProductUpdate", + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "ProductUpdate", + Mode: view.ModeExec, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "update", + PackagePath: "github.com/acme/project/shape/dev/vendor/update", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + if strings.Contains(generated, "parameter:\",kind=output,in=view\"") { + t.Fatalf("did not expect default reader output field for exec component:\n%s", generated) + } +} + +func TestComponentCodegen_ImportsNamedNonStructFieldTypes(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "user", "mysql_boolean") + + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/user-metadata", + RootView: "UserMetadata", + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "UserMetadata", + Schema: state.NewSchema(reflect.TypeOf([]struct { + ID int + IsEnabled *types.BitBool + }{})), + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "IS_ENABLED", DataType: "types.BitBool", Nullable: true}, + }, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "mysql_boolean", + PackagePath: "github.com/acme/project/shape/dev/user/mysql_boolean", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + if !strings.Contains(generated, `"github.com/viant/sqlx/types"`) { + t.Fatalf("expected generated source to import github.com/viant/sqlx/types:\n%s", generated) + } +} + +func TestComponentCodegen_EmitsNamedInputHasHelper(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "user", "mysql_boolean") + + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/user-metadata", + RootView: "UserMetadata", + Input: []*shapeplan.State{ + {Parameter: state.Parameter{Name: "Fields", In: state.NewQueryLocation("fields"), Schema: &state.Schema{DataType: "[]string"}}}, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "UserMetadata", + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "mysql_boolean", + PackagePath: "github.com/acme/project/shape/dev/user/mysql_boolean", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + if !strings.Contains(generated, "type UserMetadataInputHas struct") { + t.Fatalf("expected named UserMetadataInputHas helper declaration:\n%s", generated) + } +} + +func TestComponentCodegen_ExecWithoutStatusDoesNotImportResponse(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "team", "delete") + + component := &shapeload.Component{ + Method: "DELETE", + URI: "/v1/api/shape/dev/team/{teamID}", + RootView: "Team", + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "Team", + Mode: view.ModeExec, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "delete", + PackagePath: "github.com/acme/project/shape/dev/team/delete", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + if strings.Contains(generated, `"github.com/viant/xdatly/handler/response"`) { + t.Fatalf("did not expect response import for empty exec output:\n%s", generated) + } +} diff --git a/repository/shape/xgen/codegen_mutable_body_test.go b/repository/shape/xgen/codegen_mutable_body_test.go new file mode 100644 index 000000000..781c14114 --- /dev/null +++ b/repository/shape/xgen/codegen_mutable_body_test.go @@ -0,0 +1,189 @@ +package xgen + +import ( + "reflect" + "strings" + "testing" + + shapeload "github.com/viant/datly/repository/shape/load" + shapeplan "github.com/viant/datly/repository/shape/plan" + shapeast "github.com/viant/datly/repository/shape/velty/ast" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +type mutableBodyFoos struct { + Id *int + Name *string + FoosPerformance []*mutableBodyFoosPerformance `view:",table=FOOS_PERFORMANCE" on:"Id:ID=FooId:FOO_ID"` +} + +type mutableBodyFoosPerformance struct { + Id *int + FooId *int + Name *string +} + +func TestComponentCodegen_BuildMutableVeltyBlock_PatchOne(t *testing.T) { + codegen, inputType, support := newMutableBodyFixture("PATCH", false) + actual := renderMutableBlock(t, codegen, inputType, support) + for _, fragment := range []string{ + `$sequencer.Allocate("FOOS", $Foos, "Id")`, + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `#if($Foos)`, + `#if($CurFoosById.HasKey($Foos.Id) == true)`, + `$sql.Update($Foos, "FOOS");`, + `$sql.Insert($Foos, "FOOS");`, + } { + if !strings.Contains(actual, fragment) { + t.Fatalf("expected fragment %q in generated body:\n%s", fragment, actual) + } + } +} + +func TestComponentCodegen_BuildMutableVeltyBlock_PatchMany(t *testing.T) { + codegen, inputType, support := newMutableBodyFixture("PATCH", true) + actual := renderMutableBlock(t, codegen, inputType, support) + for _, fragment := range []string{ + `$sequencer.Allocate("FOOS", $Foos, "Id")`, + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `#foreach($RecFoos in $Foos)`, + `#if($CurFoosById.HasKey($RecFoos.Id) == true)`, + `$sql.Update($RecFoos, "FOOS");`, + `$sql.Insert($RecFoos, "FOOS");`, + } { + if !strings.Contains(actual, fragment) { + t.Fatalf("expected fragment %q in generated body:\n%s", fragment, actual) + } + } +} + +func TestComponentCodegen_BuildMutableVeltyBlock_PutOne(t *testing.T) { + codegen, inputType, support := newMutableBodyFixture("PUT", false) + actual := renderMutableBlock(t, codegen, inputType, support) + if strings.Contains(actual, `$sequencer.Allocate(`) { + t.Fatalf("did not expect sequence allocation in PUT body:\n%s", actual) + } + if strings.Contains(actual, `$sql.Insert($Foos, "FOOS");`) { + t.Fatalf("did not expect insert branch in PUT body:\n%s", actual) + } + for _, fragment := range []string{ + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `#if($Foos)`, + `#if($CurFoosById.HasKey($Foos.Id) == true)`, + `$sql.Update($Foos, "FOOS");`, + } { + if !strings.Contains(actual, fragment) { + t.Fatalf("expected fragment %q in generated body:\n%s", fragment, actual) + } + } +} + +func TestComponentCodegen_BuildMutableVeltyBlock_PostMany(t *testing.T) { + codegen, inputType, support := newMutableBodyFixture("POST", true) + actual := renderMutableBlock(t, codegen, inputType, support) + if strings.Contains(actual, `HasKey`) || strings.Contains(actual, `$sql.Update(`) { + t.Fatalf("did not expect update logic in POST body:\n%s", actual) + } + for _, fragment := range []string{ + `$sequencer.Allocate("FOOS", $Foos, "Id")`, + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `#foreach($RecFoos in $Foos)`, + `$sql.Insert($RecFoos, "FOOS");`, + } { + if !strings.Contains(actual, fragment) { + t.Fatalf("expected fragment %q in generated body:\n%s", fragment, actual) + } + } +} + +func TestComponentCodegen_BuildMutableVeltyBlock_PatchManyMany(t *testing.T) { + codegen, inputType, support := newMutableBodyFixture("PATCH", true) + actual := renderMutableBlock(t, codegen, inputType, support) + for _, fragment := range []string{ + `$sequencer.Allocate("FOOS_PERFORMANCE", $Foos, "FoosPerformance/Id")`, + `#foreach($RecFoosPerformance in $RecFoos.FoosPerformance)`, + `#set($RecFoosPerformance.FooId = $RecFoos.Id)`, + `#if($CurFoosPerformanceById.HasKey($RecFoosPerformance.Id) == true)`, + `$sql.Update($RecFoosPerformance, "FOOS_PERFORMANCE");`, + `$sql.Insert($RecFoosPerformance, "FOOS_PERFORMANCE");`, + } { + if !strings.Contains(actual, fragment) { + t.Fatalf("expected fragment %q in generated body:\n%s", fragment, actual) + } + } +} + +func TestComponentCodegen_BuildMutableVeltyBlock_PutOneMany(t *testing.T) { + codegen, inputType, support := newMutableBodyFixture("PUT", false) + actual := renderMutableBlock(t, codegen, inputType, support) + if strings.Contains(actual, `$sql.Insert($RecFoosPerformance, "FOOS_PERFORMANCE");`) { + t.Fatalf("did not expect child insert branch in PUT body:\n%s", actual) + } + for _, fragment := range []string{ + `#foreach($RecFoosPerformance in $Foos.FoosPerformance)`, + `#set($RecFoosPerformance.FooId = $Foos.Id)`, + `#if($CurFoosPerformanceById.HasKey($RecFoosPerformance.Id) == true)`, + `$sql.Update($RecFoosPerformance, "FOOS_PERFORMANCE");`, + } { + if !strings.Contains(actual, fragment) { + t.Fatalf("expected fragment %q in generated body:\n%s", fragment, actual) + } + } +} + +func newMutableBodyFixture(method string, many bool) (*ComponentCodegen, reflect.Type, *mutableComponentSupport) { + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{Name: "CurFoos", Template: &view.Template{Source: "SELECT * FROM FOOS WHERE ID IN (?)"}}, + &view.View{Name: "CurFoosPerformance", Template: &view.Template{Source: "SELECT * FROM FOOS_PERFORMANCE WHERE ID IN (?)"}}, + ) + codegen := &ComponentCodegen{Component: &shapeload.Component{ + Method: method, + Input: []*shapeplan.State{ + {Parameter: state.Parameter{Name: "CurFoos", In: state.NewViewLocation("CurFoos"), Tag: `view:"CurFoos" sql:"uri=foos/cur_foos.sql"`}}, + {Parameter: state.Parameter{Name: "CurFoosPerformance", In: state.NewViewLocation("CurFoosPerformance"), Tag: `view:"CurFoosPerformance" sql:"uri=foos/cur_foos_performance.sql"`}}, + }, + }, Resource: resource} + bodyType := reflect.TypeOf(&mutableBodyFoos{}) + if many { + bodyType = reflect.TypeOf([]*mutableBodyFoos{}) + } + inputType := reflect.StructOf([]reflect.StructField{ + {Name: "Foos", Type: bodyType}, + {Name: "CurFoos", Type: reflect.TypeOf([]*mutableBodyFoos{})}, + }) + support := &mutableComponentSupport{ + BodyFieldName: "Foos", + Helpers: []mutableIndexHelper{ + { + ViewParamName: "CurFoos", + ViewFieldName: "CurFoos", + MapFieldName: "CurFoosById", + KeyFieldName: "Id", + ItemTypeExpr: "*xgen.mutableBodyFoos", + }, + { + ViewParamName: "CurFoosPerformance", + ViewFieldName: "CurFoosPerformance", + MapFieldName: "CurFoosPerformanceById", + KeyFieldName: "Id", + ItemTypeExpr: "*xgen.mutableBodyFoosPerformance", + }, + }, + } + return codegen, inputType, support +} + +func renderMutableBlock(t *testing.T, codegen *ComponentCodegen, inputType reflect.Type, support *mutableComponentSupport) string { + t.Helper() + block, err := codegen.buildMutableVeltyBlock(inputType, support) + if err != nil { + t.Fatalf("build mutable body: %v", err) + } + builder := shapeast.NewBuilder(shapeast.Options{Lang: shapeast.LangVelty}) + if err = block.Generate(builder); err != nil { + t.Fatalf("generate mutable body: %v", err) + } + return strings.TrimSpace(builder.String()) +} diff --git a/repository/shape/xgen/codegen_mutable_helpers_test.go b/repository/shape/xgen/codegen_mutable_helpers_test.go new file mode 100644 index 000000000..a044be72e --- /dev/null +++ b/repository/shape/xgen/codegen_mutable_helpers_test.go @@ -0,0 +1,507 @@ +package xgen + +import ( + "os" + "path/filepath" + "reflect" + "regexp" + "strings" + "testing" + + shapeload "github.com/viant/datly/repository/shape/load" + shapeplan "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/shared" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +type BasicFoos struct { + Id *int + Name *string +} + +type Foos struct { + Id *int + Name *string + FoosPerformance []*FoosPerformance `view:",table=FOOS_PERFORMANCE" on:"Id:ID=FooId:FOO_ID"` +} + +type FoosPerformance struct { + Id *int + FooId *int + Name *string +} + +func TestComponentCodegen_MutableComponent_GeneratesPatchHelpers(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "generate_patch_basic_one") + + component := &shapeload.Component{ + Method: "PATCH", + URI: "/v1/api/dev/basic/foos", + RootView: "Foos", + Input: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewBodyLocation(""), + Tag: `anonymous:"true"`, + Schema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.One}, + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoosId", + In: state.NewParameterLocation("Foos"), + Schema: state.NewSchema(reflect.TypeOf(&struct{ Values []int }{})), + Tag: `codec:"structql,uri=foos/cur_foos_id.sql"`, + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoos", + In: state.NewViewLocation("CurFoos"), + Tag: `view:"CurFoos" sql:"uri=foos/cur_foos.sql"`, + Schema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.Many}, + }, + }, + }, + Output: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewOutputLocation("body"), + Tag: `anonymous:"true"`, + Schema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.One}, + }, + }, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{ + Name: "Foos", + Mode: view.ModeExec, + Schema: &state.Schema{ + Name: "Foos", + DataType: "*Foos", + Cardinality: state.One, + }, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "NAME", DataType: "string", Nullable: true}, + }, + }, + &view.View{ + Name: "CurFoos", + Mode: view.ModeQuery, + Schema: &state.Schema{ + Name: "Foos", + DataType: "*Foos", + Cardinality: state.Many, + }, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "NAME", DataType: "string", Nullable: true}, + }, + }, + ) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "generate_patch_basic_one", + PackagePath: "github.com/acme/project/shape/dev/generate_patch_basic_one", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + + inputSource := mustReadCodegenFile(t, result.InputFilePath) + if !strings.Contains(inputSource, `CurFoosById map[int]*Foos`) { + t.Fatalf("expected generated input to include indexed helper map:\n%s", inputSource) + } + + initSource := mustReadCodegenFile(t, filepath.Join(packageDir, "input_init.go")) + if !strings.Contains(initSource, `i.CurFoosById = make(map[int]*Foos, len(i.CurFoos))`) { + t.Fatalf("expected generated init helper to allocate CurFoosById:\n%s", initSource) + } + if !strings.Contains(initSource, `if item.Id == nil {`) || !strings.Contains(initSource, `i.CurFoosById[*item.Id] = item`) { + t.Fatalf("expected generated init helper to populate CurFoosById:\n%s", initSource) + } + + validateSource := mustReadCodegenFile(t, filepath.Join(packageDir, "input_validate.go")) + if !strings.Contains(validateSource, `_, err := aValidator.Validate(ctx, value, append(options, validator.WithValidation(validation))...)`) { + t.Fatalf("expected generated validate helper to call validator service:\n%s", validateSource) + } + if !strings.Contains(validateSource, `case *Foos:`) || !strings.Contains(validateSource, `if actual.Id == nil {`) || !strings.Contains(validateSource, `_, ok := i.CurFoosById[*actual.Id]`) { + t.Fatalf("expected generated validate helper to use CurFoosById marker provider:\n%s", validateSource) + } + + outputSource := mustReadCodegenFile(t, result.OutputFilePath) + if !strings.Contains(outputSource, `response.Status `+"`"+`parameter:",kind=output,in=status" json:",omitempty"`+"`") { + t.Fatalf("expected mutable output to embed response status:\n%s", outputSource) + } + if !strings.Contains(outputSource, `Violations validator.Violations `+"`"+`json:",omitempty"`+"`") { + t.Fatalf("expected mutable output to include validation violations:\n%s", outputSource) + } + if !strings.Contains(outputSource, `func (o *FoosOutput) setError(err error) {`) { + t.Fatalf("expected mutable output to include setError helper:\n%s", outputSource) + } + + if result.VeltyFilePath == "" { + t.Fatalf("expected mutable component to emit velty artifact path") + } + veltySource := mustReadCodegenFile(t, result.VeltyFilePath) + for _, fragment := range []string{ + `$sequencer.Allocate("FOOS", $Foos, "Id")`, + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `$sql.Update($Foos, "FOOS");`, + `$sql.Insert($Foos, "FOOS");`, + } { + if !strings.Contains(veltySource, fragment) { + t.Fatalf("expected generated velty body to include %q:\n%s", fragment, veltySource) + } + } + foundVelty := false + for _, generated := range result.GeneratedFiles { + if generated == result.VeltyFilePath { + foundVelty = true + break + } + } + if !foundVelty { + t.Fatalf("expected generated files to include velty artifact: %v", result.GeneratedFiles) + } + curIDsPath := filepath.Join(packageDir, "foos", "cur_foos_id.sql") + if _, err := os.Stat(curIDsPath); err != nil { + t.Fatalf("expected generated current-ids SQL at %s, files=%v", curIDsPath, result.GeneratedFiles) + } + curIDsSQL := mustReadCodegenFile(t, curIDsPath) + if !strings.Contains(curIDsSQL, `SELECT ARRAY_AGG(Id) AS Values`) { + t.Fatalf("expected generated current-ids SQL:\n%s", curIDsSQL) + } + curViewPath := filepath.Join(packageDir, "foos", "cur_foos.sql") + if _, err := os.Stat(curViewPath); err != nil { + t.Fatalf("expected generated current-view SQL at %s, files=%v", curViewPath, result.GeneratedFiles) + } + curViewSQL := mustReadCodegenFile(t, curViewPath) + if !strings.Contains(curViewSQL, `SELECT * FROM FOOS`) { + t.Fatalf("expected generated current-view SQL:\n%s", curViewSQL) + } +} + +func TestComponentCodegen_MutableComponent_DSQLParity_BasicOne(t *testing.T) { + result, packageDir := generateMutableFixture(t, mutableFixtureSpec{ + packageName: "generate_patch_basic_one", + method: "PATCH", + uri: "/v1/api/dev/basic/foos", + bodySchema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.One}, + outputSchema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.One}, + views: []*view.View{ + { + Name: "Foos", + Mode: view.ModeExec, + Connector: &view.Connector{Connection: view.Connection{DBConfig: view.DBConfig{Reference: shared.Reference{Ref: "dev"}}}}, + Schema: func() *state.Schema { + s := state.NewSchema(reflect.TypeOf(&BasicFoos{})) + s.Name, s.DataType, s.Cardinality = "Foos", "*Foos", state.One + return s + }(), + Columns: []*view.Column{{Name: "ID", DataType: "int"}, {Name: "NAME", DataType: "string", Nullable: true}}, + }, + { + Name: "CurFoos", + Mode: view.ModeQuery, + Template: &view.Template{Source: "SELECT * FROM FOOS\nWHERE $criteria.In(\"ID\", $CurFoosId.Values)"}, + Schema: func() *state.Schema { + s := state.NewSchema(reflect.TypeOf(&BasicFoos{})) + s.Name, s.DataType, s.Cardinality = "Foos", "*Foos", state.Many + return s + }(), + Columns: []*view.Column{{Name: "ID", DataType: "int"}, {Name: "NAME", DataType: "string", Nullable: true}}, + }, + }, + extraInput: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "CurFoosId", + In: state.NewParameterLocation("Foos"), + Schema: state.NewSchema(reflect.TypeOf(&struct{ Values []int }{})), + Tag: `codec:"structql,uri=foos/cur_foos_id.sql"`, + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoos", + In: state.NewViewLocation("CurFoos"), + Tag: `view:"CurFoos" sql:"uri=foos/cur_foos.sql"`, + Schema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.Many}, + }, + }, + }, + }) + assertMutableDSQLParity(t, result.VeltyFilePath, "/Users/awitas/go/src/github.com/viant/datly/e2e/local/dql/generate_patch_basic_one/patch_basic_one.sql") + assertMutableSQLFileParity(t, filepath.Join(packageDir, "foos", "cur_foos_id.sql"), "/Users/awitas/go/src/github.com/viant/datly/e2e/local/pkg/dev/generate_patch_basic_one/foos/cur_foos_id.sql") + assertMutableSQLFileParity(t, filepath.Join(packageDir, "foos", "cur_foos.sql"), "/Users/awitas/go/src/github.com/viant/datly/e2e/local/pkg/dev/generate_patch_basic_one/foos/cur_foos.sql") +} + +func TestComponentCodegen_MutableComponent_DSQLParity_BasicMany(t *testing.T) { + result, packageDir := generateMutableFixture(t, mutableFixtureSpec{ + packageName: "generate_patch_basic_many", + method: "PATCH", + uri: "/v1/api/dev/basic/foos-many", + bodySchema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.Many}, + outputSchema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.Many}, + views: []*view.View{ + { + Name: "Foos", + Mode: view.ModeExec, + Connector: &view.Connector{Connection: view.Connection{DBConfig: view.DBConfig{Reference: shared.Reference{Ref: "dev"}}}}, + Schema: func() *state.Schema { + s := state.NewSchema(reflect.TypeOf(&BasicFoos{})) + s.Name, s.DataType, s.Cardinality = "Foos", "*Foos", state.Many + return s + }(), + Columns: []*view.Column{{Name: "ID", DataType: "int"}, {Name: "NAME", DataType: "string", Nullable: true}}, + }, + { + Name: "CurFoos", + Mode: view.ModeQuery, + Template: &view.Template{Source: "SELECT * FROM FOOS\nWHERE $criteria.In(\"ID\", $CurFoosId.Values)"}, + Schema: func() *state.Schema { + s := state.NewSchema(reflect.TypeOf(&BasicFoos{})) + s.Name, s.DataType, s.Cardinality = "Foos", "*Foos", state.Many + return s + }(), + Columns: []*view.Column{{Name: "ID", DataType: "int"}, {Name: "NAME", DataType: "string", Nullable: true}}, + }, + }, + extraInput: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "CurFoosId", + In: state.NewParameterLocation("Foos"), + Schema: state.NewSchema(reflect.TypeOf(&struct{ Values []int }{})), + Tag: `codec:"structql,uri=foos/cur_foos_id.sql"`, + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoos", + In: state.NewViewLocation("CurFoos"), + Tag: `view:"CurFoos" sql:"uri=foos/cur_foos.sql"`, + Schema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.Many}, + }, + }, + }, + }) + assertMutableDSQLParity(t, result.VeltyFilePath, "/Users/awitas/go/src/github.com/viant/datly/e2e/local/dql/generate_patch_basic_many/patch_basic_many.sql") + assertMutableSQLFileParity(t, filepath.Join(packageDir, "foos", "cur_foos_id.sql"), "/Users/awitas/go/src/github.com/viant/datly/e2e/local/pkg/dev/generate_patch_basic_many/foos/cur_foos_id.sql") + assertMutableSQLFileParity(t, filepath.Join(packageDir, "foos", "cur_foos.sql"), "/Users/awitas/go/src/github.com/viant/datly/e2e/local/pkg/dev/generate_patch_basic_many/foos/cur_foos.sql") +} + +func TestComponentCodegen_MutableComponent_DSQLParity_ManyMany(t *testing.T) { + result, packageDir := generateMutableFixture(t, mutableFixtureSpec{ + packageName: "generate_patch_many_many", + method: "PATCH", + uri: "/v1/api/dev/basic/foos-many-many", + bodySchema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.Many}, + outputSchema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.Many}, + views: []*view.View{ + { + Name: "Foos", + Mode: view.ModeExec, + Connector: &view.Connector{Connection: view.Connection{DBConfig: view.DBConfig{Reference: shared.Reference{Ref: "dev"}}}}, + Schema: func() *state.Schema { + s := state.NewSchema(reflect.TypeOf(&Foos{})) + s.Name, s.DataType, s.Cardinality = "Foos", "*Foos", state.Many + return s + }(), + Columns: []*view.Column{{Name: "ID", DataType: "int"}, {Name: "NAME", DataType: "string", Nullable: true}}, + }, + { + Name: "CurFoos", + Mode: view.ModeQuery, + Template: &view.Template{Source: "SELECT * FROM FOOS\nWHERE $criteria.In(\"ID\", $CurFoosId.Values)"}, + Schema: func() *state.Schema { + s := state.NewSchema(reflect.TypeOf(&Foos{})) + s.Name, s.DataType, s.Cardinality = "Foos", "*Foos", state.Many + return s + }(), + Columns: []*view.Column{{Name: "ID", DataType: "int"}, {Name: "NAME", DataType: "string", Nullable: true}}, + }, + { + Name: "CurFoosPerformance", + Mode: view.ModeQuery, + Template: &view.Template{Source: "SELECT * FROM FOOS_PERFORMANCE\nWHERE $criteria.In(\"ID\", $CurFoosFoosPerformanceId.Values)"}, + Schema: func() *state.Schema { + s := state.NewSchema(reflect.TypeOf(&FoosPerformance{})) + s.Name, s.DataType, s.Cardinality = "FoosPerformance", "*FoosPerformance", state.Many + return s + }(), + Columns: []*view.Column{{Name: "ID", DataType: "int"}, {Name: "FOO_ID", DataType: "int"}, {Name: "NAME", DataType: "string", Nullable: true}}, + }, + }, + extraInput: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "CurFoosId", + In: state.NewParameterLocation("Foos"), + Schema: state.NewSchema(reflect.TypeOf(&struct{ Values []int }{})), + Tag: `codec:"structql,uri=foos/cur_foos_id.sql"`, + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoos", + In: state.NewViewLocation("CurFoos"), + Tag: `view:"CurFoos" sql:"uri=foos/cur_foos.sql"`, + Schema: &state.Schema{Name: "Foos", DataType: "*Foos", Cardinality: state.Many}, + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoosFoosPerformanceId", + In: state.NewParameterLocation("Foos"), + Schema: state.NewSchema(reflect.TypeOf(&struct{ Values []int }{})), + Tag: `codec:"structql,uri=foos/cur_foos_foos_performance_id.sql"`, + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoosPerformance", + In: state.NewViewLocation("CurFoosPerformance"), + Tag: `view:"CurFoosPerformance" sql:"uri=foos/cur_foos_performance.sql"`, + Schema: &state.Schema{Name: "FoosPerformance", DataType: "*FoosPerformance", Cardinality: state.Many}, + }, + }, + }, + }) + assertMutableDSQLParity(t, result.VeltyFilePath, "/Users/awitas/go/src/github.com/viant/datly/e2e/local/dql/generate_patch_many_many/patch_basic_many_many.sql") + assertMutableSQLFileParity(t, filepath.Join(packageDir, "foos", "cur_foos_id.sql"), "/Users/awitas/go/src/github.com/viant/datly/e2e/local/pkg/dev/generate_patch_basic_many/foos/cur_foos_id.sql") + assertMutableSQLFileParity(t, filepath.Join(packageDir, "foos", "cur_foos.sql"), "/Users/awitas/go/src/github.com/viant/datly/e2e/local/pkg/dev/generate_patch_basic_many/foos/cur_foos.sql") + if !strings.Contains(mustReadCodegenFile(t, filepath.Join(packageDir, "foos", "cur_foos_foos_performance_id.sql")), "SELECT ARRAY_AGG(Id) AS Values FROM `/FoosPerformance` LIMIT 1") { + t.Fatalf("expected nested current-ids helper SQL") + } + if !strings.Contains(mustReadCodegenFile(t, filepath.Join(packageDir, "foos", "cur_foos_performance.sql")), "SELECT * FROM FOOS_PERFORMANCE") { + t.Fatalf("expected nested current-view helper SQL") + } +} + +func mustReadCodegenFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(data) +} + +type mutableFixtureSpec struct { + packageName string + method string + uri string + bodySchema *state.Schema + outputSchema *state.Schema + views []*view.View + extraInput []*shapeplan.State +} + +func generateMutableFixture(t *testing.T, spec mutableFixtureSpec) (*ComponentCodegenResult, string) { + t.Helper() + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", spec.packageName) + component := &shapeload.Component{ + Method: spec.method, + URI: spec.uri, + RootView: "Foos", + Input: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewBodyLocation(""), + Tag: `anonymous:"true"`, + Schema: spec.bodySchema, + }, + }, + }, + Output: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewOutputLocation("body"), + Tag: `anonymous:"true"`, + Schema: spec.outputSchema, + }, + }, + }, + } + component.Input = append(component.Input, spec.extraInput...) + resource := view.EmptyResource() + resource.Views = append(resource.Views, spec.views...) + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: spec.packageName, + PackagePath: "github.com/acme/project/shape/dev/" + spec.packageName, + } + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + return result, packageDir +} + +func assertMutableDSQLParity(t *testing.T, actualPath, expectedPath string) { + t.Helper() + actual := normalizeMutableSQL(mustReadCodegenFile(t, actualPath)) + expected := normalizeMutableSQL(mustReadCodegenFile(t, expectedPath)) + if actual != expected { + t.Fatalf("mutable DSQL mismatch\nexpected:\n%s\n\nactual:\n%s", mustReadCodegenFile(t, expectedPath), mustReadCodegenFile(t, actualPath)) + } +} + +func assertMutableSQLFileParity(t *testing.T, actualPath, expectedPath string) { + t.Helper() + actual := normalizeMutableSQL(mustReadCodegenFile(t, actualPath)) + expected := normalizeMutableSQL(mustReadCodegenFile(t, expectedPath)) + if actual != expected { + t.Fatalf("mutable helper SQL mismatch for %s\nexpected:\n%s\n\nactual:\n%s", actualPath, mustReadCodegenFile(t, expectedPath), mustReadCodegenFile(t, actualPath)) + } +} + +func normalizeMutableSQL(value string) string { + value = strings.ReplaceAll(value, "\r\n", "\n") + lines := strings.Split(value, "\n") + out := make([]string, 0, len(lines)) + ws := regexp.MustCompile(`\s+`) + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + line = ws.ReplaceAllString(line, " ") + out = append(out, line) + } + return strings.Join(out, "\n") +} diff --git a/repository/shape/xgen/codegen_output_view_test.go b/repository/shape/xgen/codegen_output_view_test.go new file mode 100644 index 000000000..d93dbf53e --- /dev/null +++ b/repository/shape/xgen/codegen_output_view_test.go @@ -0,0 +1,79 @@ +package xgen + +import ( + "os" + "path/filepath" + "strings" + "testing" + + shapeload "github.com/viant/datly/repository/shape/load" + shapeplan "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +func TestComponentCodegen_PreservesExplicitOutputViewOneCardinality(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "user_acl") + + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/auth/user-acl", + RootView: "user_acl", + Output: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Data", + In: state.NewOutputLocation("view"), + Tag: `anonymous:"true"`, + Schema: &state.Schema{ + Cardinality: state.One, + }, + }, + }, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{ + Name: "user_acl", + Table: "USER_ACL", + Template: &view.Template{SourceURL: "user_acl/user_acl.sql"}, + Schema: &state.Schema{Name: "UserAclView", DataType: "*UserAclView", Cardinality: state.Many}, + Columns: []*view.Column{ + {Name: "UserID", DataType: "int"}, + }, + }) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "user_acl", + PackagePath: "github.com/acme/project/shape/dev/vendor/user_acl", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: false, + WithContract: false, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + if !strings.Contains(generated, "Data *UserAclView `parameter:\",kind=output,in=view\" view:\"user_acl\" sql:\"uri=user_acl/user_acl.sql\" anonymous:\"true\"`") { + t.Fatalf("expected one-cardinality output view to generate pointer field, got:\n%s", generated) + } + if strings.Contains(generated, "Data []*UserAclView") { + t.Fatalf("expected output view not to generate slice field, got:\n%s", generated) + } +} diff --git a/repository/shape/xgen/codegen_placeholder_view_test.go b/repository/shape/xgen/codegen_placeholder_view_test.go new file mode 100644 index 000000000..cf417a6f3 --- /dev/null +++ b/repository/shape/xgen/codegen_placeholder_view_test.go @@ -0,0 +1,83 @@ +package xgen + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +func TestRebuildResourceViewStructType_ReplacesPlaceholderColumnsPreservesRelations(t *testing.T) { + type placeholderProducts struct { + ID int `sqlx:"ID"` + } + type placeholderVendor struct { + Col1 string `sqlx:"name=col_1"` + Col2 string `sqlx:"name=col_2"` + Products []*placeholderProducts `view:",table=PRODUCT" sql:"uri=vendor/products.sql" sqlx:"-"` + } + + cols := []columnDescriptor{ + {name: "ID", dataType: "int", primaryKey: true}, + {name: "NAME", dataType: "string"}, + } + + rType := rebuildResourceViewStructType(reflect.TypeOf(placeholderVendor{}), cols, false) + require.NotNil(t, rType) + require.Equal(t, reflect.Struct, rType.Kind()) + + field, ok := rType.FieldByName("Id") + require.True(t, ok) + require.Equal(t, "ID", sqlxTagName(field.Tag.Get("sqlx"))) + + field, ok = rType.FieldByName("Name") + require.True(t, ok) + require.Equal(t, "NAME", sqlxTagName(field.Tag.Get("sqlx"))) + + _, ok = rType.FieldByName("Col1") + require.False(t, ok) + + field, ok = rType.FieldByName("Products") + require.True(t, ok) + require.Equal(t, `uri=vendor/products.sql`, field.Tag.Get("sql")) + require.Equal(t, ",table=PRODUCT", field.Tag.Get("view")) +} + +func TestComponentCodegen_UsesDiscoveredColumnsWhenRootTypeIsPlaceholder(t *testing.T) { + type placeholderProducts struct { + ID int `sqlx:"ID"` + } + type placeholderVendor struct { + Col1 string `sqlx:"name=col_1"` + Products []*placeholderProducts `view:",table=PRODUCT" sql:"uri=vendor/products.sql" sqlx:"-"` + } + + resource := view.EmptyResource() + resource.Views = view.Views{ + { + Name: "vendor", + Schema: &state.Schema{ + Name: "VendorView", + }, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "NAME", DataType: "string"}, + }, + }, + } + resource.Views[0].Schema.SetType(reflect.TypeOf([]placeholderVendor{})) + + codegen := &ComponentCodegen{Resource: resource} + rType := codegen.resourceViewStructType("vendor") + require.NotNil(t, rType) + _, ok := rType.FieldByName("Id") + require.True(t, ok) + _, ok = rType.FieldByName("Name") + require.True(t, ok) + _, ok = rType.FieldByName("Products") + require.True(t, ok) + _, ok = rType.FieldByName("Col1") + require.False(t, ok) +} diff --git a/repository/shape/xgen/codegen_relation_view_test.go b/repository/shape/xgen/codegen_relation_view_test.go new file mode 100644 index 000000000..4ee33efd1 --- /dev/null +++ b/repository/shape/xgen/codegen_relation_view_test.go @@ -0,0 +1,101 @@ +package xgen + +import ( + "os" + "path/filepath" + "strings" + "testing" + + shapeload "github.com/viant/datly/repository/shape/load" + shapeplan "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +func TestComponentCodegen_UsesMaterializedViewTypeForRelationHolders(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "details") + + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/vendors/{vendorID}", + RootView: "vendor", + Output: []*shapeplan.State{ + {Parameter: state.Parameter{Name: "Data", In: state.NewOutputLocation("view"), Schema: &state.Schema{Cardinality: state.Many}}}, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{ + Name: "vendor", + Table: "VENDOR", + Template: &view.Template{SourceURL: "wrapper/vendor.sql"}, + Schema: &state.Schema{Name: "VendorView", DataType: "*VendorView", Cardinality: state.Many}, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + }, + With: []*view.Relation{ + { + Holder: "Products", + Cardinality: state.Many, + On: view.Links{&view.Link{Field: "Id", Column: "ID"}}, + Of: &view.ReferenceView{ + View: view.View{ + Name: "products", + Table: "PRODUCT", + Template: &view.Template{SourceURL: "wrapper/products.sql"}, + Schema: &state.Schema{Name: "ProductsView", DataType: "*ProductsView", Cardinality: state.Many}, + }, + On: view.Links{&view.Link{Field: "VendorId", Column: "VENDOR_ID"}}, + }, + }, + }, + }, + &view.View{ + Name: "products", + Table: "PRODUCT", + Template: &view.Template{SourceURL: "wrapper/products.sql"}, + Schema: &state.Schema{Name: "ProductsView", DataType: "*ProductsView", Cardinality: state.Many}, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "VENDOR_ID", DataType: "*int", Tag: `internal:"true"`}, + }, + }, + ) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "details", + PackagePath: "github.com/acme/project/shape/dev/vendor/details", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: false, + WithContract: false, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + if !strings.Contains(generated, "Products []*ProductsView `view:\",table=PRODUCT\" on:\"Id:ID=VendorId:VENDOR_ID\" sql:\"uri=wrapper/products.sql\"`") { + t.Fatalf("expected generated VendorView to use named relation holder field, got:\n%s", generated) + } + if strings.Contains(generated, "*struct {") { + t.Fatalf("expected no anonymous relation structs, got:\n%s", generated) + } + if strings.Contains(generated, "table=(SELECT") { + t.Fatalf("expected no raw subquery text in relation view tag, got:\n%s", generated) + } +} diff --git a/repository/shape/xgen/codegen_typespec_test.go b/repository/shape/xgen/codegen_typespec_test.go new file mode 100644 index 000000000..41c7910ac --- /dev/null +++ b/repository/shape/xgen/codegen_typespec_test.go @@ -0,0 +1,87 @@ +package xgen + +import ( + "os" + "path/filepath" + "strings" + "testing" + + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + shapeload "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" +) + +func TestComponentCodegen_TypeSpecs_InputOutputAndDest(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "pkg", "dev", "vendor") + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/vendors/", + RootView: "vendor", + Directives: &dqlshape.Directives{ + Dest: "all.go", + RouterDest: "vendor_router.go", + }, + TypeSpecs: map[string]*shapeload.TypeSpec{ + "input": {Key: "input", Role: shapeload.TypeRoleInput, TypeName: "VendorReq"}, + "output": {Key: "output", Role: shapeload.TypeRoleOutput, TypeName: "VendorResp", Dest: "vendor_output.go"}, + "view:vendor": {Key: "view:vendor", Role: shapeload.TypeRoleView, Alias: "vendor", TypeName: "Vendor"}, + }, + } + resource := &view.Resource{ + Views: []*view.View{ + { + Name: "vendor", + Connector: &view.Connector{Connection: view.Connection{DBConfig: view.DBConfig{Name: "dev"}}}, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + }, + }, + }, + } + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "vendor", + PackagePath: "github.com/acme/project/pkg/dev/vendor", + } + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + if filepath.Base(result.FilePath) != "vendor_output.go" { + t.Fatalf("expected destination override vendor_output.go, got %s", filepath.Base(result.FilePath)) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + source := string(data) + expectContainsTypeSpec(t, source, "type VendorReq struct") + expectContainsTypeSpec(t, source, "type VendorResp struct") + expectContainsTypeSpec(t, source, "Data []*Vendor") + expectContainsTypeSpec(t, source, "reflect.TypeOf(VendorReq{})") + expectContainsTypeSpec(t, source, "reflect.TypeOf(VendorResp{})") + routerData, err := os.ReadFile(filepath.Join(packageDir, "vendor_router.go")) + if err != nil { + t.Fatalf("read generated router file: %v", err) + } + routerSource := string(routerData) + expectContainsTypeSpec(t, routerSource, "type VendorRouter struct") + expectContainsTypeSpec(t, routerSource, "Vendor xdatly.Component[VendorReq, VendorResp]") +} + +func expectContainsTypeSpec(t *testing.T, source string, fragment string) { + t.Helper() + if !strings.Contains(source, fragment) { + t.Fatalf("expected generated source to contain %q\nsource:\n%s", fragment, source) + } +} diff --git a/repository/shape/xgen/generator.go b/repository/shape/xgen/generator.go index bfeb2bd33..0d0d59110 100644 --- a/repository/shape/xgen/generator.go +++ b/repository/shape/xgen/generator.go @@ -12,6 +12,7 @@ import ( "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/typectx" "github.com/viant/datly/repository/shape/typectx/source" + "github.com/viant/datly/view" "github.com/viant/x" xreflectloader "github.com/viant/x/loader/xreflect" "github.com/viant/x/syntetic" @@ -53,12 +54,13 @@ func GenerateFromDQLShape(doc *shape.Document, cfg *Config) (*Result, error) { } typeNames := make([]string, 0, len(views)+len(routeTypes)) registered := map[string]bool{} + includeVelty := documentUsesVelty(doc) for _, view := range views { typeName := viewTypeName(cfg, view) if registered[typeName] { continue } - structType := buildStructType(view.columns) + structType := buildStructType(view.columns, includeVelty) if structType == nil { continue } @@ -88,7 +90,7 @@ func GenerateFromDQLShape(doc *shape.Document, cfg *Config) (*Result, error) { if typeName == "" || registered[typeName] { continue } - structType := buildStructType(ioType.fields) + structType := buildStructType(ioType.fields, includeVelty) if structType == nil { continue } @@ -627,7 +629,15 @@ func hasExplicitTypeOverride(field reflect.StructField) bool { return false } -func buildStructType(columns []columnDescriptor) reflect.Type { +func buildStructType(columns []columnDescriptor, includeVelty bool) reflect.Type { + fields := buildStructFields(columns, includeVelty) + if len(fields) == 0 { + return nil + } + return reflect.StructOf(fields) +} + +func buildStructFields(columns []columnDescriptor, includeVelty bool) []reflect.StructField { if len(columns) == 0 { return nil } @@ -660,13 +670,49 @@ func buildStructType(columns []columnDescriptor) reflect.Type { if isPK || isAutoInc { tag = fmt.Sprintf(`sqlx:"%s"`, sqlxTag) } + if includeVelty { + veltyNames := []string{column.name} + if fieldName != "" && fieldName != column.name { + veltyNames = append(veltyNames, fieldName) + } + veltyTag := fmt.Sprintf(`velty:"names=%s"`, strings.Join(veltyNames, "|")) + tag = tag + " " + veltyTag + } fields = append(fields, reflect.StructField{ Name: fieldName, Type: fieldType, Tag: reflect.StructTag(tag), }) } - return reflect.StructOf(fields) + return fields +} + +func documentUsesVelty(doc *shape.Document) bool { + if doc == nil || doc.Root == nil { + return false + } + var visit func(value any) bool + visit = func(value any) bool { + switch actual := value.(type) { + case map[string]any: + if mode := strings.TrimSpace(asString(actual["Mode"])); mode == string(view.ModeExec) { + return true + } + for _, item := range actual { + if visit(item) { + return true + } + } + case []any: + for _, item := range actual { + if visit(item) { + return true + } + } + } + return false + } + return visit(doc.Root) } // buildHasType creates a marker struct with bool fields for each column. diff --git a/repository/shape/xgen/generator_velty_tag_test.go b/repository/shape/xgen/generator_velty_tag_test.go new file mode 100644 index 000000000..2bf0afbfd --- /dev/null +++ b/repository/shape/xgen/generator_velty_tag_test.go @@ -0,0 +1,48 @@ +package xgen + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildStructType_AddsVeltyNamesFromSQLColumns(t *testing.T) { + rType := buildStructType([]columnDescriptor{ + {name: "IS_AUTH", dataType: "int"}, + }, true) + require.NotNil(t, rType) + if rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + field, ok := rType.FieldByName("IsAuth") + require.True(t, ok) + require.Equal(t, `names=IS_AUTH|IsAuth`, field.Tag.Get("velty")) + require.Equal(t, `IS_AUTH`, field.Tag.Get("sqlx")) +} + +func TestBuildStructType_DedupesVeltyNamesWhenGoFieldMatchesColumn(t *testing.T) { + rType := buildStructType([]columnDescriptor{ + {name: "UserID", dataType: "int"}, + }, true) + require.NotNil(t, rType) + if rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + field, ok := rType.FieldByName("UserID") + require.True(t, ok) + require.Equal(t, `names=UserID`, field.Tag.Get("velty")) +} + +func TestBuildStructType_OmitsVeltyWhenDisabled(t *testing.T) { + rType := buildStructType([]columnDescriptor{ + {name: "USER_ID", dataType: "int"}, + }, false) + require.NotNil(t, rType) + if rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + field := rType.Field(0) + require.Equal(t, "", field.Tag.Get("velty")) + require.Equal(t, `USER_ID`, field.Tag.Get("sqlx")) +} diff --git a/repository/shape/xgen/mutable_body.go b/repository/shape/xgen/mutable_body.go new file mode 100644 index 000000000..31b5085c8 --- /dev/null +++ b/repository/shape/xgen/mutable_body.go @@ -0,0 +1,927 @@ +package xgen + +import ( + "path/filepath" + "reflect" + "sort" + "strings" + + shapeast "github.com/viant/datly/repository/shape/velty/ast" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/tagly/format/text" +) + +func (g *ComponentCodegen) renderMutableVeltyBody(inputType reflect.Type) (string, bool, error) { + if inputType == nil { + var err error + inputType, err = g.mutableInputType() + if err != nil { + return "", false, err + } + } + support := g.mutableSupport(inputType) + if support == nil { + return "", false, nil + } + block, err := g.buildMutableVeltyBlock(inputType, support) + if err != nil { + return "", false, err + } + if block == nil { + return "", false, nil + } + builder := shapeast.NewBuilder(shapeast.Options{Lang: shapeast.LangVelty}) + if err = block.Generate(builder); err != nil { + return "", false, err + } + return strings.TrimSpace(builder.String()) + "\n", true, nil +} + +func (g *ComponentCodegen) renderMutableDSQL(inputType reflect.Type) (string, bool, error) { + body, ok, err := g.renderMutableVeltyBody(inputType) + if err != nil || !ok { + return "", ok, err + } + support := g.mutableSupport(inputType) + if support == nil { + return "", false, nil + } + var builder strings.Builder + builder.WriteString("/* ") + builder.WriteString(g.mutableRouteOptionJSON()) + builder.WriteString(" */\n\n\n") + if imports := g.mutableTypeImports(support, inputType); len(imports) > 0 { + builder.WriteString("import (\n") + for _, item := range imports { + builder.WriteString("\t") + builder.WriteString(strconvQuote(item)) + builder.WriteString("\n") + } + builder.WriteString("\t)\n\n\n") + } + builder.WriteString(g.mutableBodyDeclaration(inputType, support)) + for _, helper := range g.mutableIDHelpers(support) { + builder.WriteString(g.mutableIDsDeclaration(helper)) + } + for _, helper := range g.mutableViewHelpers(support) { + builder.WriteString(g.mutableViewDeclaration(helper)) + } + builder.WriteString(g.mutableOutputDeclaration(inputType, support)) + builder.WriteString("\n\n") + builder.WriteString(body) + return builder.String(), true, nil +} + +func (g *ComponentCodegen) mutableTypeImports(support *mutableComponentSupport, inputType reflect.Type) []string { + items := map[string]struct{}{} + add := func(typeName string) { + typeName = strings.TrimSpace(typeName) + if typeName == "" { + return + } + pkg := strings.TrimSpace(g.PackageName) + if pkg == "" && g.TypeContext != nil { + pkg = strings.TrimSpace(g.TypeContext.PackageName) + } + if pkg == "" { + return + } + items[pkg+"."+typeName] = struct{}{} + } + if bodyField, ok := inputType.FieldByName(support.BodyFieldName); ok { + if itemType, _ := mutableBodyItemType(bodyField.Type); itemType != nil { + typeName := strings.TrimSpace(support.BodyTypeName) + if typeName == "" { + typeName = itemType.Name() + } + add(typeName) + } + } + for _, helper := range support.Helpers { + typeName := strings.TrimSpace(helper.TypeName) + if typeName == "" && helper.ItemStruct != nil { + typeName = helper.ItemStruct.Name() + } + if typeName != "" { + add(typeName) + } + } + result := make([]string, 0, len(items)) + for item := range items { + result = append(result, item) + } + sort.Strings(result) + return result +} + +func (g *ComponentCodegen) mutableRouteOptionJSON() string { + connector := g.rootConnectorRef() + parts := []string{ + `"URI":"` + escapeJSON(strings.TrimSpace(g.Component.URI)) + `"`, + `"Method":"` + escapeJSON(strings.ToUpper(strings.TrimSpace(g.Component.Method))) + `"`, + } + if connector != "" { + parts = append(parts, `"Connector":"`+escapeJSON(connector)+`"`) + } + return "{" + strings.Join(parts, ",") + "}" +} + +func (g *ComponentCodegen) mutableBodyDeclaration(inputType reflect.Type, support *mutableComponentSupport) string { + bodyField, ok := inputType.FieldByName(support.BodyFieldName) + if !ok { + return "" + } + itemType, many := mutableBodyItemType(bodyField.Type) + if itemType == nil { + return "" + } + typeName := strings.TrimSpace(support.BodyTypeName) + if typeName == "" { + typeName = itemType.Name() + } + typeExpr := typeName + if many { + typeExpr = "[]" + typeName + } + return "#set($_ = $" + support.BodyFieldName + "<" + typeExpr + ">(body/).WithTag('anonymous:\"true\"').Required())\n" +} + +func (g *ComponentCodegen) mutableIDsDeclaration(helper mutableIndexHelper) string { + paramName := g.mutableIDsParamName(helper) + sqlText := g.mutableIDSQL(helper) + if paramName == "" || sqlText == "" { + return "" + } + return "\t#set($_ = $" + paramName + "(param/" + g.supportBodyFieldName(helper) + ") /*\n" + sqlText + "\n*/\n)\n" +} + +func (g *ComponentCodegen) mutableViewDeclaration(helper mutableIndexHelper) string { + typeName := strings.TrimSpace(helper.TypeName) + if typeName == "" && helper.ItemStruct != nil { + typeName = strings.TrimSpace(helper.ItemStruct.Name()) + } + viewType := "[]*" + typeName + if typeName == "" || helper.ViewFieldName == "" { + return "" + } + sqlText := g.mutableDeclarationViewSQL(helper) + if sqlText == "" { + return "" + } + return "\t#set($_ = $" + helper.ViewFieldName + "<" + viewType + ">(view/" + helper.ViewFieldName + ") /*\n" + sqlText + "\n*/\n)\n" +} + +func (g *ComponentCodegen) mutableOutputDeclaration(inputType reflect.Type, support *mutableComponentSupport) string { + bodyField, ok := inputType.FieldByName(support.BodyFieldName) + if !ok { + return "" + } + _, many := mutableBodyItemType(bodyField.Type) + typeExpr := "" + if many { + typeExpr = "[]" + } + typeName := strings.TrimSpace(support.BodyTypeName) + if typeName == "" { + if itemType, _ := mutableBodyItemType(bodyField.Type); itemType != nil { + typeName = itemType.Name() + } + } + tag := `anonymous:"true"` + if typeName != "" { + tag += ` typeName:"` + typeName + `"` + } + return "#set($_ = $" + support.BodyFieldName + "<" + typeExpr + ">(body/).WithTag('" + tag + "').Required().Output())\n" +} + +func (g *ComponentCodegen) mutableIDSQL(helper mutableIndexHelper) string { + key := strings.TrimSpace(helper.KeyFieldName) + if key == "" { + key = "Id" + } + path := "/" + if helper.RelationPath != "" { + path += helper.RelationPath + } + return "? SELECT ARRAY_AGG(" + key + ") AS Values FROM `" + path + "` LIMIT 1" +} + +func (g *ComponentCodegen) mutableViewSQL(helper mutableIndexHelper) string { + if g == nil || g.Resource == nil { + return g.mutableFallbackViewSQL(helper) + } + for _, aView := range g.Resource.Views { + if aView == nil || !strings.EqualFold(strings.TrimSpace(aView.Name), strings.TrimSpace(helper.ViewParamName)) { + continue + } + if aView.Template == nil { + return g.mutableFallbackViewSQL(helper) + } + sqlText := strings.TrimSpace(aView.Template.Source) + if sqlText != "" { + return sqlText + } + return g.mutableFallbackViewSQL(helper) + } + return g.mutableFallbackViewSQL(helper) +} + +func (g *ComponentCodegen) mutableDeclarationViewSQL(helper mutableIndexHelper) string { + sqlText := strings.TrimSpace(g.mutableViewSQL(helper)) + if sqlText == "" { + return "" + } + if strings.HasPrefix(sqlText, "?") { + return sqlText + } + return "? " + sqlText +} + +func (g *ComponentCodegen) mutableFallbackViewSQL(helper mutableIndexHelper) string { + tableName := "" + if helper.ItemStruct != nil && helper.ItemStruct.Name() != "" { + tableName = tableNameFromType(helper.ItemStruct.Name()) + } + if tableName == "" { + typeName := strings.TrimPrefix(strings.TrimPrefix(strings.TrimSpace(helper.ItemTypeExpr), "[]"), "*") + tableName = tableNameFromType(typeName) + } + if tableName == "" { + return "" + } + idParam := g.mutableIDsParamName(helper) + key := strings.TrimSpace(helper.KeyFieldName) + if key == "" { + key = "Id" + } + return "SELECT * FROM " + tableName + "\nWHERE $criteria.In(\"" + key + "\", $" + idParam + ".Values)" +} + +func (g *ComponentCodegen) supportBodyFieldName(helper mutableIndexHelper) string { + if g == nil { + return "" + } + inputType, err := g.mutableInputType() + if err != nil || inputType == nil { + return "" + } + if support := g.mutableSupport(inputType); support != nil { + return support.BodyFieldName + } + return "" +} + +func strconvQuote(s string) string { + return `"` + strings.ReplaceAll(s, `"`, `\"`) + `"` +} + +func escapeJSON(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `"`, `\"`) + return s +} + +func (g *ComponentCodegen) mutableIDHelpers(support *mutableComponentSupport) []mutableIndexHelper { + if support == nil || len(support.Helpers) == 0 { + return nil + } + ret := append([]mutableIndexHelper{}, support.Helpers...) + sort.SliceStable(ret, func(i, j int) bool { + leftDepth := mutableRelationDepth(ret[i].RelationPath) + rightDepth := mutableRelationDepth(ret[j].RelationPath) + if leftDepth != rightDepth { + return leftDepth < rightDepth + } + return ret[i].ViewFieldName < ret[j].ViewFieldName + }) + return ret +} + +func (g *ComponentCodegen) mutableViewHelpers(support *mutableComponentSupport) []mutableIndexHelper { + if support == nil || len(support.Helpers) == 0 { + return nil + } + ret := append([]mutableIndexHelper{}, support.Helpers...) + sort.SliceStable(ret, func(i, j int) bool { + leftDepth := mutableRelationDepth(ret[i].RelationPath) + rightDepth := mutableRelationDepth(ret[j].RelationPath) + if leftDepth != rightDepth { + return leftDepth > rightDepth + } + return ret[i].ViewFieldName > ret[j].ViewFieldName + }) + return ret +} + +func mutableRelationDepth(path string) int { + path = strings.Trim(path, "/") + if path == "" { + return 0 + } + return strings.Count(path, "/") + 1 +} + +type mutableGeneratedFile struct { + Path string + Content string +} + +func (g *ComponentCodegen) mutableHelperSQLFiles(support *mutableComponentSupport) []mutableGeneratedFile { + if g == nil || g.Component == nil || support == nil { + return nil + } + packageDir := strings.TrimSpace(g.PackageDir) + if packageDir == "" && g.TypeContext != nil { + packageDir = strings.TrimSpace(g.TypeContext.PackageDir) + } + if packageDir == "" { + return nil + } + var result []mutableGeneratedFile + helperByView := map[string]mutableIndexHelper{} + helperByID := map[string]mutableIndexHelper{} + for _, helper := range support.Helpers { + helperByView[strings.TrimSpace(helper.ViewFieldName)] = helper + helperByID[g.mutableIDsParamName(helper)] = helper + } + for _, input := range g.Component.Input { + if input == nil { + continue + } + switch { + case input.In != nil && input.In.Kind == state.KindView: + helper, ok := helperByView[strings.TrimSpace(input.Name)] + if !ok { + continue + } + rel := tagURIValue(input.Tag, "sql") + content := g.mutableViewSQL(helper) + if strings.TrimSpace(content) == "" { + continue + } + result = append(result, g.mutableGeneratedSQLFiles(packageDir, rel, g.mutableHelperViewRelPath(helper), content)...) + case input.In != nil && input.In.Kind == state.KindParam: + helper, ok := helperByID[strings.TrimSpace(input.Name)] + if !ok { + continue + } + rel := tagURIValue(input.Tag, "codec") + content := g.mutableIDSQL(helper) + if strings.TrimSpace(content) == "" { + continue + } + result = append(result, g.mutableGeneratedSQLFiles(packageDir, rel, g.mutableHelperIDsRelPath(helper), content)...) + } + } + for _, helper := range support.Helpers { + if content := g.mutableViewSQL(helper); strings.TrimSpace(content) != "" { + result = append(result, g.mutableGeneratedSQLFiles(packageDir, "", g.mutableHelperViewRelPath(helper), content)...) + } + if content := g.mutableIDSQL(helper); strings.TrimSpace(content) != "" { + result = append(result, g.mutableGeneratedSQLFiles(packageDir, "", g.mutableHelperIDsRelPath(helper), content)...) + } + } + return result +} + +func (g *ComponentCodegen) mutableGeneratedSQLFiles(packageDir, primaryRel, fallbackRel, content string) []mutableGeneratedFile { + seen := map[string]struct{}{} + var result []mutableGeneratedFile + appendFile := func(rel string) { + rel = strings.TrimSpace(rel) + if rel == "" { + return + } + abs := filepath.Join(packageDir, filepath.FromSlash(rel)) + if _, ok := seen[abs]; ok { + return + } + seen[abs] = struct{}{} + result = append(result, mutableGeneratedFile{Path: abs, Content: content}) + } + appendFile(primaryRel) + appendFile(fallbackRel) + return result +} + +func tagURIValue(tag, key string) string { + tag = strings.TrimSpace(tag) + if tag == "" { + return "" + } + needle := key + `:"` + start := strings.Index(tag, needle) + if start == -1 { + return "" + } + rest := tag[start+len(needle):] + end := strings.Index(rest, `"`) + if end == -1 { + return "" + } + value := rest[:end] + if idx := strings.Index(value, "uri="); idx >= 0 { + value = value[idx+4:] + if cut := strings.IndexAny(value, ", "); cut >= 0 { + value = value[:cut] + } + } + return strings.TrimSpace(value) +} + +func (g *ComponentCodegen) mutableIDsParamName(helper mutableIndexHelper) string { + name := "Cur" + support := g.mutableSupportMust() + if support != nil { + name += support.BodyFieldName + } + if helper.RelationPath != "" { + name += strings.ReplaceAll(helper.RelationPath, "/", "") + } + name += helper.KeyFieldName + return name +} + +func (g *ComponentCodegen) mutableHelperViewRelPath(helper mutableIndexHelper) string { + componentDir := text.CaseFormatUpperCamel.Format(strings.TrimSpace(g.componentName()), text.CaseFormatLowerUnderscore) + name := text.CaseFormatUpperCamel.Format(strings.TrimSpace(helper.ViewFieldName), text.CaseFormatLowerUnderscore) + if componentDir == "" || name == "" { + return "" + } + return filepath.ToSlash(filepath.Join(componentDir, name+".sql")) +} + +func (g *ComponentCodegen) mutableHelperIDsRelPath(helper mutableIndexHelper) string { + componentDir := text.CaseFormatUpperCamel.Format(strings.TrimSpace(g.componentName()), text.CaseFormatLowerUnderscore) + name := text.CaseFormatUpperCamel.Format(strings.TrimSpace(g.mutableIDsParamName(helper)), text.CaseFormatLowerUnderscore) + if componentDir == "" || name == "" { + return "" + } + return filepath.ToSlash(filepath.Join(componentDir, name+".sql")) +} + +func (g *ComponentCodegen) mutableSupportMust() *mutableComponentSupport { + inputType, err := g.mutableInputType() + if err != nil || inputType == nil { + return nil + } + return g.mutableSupport(inputType) +} + +func (g *ComponentCodegen) mutableInputType() (reflect.Type, error) { + if g == nil || g.Component == nil { + return nil, nil + } + params := normalizeInputParametersForCodegen(g.Component.InputParameters(), g.Resource, g.Component.URI) + opts := []state.ReflectOption{state.WithSetMarker(), state.WithTypeName(g.inputTypeName(g.componentName()))} + if g.componentUsesVelty() { + opts = append(opts, state.WithVelty(true)) + } + pkgPath := "" + if g.TypeContext != nil { + pkgPath = g.TypeContext.PackagePath + } + return params.ReflectType(pkgPath, g.componentLookupType(pkgPath), opts...) +} + +func (g *ComponentCodegen) buildMutableVeltyBlock(inputType reflect.Type, support *mutableComponentSupport) (shapeast.Block, error) { + var block shapeast.Block + bodyField, ok := inputType.FieldByName(support.BodyFieldName) + if !ok { + return nil, nil + } + bodyItemType, bodyIsMany := mutableBodyItemType(bodyField.Type) + if bodyItemType == nil { + return nil, nil + } + bodyKeyField, ok := lookupGeneratedIndexField(bodyItemType) + if !ok { + return nil, nil + } + bodyTable := g.mutableBodyTableName(support, bodyItemType) + if bodyTable == "" { + return nil, nil + } + + g.appendMutableSequence(&block, shapeast.NewIdent(support.BodyFieldName), "", bodyItemType, bodyTable, bodyKeyField) + g.appendMutableRelationSequences(&block, shapeast.NewIdent(support.BodyFieldName), "", bodyItemType) + + for _, helper := range support.Helpers { + block.Append(shapeast.NewAssign( + shapeast.NewIdent(helper.MapFieldName), + shapeast.NewCallExpr(shapeast.NewIdent(helper.ViewFieldName), "IndexBy", shapeast.NewQuotedLiteral(helper.KeyFieldName)), + )) + } + if len(support.Helpers) > 0 { + block.AppendEmptyLine() + } + + rootHelper := support.rootHelper() + bodyExpr := shapeast.NewIdent(support.BodyFieldName) + if bodyIsMany { + recordName := mutableRecordName(support.BodyFieldName) + forEach := shapeast.NewForEach(shapeast.NewIdent(recordName), bodyExpr, shapeast.Block{}) + g.appendMutableWriteLogic(&forEach.Body, shapeast.NewIdent(recordName), "", bodyItemType, bodyTable, support, rootHelper, bodyKeyField) + block.Append(forEach) + return block, nil + } + + condition := shapeast.NewCondition(bodyExpr, shapeast.Block{}, nil) + g.appendMutableWriteLogic(&condition.IFBlock, bodyExpr, "", bodyItemType, bodyTable, support, rootHelper, bodyKeyField) + block.Append(condition) + return block, nil +} + +func (g *ComponentCodegen) appendMutableWriteLogic(block *shapeast.Block, recordExpr *shapeast.Ident, logicalPath string, recordType reflect.Type, tableName string, support *mutableComponentSupport, rootHelper *mutableIndexHelper, keyField reflect.StructField) { + method := strings.ToUpper(strings.TrimSpace(g.Component.Method)) + hasCurrent := rootHelper != nil + writeUpdate := method == "PATCH" || method == "PUT" + writeInsert := method == "PATCH" || method == "POST" + keyFieldName := keyField.Name + + if hasCurrent && writeUpdate { + hasKey := shapeast.NewBinary( + shapeast.NewCallExpr(shapeast.NewIdent(rootHelper.MapFieldName), "HasKey", shapeast.NewIdent(recordExpr.Name+"."+keyFieldName)), + "==", + shapeast.NewLiteral("true"), + ) + condition := shapeast.NewCondition(hasKey, shapeast.Block{}, nil) + condition.IFBlock.Append(shapeast.NewStatementExpression(shapeast.NewTerminatorExpression(shapeast.NewCallExpr( + shapeast.NewIdent("sql"), "Update", recordExpr, shapeast.NewQuotedLiteral(tableName), + )))) + if writeInsert { + condition.ElseBlock = shapeast.Block{ + shapeast.NewStatementExpression(shapeast.NewTerminatorExpression(shapeast.NewCallExpr( + shapeast.NewIdent("sql"), "Insert", recordExpr, shapeast.NewQuotedLiteral(tableName), + ))), + } + } + block.Append(condition) + g.appendChildMutableWriteLogic(block, recordExpr, logicalPath, recordType, support) + return + } + + if writeInsert { + block.Append(shapeast.NewStatementExpression(shapeast.NewTerminatorExpression(shapeast.NewCallExpr( + shapeast.NewIdent("sql"), "Insert", recordExpr, shapeast.NewQuotedLiteral(tableName), + )))) + } + g.appendChildMutableWriteLogic(block, recordExpr, logicalPath, recordType, support) +} + +func (g *ComponentCodegen) appendMutableSequence(block *shapeast.Block, bodyExpr *shapeast.Ident, path string, itemType reflect.Type, tableName string, keyField reflect.StructField) { + if !g.needsMutableSequence(keyField.Type) { + return + } + block.Append(shapeast.NewStatementExpression(shapeast.NewCallExpr( + shapeast.NewIdent("sequencer"), + "Allocate", + shapeast.NewQuotedLiteral(tableName), + bodyExpr, + shapeast.NewQuotedLiteral(mutableSequencePath(path, keyField.Name)), + ))) + block.AppendEmptyLine() +} + +func mutableSequencePath(path, key string) string { + path = strings.Trim(path, "/") + if path == "" { + return key + } + return path + "/" + key +} + +func (g *ComponentCodegen) appendChildMutableWriteLogic(block *shapeast.Block, parentExpr *shapeast.Ident, logicalPath string, parentType reflect.Type, support *mutableComponentSupport) { + parentType = unwrapNamedStructType(parentType) + if parentType == nil { + return + } + for i := 0; i < parentType.NumField(); i++ { + field := parentType.Field(i) + if !isMutableRelationField(field) { + continue + } + childItemType, childMany := mutableBodyItemType(field.Type) + if childItemType == nil { + continue + } + childKeyField, ok := lookupGeneratedIndexField(childItemType) + if !ok { + continue + } + childTable := mutableRelationTableName(field) + if childTable == "" { + childTable = tableNameFromType(childItemType.Name()) + } + if childTable == "" { + continue + } + childPath := mutableSequencePath(logicalPath, field.Name) + assignments := mutableRelationAssignments(field) + childHelper := support.findHelper(field.Name, field) + + childExprName := field.Name + if childMany { + recordName := mutableRecordName(field.Name) + forEach := shapeast.NewForEach(shapeast.NewIdent(recordName), shapeast.NewIdent(parentExpr.Name+"."+field.Name), shapeast.Block{}) + appendMutableRelationAssignments(&forEach.Body, shapeast.NewIdent(recordName), parentExpr, assignments, parentType, childItemType) + g.appendMutableWriteLogic(&forEach.Body, shapeast.NewIdent(recordName), childPath, childItemType, childTable, support, childHelper, childKeyField) + block.AppendEmptyLine() + block.Append(forEach) + continue + } + condition := shapeast.NewCondition(shapeast.NewIdent(parentExpr.Name+"."+childExprName), shapeast.Block{}, nil) + childExpr := shapeast.NewIdent(parentExpr.Name + "." + childExprName) + appendMutableRelationAssignments(&condition.IFBlock, childExpr, parentExpr, assignments, parentType, childItemType) + g.appendMutableWriteLogic(&condition.IFBlock, childExpr, childPath, childItemType, childTable, support, childHelper, childKeyField) + block.AppendEmptyLine() + block.Append(condition) + } +} + +func (g *ComponentCodegen) appendMutableRelationSequences(block *shapeast.Block, rootExpr *shapeast.Ident, logicalPath string, parentType reflect.Type) { + parentType = unwrapNamedStructType(parentType) + if parentType == nil { + return + } + for i := 0; i < parentType.NumField(); i++ { + field := parentType.Field(i) + if !isMutableRelationField(field) { + continue + } + childItemType, _ := mutableBodyItemType(field.Type) + if childItemType == nil { + continue + } + childKeyField, ok := lookupGeneratedIndexField(childItemType) + if !ok { + continue + } + childTable := mutableRelationTableName(field) + if childTable == "" { + childTable = tableNameFromType(childItemType.Name()) + } + if childTable == "" { + continue + } + childPath := mutableSequencePath(logicalPath, field.Name) + g.appendMutableSequence(block, rootExpr, childPath, childItemType, childTable, childKeyField) + g.appendMutableRelationSequences(block, rootExpr, childPath, childItemType) + } +} + +func (g *ComponentCodegen) mutableBodyTableName(support *mutableComponentSupport, bodyItemType reflect.Type) string { + if support != nil { + if rootHelper := support.rootHelper(); rootHelper != nil { + if name := g.mutableTableFromViewState(rootHelper.ViewParamName); name != "" { + return name + } + } + } + if g != nil && g.Resource != nil && g.Component != nil { + rootViewName := strings.TrimSpace(g.Component.RootView) + if rootViewName != "" { + if rootView, err := g.Resource.View(rootViewName); err == nil && rootView != nil { + if rootView.Table != "" { + return strings.TrimSpace(rootView.Table) + } + if rootView.Template != nil { + if name := tableNameFromSQL(rootView.Template.Source); name != "" { + return name + } + } + if name := tableNameFromType(rootView.Name); name != "" { + return name + } + } + } + } + if name := tableNameFromType(bodyItemType.Name()); name != "" { + return name + } + return "" +} + +func (g *ComponentCodegen) mutableTableFromViewState(viewParamName string) string { + if g == nil || g.Resource == nil { + return "" + } + for _, input := range g.Component.Input { + if input == nil || strings.TrimSpace(input.Name) != strings.TrimSpace(viewParamName) { + continue + } + viewName := mutableViewNameFromTag(input.Tag) + if viewName == "" { + viewName = strings.TrimSpace(input.Name) + } + if viewName == "" { + continue + } + aView, err := g.Resource.View(viewName) + if err != nil || aView == nil || aView.Template == nil { + continue + } + if table := tableNameFromSQL(aView.Template.Source); table != "" { + return table + } + } + return "" +} + +func mutableViewNameFromTag(tag string) string { + if tag == "" { + return "" + } + idx := strings.Index(tag, `view:"`) + if idx == -1 { + return "" + } + rest := tag[idx+len(`view:"`):] + end := strings.Index(rest, `"`) + if end == -1 { + return "" + } + return strings.TrimSpace(rest[:end]) +} + +func tableNameFromSQL(sql string) string { + fields := strings.Fields(sql) + for i := 0; i < len(fields)-1; i++ { + if strings.EqualFold(fields[i], "FROM") { + candidate := strings.TrimSpace(fields[i+1]) + candidate = strings.Trim(candidate, "`()") + candidate = strings.TrimRight(candidate, ",;") + if candidate != "" { + return candidate + } + } + } + return "" +} + +func tableNameFromType(typeName string) string { + typeName = strings.TrimSpace(typeName) + if typeName == "" { + return "" + } + return text.CaseFormatUpperCamel.Format(typeName, text.CaseFormatUpperUnderscore) +} + +func mutableBodyItemType(rType reflect.Type) (reflect.Type, bool) { + for rType != nil && rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + if rType == nil { + return nil, false + } + switch rType.Kind() { + case reflect.Slice, reflect.Array: + return unwrapNamedStructType(rType.Elem()), true + case reflect.Struct: + return rType, false + default: + return nil, false + } +} + +func mutableRecordName(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "Rec" + } + return "Rec" + name +} + +func isMutableRelationField(field reflect.StructField) bool { + return strings.Contains(field.Tag.Get("view"), "table=") || field.Tag.Get("on") != "" +} + +func mutableRelationTableName(field reflect.StructField) string { + viewTag := field.Tag.Get("view") + for _, part := range strings.Split(viewTag, ",") { + part = strings.TrimSpace(part) + if strings.HasPrefix(strings.ToLower(part), "table=") { + return strings.TrimSpace(strings.TrimPrefix(part, "table=")) + } + } + return "" +} + +type mutableRelationAssignment struct { + ParentField string + ChildField string +} + +func mutableRelationAssignments(field reflect.StructField) []mutableRelationAssignment { + raw := strings.TrimSpace(field.Tag.Get("on")) + if raw == "" { + return nil + } + var result []mutableRelationAssignment + for _, expr := range strings.Split(raw, ",") { + expr = strings.TrimSpace(expr) + if expr == "" { + continue + } + parts := strings.Split(expr, "=") + if len(parts) != 2 { + continue + } + parentField := strings.TrimSpace(strings.Split(strings.TrimSpace(parts[0]), ":")[0]) + childField := strings.TrimSpace(strings.Split(strings.TrimSpace(parts[1]), ":")[0]) + if parentField == "" || childField == "" { + continue + } + result = append(result, mutableRelationAssignment{ParentField: parentField, ChildField: childField}) + } + return result +} + +func appendMutableRelationAssignments(block *shapeast.Block, childExpr, parentExpr *shapeast.Ident, assignments []mutableRelationAssignment, parentType, childType reflect.Type) { + for _, assignment := range assignments { + src := shapeast.Expression(shapeast.NewIdent(parentExpr.Name + "." + assignment.ParentField)) + var childFieldType, parentFieldType reflect.Type + if childType != nil { + if childField, ok := childType.FieldByName(assignment.ChildField); ok { + childFieldType = childField.Type + } + } + if parentType != nil { + if parentField, ok := parentType.FieldByName(assignment.ParentField); ok { + parentFieldType = parentField.Type + } + } + if childFieldType != nil && parentFieldType != nil { + childPtr := childFieldType.Kind() == reflect.Ptr + parentPtr := parentFieldType.Kind() == reflect.Ptr + if childPtr && !parentPtr { + src = shapeast.NewRefExpression(src) + } else if !childPtr && parentPtr { + src = shapeast.NewDerefExpression(src) + } + } + block.Append(shapeast.NewAssign(shapeast.NewIdent(childExpr.Name+"."+assignment.ChildField), src)) + } +} + +func (s *mutableComponentSupport) rootHelper() *mutableIndexHelper { + if s == nil { + return nil + } + want := "Cur" + s.BodyFieldName + for i := range s.Helpers { + if s.Helpers[i].ViewFieldName == want { + return &s.Helpers[i] + } + } + if len(s.Helpers) == 1 { + return &s.Helpers[0] + } + return nil +} + +func (s *mutableComponentSupport) findHelper(fieldName string, field reflect.StructField) *mutableIndexHelper { + if s == nil { + return nil + } + itemExpr, _ := collectionItemType(field) + wantSuffix := strings.TrimSpace(fieldName) + for i := range s.Helpers { + helper := &s.Helpers[i] + if itemExpr != "" && strings.EqualFold(strings.TrimSpace(helper.ItemTypeExpr), strings.TrimSpace(itemExpr)) { + return helper + } + if wantSuffix != "" && strings.HasSuffix(strings.TrimSpace(helper.ViewFieldName), wantSuffix) { + return helper + } + } + return nil +} + +func (g *ComponentCodegen) needsMutableSequence(keyType reflect.Type) bool { + if g == nil || g.Component == nil { + return false + } + method := strings.ToUpper(strings.TrimSpace(g.Component.Method)) + if method != "PATCH" && method != "POST" { + return false + } + for keyType != nil && keyType.Kind() == reflect.Ptr { + keyType = keyType.Elem() + } + if keyType == nil { + return false + } + switch keyType.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return true + default: + return false + } +} + +func mutableVeltyOutputUsesBody(aView *view.View) bool { + return aView != nil +} diff --git a/repository/shape/xgen/mutable_helpers.go b/repository/shape/xgen/mutable_helpers.go new file mode 100644 index 000000000..c5ecd2f84 --- /dev/null +++ b/repository/shape/xgen/mutable_helpers.go @@ -0,0 +1,394 @@ +package xgen + +import ( + "fmt" + "reflect" + "strings" + + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/view/state" +) + +type mutableComponentSupport struct { + BodyFieldName string + BodyTypeName string + Helpers []mutableIndexHelper +} + +type mutableIndexHelper struct { + ViewParamName string + ViewFieldName string + TypeName string + MapFieldName string + ItemTypeExpr string + MapTypeExpr string + KeyFieldName string + KeyFieldType string + KeyReadExpr string + NeedNilCheck bool + ItemIsPointer bool + RelationPath string + ItemStruct reflect.Type +} + +func (g *ComponentCodegen) mutableSupport(inputType reflect.Type) *mutableComponentSupport { + if !g.componentUsesVelty() || g.componentUsesHandler() || g.Component == nil || inputType == nil { + return nil + } + bodyFieldName := "" + for _, input := range g.Component.Input { + if input == nil || input.In == nil || input.In.Kind != state.KindRequestBody { + continue + } + bodyFieldName = exportedCodegenParamName(input.Name) + break + } + if bodyFieldName == "" { + return nil + } + + support := &mutableComponentSupport{BodyFieldName: bodyFieldName} + for _, input := range g.Component.Input { + if input == nil || input.In == nil || input.In.Kind != state.KindRequestBody { + continue + } + if input.Schema != nil { + if bodyTypeName := strings.TrimSpace(input.Schema.Name); bodyTypeName != "" { + support.BodyTypeName = bodyTypeName + } + } + break + } + for _, input := range g.Component.Input { + if input == nil || input.In == nil || input.In.Kind != state.KindView { + continue + } + helper, ok := g.mutableIndexHelper(inputType, bodyFieldName, input) + if !ok { + continue + } + support.Helpers = append(support.Helpers, helper) + } + if len(support.Helpers) == 0 { + return nil + } + return support +} + +func (g *ComponentCodegen) mutableIndexHelper(inputType reflect.Type, bodyFieldName string, param *plan.State) (mutableIndexHelper, bool) { + fieldName := exportedCodegenParamName(param.Name) + if fieldName == "" { + return mutableIndexHelper{}, false + } + viewField, ok := inputType.FieldByName(fieldName) + if !ok { + return mutableIndexHelper{}, false + } + itemTypeExpr, itemStructType := collectionItemType(viewField) + if itemStructType == nil { + return mutableIndexHelper{}, false + } + keyField, ok := lookupGeneratedIndexField(itemStructType) + if !ok { + return mutableIndexHelper{}, false + } + keyType := keyField.Type + keyReadExpr := fmt.Sprintf("item.%s", keyField.Name) + needNilCheck := false + if keyType.Kind() == reflect.Ptr { + needNilCheck = true + keyReadExpr = "*" + keyReadExpr + keyType = keyType.Elem() + } + keyTypeExpr := sourceTypeExpr(keyType, "") + if keyTypeExpr == "" { + return mutableIndexHelper{}, false + } + mapFieldName := fieldName + "By" + keyField.Name + if _, exists := inputType.FieldByName(mapFieldName); exists { + return mutableIndexHelper{}, false + } + return mutableIndexHelper{ + ViewParamName: strings.TrimSpace(param.Name), + ViewFieldName: fieldName, + TypeName: func() string { + if param.Schema == nil { + return "" + } + return strings.TrimSpace(param.Schema.Name) + }(), + MapFieldName: mapFieldName, + ItemTypeExpr: itemTypeExpr, + MapTypeExpr: fmt.Sprintf("map[%s]%s", keyTypeExpr, itemTypeExpr), + KeyFieldName: keyField.Name, + KeyFieldType: keyTypeExpr, + KeyReadExpr: keyReadExpr, + NeedNilCheck: needNilCheck, + ItemIsPointer: viewField.Type.Kind() == reflect.Slice && viewField.Type.Elem().Kind() == reflect.Ptr, + RelationPath: mutableRelationPath(inputType, itemStructType, bodyFieldName), + ItemStruct: itemStructType, + }, true +} + +func mutableRelationPath(inputType reflect.Type, itemType reflect.Type, bodyFieldName string) string { + if inputType == nil || itemType == nil || bodyFieldName == "" { + return "" + } + bodyField, ok := inputType.FieldByName(bodyFieldName) + if !ok { + return "" + } + rootType, _ := mutableBodyItemType(bodyField.Type) + if rootType == nil { + return "" + } + if sameNamedStructType(rootType, itemType) { + return "" + } + return lookupMutableRelationPath(rootType, itemType, "") +} + +func lookupMutableRelationPath(parentType reflect.Type, itemType reflect.Type, prefix string) string { + parentType = unwrapNamedStructType(parentType) + itemType = unwrapNamedStructType(itemType) + if parentType == nil || itemType == nil { + return "" + } + for i := 0; i < parentType.NumField(); i++ { + field := parentType.Field(i) + if !isMutableRelationField(field) { + continue + } + childType, _ := mutableBodyItemType(field.Type) + if childType == nil { + continue + } + current := field.Name + if prefix != "" { + current = prefix + "/" + current + } + if sameNamedStructType(childType, itemType) { + return current + } + if nested := lookupMutableRelationPath(childType, itemType, current); nested != "" { + return nested + } + } + return "" +} + +func sameNamedStructType(left, right reflect.Type) bool { + left = unwrapNamedStructType(left) + right = unwrapNamedStructType(right) + if left == nil || right == nil { + return false + } + if left == right { + return true + } + if left.Name() != "" && right.Name() != "" && left.Name() == right.Name() && left.PkgPath() == right.PkgPath() { + return true + } + return false +} + +func (s *mutableComponentSupport) renderInputFields(builder *strings.Builder) { + if s == nil { + return + } + for _, helper := range s.Helpers { + builder.WriteString(fmt.Sprintf("\t%s %s `json:\"-\"`\n", helper.MapFieldName, helper.MapTypeExpr)) + } +} + +func (s *mutableComponentSupport) renderInputInit(inputTypeName, outputTypeName string) string { + if s == nil { + return "" + } + if strings.TrimSpace(inputTypeName) == "" { + inputTypeName = "Input" + } + if strings.TrimSpace(outputTypeName) == "" { + outputTypeName = "Output" + } + var builder strings.Builder + builder.WriteString(fmt.Sprintf("func (i *%s) Init(ctx context.Context, sess handler.Session, output *%s) error {\n", inputTypeName, outputTypeName)) + builder.WriteString("\tif err := sess.Stater().Bind(ctx, i); err != nil {\n") + builder.WriteString("\t\treturn err\n") + builder.WriteString("\t}\n") + builder.WriteString("\ti.indexSlice()\n") + builder.WriteString("\treturn nil\n") + builder.WriteString("}\n\n") + builder.WriteString(fmt.Sprintf("func (i *%s) indexSlice() {\n", inputTypeName)) + for _, helper := range s.Helpers { + builder.WriteString(fmt.Sprintf("\ti.%s = make(%s, len(i.%s))\n", helper.MapFieldName, helper.MapTypeExpr, helper.ViewFieldName)) + builder.WriteString(fmt.Sprintf("\tfor _, item := range i.%s {\n", helper.ViewFieldName)) + if helper.ItemIsPointer { + builder.WriteString("\t\tif item == nil {\n") + builder.WriteString("\t\t\tcontinue\n") + builder.WriteString("\t\t}\n") + } + if helper.NeedNilCheck { + builder.WriteString(fmt.Sprintf("\t\tif item.%s == nil {\n", helper.KeyFieldName)) + builder.WriteString("\t\t\tcontinue\n") + builder.WriteString("\t\t}\n") + } + builder.WriteString(fmt.Sprintf("\t\ti.%s[%s] = item\n", helper.MapFieldName, helper.KeyReadExpr)) + builder.WriteString("\t}\n") + } + builder.WriteString("}\n") + return builder.String() +} + +func (s *mutableComponentSupport) renderInputValidate(inputTypeName, outputTypeName string) string { + if s == nil { + return "" + } + if strings.TrimSpace(inputTypeName) == "" { + inputTypeName = "Input" + } + if strings.TrimSpace(outputTypeName) == "" { + outputTypeName = "Output" + } + var builder strings.Builder + builder.WriteString(fmt.Sprintf("func (i *%s) Validate(ctx context.Context, sess handler.Session, output *%s) error {\n", inputTypeName, outputTypeName)) + builder.WriteString("\taValidator := sess.Validator()\n") + builder.WriteString("\tsessionDb, err := sess.Db()\n") + builder.WriteString("\tif err != nil {\n") + builder.WriteString("\t\treturn err\n") + builder.WriteString("\t}\n") + builder.WriteString("\tdb, err := sessionDb.Db(ctx)\n") + builder.WriteString("\tif err != nil {\n") + builder.WriteString("\t\treturn err\n") + builder.WriteString("\t}\n") + builder.WriteString("\tvar options = []validator.Option{\n") + builder.WriteString(fmt.Sprintf("\t\tvalidator.WithLocation(%q),\n", s.BodyFieldName)) + builder.WriteString("\t\tvalidator.WithDB(db),\n") + builder.WriteString("\t\tvalidator.WithUnique(true),\n") + builder.WriteString("\t\tvalidator.WithRefCheck(true),\n") + builder.WriteString("\t\tvalidator.WithCanUseMarkerProvider(i.canUseMarkerProvider),\n") + builder.WriteString("\t}\n") + builder.WriteString("\tvalidation := validator.NewValidation()\n") + builder.WriteString(fmt.Sprintf("\terr = i.validate(ctx, aValidator, validation, options, i.%s)\n", s.BodyFieldName)) + builder.WriteString("\toutput.Violations = append(output.Violations, validation.Violations...)\n") + builder.WriteString("\tif err == nil && len(validation.Violations) > 0 {\n") + builder.WriteString("\t\tvalidation.Violations.Sort()\n") + builder.WriteString("\t}\n") + builder.WriteString("\treturn err\n") + builder.WriteString("}\n\n") + builder.WriteString(fmt.Sprintf("func (i *%s) validate(ctx context.Context, aValidator *validator.Service, validation *validator.Validation, options []validator.Option, value interface{}) error {\n", inputTypeName)) + builder.WriteString("\t_, err := aValidator.Validate(ctx, value, append(options, validator.WithValidation(validation))...)\n") + builder.WriteString("\tif err != nil {\n") + builder.WriteString("\t\treturn err\n") + builder.WriteString("\t}\n") + builder.WriteString("\treturn nil\n") + builder.WriteString("}\n\n") + builder.WriteString(fmt.Sprintf("func (i *%s) canUseMarkerProvider(v interface{}) bool {\n", inputTypeName)) + builder.WriteString("\tswitch actual := v.(type) {\n") + for _, helper := range s.Helpers { + builder.WriteString(fmt.Sprintf("\tcase %s:\n", helper.ItemTypeExpr)) + if helper.NeedNilCheck { + builder.WriteString(fmt.Sprintf("\t\tif actual.%s == nil {\n", helper.KeyFieldName)) + builder.WriteString("\t\t\treturn false\n") + builder.WriteString("\t\t}\n") + } + actualKey := fmt.Sprintf("actual.%s", helper.KeyFieldName) + if helper.NeedNilCheck { + actualKey = "*" + actualKey + } + builder.WriteString(fmt.Sprintf("\t\t_, ok := i.%s[%s]\n", helper.MapFieldName, actualKey)) + builder.WriteString("\t\treturn ok\n") + } + builder.WriteString("\tdefault:\n") + builder.WriteString("\t\treturn true\n") + builder.WriteString("\t}\n") + builder.WriteString("}\n") + return builder.String() +} + +func collectionItemType(field reflect.StructField) (string, reflect.Type) { + rType := field.Type + expr := sourceFieldTypeExpr(field) + if expr == "" || rType == nil { + return "", nil + } + switch rType.Kind() { + case reflect.Slice, reflect.Array: + return strings.TrimPrefix(expr, "[]"), unwrapNamedStructType(rType.Elem()) + default: + return "", nil + } +} + +func unwrapNamedStructType(rType reflect.Type) reflect.Type { + for rType != nil && rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + if rType == nil || rType.Kind() != reflect.Struct { + return nil + } + return rType +} + +func lookupGeneratedIndexField(structType reflect.Type) (reflect.StructField, bool) { + if structType == nil || structType.Kind() != reflect.Struct { + return reflect.StructField{}, false + } + if field, ok := structType.FieldByName("Id"); ok { + return field, true + } + for i := 0; i < structType.NumField(); i++ { + field := structType.Field(i) + if generatedSQLXFieldName(field.Tag.Get("sqlx")) == "ID" { + return field, true + } + } + for i := 0; i < structType.NumField(); i++ { + field := structType.Field(i) + if strings.Contains(strings.ToLower(field.Tag.Get("sqlx")), "primarykey") { + return field, true + } + } + return reflect.StructField{}, false +} + +func sourceTypeExpr(rType reflect.Type, typeName string) string { + if rType == nil { + return typeName + } + switch rType.Kind() { + case reflect.Ptr: + return "*" + sourceTypeExpr(rType.Elem(), typeName) + case reflect.Slice: + return "[]" + sourceTypeExpr(rType.Elem(), typeName) + case reflect.Array: + return fmt.Sprintf("[%d]%s", rType.Len(), sourceTypeExpr(rType.Elem(), typeName)) + case reflect.Map: + return "map[" + sourceTypeExpr(rType.Key(), "") + "]" + sourceTypeExpr(rType.Elem(), typeName) + default: + if typeName != "" { + return typeName + } + return rType.String() + } +} + +func generatedSQLXFieldName(tag string) string { + tag = strings.TrimSpace(tag) + if tag == "" { + return "" + } + for _, part := range strings.Split(tag, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if strings.HasPrefix(part, "name=") { + return strings.TrimSpace(strings.TrimPrefix(part, "name=")) + } + if !strings.Contains(part, "=") { + return part + } + } + return "" +} diff --git a/repository/shape/xgen/repro_xgen_shapefragment_test.go b/repository/shape/xgen/repro_xgen_shapefragment_test.go new file mode 100644 index 000000000..5837a50e3 --- /dev/null +++ b/repository/shape/xgen/repro_xgen_shapefragment_test.go @@ -0,0 +1,35 @@ +package xgen + +import ( + shapeload "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "path/filepath" + "reflect" + "testing" +) + +func TestReproShapeFragment(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "details") + component := &shapeload.Component{Method: "GET", URI: "/v1/api/shape/dev/vendors/{vendorID}", RootView: "vendor", Output: []*plan.State{{Parameter: state.Parameter{Name: "Data", In: state.NewOutputLocation("view"), Schema: &state.Schema{Cardinality: state.Many}}}}} + resource := view.EmptyResource() + resource.Views = append(resource.Views, &view.View{Name: "vendor", Schema: &state.Schema{Name: "VendorView", DataType: "*VendorView", Cardinality: state.Many}}) + resource.Views[0].Schema.SetType(reflect.TypeOf([]struct { + ID int + Products []*struct{ ID int } `view:",table=PRODUCT" json:",omitempty" sqlx:"-"` + }{})) + ctx := &typectx.Context{PackageDir: packageDir, PackageName: "details", PackagePath: "github.com/acme/project/shape/dev/vendor/details"} + codegen := &ComponentCodegen{Component: component, Resource: resource, TypeContext: ctx, ProjectDir: projectDir, WithEmbed: false, WithContract: false} + frag, err := codegen.generateShapeFragment(projectDir, packageDir, "details", ctx.PackagePath) + if err != nil { + t.Fatalf("generateShapeFragment err: %v", err) + } + if frag == nil { + t.Fatalf("nil fragment") + } + t.Logf("types=%v", frag.Types) + t.Logf("decls=%s", frag.TypeDecls) +} From e71fbd904792a64fda51759913c8c6149c3ec1bb Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 7 Mar 2026 06:20:44 -0800 Subject: [PATCH 153/279] added dynamic grouping --- .../regression/cases/010_codecs/expect.json | 14 --- .../regression/cases/010_codecs/gen.json | 5 - .../regression/cases/010_codecs/test.yaml | 14 --- .../cases/010_codecs/vendors_codec.sql | 6 -- .../010_grouping/expect_account_totals.json | 12 +++ .../expect_account_user_totals.json | 14 +++ .../cases/010_grouping/expect_empty.json | 1 + .../cases/010_grouping/expect_totals.json | 6 ++ .../010_grouping/expect_user_totals.json | 12 +++ .../regression/cases/010_grouping/gen.json | 5 + .../regression/cases/010_grouping/test.yaml | 37 ++++++++ .../cases/010_grouping/vendors_grouping.sql | 16 ++++ e2e/local/regression/regression.yaml | 2 +- internal/inference/column.go | 94 ++++++++++++++++++- internal/translator/function/init.go | 1 + internal/translator/view.go | 10 +- internal/translator/viewlet.go | 8 +- service/session/selector.go | 23 +++-- view/config.go | 2 +- view/view.go | 14 +++ 20 files changed, 241 insertions(+), 55 deletions(-) delete mode 100644 e2e/local/regression/cases/010_codecs/expect.json delete mode 100644 e2e/local/regression/cases/010_codecs/gen.json delete mode 100644 e2e/local/regression/cases/010_codecs/test.yaml delete mode 100644 e2e/local/regression/cases/010_codecs/vendors_codec.sql create mode 100644 e2e/local/regression/cases/010_grouping/expect_account_totals.json create mode 100644 e2e/local/regression/cases/010_grouping/expect_account_user_totals.json create mode 100644 e2e/local/regression/cases/010_grouping/expect_empty.json create mode 100644 e2e/local/regression/cases/010_grouping/expect_totals.json create mode 100644 e2e/local/regression/cases/010_grouping/expect_user_totals.json create mode 100644 e2e/local/regression/cases/010_grouping/gen.json create mode 100644 e2e/local/regression/cases/010_grouping/test.yaml create mode 100644 e2e/local/regression/cases/010_grouping/vendors_grouping.sql diff --git a/e2e/local/regression/cases/010_codecs/expect.json b/e2e/local/regression/cases/010_codecs/expect.json deleted file mode 100644 index 52dba1e84..000000000 --- a/e2e/local/regression/cases/010_codecs/expect.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "id": 1, - "name": "Vendor 1", - "accountId": 100, - "userCreated": 1 - }, - { - "id": 2, - "name": "Vendor 2", - "accountId": 101, - "userCreated": 2 - } -] \ No newline at end of file diff --git a/e2e/local/regression/cases/010_codecs/gen.json b/e2e/local/regression/cases/010_codecs/gen.json deleted file mode 100644 index bb1701353..000000000 --- a/e2e/local/regression/cases/010_codecs/gen.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "Name": "$tagId", - "URL": "$path/vendors_codec.sql", - "Args": "" -} \ No newline at end of file diff --git a/e2e/local/regression/cases/010_codecs/test.yaml b/e2e/local/regression/cases/010_codecs/test.yaml deleted file mode 100644 index 75d28dd2e..000000000 --- a/e2e/local/regression/cases/010_codecs/test.yaml +++ /dev/null @@ -1,14 +0,0 @@ -init: - parentPath: $parent.path - expect: $LoadJSON('${parentPath}/expect.json') - -pipeline: - - test: - action: http/runner:send - requests: - - Method: GET - URL: http://127.0.0.1:8080/v1/api/dev/vendors-codec?vendorIDs=1,2 - Expect: - Code: 200 - JSONBody: $expect \ No newline at end of file diff --git a/e2e/local/regression/cases/010_codecs/vendors_codec.sql b/e2e/local/regression/cases/010_codecs/vendors_codec.sql deleted file mode 100644 index fcef47b03..000000000 --- a/e2e/local/regression/cases/010_codecs/vendors_codec.sql +++ /dev/null @@ -1,6 +0,0 @@ -/* {"URI":"vendors-codec/"} */ - -#set( $_ = $Data(output/view).Embed()) - -SELECT vendor.* -FROM (SELECT * FROM VENDOR t WHERE t.ID IN ($vendorIDs) ) vendor \ No newline at end of file diff --git a/e2e/local/regression/cases/010_grouping/expect_account_totals.json b/e2e/local/regression/cases/010_grouping/expect_account_totals.json new file mode 100644 index 000000000..20d77e4ad --- /dev/null +++ b/e2e/local/regression/cases/010_grouping/expect_account_totals.json @@ -0,0 +1,12 @@ +[ + { + "accountId": 100, + "totalId": 4, + "maxId": 3 + }, + { + "accountId": 101, + "totalId": 2, + "maxId": 2 + } +] diff --git a/e2e/local/regression/cases/010_grouping/expect_account_user_totals.json b/e2e/local/regression/cases/010_grouping/expect_account_user_totals.json new file mode 100644 index 000000000..b0a135943 --- /dev/null +++ b/e2e/local/regression/cases/010_grouping/expect_account_user_totals.json @@ -0,0 +1,14 @@ +[ + { + "accountId": 100, + "userCreated": 1, + "totalId": 4, + "maxId": 3 + }, + { + "accountId": 101, + "userCreated": 2, + "totalId": 2, + "maxId": 2 + } +] diff --git a/e2e/local/regression/cases/010_grouping/expect_empty.json b/e2e/local/regression/cases/010_grouping/expect_empty.json new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/e2e/local/regression/cases/010_grouping/expect_empty.json @@ -0,0 +1 @@ +[] diff --git a/e2e/local/regression/cases/010_grouping/expect_totals.json b/e2e/local/regression/cases/010_grouping/expect_totals.json new file mode 100644 index 000000000..24383af3f --- /dev/null +++ b/e2e/local/regression/cases/010_grouping/expect_totals.json @@ -0,0 +1,6 @@ +[ + { + "totalId": 6, + "maxId": 3 + } +] diff --git a/e2e/local/regression/cases/010_grouping/expect_user_totals.json b/e2e/local/regression/cases/010_grouping/expect_user_totals.json new file mode 100644 index 000000000..5473ea0f4 --- /dev/null +++ b/e2e/local/regression/cases/010_grouping/expect_user_totals.json @@ -0,0 +1,12 @@ +[ + { + "userCreated": 1, + "totalId": 4, + "maxId": 3 + }, + { + "userCreated": 2, + "totalId": 2, + "maxId": 2 + } +] diff --git a/e2e/local/regression/cases/010_grouping/gen.json b/e2e/local/regression/cases/010_grouping/gen.json new file mode 100644 index 000000000..007920478 --- /dev/null +++ b/e2e/local/regression/cases/010_grouping/gen.json @@ -0,0 +1,5 @@ +{ + "Name": "$tagId", + "URL": "$path/vendors_grouping.sql", + "Args": "" +} diff --git a/e2e/local/regression/cases/010_grouping/test.yaml b/e2e/local/regression/cases/010_grouping/test.yaml new file mode 100644 index 000000000..e4d51bde5 --- /dev/null +++ b/e2e/local/regression/cases/010_grouping/test.yaml @@ -0,0 +1,37 @@ +init: + parentPath: $parent.path + +pipeline: + + test: + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/dev/vendors-grouping?vendorIDs=1,2,3&_fields=accountId,totalId,maxId&_orderby=accountId + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_account_totals.json') + + - Method: GET + URL: http://127.0.0.1:8080/v1/api/dev/vendors-grouping?vendorIDs=1,2,3&_fields=accountId,userCreated,totalId,maxId&_orderby=accountId + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_account_user_totals.json') + + - Method: GET + URL: http://127.0.0.1:8080/v1/api/dev/vendors-grouping?vendorIDs=1,2,3&_fields=userCreated,totalId,maxId&_orderby=userCreated + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_user_totals.json') + + - Method: GET + URL: http://127.0.0.1:8080/v1/api/dev/vendors-grouping?vendorIDs=1,2,3&_fields=totalId,maxId + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_totals.json') + + - Method: GET + URL: http://127.0.0.1:8080/v1/api/dev/vendors-grouping?vendorIDs=1,2,3&_fields=accountId,totalId,maxId&_orderby=accountId&_limit=1&_offset=2 + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_empty.json') diff --git a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql new file mode 100644 index 000000000..5e6014969 --- /dev/null +++ b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql @@ -0,0 +1,16 @@ +/* {"URI":"vendors-grouping/"} */ + +#set( $_ = $Data(output/view).Embed()) + +SELECT vendor.*, + groupable(vendor), + allowed_order_by_columns(vendor, 'accountId:ACCOUNT_ID,userCreated:USER_CREATED,totalId:TOTAL_ID,maxId:MAX_ID') +FROM ( + SELECT ACCOUNT_ID, + USER_CREATED, + SUM(ID) AS TOTAL_ID, + MAX(ID) AS MAX_ID + FROM VENDOR t + WHERE t.ID IN ($vendorIDs) + GROUP BY 1, 2 +) vendor diff --git a/e2e/local/regression/regression.yaml b/e2e/local/regression/regression.yaml index a8b575fae..28d26c353 100644 --- a/e2e/local/regression/regression.yaml +++ b/e2e/local/regression/regression.yaml @@ -30,7 +30,7 @@ pipeline: '[]gen': '@gen' subPath: 'cases/${index}_*' - range: 11..020 + range: 10..010 template: checkSkip: action: nop diff --git a/internal/inference/column.go b/internal/inference/column.go index 503b44227..ab56ef670 100644 --- a/internal/inference/column.go +++ b/internal/inference/column.go @@ -4,17 +4,26 @@ import ( "fmt" "github.com/viant/datly/view" "github.com/viant/sqlparser" + "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/query" + "strconv" + "strings" ) type ColumnParameterNamer func(column *Field) string -func ExtractColumnConfig(column *sqlparser.Column) (*view.ColumnConfig, error) { - if column.Comments == "" { +func ExtractColumnConfig(column *sqlparser.Column, groupable bool) (*view.ColumnConfig, error) { + if column.Comments == "" && !groupable { return nil, nil } columnConfig := &view.ColumnConfig{} - if err := TryUnmarshalHint(column.Comments, columnConfig); err != nil { - return nil, fmt.Errorf("invalid column %v settings: %w, %s", column.Name, err, column.Comments) + if column.Comments != "" { + if err := TryUnmarshalHint(column.Comments, columnConfig); err != nil { + return nil, fmt.Errorf("invalid column %v settings: %w, %s", column.Name, err, column.Comments) + } + } + if groupable && columnConfig.Groupable == nil { + columnConfig.Groupable = &groupable } if columnConfig.DataType != nil { column.Type = *columnConfig.DataType @@ -23,3 +32,80 @@ func ExtractColumnConfig(column *sqlparser.Column) (*view.ColumnConfig, error) { columnConfig.Alias = column.Alias return columnConfig, nil } + +func GroupableColumns(aQuery *query.Select, columns sqlparser.Columns) map[string]bool { + result := make(map[string]bool) + if aQuery == nil || len(aQuery.GroupBy) == 0 || len(columns) == 0 { + return result + } + + index := map[string]*sqlparser.Column{} + for _, column := range columns { + if column == nil { + continue + } + for _, key := range columnGroupableKeys(column) { + index[key] = column + } + } + + for _, item := range aQuery.GroupBy { + for _, column := range groupByColumns(item, columns, index) { + result[column.Identity()] = true + } + } + return result +} + +func groupByColumns(item *query.Item, columns sqlparser.Columns, index map[string]*sqlparser.Column) []*sqlparser.Column { + if item == nil || item.Expr == nil { + return nil + } + + if literal, ok := item.Expr.(*expr.Literal); ok { + if position, err := strconv.Atoi(strings.TrimSpace(literal.Value)); err == nil && position > 0 && position <= len(columns) { + return []*sqlparser.Column{columns[position-1]} + } + } + + key := normalizedGroupableKey(sqlparser.Stringify(item.Expr)) + if key == "" { + return nil + } + if column, ok := index[key]; ok { + return []*sqlparser.Column{column} + } + return nil +} + +func columnGroupableKeys(column *sqlparser.Column) []string { + result := make([]string, 0, 4) + appendKey := func(value string) { + key := normalizedGroupableKey(value) + if key == "" { + return + } + for _, existing := range result { + if existing == key { + return + } + } + result = append(result, key) + } + + appendKey(column.Identity()) + appendKey(column.Name) + if column.Namespace != "" && column.Name != "" { + appendKey(column.Namespace + "." + column.Name) + } + appendKey(column.Expression) + return result +} + +func normalizedGroupableKey(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + return strings.ToLower(value) +} diff --git a/internal/translator/function/init.go b/internal/translator/function/init.go index b0141c3d8..d3c9e9d5a 100644 --- a/internal/translator/function/init.go +++ b/internal/translator/function/init.go @@ -8,6 +8,7 @@ func init() { _registry.Register(&allowedOrderByColumns{}) _registry.Register(&cardinality{}) _registry.Register(&allownulls{}) + _registry.Register(&groupable{}) _registry.Register(&matchStrategy{}) _registry.Register(&batchSize{}) _registry.Register(&partitioner{}) diff --git a/internal/translator/view.go b/internal/translator/view.go index fc0096ca8..09732a307 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -182,9 +182,13 @@ func (v *View) buildSelector(namespace *Viewlet, rule *Rule) { Offset: true, Projection: true, } - if !v.ParameterDerived { - selector.Constraints.Filterable = []string{"*"} - } + } + setter.SetBoolIfFalse(&selector.Constraints.Criteria, true) + setter.SetBoolIfFalse(&selector.Constraints.Limit, true) + setter.SetBoolIfFalse(&selector.Constraints.Offset, true) + setter.SetBoolIfFalse(&selector.Constraints.Projection, true) + if len(selector.Constraints.Filterable) == 0 && !v.ParameterDerived { + selector.Constraints.Filterable = []string{"*"} } if querySelectors, ok := namespace.Resource.Declarations.QuerySelectors[namespace.Name]; ok { diff --git a/internal/translator/viewlet.go b/internal/translator/viewlet.go index bd31ecb50..f11fecd19 100644 --- a/internal/translator/viewlet.go +++ b/internal/translator/viewlet.go @@ -217,6 +217,12 @@ func NewViewlet(name, SQL string, join *query.Join, resource *Resource) *Viewlet func (v *Viewlet) discoverTables(ctx context.Context, db *sql.DB, SQL string) (err error) { v.Table, err = inference.NewTable(ctx, db, SQL) + groupableColumns := map[string]bool{} + if v.Table != nil { + if parsed, parseErr := sqlparser.ParseQuery(inference.TrimParenthesis(SQL)); parseErr == nil { + groupableColumns = inference.GroupableColumns(parsed, v.Table.QueryColumns) + } + } if v.Table != nil { for _, column := range v.Table.QueryColumns { name := column.Alias @@ -224,7 +230,7 @@ func (v *Viewlet) discoverTables(ctx context.Context, db *sql.DB, SQL string) (e name = column.Name } v.Whitelisted = append(v.Whitelisted, strings.ToLower(name)) - columnConfig, err := inference.ExtractColumnConfig(column) + columnConfig, err := inference.ExtractColumnConfig(column, groupableColumns[column.Identity()]) if err != nil { return err } diff --git a/service/session/selector.go b/service/session/selector.go index ca8b64c76..bfcb0c2dd 100644 --- a/service/session/selector.go +++ b/service/session/selector.go @@ -8,6 +8,7 @@ import ( "github.com/viant/datly/service/session/criteria" "github.com/viant/datly/view" + "github.com/viant/datly/view/state" "github.com/viant/tagly/format/text" "github.com/viant/xdatly/codec" "github.com/viant/xdatly/handler/response" @@ -77,22 +78,22 @@ func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, injected = resolveInjectedQuerySelector(ns, opts.locatorOpt.QuerySelectors) } if err = s.populateFieldQuerySelector(ctx, ns, opts); err != nil { - return response.NewParameterError(ns.View.Name, selectorParameters.FieldsParameter.Name, err) + return response.NewParameterError(ns.View.Name, selectorParameterName(selectorParameters.FieldsParameter, view.QueryStateParameters.FieldsParameter), err) } if err = s.populateLimitQuerySelector(ctx, ns, opts); err != nil { - return response.NewParameterError(ns.View.Name, selectorParameters.LimitParameter.Name, err) + return response.NewParameterError(ns.View.Name, selectorParameterName(selectorParameters.LimitParameter, view.QueryStateParameters.LimitParameter), err) } if err = s.populateOffsetQuerySelector(ctx, ns, opts); err != nil { - return response.NewParameterError(ns.View.Name, selectorParameters.OffsetParameter.Name, err) + return response.NewParameterError(ns.View.Name, selectorParameterName(selectorParameters.OffsetParameter, view.QueryStateParameters.OffsetParameter), err) } if err = s.populateOrderByQuerySelector(ctx, ns, opts); err != nil { - return response.NewParameterError(ns.View.Name, selectorParameters.OrderByParameter.Name, err) + return response.NewParameterError(ns.View.Name, selectorParameterName(selectorParameters.OrderByParameter, view.QueryStateParameters.OrderByParameter), err) } if err = s.populateCriteriaQuerySelector(ctx, ns, opts); err != nil { - return response.NewParameterError(ns.View.Name, selectorParameters.CriteriaParameter.Name, err) + return response.NewParameterError(ns.View.Name, selectorParameterName(selectorParameters.CriteriaParameter, view.QueryStateParameters.CriteriaParameter), err) } if err = s.populatePageQuerySelector(ctx, ns, opts); err != nil { - return response.NewParameterError(ns.View.Name, selectorParameters.PageParameter.Name, err) + return response.NewParameterError(ns.View.Name, selectorParameterName(selectorParameters.PageParameter, view.QueryStateParameters.PageParameter), err) } // Apply injected selector last so it takes precedence over request-derived values, @@ -113,6 +114,16 @@ func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, return nil } +func selectorParameterName(parameter, fallback *state.Parameter) string { + if parameter != nil && parameter.Name != "" { + return parameter.Name + } + if fallback != nil && fallback.Name != "" { + return fallback.Name + } + return "" +} + func (s *Session) applyInjectedQuerySelector(ns *view.NamespaceView, selector *view.Statelet, injected *hstate.NamedQuerySelector) error { if injected == nil || selector == nil { return nil diff --git a/view/config.go b/view/config.go index 5dd70f6de..bf1dfe2c8 100644 --- a/view/config.go +++ b/view/config.go @@ -32,7 +32,7 @@ var QueryStateParameters = &Config{ PageParameter: &state.Parameter{Name: "Page", In: state.NewQueryLocation(PageQuery), Schema: state.NewSchema(xreflect.IntType)}, FieldsParameter: &state.Parameter{Name: "Fields", In: state.NewQueryLocation(FieldsQuery), Schema: state.NewSchema(stringsType)}, OrderByParameter: &state.Parameter{Name: "OrderBy", In: state.NewQueryLocation(OrderByQuery), Schema: state.NewSchema(stringsType)}, - CriteriaParameter: &state.Parameter{Name: "Criteria", In: state.NewQueryLocation(OrderByQuery), Schema: state.NewSchema(xreflect.StringType)}, + CriteriaParameter: &state.Parameter{Name: "Criteria", In: state.NewQueryLocation(CriteriaQuery), Schema: state.NewSchema(xreflect.StringType)}, SyncFlagParameter: &state.Parameter{Name: "SyncFlag", Cacheable: &trueValue, In: state.NewState(SyncFlag), Schema: state.NewSchema(boolType)}, ContentFormatParameter: &state.Parameter{Name: "ContentFormat", In: state.NewQueryLocation(ContentFormat), Schema: state.NewSchema(xreflect.StringType)}, } diff --git a/view/view.go b/view/view.go index b297489f1..6d744a74e 100644 --- a/view/view.go +++ b/view/view.go @@ -64,6 +64,7 @@ type ( PublishParent bool `json:",omitempty"` Partitioned *Partitioned Criteria string `json:",omitempty"` + Groupable bool `json:",omitempty"` Selector *Config `json:",omitempty"` Template *Template `json:",omitempty"` @@ -1098,6 +1099,7 @@ func (v *View) inherit(view *View) error { setter.SetStringIfEmpty(&v.Module, view.Module) setter.SetStringIfEmpty(&v.Tag, view.Tag) setter.SetBoolIfFalse(&v.PublishParent, view.PublishParent) + setter.SetBoolIfFalse(&v.Groupable, view.Groupable) setter.SetStringIfEmpty(&v.Description, view.Description) @@ -1286,6 +1288,18 @@ func (v *View) IndexedColumns() NamedColumns { return v._columns } +// IsGroupable reports whether the supplied field or column name resolves to a groupable column. +func (v *View) IsGroupable(name string) bool { + if v == nil || len(v._columns) == 0 { + return false + } + column, err := v._columns.Lookup(name) + if err != nil { + return false + } + return column.Groupable +} + func (v *View) markColumnsAsFilterable() error { if len(v.Selector.Constraints.Filterable) == 1 && strings.TrimSpace(v.Selector.Constraints.Filterable[0]) == "*" { for _, column := range v.Columns { From cbe966f2406ed1e3942aef8c7fd8c828ac8d81d3 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 08:27:15 -0700 Subject: [PATCH 154/279] added dynamic grouping --- .gcloudignore | 3 +- .gitignore | 4 +- cmd/command/service.go | 3 + cmd/command/transcribe.go | 2016 ++++++++++-- cmd/command/transcribe_test.go | 513 +++ cmd/command/translate_shape.go | 39 +- cmd/command/translate_shape_ir.go | 18 +- cmd/command/validate.go | 181 ++ cmd/command/validate_test.go | 65 + cmd/options/options.go | 8 +- cmd/options/transcribe.go | 9 +- cmd/options/validate.go | 35 + .../cases/010_grouping/vendors_grouping.sql | 3 + e2e/v1/build.yaml | 12 +- e2e/v1/cases/001_one_to_many/expect.json | 32 - e2e/v1/cases/001_one_to_many/expect_2.txt | 27 - .../dbsetup/dev/PRODUCT.json | 0 .../dbsetup/dev/VENDOR.json | 0 .../001_relation_one_to_many/expect.json | 77 + .../001_relation_one_to_many/expect_2.txt | 27 + .../test.yaml | 0 .../dbsetup/dev/USER.json | 0 .../expect.json | 20 +- .../test.yaml | 0 .../test.yaml | 5 +- .../cases/004_relation_one_to_one/test.yaml | 9 + .../dbsetup/dev/PRODUCT.json | 0 .../dbsetup/dev/VENDOR.json | 0 .../expect.json | 19 +- .../test.yaml | 0 .../cases/006_kind_header_params/expect.json | 40 + .../test.yaml | 3 +- .../dbsetup/dev/PRODUCT.json | 0 .../dbsetup/dev/VENDOR.json | 0 e2e/v1/cases/007_kind_const/expect.json | 73 + .../{011_env => 007_kind_const}/test.yaml | 0 .../dbsetup/dev/PRODUCT.json | 0 .../dbsetup/dev/VENDOR.json | 0 .../test.yaml | 0 .../dbsetup/dev/PRODUCT.json | 0 .../dbsetup/dev/VENDOR.json | 0 e2e/v1/cases/009_summary_child/expect.json | 94 + .../test.yaml | 0 e2e/v1/cases/010_codecs/expect.json | 14 - .../expect.json | 48 +- .../test.yaml | 3 +- e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json | 24 - e2e/v1/cases/011_env/expect.json | 52 - .../dbsetup/dev/CITY.json | 0 .../dbsetup/dev/DISTRICT.json | 0 .../expect.json | 0 .../test.yaml | 0 .../dbsetup/dev/PRODUCT.json | 0 .../dbsetup/dev/VENDOR.json | 0 .../{003_oauth => 012_auth_oauth}/expect.json | 0 .../{003_oauth => 012_auth_oauth}/test.yaml | 0 .../012_meta_format/dbsetup/dev/PRODUCT.json | 44 - .../012_meta_format/dbsetup/dev/VENDOR.json | 24 - e2e/v1/cases/012_meta_format/expect.json | 68 - .../cases/013_col_in/dbsetup/dev/PRODUCT.json | 44 - .../cases/013_col_in/dbsetup/dev/VENDOR.json | 24 - .../cases/013_kind_mysql_boolean/expect.json | 17 + e2e/v1/cases/013_kind_mysql_boolean/test.yaml | 10 + .../dbsetup/dev/PRODUCT.json | 0 .../dbsetup/dev/VENDOR.json | 0 .../expect.json | 0 .../test.yaml | 0 .../dbsetup/dev/PRODUCT.json | 44 - .../014_header_params/dbsetup/dev/VENDOR.json | 24 - e2e/v1/cases/014_header_params/expect.json | 26 - .../dbsetup/dev/PRODUCT.json | 0 .../dbsetup/dev/PRODUCT_JN.json | 0 .../dbsetup/dev/VENDOR.json | 0 .../expect.json | 0 .../expect/PRODUCT.json | 0 .../{004_update => 015_dml_update}/test.yaml | 0 .../015_index_by/dbsetup/dev/USER_TEAM.json | 18 - .../test.yaml | 6 +- .../dbsetup/dev/EVENTS.json | 3 - .../expect_t0.json | 3 - .../expect_t1.json | 5 - .../017_generate_post_basic_one/test.yaml | 31 - e2e/v1/cases/017_kind_variables/expect.json | 7 + .../test.yaml | 6 +- .../dbsetup/dev/TEAM.json | 7 +- .../dbsetup/dev/USER_TEAM.json | 13 + .../expect}/TEAM.json | 9 +- .../test.yaml | 7 +- .../dbsetup/dev/EVENTS.json | 3 - .../expect_t0.json | 5 - .../expect_t1.json | 10 - .../expect_t2.json | 10 - .../018_generate_post_basic_many/test.yaml | 40 - .../expect_admin.json | 9 + .../expect_readonly.json | 9 + .../cases/019_component_dependency/test.yaml | 49 + .../dbsetup/dev/EVENTS.json | 3 - .../expect_t0.json | 8 - .../expect_t1.json | 11 - .../expect_t2.json | 13 - .../test.yaml | 43 - .../dbsetup/dev/FOOS.json | 23 + .../expect_t0.json | 5 + .../expect_t1.json | 5 + .../020_generate_patch_basic_one/test.yaml | 41 + .../dbsetup/dev/EVENTS.json | 3 - .../020_generate_post_except/expect_t0.json | 4 - .../020_generate_post_except/expect_t1.json | 4 - .../cases/020_generate_post_except/test.yaml | 30 - e2e/v1/config.json | 18 + .../dql/dev/district/district_pagination.sql | 7 +- e2e/v1/dql/dev/events/basic_one_one.dql | 12 + e2e/v1/dql/dev/events/patch_basic_one.dql | 12 + e2e/v1/dql/dev/events/post_basic_many.dql | 2 +- e2e/v1/dql/dev/events/post_basic_one.dql | 2 +- .../dev/events/post_comprehensive_many.dql | 2 +- e2e/v1/dql/dev/events/post_except.dql | 2 +- e2e/v1/dql/dev/team/team.dql | 2 +- e2e/v1/dql/dev/team/user_team.dql | 21 +- e2e/v1/dql/dev/user/user_metadata.dql | 10 + e2e/v1/dql/dev/user/user_tree.sql | 5 +- e2e/v1/go_bootstrap.yaml | 227 ++ e2e/v1/regression/app.yaml | 2 +- e2e/v1/regression/regression.yaml | 6 +- e2e/v1/routerapp/main.go | 43 + e2e/v1/routerpkg/dev/linkedauth/handler.go | 61 + e2e/v1/run.yaml | 5 + e2e/v1/shapes.yaml | 78 +- gateway/dql_bootstrap.go | 219 +- gateway/dql_bootstrap_test.go | 317 +- gateway/option.go | 23 +- gateway/patch_basic_one_e2e_test.go | 142 + gateway/route_struct.go | 6 + gateway/route_struct_test.go | 35 + gateway/service.go | 1 + go.mod | 5 +- go.sum | 6 +- internal/translator/function/groupable.go | 37 + .../translator/function/groupable_test.go | 38 + internal/translator/view_selector_test.go | 58 + internal/translator/viewlet_groupable_test.go | 60 + repository/option.go | 9 + repository/option_shape_test.go | 11 + repository/path/service.go | 16 +- repository/path/service_test.go | 27 + repository/service.go | 5 + repository/service_refresh_test.go | 19 + repository/shape/compile/compiler.go | 30 +- repository/shape/compile/compiler_test.go | 37 + .../shape/compile/component_route_shape.go | 59 + repository/shape/compile/hints.go | 109 +- repository/shape/compile/hints_strip.go | 18 +- repository/shape/compile/hints_test.go | 45 +- repository/shape/compile/pipeline/infer.go | 88 +- .../shape/compile/pipeline/infer_test.go | 29 + repository/shape/compile/pipeline/read.go | 146 +- .../shape/compile/pipeline/read_normalize.go | 5 + .../shape/compile/pipeline/read_test.go | 71 + repository/shape/compile/statedecl.go | 57 +- repository/shape/compile/statedecl_test.go | 31 + repository/shape/compile/type_support.go | 256 ++ .../compile/type_support_summary_test.go | 241 ++ repository/shape/compile/viewdecl.go | 5 +- repository/shape/compile/viewdecl_append.go | 124 +- repository/shape/compile/viewdecl_options.go | 16 + repository/shape/compile/viewdecl_test.go | 8 +- repository/shape/dql/preprocess/preprocess.go | 3 +- .../shape/dql/preprocess/preprocess_test.go | 2 + .../dql/preprocess/settings_directives.go | 53 +- repository/shape/dql/shape/model.go | 1 + repository/shape/dql_engine_test.go | 300 ++ repository/shape/improvement.md | 267 ++ repository/shape/load/loader.go | 2840 ++++++++++++++++- repository/shape/load/loader_dql_test.go | 108 + repository/shape/load/loader_test.go | 1604 +++++++++- repository/shape/model.go | 5 + repository/shape/parity_test.go | 18 + repository/shape/plan/model.go | 29 +- repository/shape/plan/planner.go | 34 +- repository/shape/plan/planner_test.go | 48 +- .../shape/plan/testdata/report_summary.sql | 1 + repository/shape/scan/model.go | 1 + repository/shape/scan/scanner.go | 30 +- repository/shape/scan/scanner_test.go | 29 +- repository/shape/shape.go | 43 + repository/shape/xgen/codegen.go | 724 ++++- .../shape/xgen/codegen_groupable_test.go | 337 ++ .../shape/xgen/codegen_input_view_test.go | 102 + .../shape/xgen/codegen_mutable_body_test.go | 14 +- .../xgen/codegen_mutable_helpers_test.go | 143 +- repository/shape/xgen/mutable_body.go | 229 +- repository/shape/xgen/mutable_helpers.go | 22 +- service/executor/expand/data_unit.go | 14 + service/executor/expand/parent.go | 8 + service/operator/reader.go | 4 + service/reader/handler/handler.go | 11 + service/reader/service.go | 36 +- service/reader/sql.go | 132 +- service/session/reader.go | 11 + service/session/state.go | 66 + service/session/state_test.go | 32 + service/session/stater.go | 4 + testutil/shapeparity/bridge.go | 93 - testutil/shapeparity/scan.go | 100 + view/column.go | 28 + view/config_test.go | 9 + view/groupable_test.go | 46 + view/option.go | 50 + view/option_test.go | 83 + view/state/kind/locator/data.go | 47 +- view/state/kind/locator/data_test.go | 62 + view/state/parameter.go | 6 + view/state/parameter_test.go | 97 + view/state/parameters.go | 4 +- view/state/type.go | 15 +- view/tags/query_selector.go | 23 + view/tags/view.go | 123 + view/tags/view_test.go | 27 + view/template.go | 44 +- view/view.go | 3 + 220 files changed, 13547 insertions(+), 1788 deletions(-) create mode 100644 cmd/command/transcribe_test.go create mode 100644 cmd/command/validate.go create mode 100644 cmd/command/validate_test.go create mode 100644 cmd/options/validate.go delete mode 100644 e2e/v1/cases/001_one_to_many/expect.json delete mode 100644 e2e/v1/cases/001_one_to_many/expect_2.txt rename e2e/v1/cases/{001_one_to_many => 001_relation_one_to_many}/dbsetup/dev/PRODUCT.json (100%) rename e2e/v1/cases/{001_one_to_many => 001_relation_one_to_many}/dbsetup/dev/VENDOR.json (100%) create mode 100644 e2e/v1/cases/001_relation_one_to_many/expect.json create mode 100644 e2e/v1/cases/001_relation_one_to_many/expect_2.txt rename e2e/v1/cases/{001_one_to_many => 001_relation_one_to_many}/test.yaml (100%) rename e2e/v1/cases/{006_tree => 002_relation_self_ref_tree}/dbsetup/dev/USER.json (100%) rename e2e/v1/cases/{006_tree => 002_relation_self_ref_tree}/expect.json (90%) rename e2e/v1/cases/{006_tree => 002_relation_self_ref_tree}/test.yaml (100%) rename e2e/v1/cases/{013_col_in => 003_relation_parent_join_optimization}/test.yaml (68%) create mode 100644 e2e/v1/cases/004_relation_one_to_one/test.yaml rename e2e/v1/cases/{002_uri_param => 005_kind_uri_param}/dbsetup/dev/PRODUCT.json (100%) rename e2e/v1/cases/{002_uri_param => 005_kind_uri_param}/dbsetup/dev/VENDOR.json (100%) rename e2e/v1/cases/{002_uri_param => 005_kind_uri_param}/expect.json (60%) rename e2e/v1/cases/{002_uri_param => 005_kind_uri_param}/test.yaml (100%) create mode 100644 e2e/v1/cases/006_kind_header_params/expect.json rename e2e/v1/cases/{014_header_params => 006_kind_header_params}/test.yaml (80%) rename e2e/v1/cases/{003_oauth => 007_kind_const}/dbsetup/dev/PRODUCT.json (100%) rename e2e/v1/cases/{003_oauth => 007_kind_const}/dbsetup/dev/VENDOR.json (100%) create mode 100644 e2e/v1/cases/007_kind_const/expect.json rename e2e/v1/cases/{011_env => 007_kind_const}/test.yaml (100%) rename e2e/v1/cases/{004_update => 008_summary_root}/dbsetup/dev/PRODUCT.json (100%) rename e2e/v1/cases/{004_update => 008_summary_root}/dbsetup/dev/VENDOR.json (100%) rename e2e/v1/cases/{005_sumary => 008_summary_root}/test.yaml (100%) rename e2e/v1/cases/{005_sumary => 009_summary_child}/dbsetup/dev/PRODUCT.json (100%) rename e2e/v1/cases/{005_sumary => 009_summary_child}/dbsetup/dev/VENDOR.json (100%) create mode 100644 e2e/v1/cases/009_summary_child/expect.json rename e2e/v1/cases/{007_child_meta => 009_summary_child}/test.yaml (100%) delete mode 100644 e2e/v1/cases/010_codecs/expect.json rename e2e/v1/cases/{007_child_meta => 010_summary_multi}/expect.json (51%) rename e2e/v1/cases/{012_meta_format => 010_summary_multi}/test.yaml (78%) delete mode 100644 e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json delete mode 100644 e2e/v1/cases/011_env/expect.json rename e2e/v1/cases/{008_record_pagination => 011_summary_pagination}/dbsetup/dev/CITY.json (100%) rename e2e/v1/cases/{008_record_pagination => 011_summary_pagination}/dbsetup/dev/DISTRICT.json (100%) rename e2e/v1/cases/{008_record_pagination => 011_summary_pagination}/expect.json (100%) rename e2e/v1/cases/{008_record_pagination => 011_summary_pagination}/test.yaml (100%) rename e2e/v1/cases/{007_child_meta => 012_auth_oauth}/dbsetup/dev/PRODUCT.json (100%) rename e2e/v1/cases/{007_child_meta => 012_auth_oauth}/dbsetup/dev/VENDOR.json (100%) rename e2e/v1/cases/{003_oauth => 012_auth_oauth}/expect.json (100%) rename e2e/v1/cases/{003_oauth => 012_auth_oauth}/test.yaml (100%) delete mode 100644 e2e/v1/cases/012_meta_format/dbsetup/dev/PRODUCT.json delete mode 100644 e2e/v1/cases/012_meta_format/dbsetup/dev/VENDOR.json delete mode 100644 e2e/v1/cases/012_meta_format/expect.json delete mode 100644 e2e/v1/cases/013_col_in/dbsetup/dev/PRODUCT.json delete mode 100644 e2e/v1/cases/013_col_in/dbsetup/dev/VENDOR.json create mode 100644 e2e/v1/cases/013_kind_mysql_boolean/expect.json create mode 100644 e2e/v1/cases/013_kind_mysql_boolean/test.yaml rename e2e/v1/cases/{009_apikey => 014_cache_sql_apikey}/dbsetup/dev/PRODUCT.json (100%) rename e2e/v1/cases/{009_apikey => 014_cache_sql_apikey}/dbsetup/dev/VENDOR.json (100%) rename e2e/v1/cases/{004_update => 014_cache_sql_apikey}/expect.json (100%) rename e2e/v1/cases/{009_apikey => 014_cache_sql_apikey}/test.yaml (100%) delete mode 100644 e2e/v1/cases/014_header_params/dbsetup/dev/PRODUCT.json delete mode 100644 e2e/v1/cases/014_header_params/dbsetup/dev/VENDOR.json delete mode 100644 e2e/v1/cases/014_header_params/expect.json rename e2e/v1/cases/{011_env => 015_dml_update}/dbsetup/dev/PRODUCT.json (100%) rename e2e/v1/cases/{004_update => 015_dml_update}/dbsetup/dev/PRODUCT_JN.json (100%) rename e2e/v1/cases/{010_codecs => 015_dml_update}/dbsetup/dev/VENDOR.json (100%) rename e2e/v1/cases/{009_apikey => 015_dml_update}/expect.json (100%) rename e2e/v1/cases/{004_update => 015_dml_update}/expect/PRODUCT.json (100%) rename e2e/v1/cases/{004_update => 015_dml_update}/test.yaml (100%) delete mode 100644 e2e/v1/cases/015_index_by/dbsetup/dev/USER_TEAM.json rename e2e/v1/cases/{016_team_delete => 016_dml_delete}/test.yaml (76%) delete mode 100644 e2e/v1/cases/017_generate_post_basic_one/dbsetup/dev/EVENTS.json delete mode 100644 e2e/v1/cases/017_generate_post_basic_one/expect_t0.json delete mode 100644 e2e/v1/cases/017_generate_post_basic_one/expect_t1.json delete mode 100644 e2e/v1/cases/017_generate_post_basic_one/test.yaml create mode 100644 e2e/v1/cases/017_kind_variables/expect.json rename e2e/v1/cases/{010_codecs => 017_kind_variables}/test.yaml (52%) rename e2e/v1/cases/{016_team_delete => 018_exec_index_by}/dbsetup/dev/TEAM.json (72%) create mode 100644 e2e/v1/cases/018_exec_index_by/dbsetup/dev/USER_TEAM.json rename e2e/v1/cases/{015_index_by/dbsetup/dev => 018_exec_index_by/expect}/TEAM.json (65%) rename e2e/v1/cases/{015_index_by => 018_exec_index_by}/test.yaml (83%) delete mode 100644 e2e/v1/cases/018_generate_post_basic_many/dbsetup/dev/EVENTS.json delete mode 100644 e2e/v1/cases/018_generate_post_basic_many/expect_t0.json delete mode 100644 e2e/v1/cases/018_generate_post_basic_many/expect_t1.json delete mode 100644 e2e/v1/cases/018_generate_post_basic_many/expect_t2.json delete mode 100644 e2e/v1/cases/018_generate_post_basic_many/test.yaml create mode 100644 e2e/v1/cases/019_component_dependency/expect_admin.json create mode 100644 e2e/v1/cases/019_component_dependency/expect_readonly.json create mode 100644 e2e/v1/cases/019_component_dependency/test.yaml delete mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/dbsetup/dev/EVENTS.json delete mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/expect_t0.json delete mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/expect_t1.json delete mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/expect_t2.json delete mode 100644 e2e/v1/cases/019_generate_post_comprehensive_many/test.yaml create mode 100644 e2e/v1/cases/020_generate_patch_basic_one/dbsetup/dev/FOOS.json create mode 100644 e2e/v1/cases/020_generate_patch_basic_one/expect_t0.json create mode 100644 e2e/v1/cases/020_generate_patch_basic_one/expect_t1.json create mode 100644 e2e/v1/cases/020_generate_patch_basic_one/test.yaml delete mode 100644 e2e/v1/cases/020_generate_post_except/dbsetup/dev/EVENTS.json delete mode 100644 e2e/v1/cases/020_generate_post_except/expect_t0.json delete mode 100644 e2e/v1/cases/020_generate_post_except/expect_t1.json delete mode 100644 e2e/v1/cases/020_generate_post_except/test.yaml create mode 100644 e2e/v1/config.json create mode 100644 e2e/v1/dql/dev/events/basic_one_one.dql create mode 100644 e2e/v1/dql/dev/events/patch_basic_one.dql create mode 100644 e2e/v1/dql/dev/user/user_metadata.dql create mode 100644 e2e/v1/go_bootstrap.yaml create mode 100644 e2e/v1/routerapp/main.go create mode 100644 e2e/v1/routerpkg/dev/linkedauth/handler.go create mode 100644 gateway/patch_basic_one_e2e_test.go create mode 100644 gateway/route_struct_test.go create mode 100644 internal/translator/function/groupable.go create mode 100644 internal/translator/function/groupable_test.go create mode 100644 internal/translator/view_selector_test.go create mode 100644 internal/translator/viewlet_groupable_test.go create mode 100644 repository/service_refresh_test.go create mode 100644 repository/shape/compile/component_route_shape.go create mode 100644 repository/shape/compile/type_support_summary_test.go create mode 100644 repository/shape/improvement.md create mode 100644 repository/shape/load/loader_dql_test.go create mode 100644 repository/shape/plan/testdata/report_summary.sql create mode 100644 repository/shape/xgen/codegen_groupable_test.go delete mode 100644 testutil/shapeparity/bridge.go create mode 100644 testutil/shapeparity/scan.go create mode 100644 view/config_test.go create mode 100644 view/groupable_test.go create mode 100644 view/option_test.go create mode 100644 view/state/kind/locator/data_test.go create mode 100644 view/state/parameter_test.go create mode 100644 view/tags/query_selector.go diff --git a/.gcloudignore b/.gcloudignore index 9fdaf41b4..cc86cc624 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -6,4 +6,5 @@ e2e/ .gitignore *.yaml *.md -secrets.json \ No newline at end of file +secrets.json +.meta \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2bebf504f..711ab3c41 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ secrets.json *.db local_test.go -vendor +vendor/ datly other mydb @@ -16,4 +16,4 @@ logs .extension .datly *.zip -local \ No newline at end of file +v1/ diff --git a/cmd/command/service.go b/cmd/command/service.go index 83609b9e3..88cd4962c 100644 --- a/cmd/command/service.go +++ b/cmd/command/service.go @@ -65,6 +65,9 @@ func (s *Service) Exec(ctx context.Context, opts *options.Options) error { if opts.Transcribe != nil { return s.Transcribe(ctx, opts) } + if opts.Validate != nil { + return s.Validate(ctx, opts) + } if opts.Mcp != nil { return s.Mcp(ctx, opts) diff --git a/cmd/command/transcribe.go b/cmd/command/transcribe.go index 5ac4a86d0..8eec70e0a 100644 --- a/cmd/command/transcribe.go +++ b/cmd/command/transcribe.go @@ -4,28 +4,37 @@ import ( "context" "encoding/json" "fmt" + "os" "path" "path/filepath" + "reflect" + "regexp" "strings" - "unicode" "github.com/viant/afs" "github.com/viant/afs/file" "github.com/viant/afs/url" "github.com/viant/datly/cmd/options" - pathpkg "github.com/viant/datly/repository/path" + "github.com/viant/datly/gateway" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" "github.com/viant/datly/repository/shape" shapeColumn "github.com/viant/datly/repository/shape/column" shapeCompile "github.com/viant/datly/repository/shape/compile" shapeLoad "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/repository/shape/xgen" + "github.com/viant/datly/shared" "github.com/viant/datly/view" - viewpkg "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/scy" + "github.com/viant/scy/auth/jwt/signer" + "github.com/viant/scy/auth/jwt/verifier" + "github.com/viant/tagly/format/text" + "github.com/viant/xreflect" "gopkg.in/yaml.v3" ) -// Transcribe runs the shape-only pipeline (compile → plan → load) for each -// DQL source. It does NOT depend on internal/translator. func (s *Service) Transcribe(ctx context.Context, opts *options.Options) error { transcribe := opts.Transcribe if transcribe == nil { @@ -33,30 +42,24 @@ func (s *Service) Transcribe(ctx context.Context, opts *options.Options) error { } compiler := shapeCompile.New() loader := shapeLoad.New() + var sources []string for _, sourceURL := range transcribe.Source { _, name := url.Split(sourceURL, file.Scheme) - fmt.Printf("transcribing %v\n", name) dql, err := s.readSource(ctx, sourceURL) if err != nil { return fmt.Errorf("failed to read %s: %w", sourceURL, err) } - dql = strings.TrimSpace(dql) - if dql == "" { - return fmt.Errorf("source %s was empty", sourceURL) - } - connectorName := transcribe.DefaultConnectorName() shapeSource := &shape.Source{ Name: strings.TrimSuffix(name, path.Ext(name)), Path: url.Path(sourceURL), - DQL: dql, - Connector: connectorName, + DQL: strings.TrimSpace(dql), + Connector: transcribe.DefaultConnectorName(), } - compileOpts := transcribeCompileOptions(transcribe) - planResult, err := compiler.Compile(ctx, shapeSource, compileOpts...) + planResult, err := compiler.Compile(ctx, shapeSource, transcribeCompileOptions(transcribe)...) if err != nil { return fmt.Errorf("failed to compile %s: %w", sourceURL, err) } - componentArtifact, err := loader.LoadComponent(ctx, planResult) + componentArtifact, err := loader.LoadComponent(ctx, planResult, shape.WithLoadTypeContextPackages(true)) if err != nil { return fmt.Errorf("failed to load %s: %w", sourceURL, err) } @@ -64,92 +67,168 @@ func (s *Service) Transcribe(ctx context.Context, opts *options.Options) error { if !ok { return fmt.Errorf("unexpected component artifact for %s", sourceURL) } - // Register connectors on resource first, then discover columns from DB if componentArtifact.Resource != nil && len(transcribe.Connectors) > 0 { applyConnectorsToResource(componentArtifact.Resource, transcribe.Connectors) discoverColumns(ctx, componentArtifact.Resource) + shapeLoad.RefineSummarySchemas(componentArtifact.Resource) } - if err = s.persistTranscribeRoute(ctx, transcribe, sourceURL, dql, componentArtifact.Resource, component); err != nil { + prepareResourceForTranscribeCodegen(componentArtifact.Resource, component) + codegenResult, err := s.generateTranscribeTypes(sourceURL, dql, transcribe, componentArtifact.Resource, component) + if err != nil { return err } - } - // Persist dependencies (connections.yaml, config.json) - if len(transcribe.Connectors) > 0 { - if err := s.persistTranscribeDependencies(ctx, transcribe); err != nil { - return err + if codegenResult != nil { + alignGeneratedPackageAliases(componentArtifact.Resource, component, codegenResult.PackageDir, codegenResult.PackagePath, codegenResult.PackageName) + } + if !transcribe.SkipYAML { + if err = s.persistTranscribeRoute(ctx, transcribe, sourceURL, dql, componentArtifact.Resource, component, codegenResult); err != nil { + return err + } } + sources = append(sources, filepath.Clean(url.Path(sourceURL))) } - return nil + return s.persistTranscribeDependencies(ctx, transcribe, sources) } -func (s *Service) persistTranscribeDependencies(ctx context.Context, transcribe *options.Transcribe) error { +func (s *Service) persistTranscribeDependencies(ctx context.Context, transcribe *options.Transcribe, sources []string) error { depURL := url.Join(transcribe.Repository, "Datly", "dependencies") - - // connections.yaml — use flat format matching legacy translator output - var connectors []connEntry - for _, c := range transcribe.Connectors { - parts := strings.SplitN(c, "|", 4) - if len(parts) >= 3 { - connectors = append(connectors, connEntry{Name: parts[0], Driver: parts[1], DSN: parts[2]}) - } - } - if len(connectors) > 0 { - connURL := url.Join(depURL, "connections.yaml") - // Merge with existing connections if file exists - existing := loadExistingConnectors(ctx, s.fs, connURL) - merged := mergeConnectors(existing, connectors) - connMap := map[string]any{"Connectors": merged} - data, err := yaml.Marshal(connMap) - if err != nil { - return err + depURL = url.Normalize(depURL, file.Scheme) + if len(transcribe.Connectors) > 0 { + var connectors []connEntry + for _, c := range transcribe.Connectors { + parts := strings.SplitN(c, "|", 4) + if len(parts) >= 3 { + connectors = append(connectors, connEntry{Name: parts[0], Driver: parts[1], DSN: parts[2]}) + } } - if err = s.fs.Upload(ctx, connURL, file.DefaultFileOsMode, strings.NewReader(string(data))); err != nil { - return fmt.Errorf("failed to persist connections: %w", err) + if len(connectors) > 0 { + connURL := url.Join(depURL, "connectors.yaml") + existing := loadExistingConnectors(ctx, s.fs, connURL) + merged := mergeConnectors(existing, connectors) + connMap := map[string]any{"Connectors": merged} + data, err := yaml.Marshal(connMap) + if err != nil { + return err + } + if err = s.fs.Upload(ctx, connURL, file.DefaultFileOsMode, strings.NewReader(string(data))); err != nil { + return fmt.Errorf("failed to persist connections: %w", err) + } } } - // config.json - routeURL := url.Join(transcribe.Repository, "Datly", "routes") - cfg := map[string]any{ - "APIPrefix": transcribe.APIPrefix, - "RouteURL": routeURL, + cfgURL := url.Join(transcribe.Repository, "Datly", "config.json") + cfg := s.seedTranscribeConfig(ctx, transcribe) + if cfg.SyncFrequencyMs == 0 { + cfg.SyncFrequencyMs = 2000 + } + cfg.Meta.Init() + if cfg.Meta.StatusURI == "" { + cfg.Meta.StatusURI = "/v1/api/status" + } + payload := map[string]any{ + "APIPrefix": cfg.APIPrefix, "DependencyURL": depURL, "Endpoint": map[string]any{"Port": 8080}, - "SyncFrequencyMs": 2000, - "Meta": map[string]any{"StatusURI": "/v1/api/status"}, + "SyncFrequencyMs": cfg.SyncFrequencyMs, + "Meta": cfg.Meta, + } + if transcribe.APIPrefix != "" { + payload["APIPrefix"] = transcribe.APIPrefix } - cfgData, err := json.MarshalIndent(cfg, "", " ") + if len(cfg.APIKeys) > 0 { + payload["APIKeys"] = cfg.APIKeys + } + if cfg.JWTValidator != nil { + payload["JWTValidator"] = cfg.JWTValidator + } + if cfg.JwtSigner != nil { + payload["JwtSigner"] = cfg.JwtSigner + } + if transcribe.SkipYAML { + payload["DQLBootstrap"] = map[string]any{"Sources": mergeStrings(existingBootstrapSources(ctx, s.fs, cfgURL), sources)} + } else { + payload["RouteURL"] = url.Normalize(url.Join(transcribe.Repository, "Datly", "routes"), file.Scheme) + } + cfgData, err := json.MarshalIndent(payload, "", " ") if err != nil { return err } - cfgURL := url.Join(transcribe.Repository, "Datly", "config.json") if err = s.fs.Upload(ctx, cfgURL, file.DefaultFileOsMode, strings.NewReader(string(cfgData))); err != nil { return fmt.Errorf("failed to persist config: %w", err) } return nil } -func buildPathResource(resource *viewpkg.Resource, component *shapeLoad.Component) *pathpkg.Resource { - if resource == nil { - return nil +func (s *Service) seedTranscribeConfig(ctx context.Context, transcribe *options.Transcribe) *gateway.Config { + seed := &gateway.Config{} + projectCfg := filepath.Join(transcribe.Project, "config.json") + if data, err := s.fs.DownloadWithURL(ctx, projectCfg); err == nil { + _ = json.Unmarshal(data, seed) } - var params []*pathpkg.Parameter - if component != nil { - for _, s := range component.Input { - if s != nil { - params = append(params, &pathpkg.Parameter{ - Name: s.Name, - In: s.In, - Required: s.Required != nil && *s.Required, - Schema: s.Schema, - }) - } + applyAuth(seed, &transcribe.Auth) + return seed +} + +func applyAuth(cfg *gateway.Config, auth *options.Auth) { + if cfg == nil || auth == nil { + return + } + if strings.TrimSpace(auth.RSA) != "" { + cfg.JWTValidator = &verifier.Config{RSA: getScyResources(auth.RSA)} + cfg.JwtSigner = &signer.Config{RSA: getScyResource(strings.Split(auth.RSA, ";")[0])} + } + if strings.TrimSpace(auth.HMAC) != "" { + cfg.JWTValidator = &verifier.Config{HMAC: getScyResource(auth.HMAC)} + cfg.JwtSigner = &signer.Config{HMAC: getScyResource(auth.HMAC)} + } +} + +func getScyResource(location string) *scy.Resource { + pair := strings.Split(location, "|") + res := &scy.Resource{URL: pair[0]} + if len(pair) > 1 { + res.Key = pair[1] + } + res.URL = url.Normalize(res.URL, file.Scheme) + return res +} + +func getScyResources(location string) []*scy.Resource { + var result []*scy.Resource + for _, item := range strings.Split(location, "-") { + item = strings.TrimSpace(item) + if item == "" { + continue } + result = append(result, getScyResource(item)) } - if len(params) == 0 { + return result +} + +func existingBootstrapSources(ctx context.Context, fs afs.Service, cfgURL string) []string { + data, err := fs.DownloadWithURL(ctx, cfgURL) + if err != nil { + return nil + } + cfg := &gateway.Config{} + if err = json.Unmarshal(data, cfg); err != nil || cfg.DQLBootstrap == nil { return nil } - return &pathpkg.Resource{Parameters: params} + return append([]string{}, cfg.DQLBootstrap.Sources...) +} + +func mergeStrings(existing, incoming []string) []string { + seen := map[string]bool{} + var result []string + for _, item := range append(existing, incoming...) { + item = strings.TrimSpace(item) + if item == "" || seen[item] { + continue + } + seen[item] = true + result = append(result, item) + } + return result } type connEntry struct { @@ -185,7 +264,7 @@ func mergeConnectors(existing, incoming []connEntry) []connEntry { if _, ok := byName[c.Name]; !ok { order = append(order, c.Name) } - byName[c.Name] = c // incoming overrides existing for same name + byName[c.Name] = c } result := make([]connEntry, 0, len(order)) for _, name := range order { @@ -202,21 +281,45 @@ func (s *Service) readSource(ctx context.Context, sourceURL string) (string, err return string(payload), nil } -func (s *Service) persistTranscribeRoute(ctx context.Context, transcribe *options.Transcribe, sourceURL, dql string, resource *view.Resource, component *shapeLoad.Component) error { +func (s *Service) persistTranscribeRoute(ctx context.Context, transcribe *options.Transcribe, sourceURL, dql string, resource *view.Resource, component *shapeLoad.Component, codegenResult *xgen.ComponentCodegenResult) error { sourcePath := filepath.Clean(url.Path(sourceURL)) stem := strings.TrimSuffix(filepath.Base(sourcePath), filepath.Ext(sourcePath)) - - // Determine generated file stem: --type-file flag, or root view name in lower_underscore, or DQL filename - typeStem := transcribeTypeStem(transcribe, stem, component) - routeRoot := url.Join(transcribe.Repository, "Datly", "routes") routeYAML := url.Join(routeRoot, stem+".yaml") - + if err := s.applyGeneratedMutableArtifacts(ctx, routeRoot, resource, component, codegenResult); err != nil { + return err + } + if resource != nil && codegenResult != nil && strings.TrimSpace(codegenResult.VeltyFilePath) != "" { + if source, err := os.ReadFile(filepath.Clean(codegenResult.VeltyFilePath)); err == nil { + rootView := "" + if component != nil { + rootView = strings.TrimSpace(component.RootView) + } + root := lookupNamedView(resource, rootView) + if root == nil && len(resource.Views) > 0 { + root = resource.Views[0] + } + if root != nil { + if root.Template == nil { + root.Template = view.NewTemplate(stripLeadingRouteDirective(string(source))) + } else { + root.Template.Source = stripLeadingRouteDirective(string(source)) + } + if rel, err := filepath.Rel(filepath.Clean(codegenResult.PackageDir), filepath.Clean(codegenResult.VeltyFilePath)); err == nil { + root.Template.SourceURL = filepath.ToSlash(rel) + } + } + } + } if resource != nil { + normalizeResourceSchemaPackages(resource) for _, item := range resource.Views { if item == nil || item.Template == nil || strings.TrimSpace(item.Template.Source) == "" { continue } + if strings.HasSuffix(strings.TrimSpace(item.Template.SourceURL), "/patch.sql") || strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(component.RootView)) { + item.Template.Source = stripLeadingRouteDirective(item.Template.Source) + } sqlRel := strings.TrimSpace(item.Template.SourceURL) if sqlRel == "" { sqlRel = path.Join(stem, item.Name+".sql") @@ -228,7 +331,6 @@ func (s *Service) persistTranscribeRoute(ctx context.Context, transcribe *option item.Template.SourceURL = sqlRel } } - rootView := "" if component != nil { rootView = strings.TrimSpace(component.RootView) @@ -237,221 +339,640 @@ func (s *Service) persistTranscribeRoute(ctx context.Context, transcribe *option rootView = resource.Views[0].Name } method, uri := transcribeRulePath(dql, stem, transcribe.APIPrefix, component) - - // Build route YAML as map to control key casing (runtime expects PascalCase YAML keys) - routeEntry := map[string]any{ - "URI": uri, - "Method": method, + routeView := &view.View{ + Reference: shared.Reference{Ref: rootView}, + Name: rootView, } - if rootView != "" { - routeEntry["View"] = map[string]any{"Ref": rootView} + if root := lookupNamedView(resource, rootView); root != nil { + viewCopy := *root + routeView = &viewCopy + routeView.Reference = shared.Reference{Ref: rootView} } - payload := map[string]any{ - "Routes": []any{routeEntry}, - "Resource": sanitizeResourceForRouteYAML(resource), + route := &repository.Component{ + Path: contract.Path{ + Method: method, + URI: uri, + }, + Contract: contract.Contract{ + Service: serviceTypeForMethod(method), + Output: contract.Output{ + CaseFormat: text.CaseFormatLowerCamel, + }, + }, + View: routeView, + } + if root := lookupNamedView(resource, rootView); root != nil && root.Connector != nil { + ref := strings.TrimSpace(root.Connector.Ref) + if ref == "" { + ref = strings.TrimSpace(root.Connector.Name) + } + if ref != "" { + route.View.Connector = view.NewRefConnector(ref) + route.View.Connector.Name = ref + } + } + if component != nil { + route.TypeContext = component.TypeContext + if output := component.OutputParameters(); len(output) > 0 { + route.Contract.Output = contract.Output{ + Cardinality: state.Many, + CaseFormat: text.CaseFormatLowerCamel, + Type: state.Type{ + Parameters: output, + }, + } + } + if component.Directives != nil && component.Directives.MCP != nil { + route.Name = strings.TrimSpace(component.Directives.MCP.Name) + route.Description = strings.TrimSpace(component.Directives.MCP.Description) + route.DescriptionURI = strings.TrimSpace(component.Directives.MCP.DescriptionPath) + } + } + if component != nil && (len(component.Input) > 0 || len(component.Meta) > 0) { + params := transcribeInputParameters(component, resource) + if len(params) > 0 { + normalizeParameterSchemas(params) + route.Contract.Input.Type.Parameters = normalizeParameterTypeNameTags(params) + } + } + if component != nil { + normalizeComponentStateSchemas(component) + } + payload := &shapeRuleFile{ + Routes: []*repository.Component{route}, + Resource: sanitizeResourceForRouteYAML(resource), + With: transcribeSharedResourceRefs(resource), + } + if payload.Resource != nil { + normalizeResourceSchemaPackages(payload.Resource) + promoteAnonymousParameterTypeDefinitions(payload.Resource) + backfillResourceColumnDataTypes(payload.Resource) + canonicalizeResourceTypeDefinitions(payload.Resource) + } + if len(payload.Routes) > 0 { + if payload.Resource != nil && payload.Routes[0] != nil && payload.Routes[0].View != nil { + alignViewParameterSchemasToResourceTypes(payload.Routes[0].View, payload.Resource) + } + normalizeParameterSchemas(payload.Routes[0].Contract.Input.Type.Parameters) + normalizeParameterSchemas(payload.Routes[0].Contract.Output.Type.Parameters) + } + if component != nil && component.TypeContext != nil { + payload.TypeContext = component.TypeContext + } + if payload.Resource != nil && codegenResult != nil && strings.TrimSpace(codegenResult.VeltyFilePath) != "" { + if source, err := os.ReadFile(filepath.Clean(codegenResult.VeltyFilePath)); err == nil { + rootView := "" + if component != nil { + rootView = strings.TrimSpace(component.RootView) + } + root := lookupNamedView(payload.Resource, rootView) + if root == nil && len(payload.Resource.Views) > 0 { + root = payload.Resource.Views[0] + } + if root != nil { + if root.Template == nil { + root.Template = view.NewTemplate(stripLeadingRouteDirective(string(source))) + } else { + root.Template.Source = stripLeadingRouteDirective(string(source)) + } + if rel, err := filepath.Rel(filepath.Clean(codegenResult.PackageDir), filepath.Clean(codegenResult.VeltyFilePath)); err == nil { + root.Template.SourceURL = filepath.ToSlash(rel) + } + } + } } data, err := yaml.Marshal(payload) if err != nil { return err } + data, err = ensureSharedResourceRefsYAML(data, payload.With) + if err != nil { + return err + } + data, err = normalizeConnectorRefsYAML(data) + if err != nil { + return err + } + data, err = normalizeRouteViewRefsYAML(data) + if err != nil { + return err + } + data, err = normalizeRouteComponentEmbeddingYAML(data) + if err != nil { + return err + } if err = s.fs.Upload(ctx, routeYAML, file.DefaultFileOsMode, strings.NewReader(string(data))); err != nil { return fmt.Errorf("failed to persist route yaml %s: %w", routeYAML, err) } - _ = typeStem - // Generate Go types directly from in-memory resource (no YAML roundtrip) - if component != nil && component.TypeContext != nil && resource != nil { - generateTranscribeTypes(url.Path(sourceURL), resource, component) - } return nil } -func generateTranscribeTypes(sourceAbsPath string, resource *view.Resource, component *shapeLoad.Component) { - ctx := component.TypeContext - if ctx == nil || strings.TrimSpace(ctx.PackageDir) == "" { - return +func transcribeSharedResourceRefs(resource *view.Resource) []string { + if resource == nil { + return nil + } + var result []string + if len(collectResourceConnectorRefs(resource)) > 0 { + result = append(result, view.ResourceConnectors) } + if len(resource.CacheProviders) > 0 { + result = append(result, "cache") + } + return result +} + +func (s *Service) generateTranscribeTypes(sourceAbsPath, dql string, transcribe *options.Transcribe, resource *view.Resource, component *shapeLoad.Component) (*xgen.ComponentCodegenResult, error) { + if component == nil || component.TypeContext == nil || resource == nil { + return nil, nil + } + ctx := component.TypeContext projectDir := findProjectDir(sourceAbsPath) if projectDir == "" { - fmt.Printf("WARNING: shape codegen: cannot locate go.mod from %s, skipping type generation\n", sourceAbsPath) - return + projectDir = transcribe.Project } codegen := &xgen.ComponentCodegen{ Component: component, Resource: resource, TypeContext: ctx, ProjectDir: projectDir, - WithEmbed: true, - WithContract: false, + WithEmbed: !transcribe.SkipYAML, + WithContract: true, } - result, err := codegen.Generate() - if err != nil { - fmt.Printf("WARNING: shape codegen: type generation skipped for %s: %v\n", filepath.Base(sourceAbsPath), err) - return + if pkgPath, pkgDir, pkgName := resolvedTranscribeTypeOutput(projectDir, ctx.PackagePath); pkgPath != "" { + codegen.PackagePath = pkgPath + codegen.PackageDir = pkgDir + codegen.PackageName = pkgName + } + if method, uri := resolvedTranscribeRoute(sourceAbsPath, dql, transcribe.APIPrefix); uri != "" { + component.Method = method + component.URI = uri } - fmt.Printf("generated component %s → %s\n", strings.Join(result.Types, ", "), result.FilePath) + return codegen.Generate() } -func transcribeCompileOptions(transcribe *options.Transcribe) []shape.CompileOption { - var opts []shape.CompileOption - if transcribe.Strict { - opts = append(opts, shape.WithCompileStrict(true)) - } - namespace := strings.TrimSpace(transcribe.Namespace) - module := strings.TrimSpace(transcribe.Module) - typeOutput := strings.TrimSpace(transcribe.TypeOutput) - if typeOutput == "" || typeOutput == "." { - typeOutput = module +func (s *Service) applyGeneratedMutableArtifacts(ctx context.Context, routeRoot string, resource *view.Resource, component *shapeLoad.Component, codegenResult *xgen.ComponentCodegenResult) error { + if resource == nil || component == nil || codegenResult == nil || codegenResult.PackageDir == "" { + return nil } - if namespace != "" { - pkgDir := filepath.Join(typeOutput, namespace) - pkgName := filepath.Base(namespace) - opts = append(opts, shape.WithTypeContextPackageDir(pkgDir)) - opts = append(opts, shape.WithTypeContextPackageName(pkgName)) + uploaded := map[string]string{} + packageDir := filepath.Clean(codegenResult.PackageDir) + for _, generated := range codegenResult.GeneratedFiles { + if strings.TrimSpace(generated) == "" || !strings.HasSuffix(strings.TrimSpace(generated), ".sql") { + continue + } + absPath := filepath.Clean(generated) + rel, err := filepath.Rel(packageDir, absPath) + if err != nil { + continue + } + rel = filepath.ToSlash(rel) + if strings.HasPrefix(rel, "../") { + continue + } + data, err := os.ReadFile(absPath) + if err != nil { + return fmt.Errorf("failed to read generated sql %s: %w", absPath, err) + } + content := string(data) + if strings.TrimSpace(codegenResult.VeltyFilePath) != "" && filepath.Clean(codegenResult.VeltyFilePath) == absPath { + content = stripLeadingRouteDirective(content) + } + dest := path.Join(url.Path(routeRoot), rel) + if err = s.fs.Upload(ctx, dest, file.DefaultFileOsMode, strings.NewReader(content)); err != nil { + return fmt.Errorf("failed to persist generated sql %s: %w", dest, err) + } + uploaded[rel] = content } - return opts -} - -// transcribeTypeStem determines the Go file name stem. -// Priority: --type-file flag > root view name (lower_underscore) > DQL filename -func transcribeTypeStem(transcribe *options.Transcribe, dqlStem string, component *shapeLoad.Component) string { - if tf := strings.TrimSpace(transcribe.TypeFile); tf != "" { - return strings.TrimSuffix(tf, ".go") + if len(uploaded) == 0 { + return nil } + rootView := "" if component != nil { - if rootView := strings.TrimSpace(component.RootView); rootView != "" { - return toLowerUnderscore(rootView) - } + rootView = strings.TrimSpace(component.RootView) } - return dqlStem -} - -// toLowerUnderscore converts CamelCase or PascalCase to lower_underscore. -func toLowerUnderscore(s string) string { - var buf strings.Builder - for i, r := range s { - if unicode.IsUpper(r) { - if i > 0 { - prev := rune(s[i-1]) - if unicode.IsLower(prev) || unicode.IsDigit(prev) { - buf.WriteByte('_') + root := lookupNamedView(resource, rootView) + if root == nil && resource != nil && len(resource.Views) > 0 { + root = resource.Views[0] + } + if root != nil && strings.TrimSpace(codegenResult.VeltyFilePath) != "" { + if rel, err := filepath.Rel(packageDir, filepath.Clean(codegenResult.VeltyFilePath)); err == nil { + rel = filepath.ToSlash(rel) + if source, ok := uploaded[rel]; ok { + if root.Template == nil { + root.Template = view.NewTemplate(source) + } else { + root.Template.Source = source } + root.Template.SourceURL = rel + preserveTemplateParameters(root, component.InputParameters()) } - buf.WriteRune(unicode.ToLower(r)) - } else { - buf.WriteRune(r) } } - return buf.String() -} - -func transcribeRulePath(dql, ruleName, apiPrefix string, component *shapeLoad.Component) (string, string) { - method := "GET" - uri := "/" + strings.Trim(strings.TrimSpace(ruleName), "/") - if prefix := strings.TrimSpace(apiPrefix); prefix != "" { - uri = strings.TrimRight(prefix, "/") + uri - } - if component != nil && component.Directives != nil && component.Directives.Route != nil { - rd := component.Directives.Route - if u := strings.TrimSpace(rd.URI); u != "" { - uri = u + for _, item := range resource.Views { + if item == nil || item.Template == nil { + continue } - if len(rd.Methods) > 0 { - if m := strings.TrimSpace(strings.ToUpper(rd.Methods[0])); m != "" { - method = m - } + rel := strings.TrimSpace(item.Template.SourceURL) + if item.Template.DeclaredParametersOnly { + item.Template.Parameters = append(state.Parameters{}, resource.Parameters.UsedBy(item.Template.Source)...) + } else { + preserveTemplateParameters(item, resource.Parameters.UsedBy(item.Template.Source)) + preserveTemplateParameters(item, dependentTemplateParameters(item.Template.Parameters, resource.Parameters)) + } + if len(item.Template.Parameters) > 0 { + item.Template.UseParameterStateType = true + } + if rel == "" { + continue + } + if source, ok := uploaded[rel]; ok { + item.Template.Source = source } } - return method, uri + return nil } -// discoverColumns resolves wildcard columns from DB for all views in the resource. -func discoverColumns(ctx context.Context, resource *view.Resource) { - if resource == nil { +func preserveTemplateParameters(aView *view.View, params state.Parameters) { + if aView == nil || aView.Template == nil || len(params) == 0 { return } - detector := shapeColumn.New() - for _, aView := range resource.Views { - if aView == nil { + if aView.Template.DeclaredParametersOnly { + return + } + seen := map[string]bool{} + for _, item := range aView.Template.Parameters { + if item == nil || strings.TrimSpace(item.Name) == "" { continue } - columns, err := detector.Resolve(ctx, resource, aView) - if err != nil { - fmt.Printf(" column discovery skipped for %s: %v\n", aView.Name, err) + seen[strings.ToLower(strings.TrimSpace(item.Name))] = true + } + for _, param := range params { + if param == nil || strings.TrimSpace(param.Name) == "" { continue } - if len(columns) > 0 { - aView.Columns = columns + switch param.In.Kind { + case state.KindOutput, state.KindMeta, state.KindAsync: + continue } + key := strings.ToLower(strings.TrimSpace(param.Name)) + if seen[key] { + continue + } + aView.Template.Parameters = append(aView.Template.Parameters, param) + seen[key] = true } } -// applyConnectorsToResource registers connectors on the resource and sets refs on views. -// Connector format: name|driver|dsn (same encoding as datly translate -c flag). -func applyConnectorsToResource(resource *view.Resource, connectors []string) { - if resource == nil || len(connectors) == 0 { - return +func transcribeInputParameters(component *shapeLoad.Component, resource *view.Resource) state.Parameters { + if component == nil { + return nil } - defaultName := "" - for _, c := range connectors { - parts := strings.SplitN(c, "|", 4) - if len(parts) < 1 { - continue + params := make(state.Parameters, 0, len(component.Input)+len(component.Meta)+4) + seen := map[string]bool{} + declared := map[string]bool{} + appendParam := func(param *state.Parameter) { + if param == nil { + return } - name := strings.TrimSpace(parts[0]) - if name == "" { - continue + key := strings.ToLower(strings.TrimSpace(param.Name)) + if key == "" || seen[key] { + return } - if defaultName == "" { - defaultName = name + cloned := *param + if param.Schema != nil { + cloned.Schema = param.Schema.Clone() } - if len(parts) >= 3 { - driver := strings.TrimSpace(parts[1]) - dsn := strings.TrimSpace(parts[2]) - resource.AddConnectors(view.NewConnector(name, driver, dsn)) + if param.Output != nil { + output := *param.Output + if param.Output.Schema != nil { + output.Schema = param.Output.Schema.Clone() + } + cloned.Output = &output } + params = append(params, &cloned) + seen[key] = true } - if defaultName == "" { + for _, item := range component.Input { + if item != nil { + if name := strings.ToLower(strings.TrimSpace(item.Name)); name != "" { + declared[name] = true + } + appendParam(&item.Parameter) + } + } + rootView := lookupNamedView(resource, strings.TrimSpace(component.RootView)) + if rootView != nil && rootView.Template != nil { + for _, item := range rootView.Template.Parameters { + if item == nil { + continue + } + if !declared[strings.ToLower(strings.TrimSpace(item.Name))] { + continue + } + appendParam(item) + } + } + for _, item := range component.Meta { + if item != nil { + if name := strings.ToLower(strings.TrimSpace(item.Name)); name != "" { + declared[name] = true + } + appendParam(&item.Parameter) + } + } + return params +} + +func prepareResourceForTranscribeCodegen(resource *view.Resource, component *shapeLoad.Component) { + if resource == nil || component == nil { return } - for _, v := range resource.Views { - if v != nil && v.Connector == nil { - v.Connector = view.NewRefConnector(defaultName) + rootView := "" + if component != nil { + rootView = strings.TrimSpace(component.RootView) + } + root := lookupNamedView(resource, rootView) + if root == nil && len(resource.Views) > 0 { + root = resource.Views[0] + } + if root != nil && root.Template != nil { + preserveTemplateParameters(root, component.InputParameters()) + preserveTemplateParameters(root, resource.Parameters) + if len(root.Template.Parameters) > 0 { + root.Template.UseParameterStateType = true + } + } + for _, item := range resource.Views { + if item == nil || item.Template == nil { + continue + } + if item.Template.DeclaredParametersOnly { + item.Template.Parameters = append(state.Parameters{}, resource.Parameters.UsedBy(item.Template.Source)...) + } else { + preserveTemplateParameters(item, resource.Parameters.UsedBy(item.Template.Source)) + preserveTemplateParameters(item, dependentTemplateParameters(item.Template.Parameters, resource.Parameters)) + } + if len(item.Template.Parameters) > 0 { + item.Template.UseParameterStateType = true } } } -// sanitizeResourceForRouteYAML returns a serialization-safe copy of resource -// with connector config stripped to references only. This keeps DSN/driver -// details out of route YAML; dependencies/connections.yaml remains the source -// of truth for connector definitions. -func sanitizeResourceForRouteYAML(resource *view.Resource) *view.Resource { - if resource == nil { +func dependentTemplateParameters(params state.Parameters, resourceParams state.Parameters) state.Parameters { + if len(params) == 0 || len(resourceParams) == 0 { return nil } - - cloned := *resource - - if len(resource.Connectors) > 0 { - cloned.Connectors = make([]*view.Connector, 0, len(resource.Connectors)) - for _, connector := range resource.Connectors { - if connector == nil { + seen := map[string]bool{} + result := make(state.Parameters, 0) + for _, param := range params { + if param == nil || param.In == nil { + continue + } + switch param.In.Kind { + case state.KindParam: + name := strings.TrimSpace(param.In.Name) + if name == "" { continue } - ref := strings.TrimSpace(connector.Ref) - if ref == "" { - ref = strings.TrimSpace(connector.Name) - } - if ref == "" { + key := strings.ToLower(name) + if seen[key] { continue } - refConnector := view.NewRefConnector(ref) - refConnector.Name = ref - cloned.Connectors = append(cloned.Connectors, refConnector) + if dep := resourceParams.Lookup(name); dep != nil { + result = append(result, dep) + seen[key] = true + } + } + } + return result +} + +func stripLeadingRouteDirective(content string) string { + trimmed := strings.TrimSpace(content) + if !strings.HasPrefix(trimmed, "/*") { + return content + } + end := strings.Index(trimmed, "*/") + if end == -1 { + return content + } + header := trimmed[2:end] + if !strings.Contains(header, `"URI"`) || !strings.Contains(header, `"Method"`) { + return content + } + return strings.TrimSpace(trimmed[end+2:]) + "\n" +} + +func lookupNamedView(resource *view.Resource, name string) *view.View { + if resource == nil || strings.TrimSpace(name) == "" { + return nil + } + for _, item := range resource.Views { + if item != nil && strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) { + return item + } + } + return nil +} + +var routeDirectivePattern = regexp.MustCompile(`\$route\(\s*['"]([^'"]+)['"](?:\s*,\s*['"]([^'"]+)['"])?`) + +func resolvedTranscribeRoute(sourcePath, dql, apiPrefix string) (string, string) { + matches := routeDirectivePattern.FindStringSubmatch(dql) + if len(matches) > 0 { + method := strings.ToUpper(strings.TrimSpace(matches[2])) + if method == "" { + method = "GET" + } + return method, strings.TrimSpace(matches[1]) + } + stem := strings.TrimSuffix(filepath.Base(sourcePath), filepath.Ext(sourcePath)) + uri := "/" + strings.Trim(stem, "/") + if prefix := strings.TrimSpace(apiPrefix); prefix != "" { + uri = strings.TrimRight(prefix, "/") + uri + } + return "GET", uri +} +func resolvedTranscribeTypeOutput(projectDir, packagePath string) (string, string, string) { + projectDir = strings.TrimSpace(projectDir) + packagePath = strings.TrimSpace(packagePath) + if projectDir == "" || packagePath == "" { + return "", "", "" + } + modulePath, err := transcribeModulePath(filepath.Join(projectDir, "go.mod")) + if err != nil || modulePath == "" { + return "", "", "" + } + prefix := strings.TrimRight(modulePath, "/") + "/" + if !strings.HasPrefix(packagePath, prefix) { + return "", "", "" + } + rel := strings.TrimPrefix(packagePath, prefix) + rel = sanitizeTypeNamespace(rel) + if rel == "" { + return "", "", "" + } + pkgDir := filepath.Join(projectDir, filepath.FromSlash(rel)) + pkgName := filepath.Base(rel) + return strings.TrimRight(modulePath, "/") + "/" + rel, pkgDir, pkgName +} + +func transcribeModulePath(goModPath string) (string, error) { + data, err := os.ReadFile(goModPath) + if err != nil { + return "", err + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "module ") { + return strings.TrimSpace(strings.TrimPrefix(line, "module ")), nil + } + } + return "", fmt.Errorf("module path not found in %s", goModPath) +} +func transcribeCompileOptions(transcribe *options.Transcribe) []shape.CompileOption { + var opts []shape.CompileOption + if transcribe.Strict { + opts = append(opts, shape.WithCompileStrict(true)) + } + opts = append(opts, shape.WithLinkedTypes(false)) + namespace := strings.TrimSpace(transcribe.Namespace) + module := strings.TrimSpace(transcribe.Module) + typeOutput := strings.TrimSpace(transcribe.TypeOutput) + if typeOutput == "" || typeOutput == "." { + typeOutput = module + } + if namespace != "" { + sanitizedNamespace := sanitizeTypeNamespace(namespace) + pkgDir := filepath.Join(typeOutput, sanitizedNamespace) + pkgName := filepath.Base(sanitizedNamespace) + opts = append(opts, shape.WithTypeContextPackageDir(pkgDir)) + opts = append(opts, shape.WithTypeContextPackageName(pkgName)) + } + return opts +} + +func sanitizeTypeNamespace(namespace string) string { + parts := strings.Split(strings.ReplaceAll(strings.TrimSpace(namespace), "\\", "/"), "/") + for i, part := range parts { + part = strings.TrimSpace(part) + switch part { + case "": + continue + case "vendor": + part = "vendorsrc" + default: + part = sanitizeTypeNamespaceSegment(part) + } + parts[i] = part + } + return path.Join(parts...) +} + +func sanitizeTypeNamespaceSegment(segment string) string { + var b strings.Builder + for _, r := range segment { + switch { + case r >= 'a' && r <= 'z': + b.WriteRune(r) + case r >= 'A' && r <= 'Z': + b.WriteRune(r + ('a' - 'A')) + case r >= '0' && r <= '9': + b.WriteRune(r) + case r == '_' || r == '-': + b.WriteRune('_') + } + } + if b.Len() == 0 { + return "generated" + } + ret := b.String() + if ret[0] >= '0' && ret[0] <= '9' { + return "p" + ret + } + return ret +} + +func transcribeRulePath(_ string, ruleName, apiPrefix string, component *shapeLoad.Component) (string, string) { + method := "GET" + uri := "/" + strings.Trim(strings.TrimSpace(ruleName), "/") + if prefix := strings.TrimSpace(apiPrefix); prefix != "" { + uri = strings.TrimRight(prefix, "/") + uri + } + if component != nil { + if u := strings.TrimSpace(component.URI); u != "" { + uri = u + } + if m := strings.TrimSpace(strings.ToUpper(component.Method)); m != "" { + method = m + } + } + return method, uri +} + +func discoverColumns(ctx context.Context, resource *view.Resource) { + if resource == nil { + return + } + detector := shapeColumn.New() + for _, aView := range resource.Views { + if aView == nil { + continue + } + columns, err := detector.Resolve(ctx, resource, aView) + if err == nil && len(columns) > 0 { + aView.Columns = columns + } + } +} + +func applyConnectorsToResource(resource *view.Resource, connectors []string) { + if resource == nil || len(connectors) == 0 { + return + } + defaultName := "" + for _, c := range connectors { + parts := strings.SplitN(c, "|", 4) + if len(parts) < 1 { + continue + } + name := strings.TrimSpace(parts[0]) + if name == "" { + continue + } + if defaultName == "" { + defaultName = name + } + if len(parts) >= 3 { + resource.AddConnectors(view.NewConnector(name, strings.TrimSpace(parts[1]), strings.TrimSpace(parts[2]))) + } + } + if defaultName == "" { + return + } + for _, v := range resource.Views { + if v != nil && v.Connector == nil { + v.Connector = view.NewRefConnector(defaultName) + } + } +} + +func sanitizeResourceForRouteYAML(resource *view.Resource) *view.Resource { + if resource == nil { + return nil + } + cloned := *resource + cloned.Parameters = normalizeParameterTypeNameTags(cloneParameters(resource.Parameters)) + if refs := collectResourceConnectorRefs(resource); len(refs) > 0 { + cloned.Connectors = make([]*view.Connector, 0, len(refs)) + for _, ref := range refs { + refConnector := view.NewRefConnector(ref) + refConnector.Name = ref + cloned.Connectors = append(cloned.Connectors, refConnector) } } else { cloned.Connectors = nil } - if len(resource.Views) > 0 { cloned.Views = make(view.Views, 0, len(resource.Views)) for _, item := range resource.Views { @@ -477,6 +998,1057 @@ func sanitizeResourceForRouteYAML(resource *view.Resource) *view.Resource { } else { cloned.Views = nil } - return &cloned } + +func collectResourceConnectorRefs(resource *view.Resource) []string { + if resource == nil { + return nil + } + seen := map[string]bool{} + var result []string + appendRef := func(connector *view.Connector) { + if connector == nil { + return + } + ref := strings.TrimSpace(connector.Ref) + if ref == "" { + ref = strings.TrimSpace(connector.Name) + } + if ref == "" || seen[ref] { + return + } + seen[ref] = true + result = append(result, ref) + } + var visitView func(aView *view.View) + visitView = func(aView *view.View) { + if aView == nil { + return + } + appendRef(aView.Connector) + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + visitView(&rel.Of.View) + } + } + for _, connector := range resource.Connectors { + appendRef(connector) + } + for _, aView := range resource.Views { + visitView(aView) + } + return result +} + +func cloneParameters(params state.Parameters) state.Parameters { + if len(params) == 0 { + return nil + } + result := make(state.Parameters, 0, len(params)) + for _, item := range params { + if item == nil { + continue + } + cloned := *item + if item.Schema != nil { + cloned.Schema = item.Schema.Clone() + } + if item.Output != nil { + output := *item.Output + if item.Output.Schema != nil { + output.Schema = item.Output.Schema.Clone() + } + cloned.Output = &output + } + result = append(result, &cloned) + } + return result +} + +func normalizeParameterTypeNameTags(params state.Parameters) state.Parameters { + if len(params) == 0 { + return params + } + for _, item := range params { + if item == nil || item.Schema == nil { + continue + } + typeName := strings.TrimSpace(item.Schema.Name) + if typeName == "" { + continue + } + item.Tag = ensureTypeNameTag(item.Tag, typeName) + } + return params +} + +func normalizeComponentStateSchemas(component *shapeLoad.Component) { + if component == nil { + return + } + for _, item := range component.Input { + if item != nil { + normalizeSchemaPackage(item.Schema) + } + } + for _, item := range component.Output { + if item != nil { + normalizeSchemaPackage(item.Schema) + } + } + for _, item := range component.Meta { + if item != nil { + normalizeSchemaPackage(item.Schema) + } + } +} + +func normalizeResourceSchemaPackages(resource *view.Resource) { + if resource == nil { + return + } + normalizeParameterSchemas(resource.Parameters) + for _, aView := range resource.Views { + normalizeViewSchemaPackages(aView) + } + for _, item := range resource.Types { + if item == nil { + continue + } + item.Package = normalizedSchemaPackage(item.Package, item.ModulePath) + for _, field := range item.Fields { + if field == nil { + continue + } + normalizeSchemaPackage(field.Schema) + } + } +} + +func backfillResourceColumnDataTypes(resource *view.Resource) { + if resource == nil { + return + } + typeDefs := map[string]*view.TypeDefinition{} + for _, item := range resource.Types { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + typeDefs[strings.ToLower(strings.TrimSpace(item.Name))] = item + } + var visitView func(aView *view.View) + visitView = func(aView *view.View) { + if aView == nil { + return + } + backfillViewColumnDataTypes(aView, typeDefs) + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + visitView(&rel.Of.View) + } + } + for _, aView := range resource.Views { + visitView(aView) + } +} + +func backfillViewColumnDataTypes(aView *view.View, defs map[string]*view.TypeDefinition) { + if aView == nil || len(aView.Columns) == 0 || aView.Schema == nil { + return + } + typeName := strings.TrimSpace(aView.Schema.Name) + if typeName == "" { + return + } + def := defs[strings.ToLower(typeName)] + if def == nil { + return + } + fieldTypes := map[string]string{} + for _, field := range def.Fields { + if field == nil || field.Schema == nil { + continue + } + dataType := strings.TrimSpace(firstNonEmpty(field.Schema.DataType, field.Schema.Name)) + if dataType == "" { + continue + } + for _, key := range []string{ + strings.ToUpper(strings.TrimSpace(field.Name)), + strings.ToUpper(strings.TrimSpace(field.Column)), + strings.ToUpper(strings.TrimSpace(field.FromName)), + } { + if key != "" { + fieldTypes[key] = dataType + } + } + } + for _, column := range aView.Columns { + if column == nil || strings.TrimSpace(column.DataType) != "" { + continue + } + for _, key := range []string{ + strings.ToUpper(strings.TrimSpace(column.Name)), + strings.ToUpper(strings.TrimSpace(column.DatabaseColumn)), + strings.ToUpper(strings.TrimSpace(column.FieldName())), + } { + if dataType := strings.TrimSpace(fieldTypes[key]); dataType != "" { + column.DataType = dataType + break + } + } + } +} + +func canonicalizeResourceTypeDefinitions(resource *view.Resource) { + if resource == nil { + return + } + for _, def := range resource.Types { + canonicalizeTypeDefinition(def) + } +} + +func promoteAnonymousParameterTypeDefinitions(resource *view.Resource) { + if resource == nil { + return + } + existing := map[string]bool{} + for _, def := range resource.Types { + if def == nil || strings.TrimSpace(def.Name) == "" { + continue + } + existing[strings.ToLower(strings.TrimSpace(def.Name))] = true + } + promoted := map[string]string{} + for _, param := range resource.Parameters { + if param == nil || param.Schema == nil { + continue + } + typeName := promotedParameterTypeName(param) + if typeName == "" { + continue + } + key := strings.ToLower(typeName) + if !existing[key] { + def := typeDefinitionFromAnonymousParameter(typeName, param) + if def == nil { + continue + } + resource.Types = append(resource.Types, def) + existing[key] = true + } + promoted[strings.ToLower(strings.TrimSpace(param.Name))] = typeName + rewritePromotedParameterSchema(param, typeName) + } + if len(promoted) == 0 { + return + } + visitResourceParameters(resource, func(param *state.Parameter) { + if param == nil || param.Schema == nil { + return + } + typeName := promoted[strings.ToLower(strings.TrimSpace(param.Name))] + if typeName == "" { + return + } + rewritePromotedParameterSchema(param, typeName) + }) +} + +func promotedParameterTypeName(param *state.Parameter) string { + if param == nil || param.Schema == nil { + return "" + } + if strings.TrimSpace(param.Name) == "" { + return "" + } + if strings.TrimSpace(param.Schema.Name) != "" && !strings.Contains(strings.TrimSpace(param.Schema.Name), "struct {") { + return "" + } + dataType := strings.TrimSpace(param.Schema.DataType) + rType := param.Schema.Type() + if !strings.Contains(dataType, "struct {") { + if rType == nil { + return "" + } + base := rType + for base.Kind() == reflect.Ptr || base.Kind() == reflect.Slice || base.Kind() == reflect.Array { + base = base.Elem() + } + if base.Kind() != reflect.Struct || base.Name() != "" { + return "" + } + } + return state.SanitizeTypeName(strings.TrimSpace(param.Name)) +} + +func typeDefinitionFromAnonymousParameter(typeName string, param *state.Parameter) *view.TypeDefinition { + if param == nil || param.Schema == nil || typeName == "" { + return nil + } + fields := typeDefinitionFieldsFromReflectType(param.Schema.Type()) + if len(fields) == 0 { + return nil + } + for _, field := range fields { + if field != nil { + field.Tag = "" + } + } + def := &view.TypeDefinition{Name: typeName, Fields: dedupeTypeDefinitionFields(fields)} + canonicalizeTypeDefinition(def) + return def +} + +func rewritePromotedParameterSchema(param *state.Parameter, typeName string) { + if param == nil || param.Schema == nil || typeName == "" { + return + } + param.Schema.Name = typeName + param.Schema.DataType = typeName + param.Schema.Package = "" + param.Schema.PackagePath = "" + param.Schema.ModulePath = "" +} + +func visitResourceParameters(resource *view.Resource, visitor func(param *state.Parameter)) { + if resource == nil || visitor == nil { + return + } + for _, param := range resource.Parameters { + visitor(param) + } + var visitView func(aView *view.View) + visitView = func(aView *view.View) { + if aView == nil { + return + } + if aView.Template != nil { + for _, param := range aView.Template.Parameters { + visitor(param) + } + } + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + visitView(&rel.Of.View) + } + } + for _, aView := range resource.Views { + visitView(aView) + } +} + +func alignViewParameterSchemasToResourceTypes(aView *view.View, resource *view.Resource) { + if aView == nil || resource == nil { + return + } + typeNames := map[string]string{} + for _, def := range resource.Types { + if def == nil || strings.TrimSpace(def.Name) == "" { + continue + } + typeNames[strings.ToLower(strings.TrimSpace(def.Name))] = strings.TrimSpace(def.Name) + } + var visitView func(current *view.View) + visitView = func(current *view.View) { + if current == nil { + return + } + if current.Template != nil { + for _, param := range current.Template.Parameters { + if param == nil || param.Schema == nil { + continue + } + typeName := typeNames[strings.ToLower(strings.TrimSpace(param.Name))] + if typeName == "" { + continue + } + rewritePromotedParameterSchema(param, typeName) + } + } + for _, rel := range current.With { + if rel == nil || rel.Of == nil { + continue + } + visitView(&rel.Of.View) + } + } + visitView(aView) +} + +func canonicalizeTypeDefinition(def *view.TypeDefinition) { + if def == nil || len(def.Fields) == 0 { + return + } + fields := dedupeTypeDefinitionFields(def.Fields) + if len(fields) == 0 { + return + } + def.DataType = inlineStructDataType(fields) + def.Fields = nil + def.Schema = nil +} + +func dedupeTypeDefinitionFields(fields []*view.Field) []*view.Field { + type keyedField struct { + key string + field *view.Field + } + var ordered []keyedField + index := map[string]int{} + for _, field := range fields { + if field == nil { + continue + } + key := canonicalTypeFieldKey(field) + if key == "" { + continue + } + if pos, ok := index[key]; ok { + merged := mergeTypeFields(ordered[pos].field, field) + if preferTypeField(field, ordered[pos].field) { + ordered[pos].field = merged + } else { + ordered[pos].field = merged + } + continue + } + index[key] = len(ordered) + ordered = append(ordered, keyedField{key: key, field: cloneTypeField(field)}) + } + result := make([]*view.Field, 0, len(ordered)) + for _, item := range ordered { + if item.field != nil { + item.field.Tag = sanitizeTypeFieldTag(item.field.Tag, item.field) + result = append(result, item.field) + } + } + return result +} + +func mergeTypeFields(primary, secondary *view.Field) *view.Field { + result := cloneTypeField(primary) + if result == nil { + return cloneTypeField(secondary) + } + if secondary == nil { + return result + } + if strings.TrimSpace(result.Column) == "" { + result.Column = strings.TrimSpace(secondary.Column) + } + if strings.TrimSpace(result.FromName) == "" { + result.FromName = strings.TrimSpace(secondary.FromName) + } + if strings.TrimSpace(result.Tag) == "" { + result.Tag = strings.TrimSpace(secondary.Tag) + } + if result.Schema == nil && secondary.Schema != nil { + result.Schema = secondary.Schema.Clone() + } + return result +} + +func cloneTypeField(field *view.Field) *view.Field { + if field == nil { + return nil + } + cloned := *field + if field.Schema != nil { + cloned.Schema = field.Schema.Clone() + } + return &cloned +} + +func canonicalTypeFieldKey(field *view.Field) string { + for _, candidate := range []string{ + strings.TrimSpace(field.Column), + strings.TrimSpace(field.FromName), + strings.TrimSpace(field.Name), + } { + if candidate != "" { + return strings.ToUpper(candidate) + } + } + return "" +} + +func preferTypeField(candidate, existing *view.Field) bool { + if existing == nil { + return true + } + candidateScore := typeFieldPreferenceScore(candidate) + existingScore := typeFieldPreferenceScore(existing) + if candidateScore != existingScore { + return candidateScore > existingScore + } + return strings.TrimSpace(candidate.Name) < strings.TrimSpace(existing.Name) +} + +func typeFieldPreferenceScore(field *view.Field) int { + if field == nil { + return -1 + } + score := 0 + name := strings.TrimSpace(field.Name) + if name != "" && name != strings.ToUpper(name) { + score += 10 + } + if strings.TrimSpace(field.Column) == "" { + score += 3 + } + if strings.TrimSpace(field.FromName) == name { + score += 2 + } + if strings.EqualFold(name, "Has") { + score += 5 + } + return score +} + +func inlineStructDataType(fields []*view.Field) string { + parts := make([]string, 0, len(fields)) + for _, field := range fields { + if field == nil || field.Schema == nil { + continue + } + typeName := strings.TrimSpace(firstNonEmpty(field.Schema.DataType, field.Schema.Name)) + if typeName == "" { + continue + } + tag := strings.TrimSpace(stripVeltyTag(field.Tag)) + if tag != "" { + parts = append(parts, fmt.Sprintf(`%s %s %q`, strings.TrimSpace(field.Name), typeName, tag)) + continue + } + parts = append(parts, fmt.Sprintf(`%s %s`, strings.TrimSpace(field.Name), typeName)) + } + return "struct { " + strings.Join(parts, "; ") + " }" +} + +func typeDefinitionFieldsFromReflectType(rType reflect.Type) []*view.Field { + if rType == nil { + return nil + } + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return nil + } + result := make([]*view.Field, 0, rType.NumField()) + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if !field.IsExported() { + continue + } + result = append(result, &view.Field{ + Name: field.Name, + Schema: schemaFromReflectType(field.Type), + Tag: string(field.Tag), + FromName: field.Name, + }) + } + return result +} + +func schemaFromReflectType(rType reflect.Type) *state.Schema { + if rType == nil { + return nil + } + schema := state.NewSchema(rType) + if schema == nil { + return nil + } + if schema.Name == "" && schema.DataType == "" { + schema.DataType = rType.String() + if schema.Cardinality == "" { + schema.Cardinality = state.One + } + } + if schema.Cardinality == state.Many && schema.DataType == "" { + schema.DataType = rType.String() + } + return schema +} + +func stripVeltyTag(tag string) string { + tag = strings.TrimSpace(tag) + if tag == "" { + return "" + } + updated, _ := xreflect.RemoveTag(tag, "velty") + return strings.TrimSpace(updated) +} + +func sanitizeTypeFieldTag(tag string, field *view.Field) string { + tag = stripVeltyTag(tag) + if field == nil || field.Schema == nil { + return tag + } + dataType := strings.TrimSpace(firstNonEmpty(field.Schema.DataType, field.Schema.Name)) + if strings.HasPrefix(dataType, "*struct {") || strings.HasPrefix(dataType, "struct {") { + updated, _ := xreflect.RemoveTag(tag, "typeName") + tag = strings.TrimSpace(updated) + } + return tag +} + +func normalizeViewSchemaPackages(aView *view.View) { + if aView == nil { + return + } + normalizeSchemaPackage(aView.Schema) + if aView.Template != nil { + normalizeParameterSchemas(aView.Template.Parameters) + if aView.Template.Summary != nil { + normalizeSchemaPackage(aView.Template.Summary.Schema) + } + } + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + normalizeViewSchemaPackages(&rel.Of.View) + } +} + +func normalizeParameterSchemas(params state.Parameters) { + for _, param := range params { + if param == nil { + continue + } + normalizeSchemaPackage(param.Schema) + if param.Output != nil { + normalizeSchemaPackage(param.Output.Schema) + } + } +} + +func normalizeSchemaPackage(schema *state.Schema) { + if schema == nil { + return + } + schema.Package = normalizedSchemaPackage(schema.Package, firstNonEmpty(schema.PackagePath, schema.ModulePath)) + if strings.TrimSpace(schema.PackagePath) == "" && strings.Contains(strings.TrimSpace(schema.Package), "/") { + schema.PackagePath = strings.TrimSpace(schema.Package) + } +} + +func normalizedSchemaPackage(pkg, pkgPath string) string { + pkg = strings.TrimSpace(pkg) + pkgPath = strings.TrimSpace(pkgPath) + if pkgPath == "" && strings.Contains(pkg, "/") { + pkgPath = pkg + } + if strings.Contains(pkg, "/") { + return path.Base(pkg) + } + if pkg == "" && pkgPath != "" { + return path.Base(pkgPath) + } + return pkg +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func alignGeneratedPackageAliases(resource *view.Resource, component *shapeLoad.Component, packageDir, packagePath, packageName string) { + packageDir = strings.TrimSpace(packageDir) + packagePath = strings.TrimSpace(packagePath) + packageName = strings.TrimSpace(packageName) + if packagePath == "" || packageName == "" { + return + } + if component != nil && component.TypeContext != nil { + if packageDir != "" { + packageDir = filepath.ToSlash(filepath.Clean(packageDir)) + } + if strings.TrimSpace(component.TypeContext.PackagePath) == packagePath { + component.TypeContext.PackageName = packageName + if packageDir != "" { + component.TypeContext.PackageDir = packageDir + } + } + if strings.TrimSpace(component.TypeContext.PackagePath) == "" { + component.TypeContext.PackagePath = packagePath + } + if strings.TrimSpace(component.TypeContext.PackageName) == "" { + component.TypeContext.PackageName = packageName + } + if packageDir != "" && strings.TrimSpace(component.TypeContext.PackageDir) == "" { + component.TypeContext.PackageDir = packageDir + } + for _, group := range [][]*plan.State{component.Input, component.Output, component.Meta, component.Async, component.Other} { + for _, item := range group { + if item == nil { + continue + } + alignSchemaPackageAlias(item.Schema, packagePath, packageName) + alignSchemaPackageAlias(item.OutputSchema(), packagePath, packageName) + } + } + } + if resource == nil { + return + } + for _, item := range resource.Parameters { + if item == nil { + continue + } + alignSchemaPackageAlias(item.Schema, packagePath, packageName) + alignSchemaPackageAlias(item.OutputSchema(), packagePath, packageName) + } + for _, aView := range resource.Views { + if aView == nil { + continue + } + alignSchemaPackageAlias(aView.Schema, packagePath, packageName) + if aView.Template != nil { + alignSchemaPackageAlias(aView.Template.Schema, packagePath, packageName) + alignParameterPackages(aView.Template.Parameters, packagePath, packageName) + if aView.Template.Summary != nil { + alignSchemaPackageAlias(aView.Template.Summary.Schema, packagePath, packageName) + } + } + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + alignSchemaPackageAlias(rel.Of.Schema, packagePath, packageName) + alignSchemaPackageAlias(rel.Of.View.Schema, packagePath, packageName) + if rel.Of.View.Template != nil && rel.Of.View.Template.Summary != nil { + alignSchemaPackageAlias(rel.Of.View.Template.Summary.Schema, packagePath, packageName) + } + } + } + for _, item := range resource.Types { + if item == nil { + continue + } + if firstNonEmpty(strings.TrimSpace(item.ModulePath), schemaPackagePath(item.Schema)) == packagePath { + item.Package = packageName + } + alignSchemaPackageAlias(item.Schema, packagePath, packageName) + for _, field := range item.Fields { + if field == nil { + continue + } + alignSchemaPackageAlias(field.Schema, packagePath, packageName) + } + } +} + +func alignParameterPackages(params state.Parameters, packagePath, packageName string) { + for _, item := range params { + if item == nil { + continue + } + alignSchemaPackageAlias(item.Schema, packagePath, packageName) + alignSchemaPackageAlias(item.OutputSchema(), packagePath, packageName) + } +} + +func alignSchemaPackageAlias(schema *state.Schema, packagePath, packageName string) { + if schema == nil { + return + } + if schemaPackagePath(schema) == packagePath { + schema.Package = packageName + qualifyGeneratedSchemaDataType(schema, packageName) + } +} + +func schemaPackagePath(schema *state.Schema) string { + if schema == nil { + return "" + } + return firstNonEmpty(strings.TrimSpace(schema.PackagePath), strings.TrimSpace(schema.ModulePath)) +} + +func qualifyGeneratedSchemaDataType(schema *state.Schema, packageName string) { + if schema == nil { + return + } + packageName = strings.TrimSpace(packageName) + if packageName == "" { + return + } + dataType := strings.TrimSpace(schema.DataType) + typeName := strings.TrimLeft(strings.TrimSpace(schema.Name), "*") + if dataType == "" || typeName == "" || strings.Contains(dataType, ".") { + return + } + replacements := map[string]string{ + typeName: packageName + "." + typeName, + "*" + typeName: "*" + packageName + "." + typeName, + "[]" + typeName: "[]" + packageName + "." + typeName, + "[]*" + typeName: "[]*" + packageName + "." + typeName, + } + if qualified, ok := replacements[dataType]; ok { + schema.DataType = qualified + } +} + +func ensureTypeNameTag(tag string, typeName string) string { + typeName = strings.TrimSpace(typeName) + if typeName == "" { + return strings.TrimSpace(tag) + } + tag = strings.TrimSpace(tag) + if strings.Contains(tag, `typeName:"`) { + return tag + } + if tag == "" { + return fmt.Sprintf(`typeName:"%s"`, typeName) + } + return tag + ` typeName:"` + typeName + `"` +} + +func normalizeConnectorRefsYAML(data []byte) ([]byte, error) { + var node yaml.Node + if err := yaml.Unmarshal(data, &node); err != nil { + return nil, err + } + rewriteConnectorNode(&node) + return yaml.Marshal(&node) +} + +func normalizeRouteViewRefsYAML(data []byte) ([]byte, error) { + var node yaml.Node + if err := yaml.Unmarshal(data, &node); err != nil { + return nil, err + } + rewriteRouteViewNode(&node) + return yaml.Marshal(&node) +} + +func normalizeRouteComponentEmbeddingYAML(data []byte) ([]byte, error) { + var node yaml.Node + if err := yaml.Unmarshal(data, &node); err != nil { + return nil, err + } + if len(node.Content) == 0 || node.Content[0] == nil { + return data, nil + } + root := node.Content[0] + routes := yamlMapLookup(root, "Routes") + if routes == nil || routes.Kind != yaml.SequenceNode { + return data, nil + } + for _, item := range routes.Content { + flattenRouteComponentNode(item) + } + return yaml.Marshal(&node) +} + +func ensureSharedResourceRefsYAML(data []byte, refs []string) ([]byte, error) { + if len(refs) == 0 { + return data, nil + } + var node yaml.Node + if err := yaml.Unmarshal(data, &node); err != nil { + return nil, err + } + if len(node.Content) == 0 || node.Content[0] == nil || node.Content[0].Kind != yaml.MappingNode { + return data, nil + } + root := node.Content[0] + if existing := yamlMapLookup(root, "With"); existing != nil && existing.Kind == yaml.SequenceNode && len(existing.Content) > 0 { + return data, nil + } + seq := &yaml.Node{Kind: yaml.SequenceNode} + for _, ref := range refs { + if strings.TrimSpace(ref) == "" { + continue + } + seq.Content = append(seq.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: strings.TrimSpace(ref), Tag: "!!str"}) + } + if len(seq.Content) == 0 { + return data, nil + } + root.Content = append(root.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "With", Tag: "!!str"}, + seq, + ) + return yaml.Marshal(&node) +} + +func rewriteConnectorNode(node *yaml.Node) { + if node == nil { + return + } + switch node.Kind { + case yaml.DocumentNode: + for _, child := range node.Content { + rewriteConnectorNode(child) + } + case yaml.MappingNode: + for i := 0; i < len(node.Content)-1; i += 2 { + key := node.Content[i] + value := node.Content[i+1] + switch strings.ToLower(strings.TrimSpace(key.Value)) { + case "connector": + if ref := connectorRefFromYAMLNode(value); ref != "" { + node.Content[i+1] = connectorRefYAMLNode(ref) + value = node.Content[i+1] + } + case "connectors": + if value.Kind == yaml.SequenceNode { + for j, item := range value.Content { + if ref := connectorRefFromYAMLNode(item); ref != "" { + value.Content[j] = connectorRefYAMLNode(ref) + } + } + } + } + rewriteConnectorNode(value) + } + case yaml.SequenceNode: + for _, child := range node.Content { + rewriteConnectorNode(child) + } + } +} + +func rewriteRouteViewNode(node *yaml.Node) { + if node == nil { + return + } + switch node.Kind { + case yaml.DocumentNode: + for _, child := range node.Content { + rewriteRouteViewNode(child) + } + case yaml.MappingNode: + for i := 0; i < len(node.Content)-1; i += 2 { + key := node.Content[i] + value := node.Content[i+1] + if strings.EqualFold(strings.TrimSpace(key.Value), "view") && value.Kind == yaml.MappingNode { + if ref := routeViewRefFromYAMLNode(value); ref != "" { + node.Content[i+1] = &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "Ref", Tag: "!!str"}, + {Kind: yaml.ScalarNode, Value: ref, Tag: "!!str"}, + }, + } + value = node.Content[i+1] + } + } + rewriteRouteViewNode(value) + } + case yaml.SequenceNode: + for _, child := range node.Content { + rewriteRouteViewNode(child) + } + } +} + +func flattenRouteComponentNode(node *yaml.Node) { + if node == nil || node.Kind != yaml.MappingNode { + return + } + for _, key := range []string{"meta", "path", "contract"} { + embedded := yamlMapLookup(node, key) + if embedded == nil || embedded.Kind != yaml.MappingNode { + continue + } + removeYAMLMapKey(node, key) + node.Content = append(node.Content, embedded.Content...) + } +} + +func routeViewRefFromYAMLNode(node *yaml.Node) string { + if node == nil || node.Kind != yaml.MappingNode { + return "" + } + if ref := yamlMapLookup(node, "Ref"); ref != nil && strings.TrimSpace(ref.Value) != "" { + return strings.TrimSpace(ref.Value) + } + reference := yamlMapLookup(node, "reference") + if reference == nil { + return "" + } + ref := yamlMapLookup(reference, "ref") + if ref == nil { + return "" + } + return strings.TrimSpace(ref.Value) +} + +func connectorRefFromYAMLNode(node *yaml.Node) string { + if node == nil || node.Kind != yaml.MappingNode { + return "" + } + if ref := yamlMapLookup(node, "ref"); ref != nil && strings.TrimSpace(ref.Value) != "" { + return strings.TrimSpace(ref.Value) + } + connection := yamlMapLookup(node, "connection") + if connection == nil { + return "" + } + dbConfig := yamlMapLookup(connection, "dbconfig") + if dbConfig == nil { + return "" + } + reference := yamlMapLookup(dbConfig, "reference") + if reference == nil { + return "" + } + ref := yamlMapLookup(reference, "ref") + if ref == nil { + return "" + } + return strings.TrimSpace(ref.Value) +} + +func connectorRefYAMLNode(ref string) *yaml.Node { + return &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "ref", Tag: "!!str"}, + {Kind: yaml.ScalarNode, Value: ref, Tag: "!!str"}, + }, + } +} + +func yamlMapLookup(node *yaml.Node, key string) *yaml.Node { + if node == nil || node.Kind != yaml.MappingNode { + return nil + } + for i := 0; i < len(node.Content)-1; i += 2 { + if strings.EqualFold(strings.TrimSpace(node.Content[i].Value), key) { + return node.Content[i+1] + } + } + return nil +} + +func removeYAMLMapKey(node *yaml.Node, key string) { + if node == nil || node.Kind != yaml.MappingNode { + return + } + filtered := make([]*yaml.Node, 0, len(node.Content)) + for i := 0; i < len(node.Content)-1; i += 2 { + if strings.EqualFold(strings.TrimSpace(node.Content[i].Value), key) { + continue + } + filtered = append(filtered, node.Content[i], node.Content[i+1]) + } + node.Content = filtered +} diff --git a/cmd/command/transcribe_test.go b/cmd/command/transcribe_test.go new file mode 100644 index 000000000..73ac0786f --- /dev/null +++ b/cmd/command/transcribe_test.go @@ -0,0 +1,513 @@ +package command + +import ( + "context" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/repository/shape" + shapeCompile "github.com/viant/datly/repository/shape/compile" + shapeLoad "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" + extension "github.com/viant/datly/view/extension" + "github.com/viant/datly/view/state" + "gopkg.in/yaml.v3" +) + +func TestPatchBasicOne_LoadedComponentHasMutableExecHelpers(t *testing.T) { + source := filepath.Join("..", "..", "e2e", "v1", "dql", "dev", "events", "patch_basic_one.dql") + data, err := os.ReadFile(source) + require.NoError(t, err) + + planned, err := shapeCompile.New().Compile(context.Background(), &shape.Source{ + Name: "patch_basic_one", + Path: source, + DQL: string(data), + }) + require.NoError(t, err) + + artifact, err := shapeLoad.New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := shapeLoad.ComponentFrom(artifact) + require.True(t, ok) + require.NotNil(t, component) + + root := lookupNamedView(artifact.Resource, component.RootView) + require.NotNil(t, root) + assert.Equal(t, view.ModeExec, root.Mode) + require.NotNil(t, root.Template) + assert.True(t, root.Template.UseParameterStateType) + require.NotNil(t, root.Template.Parameters.Lookup("CurFoosId")) + require.NotNil(t, root.Template.Parameters.Lookup("CurFoos")) + assert.Equal(t, state.Many, root.Template.Parameters.Lookup("CurFoos").Schema.Cardinality) + + input := component.InputParameters() + require.Nil(t, input.Lookup("CurFoosId")) + require.Nil(t, input.Lookup("CurFoos")) + + curFoos, err := artifact.Resource.View("CurFoos") + require.NoError(t, err) + require.NotNil(t, curFoos) + require.NotNil(t, curFoos.Template) + assert.Equal(t, "foos/cur_foos.sql", curFoos.Template.SourceURL) +} + +func TestPatchBasicOne_LoadedComponentHasMutableExecHelpers_WithTypeContextPackages(t *testing.T) { + source := filepath.Join("..", "..", "e2e", "v1", "dql", "dev", "events", "patch_basic_one.dql") + data, err := os.ReadFile(source) + require.NoError(t, err) + + planned, err := shapeCompile.New().Compile(context.Background(), &shape.Source{ + Name: "patch_basic_one", + Path: source, + DQL: string(data), + }, transcribeCompileOptions(&options.Transcribe{ + Project: filepath.Join("..", "..", "e2e", "v1"), + Module: filepath.Join("..", "..", "e2e", "v1"), + TypeOutput: filepath.Join("..", "..", "e2e", "v1", "shape"), + Namespace: "dev/basic/foos", + })...) + require.NoError(t, err) + + artifact, err := shapeLoad.New().LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + + component, ok := shapeLoad.ComponentFrom(artifact) + require.True(t, ok) + require.NotNil(t, component) + + root := lookupNamedView(artifact.Resource, component.RootView) + require.NotNil(t, root) + require.NotNil(t, root.Template) + require.NotNil(t, root.Template.Parameters.Lookup("CurFoos")) + assert.Equal(t, state.Many, root.Template.Parameters.Lookup("CurFoos").Schema.Cardinality) + + curFoos, err := artifact.Resource.View("CurFoos") + require.NoError(t, err) + require.NotNil(t, curFoos) + require.NotNil(t, curFoos.Template) + require.True(t, curFoos.Template.DeclaredParametersOnly) + require.NotNil(t, curFoos.Template.Parameters.Lookup("CurFoosId")) + require.Nil(t, curFoos.Template.Parameters.Lookup("Foos")) +} + +func TestTranscribeSharedResourceRefs_IncludesConnectors(t *testing.T) { + resource := &view.Resource{ + Connectors: []*view.Connector{view.NewRefConnector("dev")}, + } + + refs := transcribeSharedResourceRefs(resource) + require.Equal(t, []string{view.ResourceConnectors}, refs) +} + +func TestEnsureSharedResourceRefsYAML_AppendsWith(t *testing.T) { + data, err := ensureSharedResourceRefsYAML([]byte("Resource: {}\nRoutes: []\n"), []string{view.ResourceConnectors}) + require.NoError(t, err) + assert.True(t, strings.Contains(string(data), "With:\n - connectors")) +} + +func TestNormalizeParameterTypeNameTags_AppendsTypeName(t *testing.T) { + params := state.Parameters{ + &state.Parameter{ + Name: "Foos", + Tag: `anonymous:"true"`, + Schema: &state.Schema{Name: "FoosView"}, + }, + } + + normalized := normalizeParameterTypeNameTags(params) + require.Len(t, normalized, 1) + assert.Equal(t, `anonymous:"true" typeName:"FoosView"`, normalized[0].Tag) +} + +func TestPreserveTemplateParameters_AppendsMutableHelpers(t *testing.T) { + aView := &view.View{ + Name: "foos", + Template: view.NewTemplate("SELECT 1", view.WithTemplateParameters( + &state.Parameter{Name: "Foos", In: state.NewBodyLocation(""), Schema: &state.Schema{Name: "FoosView"}}, + )), + } + + params := state.Parameters{ + &state.Parameter{Name: "Foos", In: state.NewBodyLocation(""), Schema: &state.Schema{Name: "FoosView"}}, + &state.Parameter{Name: "CurFoosId", In: state.NewParameterLocation("Foos"), Schema: &state.Schema{DataType: "int"}}, + &state.Parameter{Name: "CurFoos", In: state.NewViewLocation("CurFoos"), Schema: &state.Schema{Name: "FoosView"}}, + &state.Parameter{Name: "Meta", In: state.NewOutputLocation("summary"), Schema: &state.Schema{Name: "MetaView"}}, + } + + preserveTemplateParameters(aView, params) + + require.NotNil(t, aView.Template.Parameters.Lookup("Foos")) + require.NotNil(t, aView.Template.Parameters.Lookup("CurFoosId")) + require.NotNil(t, aView.Template.Parameters.Lookup("CurFoos")) + require.Nil(t, aView.Template.Parameters.Lookup("Meta")) +} + +func TestPreserveTemplateParameters_SkipsDeclaredOnlyTemplate(t *testing.T) { + aView := &view.View{ + Name: "CurFoos", + Template: view.NewTemplate("SELECT 1", + view.WithTemplateParameters( + &state.Parameter{Name: "CurFoosId", In: state.NewParameterLocation("Foos"), Schema: &state.Schema{DataType: "int"}}, + ), + view.WithTemplateDeclaredParametersOnly(true), + ), + } + + params := state.Parameters{ + &state.Parameter{Name: "Foos", In: state.NewBodyLocation(""), Schema: &state.Schema{Name: "FoosView"}}, + } + + preserveTemplateParameters(aView, params) + + require.NotNil(t, aView.Template.Parameters.Lookup("CurFoosId")) + require.Nil(t, aView.Template.Parameters.Lookup("Foos")) +} + +func TestPrepareResourceForTranscribeCodegen_DeclaredOnlyTemplateKeepsOnlyUsedParams(t *testing.T) { + resource := &view.Resource{ + Parameters: state.Parameters{ + &state.Parameter{Name: "Foos", In: state.NewBodyLocation(""), Schema: &state.Schema{Name: "FoosView"}}, + &state.Parameter{Name: "CurFoosId", In: state.NewParameterLocation("Foos"), Schema: &state.Schema{DataType: "int"}}, + }, + Views: []*view.View{ + { + Name: "foos", + Template: view.NewTemplate("SELECT 1", view.WithTemplateParameters( + &state.Parameter{Name: "Foos", In: state.NewBodyLocation(""), Schema: &state.Schema{Name: "FoosView"}}, + )), + }, + { + Name: "CurFoos", + Template: view.NewTemplate(`SELECT * FROM FOOS WHERE $criteria.In("ID", $CurFoosId.Values)`, + view.WithTemplateParameters( + &state.Parameter{Name: "CurFoosId", In: state.NewParameterLocation("Foos"), Schema: &state.Schema{DataType: "int"}}, + ), + view.WithTemplateDeclaredParametersOnly(true), + ), + }, + }, + } + component := &shapeLoad.Component{RootView: "foos"} + + prepareResourceForTranscribeCodegen(resource, component) + + curFoos := lookupNamedView(resource, "CurFoos") + require.NotNil(t, curFoos) + require.NotNil(t, curFoos.Template) + require.NotNil(t, curFoos.Template.Parameters.Lookup("CurFoosId")) + require.Nil(t, curFoos.Template.Parameters.Lookup("Foos")) +} + +func TestDependentTemplateParameters_AppendsParentSourceParameter(t *testing.T) { + params := state.Parameters{ + &state.Parameter{Name: "CurFoosId", In: state.NewParameterLocation("Foos"), Schema: &state.Schema{DataType: "int"}}, + } + resourceParams := state.Parameters{ + &state.Parameter{Name: "Foos", In: state.NewBodyLocation(""), Schema: &state.Schema{Name: "FoosView"}}, + &state.Parameter{Name: "CurFoosId", In: state.NewParameterLocation("Foos"), Schema: &state.Schema{DataType: "int"}}, + } + + deps := dependentTemplateParameters(params, resourceParams) + + require.Len(t, deps, 1) + require.Equal(t, "Foos", deps[0].Name) +} + +func TestAlignGeneratedPackageAliases_UsesGeneratedPackageName(t *testing.T) { + const pkgPath = "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one" + resource := &view.Resource{ + Views: []*view.View{ + { + Name: "foos", + Schema: &state.Schema{ + Package: "foos", + PackagePath: pkgPath, + Name: "FoosView", + DataType: "*FoosView", + Cardinality: state.Many, + }, + Template: view.NewTemplate("SELECT 1", view.WithTemplateParameters( + &state.Parameter{ + Name: "Foos", + Schema: &state.Schema{Package: "foos", PackagePath: pkgPath, Name: "FoosView", DataType: "*FoosView", Cardinality: state.One}, + }, + )), + }, + }, + Types: []*view.TypeDefinition{ + { + Name: "FoosView", + Package: "foos", + ModulePath: pkgPath, + Schema: &state.Schema{Package: "foos", PackagePath: pkgPath, Name: "FoosView", DataType: "*FoosView", Cardinality: state.Many}, + }, + }, + Parameters: state.Parameters{ + &state.Parameter{ + Name: "Foos", + Schema: &state.Schema{Package: "foos", PackagePath: pkgPath, Name: "FoosView", DataType: "*FoosView", Cardinality: state.One}, + }, + }, + } + component := &shapeLoad.Component{ + TypeContext: &typectx.Context{PackageName: "foos", PackagePath: pkgPath}, + Input: []*plan.State{ + {Parameter: state.Parameter{Name: "Foos", Schema: &state.Schema{Package: "foos", PackagePath: pkgPath, Name: "FoosView", DataType: "*FoosView", Cardinality: state.One}}}, + }, + } + + alignGeneratedPackageAliases(resource, component, filepath.Join("..", "..", "e2e", "v1", "shape", "dev", "events", "patch_basic_one"), pkgPath, "patch_basic_one") + + require.Equal(t, "patch_basic_one", component.TypeContext.PackageName) + require.Equal(t, filepath.ToSlash(filepath.Clean(filepath.Join("..", "..", "e2e", "v1", "shape", "dev", "events", "patch_basic_one"))), component.TypeContext.PackageDir) + require.Equal(t, "patch_basic_one", component.Input[0].Schema.Package) + require.Equal(t, "*patch_basic_one.FoosView", component.Input[0].Schema.DataType) + require.Equal(t, "patch_basic_one", resource.Views[0].Schema.Package) + require.Equal(t, "*patch_basic_one.FoosView", resource.Views[0].Schema.DataType) + require.Equal(t, "patch_basic_one", resource.Views[0].Template.Parameters[0].Schema.Package) + require.Equal(t, "*patch_basic_one.FoosView", resource.Views[0].Template.Parameters[0].Schema.DataType) + require.Equal(t, "patch_basic_one", resource.Parameters[0].Schema.Package) + require.Equal(t, "*patch_basic_one.FoosView", resource.Parameters[0].Schema.DataType) + require.Equal(t, "patch_basic_one", resource.Types[0].Package) + require.Equal(t, "patch_basic_one", resource.Types[0].Schema.Package) + require.Equal(t, "*patch_basic_one.FoosView", resource.Types[0].Schema.DataType) +} + +func TestGenerateTranscribeTypes_RealignsGeneratedPackageAlias(t *testing.T) { + source := filepath.Join("..", "..", "e2e", "v1", "dql", "dev", "events", "patch_basic_one.dql") + data, err := os.ReadFile(source) + require.NoError(t, err) + + planned, err := shapeCompile.New().Compile(context.Background(), &shape.Source{ + Name: "patch_basic_one", + Path: source, + DQL: string(data), + Connector: "dev", + }, transcribeCompileOptions(&options.Transcribe{ + Project: filepath.Join("..", "..", "e2e", "v1"), + Module: filepath.Join("..", "..", "e2e", "v1"), + TypeOutput: filepath.Join("..", "..", "e2e", "v1", "shape"), + Namespace: "dev/basic/foos", + })...) + require.NoError(t, err) + + artifact, err := shapeLoad.New().LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + component, ok := shapeLoad.ComponentFrom(artifact) + require.True(t, ok) + + svc := &Service{} + result, err := svc.generateTranscribeTypes(source, string(data), &options.Transcribe{ + Project: filepath.Join("..", "..", "e2e", "v1"), + Module: filepath.Join("..", "..", "e2e", "v1"), + TypeOutput: filepath.Join("..", "..", "e2e", "v1", "shape"), + Namespace: "dev/basic/foos", + }, artifact.Resource, component) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "patch_basic_one", result.PackageName) + + alignGeneratedPackageAliases(artifact.Resource, component, result.PackageDir, result.PackagePath, result.PackageName) + + root := lookupNamedView(artifact.Resource, component.RootView) + require.NotNil(t, root) + require.Equal(t, result.PackageName, root.Schema.Package) + require.Equal(t, "*patch_basic_one.FoosView", root.Schema.DataType) + require.Equal(t, result.PackageName, artifact.Resource.Parameters.Lookup("Foos").Schema.Package) + require.Equal(t, "*patch_basic_one.FoosView", artifact.Resource.Parameters.Lookup("Foos").Schema.DataType) +} + +func TestTranscribe_PatchBasicOneRouteYAMLUsesGeneratedPackageName(t *testing.T) { + cwd, err := os.Getwd() + require.NoError(t, err) + repoRoot := filepath.Clean(filepath.Join(cwd, "..", "..")) + project := filepath.Join(repoRoot, "e2e", "v1") + tempRepo := t.TempDir() + + svc := New() + err = svc.Transcribe(context.Background(), &options.Options{ + Transcribe: &options.Transcribe{ + Source: []string{filepath.Join(project, "dql", "dev", "events", "patch_basic_one.dql")}, + Repository: tempRepo, + Project: project, + Module: project, + TypeOutput: filepath.Join(project, "shape"), + Namespace: "dev/basic/foos", + APIPrefix: "/v1/api/shape", + }, + }) + require.NoError(t, err) + + routeYAML := filepath.Join(tempRepo, "Datly", "routes", "patch_basic_one.yaml") + data, err := os.ReadFile(routeYAML) + require.NoError(t, err) + text := string(data) + require.Contains(t, text, "Package: patch_basic_one") + require.Contains(t, text, "DataType: '*patch_basic_one.FoosView'") + require.Contains(t, text, "CaseFormat: lc") + require.NotContains(t, text, "Package: foos\n") +} + +func TestTranscribe_PatchBasicOneRouteYAMLBuildsNamedTemplateState(t *testing.T) { + cwd, err := os.Getwd() + require.NoError(t, err) + repoRoot := filepath.Clean(filepath.Join(cwd, "..", "..")) + project := filepath.Join(repoRoot, "e2e", "v1") + tempRepo := t.TempDir() + + svc := New() + err = svc.Transcribe(context.Background(), &options.Options{ + Transcribe: &options.Transcribe{ + Source: []string{filepath.Join(project, "dql", "dev", "events", "patch_basic_one.dql")}, + Repository: tempRepo, + Project: project, + Module: project, + TypeOutput: filepath.Join(project, "shape"), + Namespace: "dev/basic/foos", + APIPrefix: "/v1/api/shape", + }, + }) + require.NoError(t, err) + + routeYAML := filepath.Join(tempRepo, "Datly", "routes", "patch_basic_one.yaml") + data, err := os.ReadFile(routeYAML) + require.NoError(t, err) + + payload := &shapeRuleFile{} + require.NoError(t, yaml.Unmarshal(data, payload)) + require.NotNil(t, payload.Resource) + payload.Resource.Connectors = []*view.Connector{view.NewConnector("dev", "sqlite3", "file::memory:?cache=shared")} + payload.Resource.SetTypes(extension.Config.Types) + require.NoError(t, payload.Resource.Init(context.Background(), payload.Resource.TypeRegistry(), extension.Config.Codecs, nil, nil, extension.Config.Predicates)) + + root := lookupNamedView(payload.Resource, "foos") + require.NotNil(t, root) + require.NotNil(t, root.Template) + require.NotNil(t, root.Template.StateType()) + + rType := root.Template.StateType().Type() + field, ok := rType.FieldByName("Foos") + require.True(t, ok) + require.Equal(t, "*patch_basic_one.FoosView", field.Type.String()) +} + +func TestTranscribe_PatchBasicOneRouteYAMLPreservesNamedHelperParamTypes(t *testing.T) { + cwd, err := os.Getwd() + require.NoError(t, err) + repoRoot := filepath.Clean(filepath.Join(cwd, "..", "..")) + project := filepath.Join(repoRoot, "e2e", "v1") + tempRepo := t.TempDir() + + svc := New() + err = svc.Transcribe(context.Background(), &options.Options{ + Transcribe: &options.Transcribe{ + Source: []string{filepath.Join(project, "dql", "dev", "events", "patch_basic_one.dql")}, + Repository: tempRepo, + Project: project, + Module: project, + TypeOutput: filepath.Join(project, "shape"), + Namespace: "dev/basic/foos", + APIPrefix: "/v1/api/shape", + }, + }) + require.NoError(t, err) + + routeYAML := filepath.Join(tempRepo, "Datly", "routes", "patch_basic_one.yaml") + data, err := os.ReadFile(routeYAML) + require.NoError(t, err) + + payload := &shapeRuleFile{} + require.NoError(t, yaml.Unmarshal(data, payload)) + require.NotNil(t, payload.Resource) + + curFoosID := payload.Resource.Parameters.Lookup("CurFoosId") + require.NotNil(t, curFoosID) + require.NotNil(t, curFoosID.Schema) + require.Equal(t, "CurFoosId", curFoosID.Schema.DataType) + + var helperType *view.TypeDefinition + for _, item := range payload.Resource.Types { + if item != nil && item.Name == "CurFoosId" { + helperType = item + break + } + } + require.NotNil(t, helperType) + require.Equal(t, "struct { Values []int }", helperType.DataType) + + curFoos := lookupNamedView(payload.Resource, "CurFoos") + require.NotNil(t, curFoos) + require.NotNil(t, curFoos.Template) + curFoosParam := curFoos.Template.Parameters.Lookup("CurFoosId") + require.NotNil(t, curFoosParam) + require.NotNil(t, curFoosParam.Schema) + require.Equal(t, "CurFoosId", curFoosParam.Schema.DataType) +} + +func TestGenerateTranscribeTypes_MetaFormatPreservesChildSummaryType(t *testing.T) { + source := filepath.Join("..", "..", "e2e", "v1", "dql", "dev", "vendorsrv", "meta_format.dql") + data, err := os.ReadFile(source) + require.NoError(t, err) + + project := filepath.Join("..", "..", "e2e", "v1") + shapeOutput := filepath.Join(project, "shape") + transcribeOpts := &options.Transcribe{ + Project: project, + Module: project, + TypeOutput: shapeOutput, + Namespace: "dev/vendor/meta-format", + } + transcribeOpts.Connectors = []string{"dev|mysql|root:dev@tcp(localhost:3306)/dev?parseTime=true"} + + planned, err := shapeCompile.New().Compile(context.Background(), &shape.Source{ + Name: "meta_format", + Path: source, + DQL: string(data), + Connector: "dev", + }, transcribeCompileOptions(transcribeOpts)...) + require.NoError(t, err) + + artifact, err := shapeLoad.New().LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + component, ok := shapeLoad.ComponentFrom(artifact) + require.True(t, ok) + + applyConnectorsToResource(artifact.Resource, transcribeOpts.Connectors) + discoverColumns(context.Background(), artifact.Resource) + prepareResourceForTranscribeCodegen(artifact.Resource, component) + + products := lookupNamedView(artifact.Resource, "products") + require.NotNil(t, products) + require.NotNil(t, products.Template) + require.NotNil(t, products.Template.Summary) + require.NotNil(t, products.Template.Summary.Schema) + summaryType := products.Template.Summary.Schema.Type() + require.NotNil(t, summaryType) + if summaryType.Kind() == reflect.Ptr { + summaryType = summaryType.Elem() + } + field, ok := summaryType.FieldByName("VendorId") + require.True(t, ok) + require.Equal(t, reflect.TypeOf((*int)(nil)), field.Type) + + svc := &Service{} + result, err := svc.generateTranscribeTypes(source, string(data), transcribeOpts, artifact.Resource, component) + require.NoError(t, err) + require.NotNil(t, result) + + outputSource, err := os.ReadFile(result.OutputFilePath) + require.NoError(t, err) + assert.Contains(t, string(outputSource), `type ProductsMetaView struct {`) + assert.Contains(t, string(outputSource), `VendorId *int`) + assert.NotContains(t, string(outputSource), `VendorId string`) +} diff --git a/cmd/command/translate_shape.go b/cmd/command/translate_shape.go index fa12d8f03..3d6ddc205 100644 --- a/cmd/command/translate_shape.go +++ b/cmd/command/translate_shape.go @@ -70,10 +70,11 @@ func (s *Service) translateShape(ctx context.Context, opts *options.Options) err type shapeRuleFile struct { Resource *view.Resource `yaml:"Resource,omitempty"` Routes []*repository.Component `yaml:"Routes,omitempty"` + With []string `yaml:"With,omitempty"` TypeContext any `yaml:"TypeContext,omitempty"` } -func (s *Service) persistShapeRoute(ctx context.Context, opts *options.Options, sourceURL, dql string, resource *view.Resource, component *shapeLoad.Component) error { +func (s *Service) persistShapeRoute(ctx context.Context, opts *options.Options, sourceURL, _ string, resource *view.Resource, component *shapeLoad.Component) error { rule := opts.Rule() routeYAML, routeRoot, relDir, stem, err := routePathForShape(rule, opts.Repository().RepositoryURL, sourceURL) if err != nil { @@ -105,18 +106,9 @@ func (s *Service) persistShapeRoute(ctx context.Context, opts *options.Options, if rootView == "" && resource != nil && len(resource.Views) > 0 && resource.Views[0] != nil { rootView = resource.Views[0].Name } - method, uri := parseShapeRulePath(dql, rule.RuleName(), opts.Repository().APIPrefix) - // Gap 3: RouteDirective overrides method/URI when explicitly declared in DQL. - if component != nil && component.Directives != nil && component.Directives.Route != nil { - rd := component.Directives.Route - if u := strings.TrimSpace(rd.URI); u != "" { - uri = u - } - if len(rd.Methods) > 0 { - if m := strings.TrimSpace(strings.ToUpper(rd.Methods[0])); m != "" { - method = m - } - } + method, uri, err := shapeComponentPath(component) + if err != nil { + return err } route := &repository.Component{ Path: contract.Path{ @@ -172,6 +164,27 @@ func (s *Service) persistShapeRoute(ctx context.Context, opts *options.Options, return nil } +func shapeComponentPath(component *shapeLoad.Component) (string, string, error) { + if component == nil { + return "", "", fmt.Errorf("shape component was nil") + } + method := strings.TrimSpace(strings.ToUpper(component.Method)) + uri := strings.TrimSpace(component.URI) + if method == "" && len(component.ComponentRoutes) > 0 && component.ComponentRoutes[0] != nil { + method = strings.TrimSpace(strings.ToUpper(component.ComponentRoutes[0].Method)) + } + if uri == "" && len(component.ComponentRoutes) > 0 && component.ComponentRoutes[0] != nil { + uri = strings.TrimSpace(component.ComponentRoutes[0].RoutePath) + } + if method == "" { + method = "GET" + } + if uri == "" { + return "", "", fmt.Errorf("shape component route URI was empty") + } + return method, uri, nil +} + func routePathForShape(rule *options.Rule, repoURL, sourceURL string) (routeYAML string, routeRoot string, relDir string, stem string, err error) { sourcePath := filepath.Clean(url.Path(sourceURL)) basePath := filepath.Clean(rule.BaseRuleURL()) diff --git a/cmd/command/translate_shape_ir.go b/cmd/command/translate_shape_ir.go index c2f37155f..8e51812bf 100644 --- a/cmd/command/translate_shape_ir.go +++ b/cmd/command/translate_shape_ir.go @@ -92,8 +92,7 @@ func (s *Service) translateShapeIR(ctx context.Context, opts *options.Options) e return nil } -func buildShapeRulePayload(opts *options.Options, dql string, resource *view.Resource, component *shapeLoad.Component) (*shapeRuleFile, error) { - rule := opts.Rule() +func buildShapeRulePayload(_ *options.Options, _ string, resource *view.Resource, component *shapeLoad.Component) (*shapeRuleFile, error) { rootView := "" if component != nil { rootView = strings.TrimSpace(component.RootView) @@ -101,18 +100,9 @@ func buildShapeRulePayload(opts *options.Options, dql string, resource *view.Res if rootView == "" && resource != nil && len(resource.Views) > 0 && resource.Views[0] != nil { rootView = resource.Views[0].Name } - method, uri := parseShapeRulePath(dql, rule.RuleName(), opts.Repository().APIPrefix) - // Gap 3: RouteDirective overrides method/URI when explicitly declared in DQL. - if component != nil && component.Directives != nil && component.Directives.Route != nil { - rd := component.Directives.Route - if u := strings.TrimSpace(rd.URI); u != "" { - uri = u - } - if len(rd.Methods) > 0 { - if m := strings.TrimSpace(strings.ToUpper(rd.Methods[0])); m != "" { - method = m - } - } + method, uri, err := shapeComponentPath(component) + if err != nil { + return nil, err } route := &repository.Component{ Path: contract.Path{ diff --git a/cmd/command/validate.go b/cmd/command/validate.go new file mode 100644 index 000000000..9ce7657cb --- /dev/null +++ b/cmd/command/validate.go @@ -0,0 +1,181 @@ +package command + +import ( + "context" + "fmt" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/viant/afs/file" + "github.com/viant/afs/url" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/repository/shape" + shapeCompile "github.com/viant/datly/repository/shape/compile" + shapeLoad "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/plan" + shapevalidate "github.com/viant/datly/repository/shape/validate" +) + +func (s *Service) Validate(ctx context.Context, opts *options.Options) error { + validate := opts.Validate + if validate == nil { + return fmt.Errorf("validate options not set") + } + compiler := shapeCompile.New() + loader := shapeLoad.New() + var validated []string + for _, sourceURL := range validate.Source { + dql, err := s.readSource(ctx, sourceURL) + if err != nil { + return fmt.Errorf("failed to read %s: %w", sourceURL, err) + } + shapeSource := &shape.Source{ + Name: strings.TrimSuffix(filepath.Base(url.Path(sourceURL)), filepath.Ext(sourceURL)), + Path: url.Path(sourceURL), + DQL: strings.TrimSpace(dql), + Connector: validateDefaultConnectorName(validate), + } + planResult, err := compiler.Compile(ctx, shapeSource, validateCompileOptions(validate)...) + if err != nil { + return fmt.Errorf("validate %s: %w", sourceURL, err) + } + if err = validateDiagnostics(sourceURL, planResult); err != nil { + return err + } + if err = validatePlannedSQLAssets(ctx, s, shapeSource, planResult); err != nil { + return fmt.Errorf("validate %s: %w", sourceURL, err) + } + resourceArtifacts, err := loader.LoadResource(ctx, planResult, shape.WithLoadTypeContextPackages(true)) + if err != nil { + return fmt.Errorf("validate %s: %w", sourceURL, err) + } + if err = shapevalidate.ValidateRelations(resourceArtifacts.Resource); err != nil { + return fmt.Errorf("validate %s: %w", sourceURL, err) + } + validated = append(validated, filepath.Clean(url.Path(sourceURL))) + } + sort.Strings(validated) + for _, item := range validated { + fmt.Printf("validated %s\n", item) + } + return nil +} + +func validateDefaultConnectorName(v *options.Validate) string { + if v == nil || len(v.Connectors) == 0 { + return "" + } + parts := strings.SplitN(v.Connectors[0], "|", 2) + if len(parts) == 0 { + return "" + } + return strings.TrimSpace(parts[0]) +} + +func validateCompileOptions(v *options.Validate) []shape.CompileOption { + var opts []shape.CompileOption + if v != nil && v.Strict { + opts = append(opts, shape.WithCompileStrict(true)) + } + opts = append(opts, shape.WithLinkedTypes(false)) + return opts +} + +func validateDiagnostics(sourceURL string, result *shape.PlanResult) error { + planned, ok := plan.ResultFrom(result) + if !ok || planned == nil { + return nil + } + var issues []string + for _, diag := range planned.Diagnostics { + if diag == nil || diag.Severity != "error" { + continue + } + issues = append(issues, diag.Error()) + } + if len(issues) == 0 { + return nil + } + return fmt.Errorf("validate %s: %s", sourceURL, strings.Join(issues, "; ")) +} + +func validatePlannedSQLAssets(ctx context.Context, s *Service, source *shape.Source, result *shape.PlanResult) error { + planned, ok := plan.ResultFrom(result) + if !ok || planned == nil { + return nil + } + assets := collectPlannedSQLAssets(source, planned) + for _, asset := range assets { + if _, err := s.fs.DownloadWithURL(ctx, asset); err != nil { + return fmt.Errorf("missing SQL asset %s: %w", asset, err) + } + } + return nil +} + +func collectPlannedSQLAssets(source *shape.Source, planned *plan.Result) []string { + seen := map[string]bool{} + var result []string + appendAsset := func(candidate string) { + raw := strings.TrimSpace(candidate) + if !isExplicitSourceAsset(source, raw) { + return + } + candidate = raw + if candidate == "" { + return + } + if strings.Contains(candidate, "://") { + candidate = url.Path(candidate) + } + if !filepath.IsAbs(candidate) { + baseDir := "" + if source != nil { + baseDir = source.BaseDir() + } + if baseDir != "" { + candidate = filepath.Join(baseDir, filepath.FromSlash(candidate)) + } + } + candidate = filepath.Clean(candidate) + if candidate == "." || seen[candidate] { + return + } + seen[candidate] = true + result = append(result, file.Scheme+"://"+filepath.ToSlash(candidate)) + } + for _, route := range planned.Components { + if route == nil { + continue + } + appendAsset(route.SourceURL) + appendAsset(route.SummaryURL) + } + for _, item := range planned.Views { + if item == nil { + continue + } + appendAsset(item.SQLURI) + appendAsset(item.SummaryURL) + } + sort.Strings(result) + return result +} + +func isExplicitSourceAsset(source *shape.Source, candidate string) bool { + candidate = strings.TrimSpace(candidate) + if candidate == "" { + return false + } + if source == nil || strings.TrimSpace(source.DQL) == "" { + return true + } + clean := filepath.ToSlash(candidate) + if strings.Contains(source.DQL, clean) { + return true + } + base := path.Base(clean) + return base != "" && strings.Contains(source.DQL, base) +} diff --git a/cmd/command/validate_test.go b/cmd/command/validate_test.go new file mode 100644 index 000000000..7f7bb020a --- /dev/null +++ b/cmd/command/validate_test.go @@ -0,0 +1,65 @@ +package command + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" +) + +func TestCollectPlannedSQLAssets_AbsolutizesAndDedupes(t *testing.T) { + tempDir := t.TempDir() + source := &shape.Source{Path: filepath.Join(tempDir, "query.dql")} + planned := &plan.Result{ + Components: []*plan.ComponentRoute{ + {SourceURL: "foo/root.sql"}, + }, + Views: []*plan.View{ + {Name: "foo", SQLURI: "foo/root.sql"}, + {Name: "bar", SQLURI: "bar/detail.sql", SummaryURL: "bar/summary.sql"}, + }, + } + + assets := collectPlannedSQLAssets(source, planned) + + require.Len(t, assets, 3) + require.Contains(t, assets[0]+assets[1]+assets[2], filepath.ToSlash(filepath.Join(tempDir, "foo", "root.sql"))) + require.Contains(t, assets[0]+assets[1]+assets[2], filepath.ToSlash(filepath.Join(tempDir, "bar", "detail.sql"))) + require.Contains(t, assets[0]+assets[1]+assets[2], filepath.ToSlash(filepath.Join(tempDir, "bar", "summary.sql"))) +} + +func TestValidatePlannedSQLAssets_MissingPath(t *testing.T) { + tempDir := t.TempDir() + source := &shape.Source{Path: filepath.Join(tempDir, "query.dql")} + planned := &shape.PlanResult{Source: source, Plan: &plan.Result{ + Views: []*plan.View{ + {Name: "foo", SQLURI: "foo/missing.sql"}, + }, + }} + + svc := New() + err := validatePlannedSQLAssets(context.Background(), svc, source, planned) + require.Error(t, err) + require.Contains(t, err.Error(), "missing SQL asset") + require.Contains(t, err.Error(), "missing.sql") +} + +func TestValidate_PatchBasicOne(t *testing.T) { + projectDir, err := filepath.Abs(filepath.Join("..", "..", "e2e", "v1")) + require.NoError(t, err) + source := filepath.Join(projectDir, "dql", "dev", "events", "patch_basic_one.dql") + svc := New() + + err = svc.Validate(context.Background(), &options.Options{ + Validate: &options.Validate{ + Project: projectDir, + Source: []string{source}, + }, + }) + + require.NoError(t, err) +} diff --git a/cmd/options/options.go b/cmd/options/options.go index 866b332d2..4790eb1e4 100644 --- a/cmd/options/options.go +++ b/cmd/options/options.go @@ -10,7 +10,8 @@ type Options struct { Plugin *Plugin `command:"plugin" description:"build custom datly rule plugin" ` Generate *Generate `command:"gen" description:"generate dql for put,patch or post operation" ` Translate *Translate `command:"translate" description:"translate dql into datly repository rule"` - Transcribe *Transcribe `command:"transcribe" description:"transcribe dql using shape pipeline (no internal/translator)"` + Transcribe *Transcribe `command:"transcribe" description:"compile dql with shape pipeline and generate bootstrap artifacts"` + Validate *Validate `command:"validate" description:"validate DQL and referenced SQL assets with the shape pipeline"` Cache *CacheWarmup `command:"cache" description:"warmup cache"` Run *Run `command:"run" description:"start datly in standalone mode"` Mcp *Mcp `command:"mcp" description:"run mcp"` @@ -76,6 +77,9 @@ func (o *Options) Init(ctx context.Context) error { if o.Transcribe != nil { return o.Transcribe.Init(ctx) } + if o.Validate != nil { + return o.Validate.Init(ctx) + } if o.Run != nil { return o.Run.Init() } @@ -111,6 +115,8 @@ func NewOptions(args Arguments) *Options { ret.Translate = &Translate{} case "transcribe": ret.Transcribe = &Transcribe{} + case "validate": + ret.Validate = &Validate{} case "cache": ret.Cache = &CacheWarmup{} case "run": diff --git a/cmd/options/transcribe.go b/cmd/options/transcribe.go index dac96b072..8748c17e5 100644 --- a/cmd/options/transcribe.go +++ b/cmd/options/transcribe.go @@ -9,11 +9,10 @@ import ( "github.com/viant/afs/url" ) -// Transcribe defines options for the transcribe command which uses -// the shape pipeline exclusively (compile → plan → load) without -// depending on internal/translator. +// Transcribe defines options for the shape-only DQL -> code/config pipeline. type Transcribe struct { Connector + Auth Source []string `short:"s" long:"src" description:"DQL source file(s)"` Repository string `short:"r" long:"repo" description:"output repository location" default:"repo/dev"` Namespace string `short:"u" long:"namespace" description:"route namespace" default:"dev"` @@ -23,9 +22,9 @@ type Transcribe struct { TypeFile string `long:"type-file" description:"generated go file name (default: dql filename or main view in lower_underscore)"` Project string `short:"p" long:"proj" description:"project location"` APIPrefix string `short:"a" long:"api" description:"api prefix" default:"/v1/api"` + SkipYAML bool `long:"skip-yaml" description:"generate bootstrap config and shapes without route yaml"` } -// DefaultConnectorName returns the first connector name from the -c flags. func (t *Transcribe) DefaultConnectorName() string { if len(t.Connectors) == 0 { return "" @@ -38,11 +37,13 @@ func (t *Transcribe) DefaultConnectorName() string { } func (t *Transcribe) Init(ctx context.Context) error { + _ = ctx if t.Project == "" { t.Project, _ = os.Getwd() } t.Project = ensureAbsPath(t.Project) t.Connector.Init() + t.Auth.Init() if url.IsRelative(t.Repository) { t.Repository = url.Join(t.Project, t.Repository) } diff --git a/cmd/options/validate.go b/cmd/options/validate.go new file mode 100644 index 000000000..6d98850ea --- /dev/null +++ b/cmd/options/validate.go @@ -0,0 +1,35 @@ +package options + +import ( + "context" + "fmt" + "os" + + "github.com/viant/afs/url" +) + +// Validate defines options for shape-only DQL validation. +type Validate struct { + Connector + Source []string `short:"s" long:"src" description:"DQL source file(s)"` + Project string `short:"p" long:"proj" description:"project location"` + Strict bool `long:"strict" description:"enable strict compile mode"` +} + +func (v *Validate) Init(ctx context.Context) error { + _ = ctx + if v.Project == "" { + v.Project, _ = os.Getwd() + } + v.Project = ensureAbsPath(v.Project) + v.Connector.Init() + if len(v.Source) == 0 { + return fmt.Errorf("validate: at least one --src is required") + } + for i := range v.Source { + if url.IsRelative(v.Source[i]) { + expandRelativeIfNeeded(&v.Source[i], v.Project) + } + } + return nil +} diff --git a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql index 5e6014969..917eb3c74 100644 --- a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql +++ b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql @@ -1,6 +1,7 @@ /* {"URI":"vendors-grouping/"} */ #set( $_ = $Data(output/view).Embed()) +#set( $_ = $ID<[]int>(query/id)..WithPredicate(0, 'equal', 't', 'ID')) SELECT vendor.*, groupable(vendor), @@ -14,3 +15,5 @@ FROM ( WHERE t.ID IN ($vendorIDs) GROUP BY 1, 2 ) vendor + + diff --git a/e2e/v1/build.yaml b/e2e/v1/build.yaml index ac6377315..7b3fb2f87 100644 --- a/e2e/v1/build.yaml +++ b/e2e/v1/build.yaml @@ -1,5 +1,3 @@ -init: - pipeline: deploy: setPath: @@ -17,13 +15,11 @@ pipeline: package: action: exec:run - comments: build datly binary + comments: build plain datly binary for pure DQL bootstrap tests target: $target checkError: true commands: - export GO111MODULE=on - - cd ${appPath}/cmd/datly - - go mod tidy - - go mod download - - go build -ldflags "-X main.BuildTimeInS=`date +%s`" - - mv datly /tmp/datly + - export GOFLAGS=-mod=mod + - cd ${appPath} + - go build -ldflags "-X main.BuildTimeInS=`date +%s`" -o /tmp/datly ./cmd/datly diff --git a/e2e/v1/cases/001_one_to_many/expect.json b/e2e/v1/cases/001_one_to_many/expect.json deleted file mode 100644 index c83b372a9..000000000 --- a/e2e/v1/cases/001_one_to_many/expect.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "@indexBy@": "id" - }, - { - "id": 1, - "name": "Vendor 1", - "products": [ - { - "@indexBy@": "id" - }, - { - "id": 1, - "name": "V1 Product 1", - "userCreated": 1 - }, - { - "id": 2, - "name": "V1 Product 2", - "userCreated": 1 - } - ] - }, - { - "id": 2, - "name": "Vendor 2" - }, - { - "id": 3, - "name": "Vendor 3" - } -] \ No newline at end of file diff --git a/e2e/v1/cases/001_one_to_many/expect_2.txt b/e2e/v1/cases/001_one_to_many/expect_2.txt deleted file mode 100644 index 7796339fb..000000000 --- a/e2e/v1/cases/001_one_to_many/expect_2.txt +++ /dev/null @@ -1,27 +0,0 @@ -package generated - -import ( - "time" -) - -type GeneratedStruct struct { - Id int `sqlx:"ID" velty:"names=ID|Id"` - Name *string `sqlx:"NAME" velty:"names=NAME|Name"` - AccountId *int `sqlx:"ACCOUNT_ID" velty:"names=ACCOUNT_ID|AccountId"` - Created *time.Time `sqlx:"CREATED" velty:"names=CREATED|Created"` - UserCreated *int `sqlx:"USER_CREATED" velty:"names=USER_CREATED|UserCreated"` - Updated *time.Time `sqlx:"UPDATED" velty:"names=UPDATED|Updated"` - UserUpdated *int `sqlx:"USER_UPDATED" velty:"names=USER_UPDATED|UserUpdated"` - Products []*Products `view:",table=PRODUCT"` -} - -type Products struct { - Id int `sqlx:"ID" velty:"names=ID|Id"` - Name *string `sqlx:"NAME" velty:"names=NAME|Name"` - VendorId *int `sqlx:"VENDOR_ID" internal:"true" velty:"names=VENDOR_ID|VendorId"` - Status *int `sqlx:"STATUS" velty:"names=STATUS|Status"` - Created *time.Time `sqlx:"CREATED" velty:"names=CREATED|Created"` - UserCreated *int `sqlx:"USER_CREATED" velty:"names=USER_CREATED|UserCreated"` - Updated *time.Time `sqlx:"UPDATED" velty:"names=UPDATED|Updated"` - UserUpdated *int `sqlx:"USER_UPDATED" velty:"names=USER_UPDATED|UserUpdated"` -} diff --git a/e2e/v1/cases/001_one_to_many/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/001_relation_one_to_many/dbsetup/dev/PRODUCT.json similarity index 100% rename from e2e/v1/cases/001_one_to_many/dbsetup/dev/PRODUCT.json rename to e2e/v1/cases/001_relation_one_to_many/dbsetup/dev/PRODUCT.json diff --git a/e2e/v1/cases/001_one_to_many/dbsetup/dev/VENDOR.json b/e2e/v1/cases/001_relation_one_to_many/dbsetup/dev/VENDOR.json similarity index 100% rename from e2e/v1/cases/001_one_to_many/dbsetup/dev/VENDOR.json rename to e2e/v1/cases/001_relation_one_to_many/dbsetup/dev/VENDOR.json diff --git a/e2e/v1/cases/001_relation_one_to_many/expect.json b/e2e/v1/cases/001_relation_one_to_many/expect.json new file mode 100644 index 000000000..1608ce562 --- /dev/null +++ b/e2e/v1/cases/001_relation_one_to_many/expect.json @@ -0,0 +1,77 @@ +[ + { + "id": 1, + "name": "Vendor 1", + "accountId": 100, + "created": "2026-03-07T00:00:00Z", + "userCreated": 1, + "updated": null, + "userUpdated": null, + "products": [ + { + "id": 1, + "name": "V1 Product 1", + "created": "2026-03-07T00:00:00Z", + "userCreated": 1, + "updated": null, + "userUpdated": null + }, + { + "id": 2, + "name": "V1 Product 2", + "status": 1, + "created": "2026-03-07T00:00:00Z", + "userCreated": 1, + "updated": null, + "userUpdated": null + } + ] + }, + { + "id": 2, + "name": "Vendor 2", + "accountId": 101, + "created": "2026-03-07T00:00:00Z", + "userCreated": 2, + "updated": null, + "userUpdated": null, + "products": [ + { + "id": 3, + "name": "V2 Product 1", + "status": 1, + "created": "2026-03-07T00:00:00Z", + "userCreated": 2, + "updated": null, + "userUpdated": null + }, + { + "id": 4, + "name": "V2 Product 2", + "status": 1, + "created": "2026-03-07T00:00:00Z", + "userCreated": 2, + "updated": null, + "userUpdated": null + }, + { + "id": 5, + "name": "V2 Product 3", + "status": 1, + "created": "2026-03-07T00:00:00Z", + "userCreated": 2, + "updated": null, + "userUpdated": null + } + ] + }, + { + "id": 3, + "name": "Vendor 3", + "accountId": 100, + "created": "2026-03-07T00:00:00Z", + "userCreated": 1, + "updated": null, + "userUpdated": null + } +] diff --git a/e2e/v1/cases/001_relation_one_to_many/expect_2.txt b/e2e/v1/cases/001_relation_one_to_many/expect_2.txt new file mode 100644 index 000000000..e50eb7586 --- /dev/null +++ b/e2e/v1/cases/001_relation_one_to_many/expect_2.txt @@ -0,0 +1,27 @@ +package generated + +import ( + "time" +) + +type GeneratedStruct struct { + Id int `sqlx:"ID"` + Name *string `sqlx:"NAME"` + AccountId *int `sqlx:"ACCOUNT_ID"` + Created *time.Time `sqlx:"CREATED"` + UserCreated *int `sqlx:"USER_CREATED"` + Updated *time.Time `sqlx:"UPDATED"` + UserUpdated *int `sqlx:"USER_UPDATED"` + Products []*Products `view:",table=PRODUCT" json:",omitempty" sqlx:"-"` +} + +type Products struct { + Id int `sqlx:"ID"` + Name *string `sqlx:"NAME"` + VendorId *int `sqlx:"VENDOR_ID" internal:"true"` + Status *int `sqlx:"STATUS"` + Created *time.Time `sqlx:"CREATED"` + UserCreated *int `sqlx:"USER_CREATED"` + Updated *time.Time `sqlx:"UPDATED"` + UserUpdated *int `sqlx:"USER_UPDATED"` +} diff --git a/e2e/v1/cases/001_one_to_many/test.yaml b/e2e/v1/cases/001_relation_one_to_many/test.yaml similarity index 100% rename from e2e/v1/cases/001_one_to_many/test.yaml rename to e2e/v1/cases/001_relation_one_to_many/test.yaml diff --git a/e2e/v1/cases/006_tree/dbsetup/dev/USER.json b/e2e/v1/cases/002_relation_self_ref_tree/dbsetup/dev/USER.json similarity index 100% rename from e2e/v1/cases/006_tree/dbsetup/dev/USER.json rename to e2e/v1/cases/002_relation_self_ref_tree/dbsetup/dev/USER.json diff --git a/e2e/v1/cases/006_tree/expect.json b/e2e/v1/cases/002_relation_self_ref_tree/expect.json similarity index 90% rename from e2e/v1/cases/006_tree/expect.json rename to e2e/v1/cases/002_relation_self_ref_tree/expect.json index 9eec63220..d75a802c0 100644 --- a/e2e/v1/cases/006_tree/expect.json +++ b/e2e/v1/cases/002_relation_self_ref_tree/expect.json @@ -1,46 +1,44 @@ { - "status": "ok", "data": [ - {"@indexBy@": "id"}, { + "accountId": 100, "id": 1, "name": "User 1", - "accountId": 100, "team": [ - {"@indexBy@": "id"}, { + "accountId": 100, "id": 3, "name": "User 3", - "accountId": 100, "team": [ { + "accountId": 100, "id": 5, "name": "User 1", - "accountId": 100, "team": [] } ] }, { + "accountId": 101, "id": 4, "name": "User 2", - "accountId": 101, "team": [] } ] }, { + "accountId": 101, "id": 2, "name": "User 2", - "accountId": 101, "team": [ { + "accountId": 101, "id": 6, "name": "User 2", - "accountId": 101, "team": [] } ] } - ] -} \ No newline at end of file + ], + "status": "ok" +} diff --git a/e2e/v1/cases/006_tree/test.yaml b/e2e/v1/cases/002_relation_self_ref_tree/test.yaml similarity index 100% rename from e2e/v1/cases/006_tree/test.yaml rename to e2e/v1/cases/002_relation_self_ref_tree/test.yaml diff --git a/e2e/v1/cases/013_col_in/test.yaml b/e2e/v1/cases/003_relation_parent_join_optimization/test.yaml similarity index 68% rename from e2e/v1/cases/013_col_in/test.yaml rename to e2e/v1/cases/003_relation_parent_join_optimization/test.yaml index a48b060f5..0c45ed97b 100644 --- a/e2e/v1/cases/013_col_in/test.yaml +++ b/e2e/v1/cases/003_relation_parent_join_optimization/test.yaml @@ -1,12 +1,9 @@ -init: - parentPath: $parent.path pipeline: - test: + description: parent join-on optimization is expanded inside child UNION branches action: http/runner:send requests: - Method: GET URL: http://127.0.0.1:8080/v1/api/shape/dev/col/vendors/ Expect: Code: 200 - diff --git a/e2e/v1/cases/004_relation_one_to_one/test.yaml b/e2e/v1/cases/004_relation_one_to_one/test.yaml new file mode 100644 index 000000000..cecbb819e --- /dev/null +++ b/e2e/v1/cases/004_relation_one_to_one/test.yaml @@ -0,0 +1,9 @@ +pipeline: + test: + action: http/runner:send + requests: + - Method: GET + description: one-to-one relation via JOIN ... AND 1=1 hint + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-one-one + Expect: + Code: 200 diff --git a/e2e/v1/cases/002_uri_param/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/005_kind_uri_param/dbsetup/dev/PRODUCT.json similarity index 100% rename from e2e/v1/cases/002_uri_param/dbsetup/dev/PRODUCT.json rename to e2e/v1/cases/005_kind_uri_param/dbsetup/dev/PRODUCT.json diff --git a/e2e/v1/cases/002_uri_param/dbsetup/dev/VENDOR.json b/e2e/v1/cases/005_kind_uri_param/dbsetup/dev/VENDOR.json similarity index 100% rename from e2e/v1/cases/002_uri_param/dbsetup/dev/VENDOR.json rename to e2e/v1/cases/005_kind_uri_param/dbsetup/dev/VENDOR.json diff --git a/e2e/v1/cases/002_uri_param/expect.json b/e2e/v1/cases/005_kind_uri_param/expect.json similarity index 60% rename from e2e/v1/cases/002_uri_param/expect.json rename to e2e/v1/cases/005_kind_uri_param/expect.json index 29afd6592..308b3c9a5 100644 --- a/e2e/v1/cases/002_uri_param/expect.json +++ b/e2e/v1/cases/005_kind_uri_param/expect.json @@ -8,34 +8,41 @@ ], "vendor": { "accountId": 101, + "created": "2026-03-07T00:00:00Z", "id": 2, "name": "Vendor 2", "products": [ - {"@indexBy@": "id"}, { + "created": "2026-03-07T00:00:00Z", "id": 3, "name": "V2 Product 1", "status": 1, + "updated": null, "userCreated": 2, - "userUpdated": 0 + "userUpdated": null }, { + "created": "2026-03-07T00:00:00Z", "id": 4, "name": "V2 Product 2", "status": 1, + "updated": null, "userCreated": 2, - "userUpdated": 0 + "userUpdated": null }, { + "created": "2026-03-07T00:00:00Z", "id": 5, "name": "V2 Product 3", "status": 1, + "updated": null, "userCreated": 2, - "userUpdated": 0 + "userUpdated": null } ], + "updated": null, "userCreated": 2, - "userUpdated": 0 + "userUpdated": null } } -] \ No newline at end of file +] diff --git a/e2e/v1/cases/002_uri_param/test.yaml b/e2e/v1/cases/005_kind_uri_param/test.yaml similarity index 100% rename from e2e/v1/cases/002_uri_param/test.yaml rename to e2e/v1/cases/005_kind_uri_param/test.yaml diff --git a/e2e/v1/cases/006_kind_header_params/expect.json b/e2e/v1/cases/006_kind_header_params/expect.json new file mode 100644 index 000000000..d25967f48 --- /dev/null +++ b/e2e/v1/cases/006_kind_header_params/expect.json @@ -0,0 +1,40 @@ +[ + { + "accountId": 101, + "created": "2026-03-07T00:00:00Z", + "id": 2, + "name": "Vendor 2", + "products": [ + { + "created": "2026-03-07T00:00:00Z", + "id": 3, + "name": "V2 Product 1", + "status": 1, + "updated": null, + "userCreated": 2, + "userUpdated": null + }, + { + "created": "2026-03-07T00:00:00Z", + "id": 4, + "name": "V2 Product 2", + "status": 1, + "updated": null, + "userCreated": 2, + "userUpdated": null + }, + { + "created": "2026-03-07T00:00:00Z", + "id": 5, + "name": "V2 Product 3", + "status": 1, + "updated": null, + "userCreated": 2, + "userUpdated": null + } + ], + "updated": null, + "userCreated": 2, + "userUpdated": null + } +] diff --git a/e2e/v1/cases/014_header_params/test.yaml b/e2e/v1/cases/006_kind_header_params/test.yaml similarity index 80% rename from e2e/v1/cases/014_header_params/test.yaml rename to e2e/v1/cases/006_kind_header_params/test.yaml index 9fe964c2c..13ea703b1 100644 --- a/e2e/v1/cases/014_header_params/test.yaml +++ b/e2e/v1/cases/006_kind_header_params/test.yaml @@ -2,8 +2,8 @@ init: parentPath: $parent.path expect: $LoadData('${parentPath}/expect.json') pipeline: - test: + description: inline SQL header hint binds Vendor-Id into the request parameter action: http/runner:send requests: - Method: GET @@ -13,4 +13,3 @@ pipeline: Expect: Code: 200 JSONBody: $expect - diff --git a/e2e/v1/cases/003_oauth/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/007_kind_const/dbsetup/dev/PRODUCT.json similarity index 100% rename from e2e/v1/cases/003_oauth/dbsetup/dev/PRODUCT.json rename to e2e/v1/cases/007_kind_const/dbsetup/dev/PRODUCT.json diff --git a/e2e/v1/cases/003_oauth/dbsetup/dev/VENDOR.json b/e2e/v1/cases/007_kind_const/dbsetup/dev/VENDOR.json similarity index 100% rename from e2e/v1/cases/003_oauth/dbsetup/dev/VENDOR.json rename to e2e/v1/cases/007_kind_const/dbsetup/dev/VENDOR.json diff --git a/e2e/v1/cases/007_kind_const/expect.json b/e2e/v1/cases/007_kind_const/expect.json new file mode 100644 index 000000000..8322bf6db --- /dev/null +++ b/e2e/v1/cases/007_kind_const/expect.json @@ -0,0 +1,73 @@ +[ + { + "accountId": 100, + "created": "2026-03-07T00:00:00Z", + "id": 1, + "name": "Vendor 1", + "products": [ + { + "created": "2026-03-07T00:00:00Z", + "id": 1, + "name": "V1 Product 1", + "updated": null, + "userCreated": 1, + "userUpdated": null, + "vendorId": 1 + }, + { + "created": "2026-03-07T00:00:00Z", + "id": 2, + "name": "V1 Product 2", + "status": 1, + "updated": null, + "userCreated": 1, + "userUpdated": null, + "vendorId": 1 + } + ], + "updated": null, + "userCreated": 1, + "userUpdated": null + }, + { + "accountId": 101, + "created": "2026-03-07T00:00:00Z", + "id": 2, + "name": "Vendor 2", + "products": [ + { + "created": "2026-03-07T00:00:00Z", + "id": 3, + "name": "V2 Product 1", + "status": 1, + "updated": null, + "userCreated": 2, + "userUpdated": null, + "vendorId": 2 + }, + { + "created": "2026-03-07T00:00:00Z", + "id": 4, + "name": "V2 Product 2", + "status": 1, + "updated": null, + "userCreated": 2, + "userUpdated": null, + "vendorId": 2 + }, + { + "created": "2026-03-07T00:00:00Z", + "id": 5, + "name": "V2 Product 3", + "status": 1, + "updated": null, + "userCreated": 2, + "userUpdated": null, + "vendorId": 2 + } + ], + "updated": null, + "userCreated": 2, + "userUpdated": null + } +] diff --git a/e2e/v1/cases/011_env/test.yaml b/e2e/v1/cases/007_kind_const/test.yaml similarity index 100% rename from e2e/v1/cases/011_env/test.yaml rename to e2e/v1/cases/007_kind_const/test.yaml diff --git a/e2e/v1/cases/004_update/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/008_summary_root/dbsetup/dev/PRODUCT.json similarity index 100% rename from e2e/v1/cases/004_update/dbsetup/dev/PRODUCT.json rename to e2e/v1/cases/008_summary_root/dbsetup/dev/PRODUCT.json diff --git a/e2e/v1/cases/004_update/dbsetup/dev/VENDOR.json b/e2e/v1/cases/008_summary_root/dbsetup/dev/VENDOR.json similarity index 100% rename from e2e/v1/cases/004_update/dbsetup/dev/VENDOR.json rename to e2e/v1/cases/008_summary_root/dbsetup/dev/VENDOR.json diff --git a/e2e/v1/cases/005_sumary/test.yaml b/e2e/v1/cases/008_summary_root/test.yaml similarity index 100% rename from e2e/v1/cases/005_sumary/test.yaml rename to e2e/v1/cases/008_summary_root/test.yaml diff --git a/e2e/v1/cases/005_sumary/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/009_summary_child/dbsetup/dev/PRODUCT.json similarity index 100% rename from e2e/v1/cases/005_sumary/dbsetup/dev/PRODUCT.json rename to e2e/v1/cases/009_summary_child/dbsetup/dev/PRODUCT.json diff --git a/e2e/v1/cases/005_sumary/dbsetup/dev/VENDOR.json b/e2e/v1/cases/009_summary_child/dbsetup/dev/VENDOR.json similarity index 100% rename from e2e/v1/cases/005_sumary/dbsetup/dev/VENDOR.json rename to e2e/v1/cases/009_summary_child/dbsetup/dev/VENDOR.json diff --git a/e2e/v1/cases/009_summary_child/expect.json b/e2e/v1/cases/009_summary_child/expect.json new file mode 100644 index 000000000..119a9d5a7 --- /dev/null +++ b/e2e/v1/cases/009_summary_child/expect.json @@ -0,0 +1,94 @@ +{ + "data": [ + { + "accountId": 100, + "created": "2026-03-07T00:00:00Z", + "id": 1, + "name": "Vendor 1", + "products": [ + { + "created": "2026-03-07T00:00:00Z", + "id": 1, + "name": "V1 Product 1", + "updated": null, + "userCreated": 1, + "userUpdated": null + }, + { + "created": "2026-03-07T00:00:00Z", + "id": 2, + "name": "V1 Product 2", + "status": 1, + "updated": null, + "userCreated": 1, + "userUpdated": null + } + ], + "productsMeta": { + "pageCnt": 1, + "totalProducts": 2 + }, + "updated": null, + "userCreated": 1, + "userUpdated": null + }, + { + "accountId": 101, + "created": "2026-03-07T00:00:00Z", + "id": 2, + "name": "Vendor 2", + "products": [ + { + "created": "2026-03-07T00:00:00Z", + "id": 3, + "name": "V2 Product 1", + "status": 1, + "updated": null, + "userCreated": 2, + "userUpdated": null + }, + { + "created": "2026-03-07T00:00:00Z", + "id": 4, + "name": "V2 Product 2", + "status": 1, + "updated": null, + "userCreated": 2, + "userUpdated": null + }, + { + "created": "2026-03-07T00:00:00Z", + "id": 5, + "name": "V2 Product 3", + "status": 1, + "updated": null, + "userCreated": 2, + "userUpdated": null + } + ], + "productsMeta": { + "pageCnt": 1, + "totalProducts": 3 + }, + "updated": null, + "userCreated": 2, + "userUpdated": null + }, + { + "accountId": 100, + "created": "2026-03-07T00:00:00Z", + "id": 3, + "name": "Vendor 3", + "products": [], + "productsMeta": null, + "updated": null, + "userCreated": 1, + "userUpdated": null + } + ], + "meta": { + "cnt": 3, + "pageCnt": 1 + }, + "status": "ok" +} diff --git a/e2e/v1/cases/007_child_meta/test.yaml b/e2e/v1/cases/009_summary_child/test.yaml similarity index 100% rename from e2e/v1/cases/007_child_meta/test.yaml rename to e2e/v1/cases/009_summary_child/test.yaml diff --git a/e2e/v1/cases/010_codecs/expect.json b/e2e/v1/cases/010_codecs/expect.json deleted file mode 100644 index 52dba1e84..000000000 --- a/e2e/v1/cases/010_codecs/expect.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "id": 1, - "name": "Vendor 1", - "accountId": 100, - "userCreated": 1 - }, - { - "id": 2, - "name": "Vendor 2", - "accountId": 101, - "userCreated": 2 - } -] \ No newline at end of file diff --git a/e2e/v1/cases/007_child_meta/expect.json b/e2e/v1/cases/010_summary_multi/expect.json similarity index 51% rename from e2e/v1/cases/007_child_meta/expect.json rename to e2e/v1/cases/010_summary_multi/expect.json index 9d4a3c13f..85addf14a 100644 --- a/e2e/v1/cases/007_child_meta/expect.json +++ b/e2e/v1/cases/010_summary_multi/expect.json @@ -1,25 +1,38 @@ { - "status": "ok", + "meta": { + "pageCnt": 1, + "cnt": 3 + }, "data": [ { "id": 1, "name": "Vendor 1", "accountId": 100, + "created": "2026-03-07T00:00:00Z", "userCreated": 1, + "updated": null, + "userUpdated": null, "products": [ { "id": 1, "name": "V1 Product 1", - "userCreated": 1 + "created": "2026-03-07T00:00:00Z", + "userCreated": 1, + "updated": null, + "userUpdated": null }, { "id": 2, "name": "V1 Product 2", "status": 1, - "userCreated": 1 + "created": "2026-03-07T00:00:00Z", + "userCreated": 1, + "updated": null, + "userUpdated": null } ], "productsMeta": { + "vendorId": 1, "pageCnt": 1, "totalProducts": 2 } @@ -28,28 +41,41 @@ "id": 2, "name": "Vendor 2", "accountId": 101, + "created": "2026-03-07T00:00:00Z", "userCreated": 2, + "updated": null, + "userUpdated": null, "products": [ { "id": 3, "name": "V2 Product 1", "status": 1, - "userCreated": 2 + "created": "2026-03-07T00:00:00Z", + "userCreated": 2, + "updated": null, + "userUpdated": null }, { "id": 4, "name": "V2 Product 2", "status": 1, - "userCreated": 2 + "created": "2026-03-07T00:00:00Z", + "userCreated": 2, + "updated": null, + "userUpdated": null }, { "id": 5, "name": "V2 Product 3", "status": 1, - "userCreated": 2 + "created": "2026-03-07T00:00:00Z", + "userCreated": 2, + "updated": null, + "userUpdated": null } ], "productsMeta": { + "vendorId": 2, "pageCnt": 1, "totalProducts": 3 } @@ -58,12 +84,12 @@ "id": 3, "name": "Vendor 3", "accountId": 100, + "created": "2026-03-07T00:00:00Z", "userCreated": 1, + "updated": null, + "userUpdated": null, "products": [] } ], - "meta": { - "pageCnt": 1, - "cnt": 3 - } -} \ No newline at end of file + "status": "ok" +} diff --git a/e2e/v1/cases/012_meta_format/test.yaml b/e2e/v1/cases/010_summary_multi/test.yaml similarity index 78% rename from e2e/v1/cases/012_meta_format/test.yaml rename to e2e/v1/cases/010_summary_multi/test.yaml index ee94073fa..a4836c0b8 100644 --- a/e2e/v1/cases/012_meta_format/test.yaml +++ b/e2e/v1/cases/010_summary_multi/test.yaml @@ -1,10 +1,9 @@ init: parentPath: $parent.path expect: $LoadJSON('${parentPath}/expect.json') - pipeline: - test: + description: multiple summary views can be joined off the same root projection action: http/runner:send requests: - Method: GET diff --git a/e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json b/e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json deleted file mode 100644 index c4c724a74..000000000 --- a/e2e/v1/cases/011_env/dbsetup/dev/VENDOR.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - {}, - { - "ID": 1, - "NAME": "Vendor 1", - "ACCOUNT_ID": 100, - "CREATED": "", - "USER_CREATED": 1 - }, - { - "ID": 2, - "NAME": "Vendor 2", - "ACCOUNT_ID": 101, - "CREATED": "", - "USER_CREATED": 2 - }, - { - "ID": 3, - "NAME": "Vendor 3", - "ACCOUNT_ID": 100, - "CREATED": "", - "USER_CREATED": 1 - } -] \ No newline at end of file diff --git a/e2e/v1/cases/011_env/expect.json b/e2e/v1/cases/011_env/expect.json deleted file mode 100644 index 68b81fefe..000000000 --- a/e2e/v1/cases/011_env/expect.json +++ /dev/null @@ -1,52 +0,0 @@ -[ - { - "id": 1, - "name": "Vendor 1", - "accountId": 100, - "userCreated": 1, - "products": [ - { - "id": 1, - "name": "V1 Product 1", - "vendorId": 1, - "userCreated": 1 - }, - { - "id": 2, - "name": "V1 Product 2", - "vendorId": 1, - "status": 1, - "userCreated": 1 - } - ] - }, - { - "id": 2, - "name": "Vendor 2", - "accountId": 101, - "userCreated": 2, - "products": [ - { - "id": 3, - "name": "V2 Product 1", - "vendorId": 2, - "status": 1, - "userCreated": 2 - }, - { - "id": 4, - "name": "V2 Product 2", - "vendorId": 2, - "status": 1, - "userCreated": 2 - }, - { - "id": 5, - "name": "V2 Product 3", - "vendorId": 2, - "status": 1, - "userCreated": 2 - } - ] - } -] \ No newline at end of file diff --git a/e2e/v1/cases/008_record_pagination/dbsetup/dev/CITY.json b/e2e/v1/cases/011_summary_pagination/dbsetup/dev/CITY.json similarity index 100% rename from e2e/v1/cases/008_record_pagination/dbsetup/dev/CITY.json rename to e2e/v1/cases/011_summary_pagination/dbsetup/dev/CITY.json diff --git a/e2e/v1/cases/008_record_pagination/dbsetup/dev/DISTRICT.json b/e2e/v1/cases/011_summary_pagination/dbsetup/dev/DISTRICT.json similarity index 100% rename from e2e/v1/cases/008_record_pagination/dbsetup/dev/DISTRICT.json rename to e2e/v1/cases/011_summary_pagination/dbsetup/dev/DISTRICT.json diff --git a/e2e/v1/cases/008_record_pagination/expect.json b/e2e/v1/cases/011_summary_pagination/expect.json similarity index 100% rename from e2e/v1/cases/008_record_pagination/expect.json rename to e2e/v1/cases/011_summary_pagination/expect.json diff --git a/e2e/v1/cases/008_record_pagination/test.yaml b/e2e/v1/cases/011_summary_pagination/test.yaml similarity index 100% rename from e2e/v1/cases/008_record_pagination/test.yaml rename to e2e/v1/cases/011_summary_pagination/test.yaml diff --git a/e2e/v1/cases/007_child_meta/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/012_auth_oauth/dbsetup/dev/PRODUCT.json similarity index 100% rename from e2e/v1/cases/007_child_meta/dbsetup/dev/PRODUCT.json rename to e2e/v1/cases/012_auth_oauth/dbsetup/dev/PRODUCT.json diff --git a/e2e/v1/cases/007_child_meta/dbsetup/dev/VENDOR.json b/e2e/v1/cases/012_auth_oauth/dbsetup/dev/VENDOR.json similarity index 100% rename from e2e/v1/cases/007_child_meta/dbsetup/dev/VENDOR.json rename to e2e/v1/cases/012_auth_oauth/dbsetup/dev/VENDOR.json diff --git a/e2e/v1/cases/003_oauth/expect.json b/e2e/v1/cases/012_auth_oauth/expect.json similarity index 100% rename from e2e/v1/cases/003_oauth/expect.json rename to e2e/v1/cases/012_auth_oauth/expect.json diff --git a/e2e/v1/cases/003_oauth/test.yaml b/e2e/v1/cases/012_auth_oauth/test.yaml similarity index 100% rename from e2e/v1/cases/003_oauth/test.yaml rename to e2e/v1/cases/012_auth_oauth/test.yaml diff --git a/e2e/v1/cases/012_meta_format/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/012_meta_format/dbsetup/dev/PRODUCT.json deleted file mode 100644 index 9f31630e2..000000000 --- a/e2e/v1/cases/012_meta_format/dbsetup/dev/PRODUCT.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - {}, - { - "ID": 1, - "NAME": "V1 Product 1", - "VENDOR_ID": 1, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 1 - }, - { - "ID": 2, - "NAME": "V1 Product 2", - "VENDOR_ID": 1, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 1 - }, - - { - "ID": 3, - "NAME": "V2 Product 1", - "VENDOR_ID": 2, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 2 - }, - { - "ID": 4, - "NAME": "V2 Product 2", - "VENDOR_ID": 2, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 2 - }, - { - "ID": 5, - "NAME": "V2 Product 3", - "VENDOR_ID": 2, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 2 - } -] \ No newline at end of file diff --git a/e2e/v1/cases/012_meta_format/dbsetup/dev/VENDOR.json b/e2e/v1/cases/012_meta_format/dbsetup/dev/VENDOR.json deleted file mode 100644 index c4c724a74..000000000 --- a/e2e/v1/cases/012_meta_format/dbsetup/dev/VENDOR.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - {}, - { - "ID": 1, - "NAME": "Vendor 1", - "ACCOUNT_ID": 100, - "CREATED": "", - "USER_CREATED": 1 - }, - { - "ID": 2, - "NAME": "Vendor 2", - "ACCOUNT_ID": 101, - "CREATED": "", - "USER_CREATED": 2 - }, - { - "ID": 3, - "NAME": "Vendor 3", - "ACCOUNT_ID": 100, - "CREATED": "", - "USER_CREATED": 1 - } -] \ No newline at end of file diff --git a/e2e/v1/cases/012_meta_format/expect.json b/e2e/v1/cases/012_meta_format/expect.json deleted file mode 100644 index f822c8cc7..000000000 --- a/e2e/v1/cases/012_meta_format/expect.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "status": "ok", - "data": [ - { - "id": 1, - "name": "Vendor 1", - "accountId": 100, - "userCreated": 1, - "products": [ - { - "id": 1, - "name": "V1 Product 1", - "userCreated": 1 - }, - { - "id": 2, - "name": "V1 Product 2", - "userCreated": 1 - } - ], - "productsMeta": { - "pageCnt": 1, - "totalProducts": 2 - } - }, - { - "id": 2, - "name": "Vendor 2", - "accountId": 101, - "userCreated": 2, - "products": [ - { - "id": 3, - "name": "V2 Product 1", - "status": 1, - "userCreated": 2 - }, - { - "id": 4, - "name": "V2 Product 2", - "status": 1, - "userCreated": 2 - }, - { - "id": 5, - "name": "V2 Product 3", - "status": 1, - "userCreated": 2 - } - ], - "productsMeta": { - "pageCnt": 1, - "totalProducts": 3 - } - }, - { - "id": 3, - "name": "Vendor 3", - "accountId": 100, - "userCreated": 1, - "products": [] - } - ], - "meta": { - "pageCnt": 1, - "cnt": 3 - } -} \ No newline at end of file diff --git a/e2e/v1/cases/013_col_in/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/013_col_in/dbsetup/dev/PRODUCT.json deleted file mode 100644 index 9f31630e2..000000000 --- a/e2e/v1/cases/013_col_in/dbsetup/dev/PRODUCT.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - {}, - { - "ID": 1, - "NAME": "V1 Product 1", - "VENDOR_ID": 1, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 1 - }, - { - "ID": 2, - "NAME": "V1 Product 2", - "VENDOR_ID": 1, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 1 - }, - - { - "ID": 3, - "NAME": "V2 Product 1", - "VENDOR_ID": 2, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 2 - }, - { - "ID": 4, - "NAME": "V2 Product 2", - "VENDOR_ID": 2, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 2 - }, - { - "ID": 5, - "NAME": "V2 Product 3", - "VENDOR_ID": 2, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 2 - } -] \ No newline at end of file diff --git a/e2e/v1/cases/013_col_in/dbsetup/dev/VENDOR.json b/e2e/v1/cases/013_col_in/dbsetup/dev/VENDOR.json deleted file mode 100644 index c4c724a74..000000000 --- a/e2e/v1/cases/013_col_in/dbsetup/dev/VENDOR.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - {}, - { - "ID": 1, - "NAME": "Vendor 1", - "ACCOUNT_ID": 100, - "CREATED": "", - "USER_CREATED": 1 - }, - { - "ID": 2, - "NAME": "Vendor 2", - "ACCOUNT_ID": 101, - "CREATED": "", - "USER_CREATED": 2 - }, - { - "ID": 3, - "NAME": "Vendor 3", - "ACCOUNT_ID": 100, - "CREATED": "", - "USER_CREATED": 1 - } -] \ No newline at end of file diff --git a/e2e/v1/cases/013_kind_mysql_boolean/expect.json b/e2e/v1/cases/013_kind_mysql_boolean/expect.json new file mode 100644 index 000000000..b988773ac --- /dev/null +++ b/e2e/v1/cases/013_kind_mysql_boolean/expect.json @@ -0,0 +1,17 @@ +[ + { + "@indexBy@": "id" + }, + { + "id": 1, + "userId": 1, + "isEnabled": true, + "isActivated": false + }, + { + "id": 2, + "userId": 2, + "isEnabled": false, + "isActivated": true + } +] diff --git a/e2e/v1/cases/013_kind_mysql_boolean/test.yaml b/e2e/v1/cases/013_kind_mysql_boolean/test.yaml new file mode 100644 index 000000000..b727b3e19 --- /dev/null +++ b/e2e/v1/cases/013_kind_mysql_boolean/test.yaml @@ -0,0 +1,10 @@ +pipeline: + test: + description: MySQL boolean columns are projected with the expected output shape + action: http/runner:send + requests: + - Method: GET + URL: http://127.0.0.1:8080/v1/api/shape/dev/user-metadata + Expect: + Code: 200 + JSONBody: $LoadJSON('${parent.path}/expect.json') diff --git a/e2e/v1/cases/009_apikey/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/014_cache_sql_apikey/dbsetup/dev/PRODUCT.json similarity index 100% rename from e2e/v1/cases/009_apikey/dbsetup/dev/PRODUCT.json rename to e2e/v1/cases/014_cache_sql_apikey/dbsetup/dev/PRODUCT.json diff --git a/e2e/v1/cases/009_apikey/dbsetup/dev/VENDOR.json b/e2e/v1/cases/014_cache_sql_apikey/dbsetup/dev/VENDOR.json similarity index 100% rename from e2e/v1/cases/009_apikey/dbsetup/dev/VENDOR.json rename to e2e/v1/cases/014_cache_sql_apikey/dbsetup/dev/VENDOR.json diff --git a/e2e/v1/cases/004_update/expect.json b/e2e/v1/cases/014_cache_sql_apikey/expect.json similarity index 100% rename from e2e/v1/cases/004_update/expect.json rename to e2e/v1/cases/014_cache_sql_apikey/expect.json diff --git a/e2e/v1/cases/009_apikey/test.yaml b/e2e/v1/cases/014_cache_sql_apikey/test.yaml similarity index 100% rename from e2e/v1/cases/009_apikey/test.yaml rename to e2e/v1/cases/014_cache_sql_apikey/test.yaml diff --git a/e2e/v1/cases/014_header_params/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/014_header_params/dbsetup/dev/PRODUCT.json deleted file mode 100644 index 9f31630e2..000000000 --- a/e2e/v1/cases/014_header_params/dbsetup/dev/PRODUCT.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - {}, - { - "ID": 1, - "NAME": "V1 Product 1", - "VENDOR_ID": 1, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 1 - }, - { - "ID": 2, - "NAME": "V1 Product 2", - "VENDOR_ID": 1, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 1 - }, - - { - "ID": 3, - "NAME": "V2 Product 1", - "VENDOR_ID": 2, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 2 - }, - { - "ID": 4, - "NAME": "V2 Product 2", - "VENDOR_ID": 2, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 2 - }, - { - "ID": 5, - "NAME": "V2 Product 3", - "VENDOR_ID": 2, - "CREATED": "", - "STATUS": 1, - "USER_CREATED": 2 - } -] \ No newline at end of file diff --git a/e2e/v1/cases/014_header_params/dbsetup/dev/VENDOR.json b/e2e/v1/cases/014_header_params/dbsetup/dev/VENDOR.json deleted file mode 100644 index c4c724a74..000000000 --- a/e2e/v1/cases/014_header_params/dbsetup/dev/VENDOR.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - {}, - { - "ID": 1, - "NAME": "Vendor 1", - "ACCOUNT_ID": 100, - "CREATED": "", - "USER_CREATED": 1 - }, - { - "ID": 2, - "NAME": "Vendor 2", - "ACCOUNT_ID": 101, - "CREATED": "", - "USER_CREATED": 2 - }, - { - "ID": 3, - "NAME": "Vendor 3", - "ACCOUNT_ID": 100, - "CREATED": "", - "USER_CREATED": 1 - } -] \ No newline at end of file diff --git a/e2e/v1/cases/014_header_params/expect.json b/e2e/v1/cases/014_header_params/expect.json deleted file mode 100644 index b963eee2e..000000000 --- a/e2e/v1/cases/014_header_params/expect.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "id": 2, - "name": "Vendor 2", - "products": [ - { - "@indexBy@": "id" - }, - { - "id": 3, - "name": "V2 Product 1", - "userCreated": 2 - }, - { - "id": 4, - "name": "V2 Product 2", - "userCreated": 2 - }, - { - "id": 5, - "name": "V2 Product 3", - "userCreated": 2 - } - ] - } -] \ No newline at end of file diff --git a/e2e/v1/cases/011_env/dbsetup/dev/PRODUCT.json b/e2e/v1/cases/015_dml_update/dbsetup/dev/PRODUCT.json similarity index 100% rename from e2e/v1/cases/011_env/dbsetup/dev/PRODUCT.json rename to e2e/v1/cases/015_dml_update/dbsetup/dev/PRODUCT.json diff --git a/e2e/v1/cases/004_update/dbsetup/dev/PRODUCT_JN.json b/e2e/v1/cases/015_dml_update/dbsetup/dev/PRODUCT_JN.json similarity index 100% rename from e2e/v1/cases/004_update/dbsetup/dev/PRODUCT_JN.json rename to e2e/v1/cases/015_dml_update/dbsetup/dev/PRODUCT_JN.json diff --git a/e2e/v1/cases/010_codecs/dbsetup/dev/VENDOR.json b/e2e/v1/cases/015_dml_update/dbsetup/dev/VENDOR.json similarity index 100% rename from e2e/v1/cases/010_codecs/dbsetup/dev/VENDOR.json rename to e2e/v1/cases/015_dml_update/dbsetup/dev/VENDOR.json diff --git a/e2e/v1/cases/009_apikey/expect.json b/e2e/v1/cases/015_dml_update/expect.json similarity index 100% rename from e2e/v1/cases/009_apikey/expect.json rename to e2e/v1/cases/015_dml_update/expect.json diff --git a/e2e/v1/cases/004_update/expect/PRODUCT.json b/e2e/v1/cases/015_dml_update/expect/PRODUCT.json similarity index 100% rename from e2e/v1/cases/004_update/expect/PRODUCT.json rename to e2e/v1/cases/015_dml_update/expect/PRODUCT.json diff --git a/e2e/v1/cases/004_update/test.yaml b/e2e/v1/cases/015_dml_update/test.yaml similarity index 100% rename from e2e/v1/cases/004_update/test.yaml rename to e2e/v1/cases/015_dml_update/test.yaml diff --git a/e2e/v1/cases/015_index_by/dbsetup/dev/USER_TEAM.json b/e2e/v1/cases/015_index_by/dbsetup/dev/USER_TEAM.json deleted file mode 100644 index 49a3be744..000000000 --- a/e2e/v1/cases/015_index_by/dbsetup/dev/USER_TEAM.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - {}, - { - "ID": 1, - "TEAM_ID": 1, - "USER_ID": 1 - }, - { - "ID": 2, - "TEAM_ID": 1, - "USER_ID": 2 - }, - { - "ID": 3, - "TEAM_ID": 2, - "USER_ID": 1 - } -] \ No newline at end of file diff --git a/e2e/v1/cases/016_team_delete/test.yaml b/e2e/v1/cases/016_dml_delete/test.yaml similarity index 76% rename from e2e/v1/cases/016_team_delete/test.yaml rename to e2e/v1/cases/016_dml_delete/test.yaml index 4b92179e1..4b90699b2 100644 --- a/e2e/v1/cases/016_team_delete/test.yaml +++ b/e2e/v1/cases/016_dml_delete/test.yaml @@ -1,8 +1,6 @@ -init: - parentPath: $parent.path pipeline: - test: + description: delete executor removes the targeted team row by path parameter action: http/runner:send requests: - Method: DELETE @@ -11,7 +9,7 @@ pipeline: Code: 200 checkDb: - action: 'dsunit:query' + action: dsunit:query datastore: dev SQL: 'SELECT COUNT(*) AS NUM_RECORDS FROM (SELECT 1 FROM TEAM WHERE ID = 1000000) T' expect: diff --git a/e2e/v1/cases/017_generate_post_basic_one/dbsetup/dev/EVENTS.json b/e2e/v1/cases/017_generate_post_basic_one/dbsetup/dev/EVENTS.json deleted file mode 100644 index ec2649bb4..000000000 --- a/e2e/v1/cases/017_generate_post_basic_one/dbsetup/dev/EVENTS.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - {} -] \ No newline at end of file diff --git a/e2e/v1/cases/017_generate_post_basic_one/expect_t0.json b/e2e/v1/cases/017_generate_post_basic_one/expect_t0.json deleted file mode 100644 index 89d54528d..000000000 --- a/e2e/v1/cases/017_generate_post_basic_one/expect_t0.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "id": "@exists@" -} \ No newline at end of file diff --git a/e2e/v1/cases/017_generate_post_basic_one/expect_t1.json b/e2e/v1/cases/017_generate_post_basic_one/expect_t1.json deleted file mode 100644 index f555bae5c..000000000 --- a/e2e/v1/cases/017_generate_post_basic_one/expect_t1.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "id": "@exists@", - "name": "017_ Custom name", - "quantity": 25 -} \ No newline at end of file diff --git a/e2e/v1/cases/017_generate_post_basic_one/test.yaml b/e2e/v1/cases/017_generate_post_basic_one/test.yaml deleted file mode 100644 index dc474bf6e..000000000 --- a/e2e/v1/cases/017_generate_post_basic_one/test.yaml +++ /dev/null @@ -1,31 +0,0 @@ -init: - parentPath: $parent.path -pipeline: - - test: - action: http/runner:send - requests: - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events - JsonBody: - Name: '017_' - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t0.json') - - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events - JSONBody: - Name: "017_ Custom name" - Quantity: 25 - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t1.json') - - checkDB: - action: 'dsunit:query' - dataStore: dev - SQL: | - SELECT (CASE WHEN COUNT(*) = 0 THEN FALSE ELSE TRUE END) AS ADDED_NEW_ROWS FROM ( SELECT * FROM EVENTS WHERE NAME LIKE '017_%') T; - expect: - - ADDED_NEW_ROWS: true diff --git a/e2e/v1/cases/017_kind_variables/expect.json b/e2e/v1/cases/017_kind_variables/expect.json new file mode 100644 index 000000000..062d072e7 --- /dev/null +++ b/e2e/v1/cases/017_kind_variables/expect.json @@ -0,0 +1,7 @@ +[ + { + "key1": "setting1 - VENDOR", + "key2": "setting2 - PRODUCT", + "key3": true + } +] diff --git a/e2e/v1/cases/010_codecs/test.yaml b/e2e/v1/cases/017_kind_variables/test.yaml similarity index 52% rename from e2e/v1/cases/010_codecs/test.yaml rename to e2e/v1/cases/017_kind_variables/test.yaml index 3aab3ad23..ee1b80edd 100644 --- a/e2e/v1/cases/010_codecs/test.yaml +++ b/e2e/v1/cases/017_kind_variables/test.yaml @@ -3,12 +3,12 @@ init: expect: $LoadJSON('${parentPath}/expect.json') pipeline: - test: + description: constant variables are substituted into SQL expressions before execution action: http/runner:send requests: - Method: GET - URL: http://127.0.0.1:8080/v1/api/shape/dev/vendors-codec?vendorIDs=1,2 + URL: http://127.0.0.1:8080/v1/api/shape/dev/ws/vars/ Expect: Code: 200 - JSONBody: $expect \ No newline at end of file + JSONBody: $expect diff --git a/e2e/v1/cases/016_team_delete/dbsetup/dev/TEAM.json b/e2e/v1/cases/018_exec_index_by/dbsetup/dev/TEAM.json similarity index 72% rename from e2e/v1/cases/016_team_delete/dbsetup/dev/TEAM.json rename to e2e/v1/cases/018_exec_index_by/dbsetup/dev/TEAM.json index b0dc46dc4..aa65096e6 100644 --- a/e2e/v1/cases/016_team_delete/dbsetup/dev/TEAM.json +++ b/e2e/v1/cases/018_exec_index_by/dbsetup/dev/TEAM.json @@ -14,10 +14,5 @@ "ID": 3, "NAME": "Team - 3", "ACTIVE": true - }, - { - "ID": 1000000, - "NAME": "Team - 1000000", - "ACTIVE": true } -] \ No newline at end of file +] diff --git a/e2e/v1/cases/018_exec_index_by/dbsetup/dev/USER_TEAM.json b/e2e/v1/cases/018_exec_index_by/dbsetup/dev/USER_TEAM.json new file mode 100644 index 000000000..35c2f6810 --- /dev/null +++ b/e2e/v1/cases/018_exec_index_by/dbsetup/dev/USER_TEAM.json @@ -0,0 +1,13 @@ +[ + {}, + { + "ID": 1, + "USER_ID": 1, + "TEAM_ID": 1 + }, + { + "ID": 2, + "USER_ID": 2, + "TEAM_ID": 1 + } +] diff --git a/e2e/v1/cases/015_index_by/dbsetup/dev/TEAM.json b/e2e/v1/cases/018_exec_index_by/expect/TEAM.json similarity index 65% rename from e2e/v1/cases/015_index_by/dbsetup/dev/TEAM.json rename to e2e/v1/cases/018_exec_index_by/expect/TEAM.json index b0dc46dc4..308755ef4 100644 --- a/e2e/v1/cases/015_index_by/dbsetup/dev/TEAM.json +++ b/e2e/v1/cases/018_exec_index_by/expect/TEAM.json @@ -13,11 +13,6 @@ { "ID": 3, "NAME": "Team - 3", - "ACTIVE": true - }, - { - "ID": 1000000, - "NAME": "Team - 1000000", - "ACTIVE": true + "ACTIVE": false } -] \ No newline at end of file +] diff --git a/e2e/v1/cases/015_index_by/test.yaml b/e2e/v1/cases/018_exec_index_by/test.yaml similarity index 83% rename from e2e/v1/cases/015_index_by/test.yaml rename to e2e/v1/cases/018_exec_index_by/test.yaml index 40681a32f..2211c4b64 100644 --- a/e2e/v1/cases/015_index_by/test.yaml +++ b/e2e/v1/cases/018_exec_index_by/test.yaml @@ -1,8 +1,9 @@ init: parentPath: $parent.path -pipeline: +pipeline: test: + description: indexed view state is used to validate team membership before executor updates action: http/runner:send requests: - Method: PUT @@ -25,8 +26,8 @@ pipeline: Code: 200 checkDb: - action: 'dsunit:expect' - dataStore: dev + action: dsunit:expect + datastore: dev expand: true checkPolicy: 1 URL: ${parentPath}/expect diff --git a/e2e/v1/cases/018_generate_post_basic_many/dbsetup/dev/EVENTS.json b/e2e/v1/cases/018_generate_post_basic_many/dbsetup/dev/EVENTS.json deleted file mode 100644 index ec2649bb4..000000000 --- a/e2e/v1/cases/018_generate_post_basic_many/dbsetup/dev/EVENTS.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - {} -] \ No newline at end of file diff --git a/e2e/v1/cases/018_generate_post_basic_many/expect_t0.json b/e2e/v1/cases/018_generate_post_basic_many/expect_t0.json deleted file mode 100644 index 0b84b5838..000000000 --- a/e2e/v1/cases/018_generate_post_basic_many/expect_t0.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - { - "id": "@exists@" - } -] diff --git a/e2e/v1/cases/018_generate_post_basic_many/expect_t1.json b/e2e/v1/cases/018_generate_post_basic_many/expect_t1.json deleted file mode 100644 index c9a5ab883..000000000 --- a/e2e/v1/cases/018_generate_post_basic_many/expect_t1.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "id": "@exists@", - "name": "018_ " - }, - { - "id": "@exists@", - "name": "018_ " - } -] \ No newline at end of file diff --git a/e2e/v1/cases/018_generate_post_basic_many/expect_t2.json b/e2e/v1/cases/018_generate_post_basic_many/expect_t2.json deleted file mode 100644 index 00f87b1c9..000000000 --- a/e2e/v1/cases/018_generate_post_basic_many/expect_t2.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "id": "@exists@", - "name": "018_ Custom - 1" - }, - { - "id": "@exists@", - "name": "018_ Custom - 2" - } -] \ No newline at end of file diff --git a/e2e/v1/cases/018_generate_post_basic_many/test.yaml b/e2e/v1/cases/018_generate_post_basic_many/test.yaml deleted file mode 100644 index a2fa2aafb..000000000 --- a/e2e/v1/cases/018_generate_post_basic_many/test.yaml +++ /dev/null @@ -1,40 +0,0 @@ -init: - parentPath: $parent.path -pipeline: - - test: - action: http/runner:send - requests: - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-many - JSONBody: - - Name: '018_ ' - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t0.json') - - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-many - JSONBody: - - Name: '018_ ' - - Name: '018_ ' - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t1.json') - - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-many - JSONBody: - - Name: '018_ Custom - 1' - - Name: '018_ Custom - 2' - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t2.json') - - checkDB: - action: 'dsunit:query' - dataStore: dev - SQL: | - SELECT (CASE WHEN COUNT(*) = 0 THEN FALSE ELSE TRUE END) AS ADDED_NEW_ROWS FROM ( SELECT * FROM EVENTS WHERE NAME LIKE '018_%') T; - expect: - - ADDED_NEW_ROWS: true diff --git a/e2e/v1/cases/019_component_dependency/expect_admin.json b/e2e/v1/cases/019_component_dependency/expect_admin.json new file mode 100644 index 000000000..86326dd5a --- /dev/null +++ b/e2e/v1/cases/019_component_dependency/expect_admin.json @@ -0,0 +1,9 @@ +[ + { + "@indexBy@": "id" + }, + { + "id": 2, + "name": "Vendor 2" + } +] diff --git a/e2e/v1/cases/019_component_dependency/expect_readonly.json b/e2e/v1/cases/019_component_dependency/expect_readonly.json new file mode 100644 index 000000000..02513fe0b --- /dev/null +++ b/e2e/v1/cases/019_component_dependency/expect_readonly.json @@ -0,0 +1,9 @@ +[ + { + "@indexBy@": "id" + }, + { + "id": 1, + "name": "Vendor 1" + } +] diff --git a/e2e/v1/cases/019_component_dependency/test.yaml b/e2e/v1/cases/019_component_dependency/test.yaml new file mode 100644 index 000000000..d887709d6 --- /dev/null +++ b/e2e/v1/cases/019_component_dependency/test.yaml @@ -0,0 +1,49 @@ +init: + parentPath: $parent.path + expectReadOnly: $LoadData('${parentPath}/expect_readonly.json') + expectAdmin: $LoadData('${parentPath}/expect_admin.json') + +pipeline: + signReadOnly: + action: secret:signJWT + privateKey: + URL: ${appPath}/e2e/cloud/jwt/private.enc + Key: blowfish://default + claims: + userID: 1 + firstName: Tester + email: tester@viantint.com + + testReadOnly: + action: http/runner:send + requests: + - Method: GET + description: component dependency resolves a sibling DQL component and gates vendor access for read-only users + URL: http://127.0.0.1:8080/v1/api/shape/dev/vendors/component-acl + Header: + Authorization: Bearer ${signReadOnly.TokenString} + Expect: + Code: 200 + JSONBody: $expectReadOnly + + signAdmin: + action: secret:signJWT + privateKey: + URL: ${appPath}/e2e/cloud/jwt/private.enc + Key: blowfish://default + claims: + userID: 2 + firstName: Developer + email: dev@viantint.com + + testAdmin: + action: http/runner:send + requests: + - Method: GET + description: component dependency keeps query predicate filtering for non-read-only users + URL: http://127.0.0.1:8080/v1/api/shape/dev/vendors/component-acl?name=2 + Header: + Authorization: Bearer ${signAdmin.TokenString} + Expect: + Code: 200 + JSONBody: $expectAdmin diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/dbsetup/dev/EVENTS.json b/e2e/v1/cases/019_generate_post_comprehensive_many/dbsetup/dev/EVENTS.json deleted file mode 100644 index ec2649bb4..000000000 --- a/e2e/v1/cases/019_generate_post_comprehensive_many/dbsetup/dev/EVENTS.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - {} -] \ No newline at end of file diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t0.json b/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t0.json deleted file mode 100644 index bbbfbd1a1..000000000 --- a/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t0.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "status": "ok", - "data": [ - { - "id": "@exists@" - } - ] -} diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t1.json b/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t1.json deleted file mode 100644 index be52f980e..000000000 --- a/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t1.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "status": "ok", - "data": [ - { - "id": "@exists@" - }, - { - "id": "@exists@" - } - ] -} diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t2.json b/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t2.json deleted file mode 100644 index 96bc28252..000000000 --- a/e2e/v1/cases/019_generate_post_comprehensive_many/expect_t2.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "status": "ok", - "data": [ - { - "id": "@exists@", - "name": "019_ Custom - 1" - }, - { - "id": "@exists@", - "name": "019_ Custom - 2" - } - ] -} diff --git a/e2e/v1/cases/019_generate_post_comprehensive_many/test.yaml b/e2e/v1/cases/019_generate_post_comprehensive_many/test.yaml deleted file mode 100644 index e5317770e..000000000 --- a/e2e/v1/cases/019_generate_post_comprehensive_many/test.yaml +++ /dev/null @@ -1,43 +0,0 @@ -init: - parentPath: $parent.path -pipeline: - - test: - action: http/runner:send - requests: - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/comprehensive/events-many - JSONBody: - data: - - name: '019_ ' - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t0.json') - - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/comprehensive/events-many - JSONBody: - data: - - name: '019_ ' - - name: '019_ ' - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t1.json') - - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/comprehensive/events-many - JSONBody: - data: - - name: '019_ Custom - 1' - - name: '019_ Custom - 2' - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t2.json') - - checkDB: - action: 'dsunit:query' - dataStore: dev - SQL: | - SELECT (CASE WHEN COUNT(*) = 0 THEN FALSE ELSE TRUE END) AS ADDED_NEW_ROWS FROM ( SELECT * FROM EVENTS WHERE NAME LIKE '019_%') T; - expect: - - ADDED_NEW_ROWS: true diff --git a/e2e/v1/cases/020_generate_patch_basic_one/dbsetup/dev/FOOS.json b/e2e/v1/cases/020_generate_patch_basic_one/dbsetup/dev/FOOS.json new file mode 100644 index 000000000..56cc97b53 --- /dev/null +++ b/e2e/v1/cases/020_generate_patch_basic_one/dbsetup/dev/FOOS.json @@ -0,0 +1,23 @@ +[ + {}, + { + "ID": 1, + "NAME": "foo 1", + "QUANTITY": 100 + }, + { + "ID": 2, + "NAME": "foo 2", + "QUANTITY": 200 + }, + { + "ID": 3, + "NAME": "foo 3", + "QUANTITY": 300 + }, + { + "ID": 4, + "NAME": "foo 4", + "QUANTITY": 400 + } +] diff --git a/e2e/v1/cases/020_generate_patch_basic_one/expect_t0.json b/e2e/v1/cases/020_generate_patch_basic_one/expect_t0.json new file mode 100644 index 000000000..a5feed5e4 --- /dev/null +++ b/e2e/v1/cases/020_generate_patch_basic_one/expect_t0.json @@ -0,0 +1,5 @@ +{ + "id": 4, + "quantity": 2500, + "name": "changed - foo 4" +} diff --git a/e2e/v1/cases/020_generate_patch_basic_one/expect_t1.json b/e2e/v1/cases/020_generate_patch_basic_one/expect_t1.json new file mode 100644 index 000000000..caed750cf --- /dev/null +++ b/e2e/v1/cases/020_generate_patch_basic_one/expect_t1.json @@ -0,0 +1,5 @@ +{ + "id": "@exists@", + "quantity": 1234, + "name": "created" +} diff --git a/e2e/v1/cases/020_generate_patch_basic_one/test.yaml b/e2e/v1/cases/020_generate_patch_basic_one/test.yaml new file mode 100644 index 000000000..c430c00cb --- /dev/null +++ b/e2e/v1/cases/020_generate_patch_basic_one/test.yaml @@ -0,0 +1,41 @@ +init: + parentPath: $parent.path + +pipeline: + test: + description: generated PATCH component updates an existing FOOS row and inserts a new one when the primary key is absent + action: http/runner:send + requests: + - Method: PATCH + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/foos + JsonBody: + ID: 4 + Quantity: 2500 + Name: 'changed - foo 4' + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t0.json') + + - Method: PATCH + URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/foos + JsonBody: + Quantity: 1234 + Name: 'created' + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_t1.json') + + checkDb: + action: dsunit:query + datastore: dev + sql: | + SELECT + (CASE WHEN EXISTS(SELECT 1 FROM FOOS WHERE ID = 4 AND QUANTITY = 2500 AND NAME = 'changed - foo 4') THEN TRUE ELSE FALSE END) AS UPDATED_ROW, + (CASE WHEN EXISTS(SELECT 1 FROM FOOS WHERE ID <> 4 AND QUANTITY = 1234 AND NAME = 'created') THEN TRUE ELSE FALSE END) AS INSERTED_ROW, + (SELECT COUNT(*) FROM FOOS) AS TOTAL_ROWS, + (SELECT COUNT(*) FROM FOOS WHERE NAME = 'created' AND QUANTITY = 1234) AS INSERTED_COUNT + expect: + - UPDATED_ROW: true + INSERTED_ROW: true + TOTAL_ROWS: 5 + INSERTED_COUNT: 1 diff --git a/e2e/v1/cases/020_generate_post_except/dbsetup/dev/EVENTS.json b/e2e/v1/cases/020_generate_post_except/dbsetup/dev/EVENTS.json deleted file mode 100644 index ec2649bb4..000000000 --- a/e2e/v1/cases/020_generate_post_except/dbsetup/dev/EVENTS.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - {} -] \ No newline at end of file diff --git a/e2e/v1/cases/020_generate_post_except/expect_t0.json b/e2e/v1/cases/020_generate_post_except/expect_t0.json deleted file mode 100644 index 60a83114f..000000000 --- a/e2e/v1/cases/020_generate_post_except/expect_t0.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "id": "@exists@", - "quantity": -1234 -} \ No newline at end of file diff --git a/e2e/v1/cases/020_generate_post_except/expect_t1.json b/e2e/v1/cases/020_generate_post_except/expect_t1.json deleted file mode 100644 index 23195e3ca..000000000 --- a/e2e/v1/cases/020_generate_post_except/expect_t1.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "id": "@exists@", - "quantity": -2345 -} \ No newline at end of file diff --git a/e2e/v1/cases/020_generate_post_except/test.yaml b/e2e/v1/cases/020_generate_post_except/test.yaml deleted file mode 100644 index 8f97f8139..000000000 --- a/e2e/v1/cases/020_generate_post_except/test.yaml +++ /dev/null @@ -1,30 +0,0 @@ -init: - parentPath: $parent.path -pipeline: - - test: - action: http/runner:send - requests: - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-except - JsonBody: - Quantity: -1234 - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t0.json') - - - Method: POST - URL: http://127.0.0.1:8080/v1/api/shape/dev/basic/events-except - JSONBody: - Quantity: -2345 - Expect: - Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect_t1.json') - - checkDB: - action: 'dsunit:query' - dataStore: dev - SQL: | - SELECT (CASE WHEN COUNT(*) = 0 THEN FALSE ELSE TRUE END) AS ADDED_NEW_ROWS FROM ( SELECT * FROM EVENTS WHERE QUANTITY IN (-12345,-2345)) T; - expect: - - ADDED_NEW_ROWS: true diff --git a/e2e/v1/config.json b/e2e/v1/config.json new file mode 100644 index 000000000..fc6d24787 --- /dev/null +++ b/e2e/v1/config.json @@ -0,0 +1,18 @@ +{ + "APIPrefix": "/v1/api", + "APIKeys": [ + { + "Header": "App-Secret-Id", + "URI": "/v1/api/shape/dev/secured", + "Value": "changeme" + } + ], + "Endpoint": { + "Port": 8080 + }, + "Meta": { + "StatusURI": "/v1/api/status", + "StructURI": "/v1/api/meta/struct" + }, + "SyncFrequencyMs": 3600000 +} diff --git a/e2e/v1/dql/dev/district/district_pagination.sql b/e2e/v1/dql/dev/district/district_pagination.sql index 212ccbd30..c927d4883 100644 --- a/e2e/v1/dql/dev/district/district_pagination.sql +++ b/e2e/v1/dql/dev/district/district_pagination.sql @@ -1,9 +1,10 @@ -#package('github.com/viant/datly/e2e/v1/shape/dev/district') +#package('github.com/viant/datly/e2e/v1/shape/dev/district/pagination') #setting($_ = $connector('dev')) #setting($_ = $route('/v1/api/shape/dev/meta/districts', 'GET')) -#set( $_ = $Page(query/page).Optional().QuerySelector('districts')) -#set( $_ = $Data(output/view).Embed()) +#define($_ = $IDs<[]int>(query/IDs)) +#define($_ = $Page(query/page).Optional().QuerySelector('districts')) +#define($_ = $Data(output/view).Embed()) SELECT districts.*, diff --git a/e2e/v1/dql/dev/events/basic_one_one.dql b/e2e/v1/dql/dev/events/basic_one_one.dql new file mode 100644 index 000000000..061949707 --- /dev/null +++ b/e2e/v1/dql/dev/events/basic_one_one.dql @@ -0,0 +1,12 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/events/relation_one_one') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/basic/events-one-one', 'GET')) + + +#set( $_ = $Data(output/view).Embed()) + + +SELECT EVENTS.*, + EVENTS_PERFORMANCE.* +FROM (SELECT ID, QUANTITY FROM EVENTS) EVENTS +JOIN (SELECT * FROM EVENTS_PERFORMANCE) EVENTS_PERFORMANCE ON EVENTS.ID = EVENTS_PERFORMANCE.EVENT_ID AND 1=1 diff --git a/e2e/v1/dql/dev/events/patch_basic_one.dql b/e2e/v1/dql/dev/events/patch_basic_one.dql new file mode 100644 index 000000000..ef5f2eb14 --- /dev/null +++ b/e2e/v1/dql/dev/events/patch_basic_one.dql @@ -0,0 +1,12 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/basic/foos', 'PATCH')) +#setting($_ = $useTemplate('patch')) + + +#set($_ = $Foos(body/).Cardinality('One').Tag('anonymous:"true"')) +#set($_ = $Foos(body/).Output().Tag('anonymous:"true"')) + + +SELECT foos.* +FROM (SELECT * FROM FOOS) foos diff --git a/e2e/v1/dql/dev/events/post_basic_many.dql b/e2e/v1/dql/dev/events/post_basic_many.dql index 6070080f7..04a7e59c4 100644 --- a/e2e/v1/dql/dev/events/post_basic_many.dql +++ b/e2e/v1/dql/dev/events/post_basic_many.dql @@ -1,4 +1,4 @@ -#package('github.com/viant/datly/e2e/v1/shape/dev/events') +#package('github.com/viant/datly/e2e/v1/shape/dev/events/basic_many') #setting($_ = $connector('dev')) #setting($_ = $route('/v1/api/shape/dev/basic/events-many', 'POST')) diff --git a/e2e/v1/dql/dev/events/post_basic_one.dql b/e2e/v1/dql/dev/events/post_basic_one.dql index 5ba191c14..fc96181fd 100644 --- a/e2e/v1/dql/dev/events/post_basic_one.dql +++ b/e2e/v1/dql/dev/events/post_basic_one.dql @@ -1,4 +1,4 @@ -#package('github.com/viant/datly/e2e/v1/shape/dev/events') +#package('github.com/viant/datly/e2e/v1/shape/dev/events/basic_one') #setting($_ = $connector('dev')) #setting($_ = $route('/v1/api/shape/dev/basic/events', 'POST')) diff --git a/e2e/v1/dql/dev/events/post_comprehensive_many.dql b/e2e/v1/dql/dev/events/post_comprehensive_many.dql index 5b048f267..ffe8866c0 100644 --- a/e2e/v1/dql/dev/events/post_comprehensive_many.dql +++ b/e2e/v1/dql/dev/events/post_comprehensive_many.dql @@ -1,4 +1,4 @@ -#package('github.com/viant/datly/e2e/v1/shape/dev/events') +#package('github.com/viant/datly/e2e/v1/shape/dev/events/comprehensive_many') #setting($_ = $connector('dev')) #setting($_ = $route('/v1/api/shape/dev/comprehensive/events-many', 'POST')) diff --git a/e2e/v1/dql/dev/events/post_except.dql b/e2e/v1/dql/dev/events/post_except.dql index 5ef13f77d..b8763899c 100644 --- a/e2e/v1/dql/dev/events/post_except.dql +++ b/e2e/v1/dql/dev/events/post_except.dql @@ -1,4 +1,4 @@ -#package('github.com/viant/datly/e2e/v1/shape/dev/events') +#package('github.com/viant/datly/e2e/v1/shape/dev/events/except') #setting($_ = $connector('dev')) #setting($_ = $route('/v1/api/shape/dev/basic/events-except', 'POST')) diff --git a/e2e/v1/dql/dev/team/team.dql b/e2e/v1/dql/dev/team/team.dql index 2d2bb72cd..6c1320af2 100644 --- a/e2e/v1/dql/dev/team/team.dql +++ b/e2e/v1/dql/dev/team/team.dql @@ -1,4 +1,4 @@ -#package('github.com/viant/datly/e2e/v1/shape/dev/team') +#package('github.com/viant/datly/e2e/v1/shape/dev/team/delete') #setting($_ = $connector('dev')) #setting($_ = $route('/v1/api/shape/dev/team/{teamID}', 'DELETE')) diff --git a/e2e/v1/dql/dev/team/user_team.dql b/e2e/v1/dql/dev/team/user_team.dql index 2fcf7f9bc..0f9f5480c 100644 --- a/e2e/v1/dql/dev/team/user_team.dql +++ b/e2e/v1/dql/dev/team/user_team.dql @@ -1,11 +1,15 @@ -#package('github.com/viant/datly/e2e/v1/shape/dev/team') +#package('github.com/viant/datly/e2e/v1/shape/dev/team/user_team') #setting($_ = $connector('dev')) #setting($_ = $route('/v1/api/shape/dev/teams', 'PUT')) -#set($_ = $TeamIDs<[]int>(query/TeamIDs)) +#define($_ = $TeamIDs<[]int>(query/TeamIDs)) -#set($teamStatsIndex = $Unsafe.TeamStats.IndexBy("ID") /* - {"Required": false} +#set($teamStatsIndex = $Unsafe.TeamStats.IndexBy("Id")) + +#define($_ = $TeamStats(view/team_stats). + WithColumnType('ID', 'int'). + WithColumnType('TEAM_MEMBERS', 'int'). + WithColumnType('NAME', 'string') /* SELECT t.ID, ( @@ -19,17 +23,16 @@ LEFT JOIN USER_TEAM ut ON t.ID = ut.TEAM_ID WHERE t.ID IN ($TeamIDs) GROUP BY t.ID -*/) - +*/) #foreach($teamID in $Unsafe.TeamIDs) #if($teamStatsIndex.HasKey($teamID) == false) - $logger.Fatal("not found team with ID %v", $teamID) + $logger.FatalfWithCode(400, "not found team with ID %v", $teamID) #end #set($aTeam = $teamStatsIndex[$teamID]) - #if($aTeam.TEAM_MEMBERS != 0) - $logger.Fatal("can't deactivate team %v with %v members", $aTeam.NAME, $aTeam.TEAM_MEMBERS) + #if($aTeam.TeamMembers != 0) + $logger.FatalfWithCode(400, "can't deactivate team %v with %v members", $aTeam.Name, $aTeam.TeamMembers) #end UPDATE TEAM SET ACTIVE = false diff --git a/e2e/v1/dql/dev/user/user_metadata.dql b/e2e/v1/dql/dev/user/user_metadata.dql new file mode 100644 index 000000000..0f6c71def --- /dev/null +++ b/e2e/v1/dql/dev/user/user_metadata.dql @@ -0,0 +1,10 @@ +#package('github.com/viant/datly/e2e/v1/shape/dev/user/mysql_boolean') +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/user-metadata', 'GET')) + +#define($_ = $Fields<[]string>(query/fields).Optional().QuerySelector('user_metadata')) +#define($_ = $Page(query/page).Optional().QuerySelector('user_metadata')) +#define($_ = $UserMetadata(output/view).Embed()) + +SELECT user_metadata.* +FROM (SELECT * FROM USER_METADATA t) user_metadata diff --git a/e2e/v1/dql/dev/user/user_tree.sql b/e2e/v1/dql/dev/user/user_tree.sql index 42d9c5e8a..33b8e13c1 100644 --- a/e2e/v1/dql/dev/user/user_tree.sql +++ b/e2e/v1/dql/dev/user/user_tree.sql @@ -1,7 +1,10 @@ -#package('github.com/viant/datly/e2e/v1/shape/dev/user') +#package('github.com/viant/datly/e2e/v1/shape/dev/user/tree') #setting($_ = $connector('dev')) #setting($_ = $route('/v1/api/shape/dev/users/', 'GET')) +#define($_ = $Data(output/view)) +#define($_ = $Status(output/status).Embed()) + SELECT user.* EXCEPT MGR_ID, self_ref(user, 'Team', 'ID', 'MGR_ID') FROM (SELECT t.* FROM USER t ) user diff --git a/e2e/v1/go_bootstrap.yaml b/e2e/v1/go_bootstrap.yaml new file mode 100644 index 000000000..aa91fcf2d --- /dev/null +++ b/e2e/v1/go_bootstrap.yaml @@ -0,0 +1,227 @@ +init: + wildcardConfig: ${v1Path}/autogen/Datly/config_go_all.json + singleConfig: ${v1Path}/autogen/Datly/config_go_vendor_list.json + routerAppDir: ${v1Path}/routerapp + routerPkgDir: ${v1Path}/routerpkg + routerImports: ${routerAppDir}/imports_gen.go + goBootstrapPort: 8081 + wildcardExpect: $LoadData('${v1Path}/cases/001_relation_one_to_many/expect.json') + adminExpect: $LoadData('${v1Path}/cases/019_component_dependency/expect_admin.json') + +pipeline: + prepare: + action: exec:run + target: $target + checkError: true + commands: + - cd ${appPath} + - mkdir -p ${routerAppDir} + - mkdir -p ${v1Path}/autogen/Datly + - printf "package main\n\nimport (\n" > ${routerImports} + - | + find ${v1Path}/shape ${routerPkgDir} -type f -name '*.go' ! -name '*_test.go' -print \ + | xargs -n1 dirname \ + | sort -u \ + | sed "s#^${appPath}/#github.com/viant/datly/#" \ + | sed 's#^#\t_ "#; s#$#"#' >> ${routerImports} + - printf ")\n" >> ${routerImports} + - | + cat > ${wildcardConfig} <<'EOF' + { + "APIKeys": [ + { + "Header": "App-Secret-Id", + "URI": "/v1/api/shape/dev/secured", + "Value": "changeme" + } + ], + "APIPrefix": "/v1/api/shape", + "GoBootstrap": { + "Packages": [ + "github.com/viant/datly/e2e/v1/shape/dev/...", + "github.com/viant/datly/e2e/v1/routerpkg/..." + ] + }, + "DependencyURL": "${v1Path}/autogen/Datly/dependencies", + "Endpoint": { + "Port": ${goBootstrapPort} + }, + "JWTValidator": { + "RSA": [ + { + "Key": "blowfish://default", + "URL": "file://localhost${appPath}/e2e/local/jwt/public.enc" + } + ] + }, + "JwtSigner": { + "RSA": { + "URL": "file://localhost${appPath}/e2e/local/jwt/public.enc", + "Key": "blowfish://default" + } + }, + "Meta": { + "CacheURI": "/v1/api/cache/warmup", + "ConfigURI": "/v1/api/meta/config", + "MetricURI": "/v1/api/meta/metric", + "OpenApiURI": "/v1/api/meta/openapi", + "StateURI": "/v1/api/meta/state", + "StatusURI": "/v1/api/status", + "StructURI": "/v1/api/meta/struct", + "ViewURI": "/v1/api/meta/view" + }, + "SyncFrequencyMs": 2000 + } + EOF + - | + cat > ${singleConfig} <<'EOF' + { + "APIKeys": [ + { + "Header": "App-Secret-Id", + "URI": "/v1/api/shape/dev/secured", + "Value": "changeme" + } + ], + "APIPrefix": "/v1/api/shape", + "GoBootstrap": { + "Packages": [ + "github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/list" + ] + }, + "DependencyURL": "${v1Path}/autogen/Datly/dependencies", + "Endpoint": { + "Port": ${goBootstrapPort} + }, + "JWTValidator": { + "RSA": [ + { + "Key": "blowfish://default", + "URL": "file://localhost${appPath}/e2e/local/jwt/public.enc" + } + ] + }, + "JwtSigner": { + "RSA": { + "URL": "file://localhost${appPath}/e2e/local/jwt/public.enc", + "Key": "blowfish://default" + } + }, + "Meta": { + "CacheURI": "/v1/api/cache/warmup", + "ConfigURI": "/v1/api/meta/config", + "MetricURI": "/v1/api/meta/metric", + "OpenApiURI": "/v1/api/meta/openapi", + "StateURI": "/v1/api/meta/state", + "StatusURI": "/v1/api/status", + "StructURI": "/v1/api/meta/struct", + "ViewURI": "/v1/api/meta/view" + }, + "SyncFrequencyMs": 2000 + } + EOF + + build: + action: exec:run + target: $target + checkError: true + commands: + - cd ${appPath} + - export GO111MODULE=on + - export GOFLAGS=-mod=mod + - go build -ldflags "-X main.BuildTimeInS=`date +%s`" -o /tmp/datly_go_router ./e2e/v1/routerapp + + wildcardApp: + stop: + action: process:stop + target: $target + input: datly_go_router + + start: + action: process:start + sleepTimeMs: 6000 + target: $target + directory: /tmp/ + checkError: true + immuneToHangups: true + env: + TEST: 1 + command: ulimit -Sn 10000 && ./datly_go_router -c=${wildcardConfig} > /tmp/datly_v1_go_all.out 2>&1 + + wildcardJWT: + action: secret:signJWT + privateKey: + URL: ${appPath}/e2e/cloud/jwt/private.enc + Key: blowfish://default + claims: + userID: 2 + firstName: Developer + email: dev@viantint.com + + wildcardTest: + action: http/runner:send + requests: + - Method: GET + description: wildcard GoBootstrap loads vendor list route from generated router packages + URL: http://127.0.0.1:${goBootstrapPort}/v1/api/shape/dev/vendors/ + Expect: + Code: 200 + JSONBody: $wildcardExpect + - Method: GET + description: wildcard GoBootstrap loads cross-package component dependency route + URL: http://127.0.0.1:${goBootstrapPort}/v1/api/shape/dev/vendors/component-acl?name=2 + Header: + Authorization: Bearer ${wildcardJWT.TokenString} + Expect: + Code: 200 + JSONBody: $adminExpect + - Method: GET + description: wildcard GoBootstrap loads linked handler route from linked Go package + URL: http://127.0.0.1:${goBootstrapPort}/v1/api/shape/dev/linked/auth?echo=hello + Header: + Authorization: Bearer ${wildcardJWT.TokenString} + Expect: + Code: 200 + JSONBody: + status: ok + data: + userID: 2 + firstName: Developer + echo: hello + + singleApp: + stop: + action: process:stop + target: $target + input: datly_go_router + + start: + action: process:start + sleepTimeMs: 6000 + target: $target + directory: /tmp/ + checkError: true + immuneToHangups: true + env: + TEST: 1 + command: ulimit -Sn 10000 && ./datly_go_router -c=${singleConfig} > /tmp/datly_v1_go_one.out 2>&1 + + singleTest: + action: http/runner:send + requests: + - Method: GET + description: single-package GoBootstrap loads only the requested vendor/list package + URL: http://127.0.0.1:${goBootstrapPort}/v1/api/shape/dev/vendors/ + Expect: + Code: 200 + JSONBody: $wildcardExpect + - Method: GET + description: single-package GoBootstrap does not load unrelated user ACL route + URL: http://127.0.0.1:${goBootstrapPort}/v1/api/shape/dev/auth/user-acl + Expect: + Code: 404 + + cleanup: + action: process:stop + target: $target + input: datly_go_router diff --git a/e2e/v1/regression/app.yaml b/e2e/v1/regression/app.yaml index b4bf66e7a..d46066f08 100644 --- a/e2e/v1/regression/app.yaml +++ b/e2e/v1/regression/app.yaml @@ -14,4 +14,4 @@ pipeline: immuneToHangups: true env: TEST: 1 - command: ulimit -Sn 10000 && ./datly -c=${v1Path}/autogen/Datly/config.json -z=/tmp/jobs/datly_v1 > /tmp/datly_v1.out 2>&1 + command: pkill -f '${v1Path}/autogen/Datly/config.json -z=/tmp/jobs/datly_v1' >/dev/null 2>&1 || true; ulimit -Sn 10000 && ./datly -c=${v1Path}/autogen/Datly/config.json -z=/tmp/jobs/datly_v1 > /tmp/datly_v1.out 2>&1 diff --git a/e2e/v1/regression/regression.yaml b/e2e/v1/regression/regression.yaml index 0c5f00560..56c85ba6d 100644 --- a/e2e/v1/regression/regression.yaml +++ b/e2e/v1/regression/regression.yaml @@ -3,7 +3,7 @@ init: pipeline: database: action: run - request: '@db' + request: '@db.yaml' app: when: $debugger!=on @@ -16,8 +16,8 @@ pipeline: data: '[]dev_dbsetup': '@dbsetup/dev' - subPath: 'cases/${index}_*' - range: 001..020 + subPath: '../cases/${index}_*' + range: 1..022 template: checkSkip: action: nop diff --git a/e2e/v1/routerapp/main.go b/e2e/v1/routerapp/main.go new file mode 100644 index 000000000..5e4915f77 --- /dev/null +++ b/e2e/v1/routerapp/main.go @@ -0,0 +1,43 @@ +package main + +import ( + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + _ "github.com/viant/afs/embed" + _ "github.com/viant/afsc/gs" + _ "github.com/viant/bigquery" + "github.com/viant/datly/cmd" + "github.com/viant/datly/cmd/env" + "github.com/viant/datly/service/executor/expand" + _ "github.com/viant/scy/kms/blowfish" + _ "github.com/viant/sqlx/metadata/product/mysql" + _ "github.com/viant/sqlx/metadata/product/pg" + _ "github.com/viant/sqlx/metadata/product/sqlite" + "os" + "strconv" + "time" +) + +var ( + Version = "development" + BuildTimeInS string +) + +func init() { + os.Setenv("DATLY_NOPANIC", "true") + expand.SetPanicOnError(false) + + if BuildTimeInS != "" { + seconds, err := strconv.Atoi(BuildTimeInS) + if err != nil { + panic(err) + } + env.BuildTime = time.Unix(int64(seconds), 0) + } +} + +func main() { + if err := cmd.RunApp(Version, os.Args[1:]); err != nil { + panic(err) + } +} diff --git a/e2e/v1/routerpkg/dev/linkedauth/handler.go b/e2e/v1/routerpkg/dev/linkedauth/handler.go new file mode 100644 index 000000000..88e80ed3d --- /dev/null +++ b/e2e/v1/routerpkg/dev/linkedauth/handler.go @@ -0,0 +1,61 @@ +package linkedauth + +import ( + "context" + "reflect" + + "github.com/viant/scy/auth/jwt" + "github.com/viant/xdatly" + xhandler "github.com/viant/xdatly/handler" + "github.com/viant/xdatly/handler/response" + "github.com/viant/xdatly/types/core" + "github.com/viant/xdatly/types/custom/dependency/checksum" +) + +const packageName = "github.com/viant/datly/e2e/v1/routerpkg/dev/linkedauth" + +func init() { + core.RegisterType(packageName, "LinkedAuthInput", reflect.TypeOf(LinkedAuthInput{}), checksum.GeneratedTime) + core.RegisterType(packageName, "LinkedAuthOutput", reflect.TypeOf(LinkedAuthOutput{}), checksum.GeneratedTime) + core.RegisterType(packageName, "LinkedAuthPayload", reflect.TypeOf(LinkedAuthPayload{}), checksum.GeneratedTime) + core.RegisterType(packageName, "Handler", reflect.TypeOf(Handler{}), checksum.GeneratedTime) +} + +type LinkedAuthInput struct { + Jwt *jwt.Claims `parameter:",kind=header,in=Authorization,dataType=string,errorCode=401" codec:"JwtClaim"` + Echo string `parameter:",kind=query,in=echo"` +} + +type LinkedAuthPayload struct { + UserID int `json:"userID"` + FirstName string `json:"firstName,omitempty"` + Echo string `json:"echo,omitempty"` +} + +type LinkedAuthOutput struct { + response.Status `parameter:",kind=output,in=status" json:",omitempty"` + Data *LinkedAuthPayload `parameter:",kind=output,in=view" json:"data,omitempty"` +} + +type Handler struct{} + +func (h *Handler) Exec(ctx context.Context, sess xhandler.Session) (interface{}, error) { + input := &LinkedAuthInput{} + if err := sess.Stater().Bind(ctx, input); err != nil { + return nil, err + } + if input.Jwt == nil { + return nil, response.NewError(401, "unauthorized access") + } + return &LinkedAuthOutput{ + Data: &LinkedAuthPayload{ + UserID: input.Jwt.UserID, + FirstName: input.Jwt.FirstName, + Echo: input.Echo, + }, + }, nil +} + +type LinkedAuthRouter struct { + LinkedAuth xdatly.Component[LinkedAuthInput, LinkedAuthOutput] `component:",path=/v1/api/shape/dev/linked/auth,method=GET,connector=dev,input=LinkedAuthInput,output=LinkedAuthOutput,handler=Handler"` +} diff --git a/e2e/v1/run.yaml b/e2e/v1/run.yaml index 435a61291..e306b39d4 100644 --- a/e2e/v1/run.yaml +++ b/e2e/v1/run.yaml @@ -41,3 +41,8 @@ pipeline: action: run description: run v1 regression test request: '@regression/regression' + + goBootstrap: + action: run + description: run linked Go router bootstrap regression test + request: '@go_bootstrap' diff --git a/e2e/v1/shapes.yaml b/e2e/v1/shapes.yaml index f433438b7..e50ed79c6 100644 --- a/e2e/v1/shapes.yaml +++ b/e2e/v1/shapes.yaml @@ -1,7 +1,12 @@ init: shapePath: ${v1Path}/shape + shapeDevPath: ${shapePath}/dev repoPath: ${v1Path}/autogen + routerImports: ${v1Path}/routerapp/imports_gen.go conn: -c='dev|mysql|root:dev@tcp(${dbIP.mysql}:3306)/dev${qMark}parseTime=true' + jwt: -J='${appPath}/e2e/local/jwt/public.enc|blowfish://default' + api: -a='/v1/api/shape' + mode: --skip-yaml pipeline: @@ -14,7 +19,9 @@ pipeline: - mkdir -p ${repoPath} - rm -rf ${repoPath} - mkdir -p ${shapePath} + - rm -rf ${shapeDevPath} - rm -rf ${shapePath} + - rm -f ${routerImports} vendor: action: exec:run @@ -22,51 +29,26 @@ pipeline: checkError: true commands: - cd ${v1Path} - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_list.dql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_details.sql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_auth.sql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/product_update.sql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_meta.sql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/child_meta.sql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_apikey.sql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendors_codec.sql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/vendor_col_in.dql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/header_vendors.dql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/environment.dql - - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn -s dql/dev/vendor/meta_format.dql - - user: - action: exec:run - TimeoutMs: 120000 - checkError: true - commands: - - cd ${v1Path} - - /tmp/datly transcribe -u dev/user -m ${shapePath} -r ${repoPath} $conn -s dql/dev/user/user_tree.sql - - district: - action: exec:run - TimeoutMs: 120000 - checkError: true - commands: - - cd ${v1Path} - - /tmp/datly transcribe -u dev/district -m ${shapePath} -r ${repoPath} $conn -s dql/dev/district/district_pagination.sql - - events: - action: exec:run - TimeoutMs: 120000 - checkError: true - commands: - - cd ${v1Path} - - /tmp/datly transcribe -u dev/events -m ${shapePath} -r ${repoPath} $conn -s dql/dev/events/post_basic_one.dql - - /tmp/datly transcribe -u dev/events -m ${shapePath} -r ${repoPath} $conn -s dql/dev/events/post_basic_many.dql - - /tmp/datly transcribe -u dev/events -m ${shapePath} -r ${repoPath} $conn -s dql/dev/events/post_comprehensive_many.dql - - /tmp/datly transcribe -u dev/events -m ${shapePath} -r ${repoPath} $conn -s dql/dev/events/post_except.dql - - team: - action: exec:run - TimeoutMs: 120000 - checkError: true - commands: - - cd ${v1Path} - - /tmp/datly transcribe -u dev/team -m ${shapePath} -r ${repoPath} $conn -s dql/dev/team/team.dql - - /tmp/datly transcribe -u dev/team -m ${shapePath} -r ${repoPath} $conn -s dql/dev/team/user_team.dql + - /tmp/datly transcribe -u dev/vendor -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/vendor_list.dql + - /tmp/datly transcribe -u dev/vendor/details -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/vendor_details.dql + - /tmp/datly transcribe -u dev/vendor/col -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/vendor_col_in.dql + - /tmp/datly transcribe -u dev/vendor/auth -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/vendor_auth.dql + - /tmp/datly transcribe -u dev/vendor/secured -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/vendor_apikey.dql + - /tmp/datly transcribe -u dev/vendor/update -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/product_update.dql + - /tmp/datly transcribe -u dev/vendor/meta -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/vendor_meta.dql + - /tmp/datly transcribe -u dev/vendor/meta-nested -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/child_meta.dql + - /tmp/datly transcribe -u dev/vendor/meta-format -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/meta_format.dql + - /tmp/datly transcribe -u dev/vendor/header -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/header_vendors.dql + - /tmp/datly transcribe -u dev/vendor/env -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/const.dql + - /tmp/datly transcribe -u dev/vendor/variables -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/vars.dql + - /tmp/datly transcribe -u dev/vendor/user_acl -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/user_acl.dql + - /tmp/datly transcribe -u dev/vendor/component_acl -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/component_acl/vendor_acl.dql + - /tmp/datly transcribe -u dev/vendor/grouping -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/vendorsrv/vendors_grouping.dql + - /tmp/datly transcribe -u dev/district/meta -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/district/district_pagination.sql + - /tmp/datly transcribe -u dev/user/metadata -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/user/user_metadata.dql + - /tmp/datly transcribe -u dev/user -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/user/user_tree.sql + - /tmp/datly transcribe -u dev/basic/events-one-one -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/events/basic_one_one.dql + - /tmp/datly transcribe -u dev/basic/foos -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/events/patch_basic_one.dql + - /tmp/datly transcribe -u dev/basic/foos-many -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/events/patch_basic_many.dql + - /tmp/datly transcribe -u dev/team/user_team -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/team/user_team.dql + - /tmp/datly transcribe -u dev/team/delete -m ${shapePath} -r ${repoPath} $conn $jwt $api $mode -s dql/dev/team/team.dql diff --git a/gateway/dql_bootstrap.go b/gateway/dql_bootstrap.go index fe7d62dbc..fc5239f5d 100644 --- a/gateway/dql_bootstrap.go +++ b/gateway/dql_bootstrap.go @@ -2,11 +2,11 @@ package gateway import ( "context" - "encoding/json" "fmt" "os" "path" "path/filepath" + "reflect" "sort" "strings" @@ -17,6 +17,8 @@ import ( shapeLoad "github.com/viant/datly/repository/shape/load" datlyservice "github.com/viant/datly/service" "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/tagly/format/text" ) func (r *Service) applyDQLBootstrap(ctx context.Context, repo *repository.Service, cfg *DQLBootstrap) error { @@ -72,7 +74,7 @@ func (r *Service) applyDQLBootstrap(ctx context.Context, repo *repository.Servic return nil } -func compileBootstrapComponent(ctx context.Context, compiler *shapeCompile.DQLCompiler, loader *shapeLoad.Loader, repo *repository.Service, sourcePath string, cfg *DQLBootstrap, apiPrefix string) (*repository.Component, error) { +func compileBootstrapComponent(ctx context.Context, compiler *shapeCompile.DQLCompiler, loader *shapeLoad.Loader, repo *repository.Service, sourcePath string, cfg *DQLBootstrap, _ string) (*repository.Component, error) { data, err := os.ReadFile(sourcePath) if err != nil { return nil, fmt.Errorf("failed to read DQL bootstrap source %s: %w", sourcePath, err) @@ -101,22 +103,63 @@ func compileBootstrapComponent(ctx context.Context, compiler *shapeCompile.DQLCo if !ok || loaded == nil { return nil, fmt.Errorf("unexpected shape component artifact for %s", sourcePath) } + bootstrapMetadata := snapshotBootstrapViewMetadata(componentArtifact.Resource) rootView := lookupRootView(componentArtifact.Resource, loaded.RootView) if rootView == nil { return nil, fmt.Errorf("missing root view %q for %s", loaded.RootView, sourcePath) } - method, uri := resolvePathSettings(sourcePath, dql, apiPrefix) + method := strings.TrimSpace(strings.ToUpper(loaded.Method)) + uri := strings.TrimSpace(loaded.URI) + if method == "" && len(loaded.ComponentRoutes) > 0 && loaded.ComponentRoutes[0] != nil { + method = strings.TrimSpace(strings.ToUpper(loaded.ComponentRoutes[0].Method)) + } + if uri == "" && len(loaded.ComponentRoutes) > 0 && loaded.ComponentRoutes[0] != nil { + uri = strings.TrimSpace(loaded.ComponentRoutes[0].RoutePath) + } + if method == "" { + method = "GET" + } + if uri == "" { + return nil, fmt.Errorf("missing shape component route for %s", sourcePath) + } + var outputType reflect.Type + if shouldMaterializeBootstrapOutputType(loaded, rootView) { + pkgPath := bootstrapTypePackage(loaded) + lookupType := componentArtifact.Resource.LookupType() + outputType, err = loaded.OutputReflectType(pkgPath, lookupType) + if err != nil { + return nil, fmt.Errorf("failed to materialize bootstrap output type for %s: %w", sourcePath, err) + } + } componentModel := &repository.Component{ Path: contract.Path{ Method: method, URI: uri, }, Contract: contract.Contract{ + Input: contract.Input{ + Type: state.Type{ + Parameters: loaded.InputParameters(), + }, + }, + Output: contract.Output{ + CaseFormat: bootstrapOutputCaseFormat(loaded), + Cardinality: bootstrapOutputCardinality(loaded, rootView), + Type: state.Type{ + Parameters: loaded.OutputParameters(), + }, + }, Service: defaultServiceForMethod(method, rootView), }, View: rootView, TypeContext: loaded.TypeContext, } + if outputType != nil { + if componentModel.Contract.Output.Type.Schema == nil { + componentModel.Contract.Output.Type.Schema = state.NewSchema(nil) + } + componentModel.Contract.Output.Type.SetType(outputType) + } loadOptions := []repository.Option{} if repo != nil { loadOptions = append(loadOptions, repository.WithResources(repo.Resources())) @@ -129,15 +172,43 @@ func compileBootstrapComponent(ctx context.Context, compiler *shapeCompile.DQLCo if err != nil { return nil, fmt.Errorf("failed to materialize bootstrap component for %s: %w", sourcePath, err) } + mergeBootstrapViewMetadata(components.Resource, bootstrapMetadata) if err = components.Init(ctx); err != nil { return nil, fmt.Errorf("failed to initialize bootstrap component for %s: %w", sourcePath, err) } if len(components.Components) == 0 || components.Components[0] == nil { return nil, fmt.Errorf("empty initialized bootstrap component for %s", sourcePath) } + mergeBootstrapView(components.Components[0].View, lookupRootView(bootstrapMetadata, loaded.RootView)) return components.Components[0], nil } +func bootstrapTypePackage(component *shapeLoad.Component) string { + if component == nil || component.TypeContext == nil { + return "" + } + if pkgPath := strings.TrimSpace(component.TypeContext.PackagePath); pkgPath != "" { + return pkgPath + } + return strings.TrimSpace(component.TypeContext.DefaultPackage) +} + +func shouldMaterializeBootstrapOutputType(component *shapeLoad.Component, rootView *view.View) bool { + if component == nil || rootView == nil || rootView.Schema == nil || rootView.Schema.Cardinality != state.One { + return false + } + for _, item := range component.Output { + if item == nil || item.In == nil || item.In.Kind != state.KindOutput || item.In.Name != "view" { + continue + } + if !strings.Contains(item.Tag, "anonymous") || item.Schema == nil { + return false + } + return item.Schema.Cardinality == state.One + } + return false +} + func mergeBootstrapSharedResources(target *view.Resource, repo *repository.Service) { if target == nil || repo == nil || repo.Resources() == nil { return @@ -180,7 +251,7 @@ func hasRepositoryProvider(ctx context.Context, repo *repository.Service, path * _, err := repo.Registry().LookupProvider(ctx, path) if err != nil { message := strings.ToLower(strings.TrimSpace(err.Error())) - if strings.Contains(message, "not found") { + if strings.Contains(message, "not found") || strings.Contains(message, "couldn't match uri") { return false, nil } return false, err @@ -410,44 +481,128 @@ func lookupRootView(resource *view.Resource, root string) *view.View { return nil } -type bootstrapRuleSettings struct { - Method string `json:"Method"` - URI string `json:"URI"` +func bootstrapOutputCardinality(component *shapeLoad.Component, rootView *view.View) state.Cardinality { + if component != nil { + if output := component.OutputParameters(); len(output) > 0 { + if parameter := output.LookupByLocation(state.KindOutput, "view"); parameter != nil && parameter.Schema != nil && parameter.Schema.Cardinality != "" { + return parameter.Schema.Cardinality + } + } + } + if rootView != nil && rootView.Schema != nil && rootView.Schema.Cardinality != "" { + return rootView.Schema.Cardinality + } + return "" } -func resolvePathSettings(sourcePath, dql, apiPrefix string) (string, string) { - method := "GET" - uri := "" - settings := parseBootstrapRuleSettings(dql) - if settings != nil { - if candidate := strings.TrimSpace(strings.ToUpper(settings.Method)); candidate != "" { - method = candidate +func bootstrapOutputCaseFormat(component *shapeLoad.Component) text.CaseFormat { + if component != nil && component.Directives != nil { + if value := strings.TrimSpace(component.Directives.CaseFormat); value != "" { + return text.CaseFormat(value) } - uri = strings.TrimSpace(settings.URI) } - if uri == "" { - stem := strings.TrimSuffix(filepath.Base(sourcePath), filepath.Ext(sourcePath)) - uri = "/" + strings.Trim(stem, "/") - if prefix := strings.TrimSpace(apiPrefix); prefix != "" { - uri = strings.TrimRight(prefix, "/") + uri + return text.CaseFormatLowerCamel +} + +func mergeBootstrapViewMetadata(target, source *view.Resource) { + if target == nil || source == nil { + return + } + sourceViews := source.Views.Index() + for _, candidate := range target.Views { + if candidate == nil { + continue + } + original, _ := sourceViews.Lookup(candidate.Name) + if original == nil { + continue } + mergeBootstrapView(candidate, original) } - return method, uri } -func parseBootstrapRuleSettings(dql string) *bootstrapRuleSettings { - start := strings.Index(dql, "/*") - end := strings.Index(dql, "*/") - if start == -1 || end == -1 || end <= start+2 { - return nil +func mergeBootstrapView(target, source *view.View) { + if target == nil || source == nil { + return } - raw := strings.TrimSpace(dql[start+2 : end]) - if !strings.HasPrefix(raw, "{") || !strings.HasSuffix(raw, "}") { - return nil + if source.AllowNulls != nil { + value := *source.AllowNulls + target.AllowNulls = &value + } + if source.Groupable { + target.Groupable = true + } + if source.Selector != nil { + target.Selector = source.Selector } - ret := &bootstrapRuleSettings{} - if err := json.Unmarshal([]byte(raw), ret); err != nil { + if len(source.ColumnsConfig) > 0 { + target.ColumnsConfig = map[string]*view.ColumnConfig{} + for key, cfg := range source.ColumnsConfig { + if cfg == nil { + continue + } + cloned := *cfg + if cfg.DataType != nil { + value := *cfg.DataType + cloned.DataType = &value + } + if cfg.Tag != nil { + value := *cfg.Tag + cloned.Tag = &value + } + if cfg.Groupable != nil { + value := *cfg.Groupable + cloned.Groupable = &value + } + target.ColumnsConfig[key] = &cloned + } + } +} + +func snapshotBootstrapViewMetadata(resource *view.Resource) *view.Resource { + if resource == nil { return nil } - return ret + result := &view.Resource{} + for _, item := range resource.Views { + if item == nil { + continue + } + cloned := &view.View{ + Name: item.Name, + Groupable: item.Groupable, + } + cloned.Reference.Ref = item.Ref + if item.AllowNulls != nil { + value := *item.AllowNulls + cloned.AllowNulls = &value + } + if item.Selector != nil { + cloned.Selector = item.Selector + } + if len(item.ColumnsConfig) > 0 { + cloned.ColumnsConfig = map[string]*view.ColumnConfig{} + for key, cfg := range item.ColumnsConfig { + if cfg == nil { + continue + } + copied := *cfg + if cfg.DataType != nil { + value := *cfg.DataType + copied.DataType = &value + } + if cfg.Tag != nil { + value := *cfg.Tag + copied.Tag = &value + } + if cfg.Groupable != nil { + value := *cfg.Groupable + copied.Groupable = &value + } + cloned.ColumnsConfig[key] = &copied + } + } + result.Views = append(result.Views, cloned) + } + return result } diff --git a/gateway/dql_bootstrap_test.go b/gateway/dql_bootstrap_test.go index b36714cd0..87a56eb4f 100644 --- a/gateway/dql_bootstrap_test.go +++ b/gateway/dql_bootstrap_test.go @@ -2,15 +2,27 @@ package gateway import ( "context" + "net/http" "os" "path/filepath" + "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + marshalconfig "github.com/viant/datly/gateway/router/marshal/config" + marshaljson "github.com/viant/datly/gateway/router/marshal/json" "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" + shape "github.com/viant/datly/repository/shape" + shapeCompile "github.com/viant/datly/repository/shape/compile" + shapeLoad "github.com/viant/datly/repository/shape/load" + operator2 "github.com/viant/datly/service/operator" + "github.com/viant/datly/service/session" "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/datly/view/state/kind/locator" + "github.com/viant/tagly/format/text" ) func TestConfigValidate_AllowsEmptyRouteURLWithDQLBootstrap(t *testing.T) { @@ -55,14 +67,32 @@ func TestDiscoverDQLBootstrapSources(t *testing.T) { assert.Contains(t, sources, filepath.Join(root, "sql", "nested", "b.sql")) } -func TestResolvePathSettings(t *testing.T) { - method, uri := resolvePathSettings("/tmp/orders/get.dql", `/* {"Method":"POST","URI":"/v1/api/orders"} */ SELECT 1`, "/v1/api") - assert.Equal(t, "POST", method) - assert.Equal(t, "/v1/api/orders", uri) +func TestCompileBootstrapComponent_UsesShapeRouteMetadata(t *testing.T) { + ctx := context.Background() + repo, err := repository.New(ctx, repository.WithComponentURL(""), repository.WithNoPlugin()) + require.NoError(t, err) + connectors, err := repo.Resources().Lookup(view.ResourceConnectors) + require.NoError(t, err) + connectors.Connectors = append(connectors.Connectors, &view.Connector{ + Connection: view.Connection{ + DBConfig: view.DBConfig{ + Name: "test_conn", + Driver: "sqlite3", + DSN: ":memory:", + }, + }, + }) - method, uri = resolvePathSettings("/tmp/orders/get.dql", `SELECT 1`, "/v1/api") - assert.Equal(t, "GET", method) - assert.Equal(t, "/v1/api/get", uri) + root := t.TempDir() + source := filepath.Join(root, "orders.dql") + dql := "#setting($_ = $connector('test_conn'))\n#setting($_ = $route('/v1/api/orders', 'POST'))\nSELECT 1 AS id" + require.NoError(t, os.WriteFile(source, []byte(dql), 0o644)) + + component, err := compileBootstrapComponent(ctx, shapeCompile.New(), shapeLoad.New(), repo, source, &DQLBootstrap{}, "/v1/api") + require.NoError(t, err) + require.NotNil(t, component) + assert.Equal(t, "POST", component.Method) + assert.Equal(t, "/v1/api/orders", component.URI) } func TestDQLBootstrapEffectivePrecedence(t *testing.T) { @@ -92,7 +122,7 @@ func TestApplyDQLBootstrap_Precedence(t *testing.T) { root := t.TempDir() source := filepath.Join(root, "test.dql") - require.NoError(t, os.WriteFile(source, []byte(`/* {"Method":"GET","URI":"/v1/api/test","Connector":"test_conn"} */ SELECT 1 AS id`), 0o644)) + require.NoError(t, os.WriteFile(source, []byte("#setting($_ = $connector('test_conn'))\n#setting($_ = $route('/v1/api/test', 'GET'))\nSELECT 1 AS id"), 0o644)) srv := &Service{Config: &Config{ExposableConfig: ExposableConfig{APIPrefix: "/v1/api"}}} routesWins := &DQLBootstrap{ @@ -120,3 +150,274 @@ func TestApplyDQLBootstrap_Precedence(t *testing.T) { require.NotNil(t, component.View) assert.Equal(t, "test", component.View.Name) } + +func TestCompileBootstrapComponent_PreservesShapeIOAndGroupingMetadata(t *testing.T) { + ctx := context.Background() + repo, err := repository.New(ctx, repository.WithComponentURL(""), repository.WithNoPlugin()) + require.NoError(t, err) + + connectors, err := repo.Resources().Lookup(view.ResourceConnectors) + require.NoError(t, err) + connectors.Connectors = append(connectors.Connectors, &view.Connector{ + Connection: view.Connection{ + DBConfig: view.DBConfig{ + Name: "dev", + Driver: "mysql", + DSN: "root:dev@tcp(127.0.0.1:3306)/dev?parseTime=true", + }, + }, + }) + + root := t.TempDir() + source := filepath.Join(root, "vendors_grouping.dql") + dql := ` +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/vendors-grouping', 'GET')) +#define($_ = $VendorIDs<[]int>(query/vendorIDs)) +#define($_ = $Fields<[]string>(query/_fields).Optional().QuerySelector('vendor')) +#define($_ = $OrderBy(query/_orderby).Optional().QuerySelector('vendor')) +#define($_ = $Data(output/view).Embed()) +SELECT vendor.*, + groupable(vendor), + allowed_order_by_columns(vendor, 'accountId:ACCOUNT_ID,userCreated:USER_CREATED,totalId:TOTAL_ID,maxId:MAX_ID') +FROM ( + SELECT ACCOUNT_ID, + USER_CREATED, + SUM(ID) AS TOTAL_ID, + MAX(ID) AS MAX_ID + FROM VENDOR t + WHERE t.ID IN ($VendorIDs) + GROUP BY 1, 2 +) vendor` + require.NoError(t, os.WriteFile(source, []byte(dql), 0o644)) + + planResult, err := shapeCompile.New().Compile(ctx, &shape.Source{ + Name: "vendors_grouping", + Path: source, + DQL: dql, + }) + require.NoError(t, err) + artifact, err := shapeLoad.New().LoadComponent(ctx, planResult) + require.NoError(t, err) + loaded, ok := artifact.Component.(*shapeLoad.Component) + require.True(t, ok) + sourceRoot := lookupRootView(artifact.Resource, loaded.RootView) + require.NotNil(t, sourceRoot) + require.NotNil(t, sourceRoot.ColumnsConfig["ACCOUNT_ID"]) + require.NotNil(t, sourceRoot.ColumnsConfig["ACCOUNT_ID"].Groupable) + assert.True(t, *sourceRoot.ColumnsConfig["ACCOUNT_ID"].Groupable) + + component, err := compileBootstrapComponent(ctx, shapeCompile.New(), shapeLoad.New(), repo, source, &DQLBootstrap{}, "/v1/api/shape") + require.NoError(t, err) + require.NotNil(t, component) + require.NotNil(t, component.View) + require.True(t, component.View.Groupable) + require.NotNil(t, component.View.Selector) + require.NotNil(t, component.View.Selector.Constraints) + assert.True(t, component.View.Selector.Constraints.OrderBy) + assert.Equal(t, "ACCOUNT_ID", component.View.Selector.Constraints.OrderByColumn["accountId"]) + assert.Equal(t, "ACCOUNT_ID", component.View.Selector.Constraints.OrderByColumn["accountid"]) + require.NotNil(t, component.View.ColumnsConfig["ACCOUNT_ID"]) + require.NotNil(t, component.View.ColumnsConfig["ACCOUNT_ID"].Groupable) + assert.True(t, *component.View.ColumnsConfig["ACCOUNT_ID"].Groupable) + assert.Equal(t, text.CaseFormatLowerCamel, component.Output.CaseFormat) + + inputVendorIDs := component.Input.Type.Parameters.Lookup("VendorIDs") + require.NotNil(t, inputVendorIDs) + assert.Equal(t, state.KindQuery, inputVendorIDs.In.Kind) + assert.Equal(t, "vendorIDs", inputVendorIDs.In.Name) + + inputFields := component.Input.Type.Parameters.Lookup("Fields") + require.NotNil(t, inputFields) + assert.Equal(t, state.KindQuery, inputFields.In.Kind) + assert.Equal(t, "_fields", inputFields.In.Name) + + outputView := component.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view") + require.NotNil(t, outputView) + assert.Contains(t, outputView.Tag, `anonymous:"true"`) + assert.Equal(t, state.Many, component.Output.Cardinality) +} + +func TestCompileBootstrapComponent_MetaFormatOutputTypeMatchesRootView(t *testing.T) { + ctx := context.Background() + repo, err := repository.New(ctx, repository.WithComponentURL(""), repository.WithNoPlugin()) + require.NoError(t, err) + + connectors, err := repo.Resources().Lookup(view.ResourceConnectors) + require.NoError(t, err) + connectors.Connectors = append(connectors.Connectors, &view.Connector{ + Connection: view.Connection{ + DBConfig: view.DBConfig{ + Name: "dev", + Driver: "sqlite3", + DSN: ":memory:", + }, + }, + }) + + source := filepath.Join("..", "e2e", "v1", "dql", "dev", "vendorsrv", "meta_format.dql") + dqlBytes, err := os.ReadFile(source) + require.NoError(t, err) + planResult, err := shapeCompile.New().Compile(ctx, &shape.Source{ + Name: "meta_format", + Path: source, + DQL: string(dqlBytes), + }) + require.NoError(t, err) + artifact, err := shapeLoad.New().LoadComponent(ctx, planResult) + require.NoError(t, err) + loaded, ok := artifact.Component.(*shapeLoad.Component) + require.True(t, ok) + loadedRoot := lookupRootView(artifact.Resource, loaded.RootView) + require.NotNil(t, loadedRoot) + require.NotNil(t, loadedRoot.Schema) + t.Logf("loaded root view schema type: %v", loadedRoot.Schema.Type()) + + component, err := compileBootstrapComponent(ctx, shapeCompile.New(), shapeLoad.New(), repo, source, &DQLBootstrap{}, "/v1/api/shape") + require.NoError(t, err) + require.NotNil(t, component) + require.NotNil(t, component.View) + require.NotNil(t, component.View.Schema) + require.NotNil(t, component.View.Schema.Type()) + + outputView := component.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view") + require.NotNil(t, outputView) + require.NotNil(t, outputView.Schema) + require.NotNil(t, outputView.Schema.Type()) + + rootType := component.View.OutputType() + outputType := outputView.OutputType() + assert.Equal(t, rootType.Kind(), outputType.Kind()) + if rootType.Kind() == reflect.Slice { + assert.Equal(t, rootType.Elem().Kind(), outputType.Elem().Kind()) + } + + outputSummary := component.Output.Type.Parameters.LookupByLocation(state.KindOutput, "summary") + require.NotNil(t, outputSummary) + require.NotNil(t, outputSummary.Schema) + require.NotNil(t, outputSummary.Schema.Type()) + require.NotNil(t, component.View.Template) + require.NotNil(t, component.View.Template.Summary) + require.NotNil(t, component.View.Template.Summary.Schema) + require.NotNil(t, component.View.Template.Summary.Schema.Type()) + assert.Equal(t, component.View.Template.Summary.Schema.Type().String(), outputSummary.Schema.Type().String()) +} + +func TestCompileBootstrapComponent_MetaFormatLiveOutputMarshal(t *testing.T) { + ctx := context.Background() + repo, err := repository.New(ctx, repository.WithComponentURL(""), repository.WithNoPlugin()) + require.NoError(t, err) + + connectors, err := repo.Resources().Lookup(view.ResourceConnectors) + require.NoError(t, err) + connectors.Connectors = append(connectors.Connectors, &view.Connector{ + Connection: view.Connection{ + DBConfig: view.DBConfig{ + Name: "dev", + Driver: "mysql", + DSN: "root:dev@tcp(127.0.0.1:3306)/dev?parseTime=true", + }, + }, + }) + + source := filepath.Join("..", "e2e", "v1", "dql", "dev", "vendorsrv", "meta_format.dql") + component, err := compileBootstrapComponent(ctx, shapeCompile.New(), shapeLoad.New(), repo, source, &DQLBootstrap{}, "/v1/api/shape") + require.NoError(t, err) + require.NotNil(t, component.View) + require.NotNil(t, component.View.Schema) + t.Logf("root view schema type: %v", component.View.Schema.Type()) + if outputView := component.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); outputView != nil && outputView.Schema != nil { + t.Logf("output/view schema type: %v", outputView.Schema.Type()) + } + if outputSummary := component.Output.Type.Parameters.LookupByLocation(state.KindOutput, "summary"); outputSummary != nil && outputSummary.Schema != nil { + t.Logf("output/summary schema type: %v", outputSummary.Schema.Type()) + } + + svc := operator2.New() + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/v1/api/shape/dev/meta/vendors-format/", nil) + require.NoError(t, err) + sess := session.New(component.View, session.WithComponent(component), session.WithLocatorOptions(locator.WithRequest(req))) + outputValue, err := svc.Operate(ctx, sess, component) + require.NoError(t, err) + require.NotNil(t, outputValue) + t.Logf("output type: %T", outputValue) + + marshaller := marshaljson.New(&marshalconfig.IOConfig{CaseFormat: component.Output.CaseFormat}) + _, err = marshaller.Marshal(outputValue) + require.NoError(t, err) +} + +func TestCompileBootstrapComponent_UserAclMaterializesAnonymousOutputStateType(t *testing.T) { + ctx := context.Background() + repo, err := repository.New(ctx, repository.WithComponentURL(""), repository.WithNoPlugin()) + require.NoError(t, err) + + connectors, err := repo.Resources().Lookup(view.ResourceConnectors) + require.NoError(t, err) + connectors.Connectors = append(connectors.Connectors, &view.Connector{ + Connection: view.Connection{ + DBConfig: view.DBConfig{ + Name: "dev", + Driver: "sqlite3", + DSN: ":memory:", + }, + }, + }) + + source := filepath.Join("..", "e2e", "v1", "dql", "dev", "vendorsrv", "user_acl.dql") + component, err := compileBootstrapComponent(ctx, shapeCompile.New(), shapeLoad.New(), repo, source, &DQLBootstrap{}, "/v1/api/shape") + require.NoError(t, err) + require.NotNil(t, component) + require.True(t, component.Output.Type.Type().IsDefined()) + require.NotNil(t, component.Output.Type.Schema) + require.NotNil(t, component.Output.Type.Schema.Type()) + + outputView := component.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view") + require.NotNil(t, outputView) + require.NotNil(t, outputView.Schema) + require.NotNil(t, outputView.Schema.Type()) + assert.Equal(t, reflect.Pointer, outputView.OutputType().Kind()) +} + +func TestCompileBootstrapComponent_PatchBasicOneBodyParameterIsSingular(t *testing.T) { + ctx := context.Background() + repo, err := repository.New(ctx, repository.WithComponentURL(""), repository.WithNoPlugin()) + require.NoError(t, err) + + connectors, err := repo.Resources().Lookup(view.ResourceConnectors) + require.NoError(t, err) + connectors.Connectors = append(connectors.Connectors, &view.Connector{ + Connection: view.Connection{ + DBConfig: view.DBConfig{ + Name: "dev", + Driver: "sqlite3", + DSN: ":memory:", + }, + }, + }) + + source := filepath.Join("..", "e2e", "v1", "dql", "dev", "events", "patch_basic_one.dql") + component, err := compileBootstrapComponent(ctx, shapeCompile.New(), shapeLoad.New(), repo, source, &DQLBootstrap{}, "/v1/api/shape") + require.NoError(t, err) + require.NotNil(t, component) + + bodyParams := component.Input.Type.Parameters.FilterByKind(state.KindRequestBody) + require.Len(t, bodyParams, 1) + body := bodyParams[0] + require.NotNil(t, body) + require.NotNil(t, body.Schema) + assert.Equal(t, state.One, body.Schema.Cardinality) + require.NotNil(t, body.Schema.Type()) + assert.NotEqual(t, reflect.Slice, body.Schema.Type().Kind(), "body schema should not remain slice-shaped") + + inputStateType := component.Input.Type.Type() + require.NotNil(t, inputStateType) + inputType := inputStateType.Type() + require.NotNil(t, inputType) + if inputType.Kind() == reflect.Ptr { + inputType = inputType.Elem() + } + field, ok := inputType.FieldByName("Foos") + require.True(t, ok) + assert.NotEqual(t, reflect.Slice, field.Type.Kind(), "input Foos field should not remain slice-shaped") +} diff --git a/gateway/option.go b/gateway/option.go index 2e77d159d..2b113a008 100644 --- a/gateway/option.go +++ b/gateway/option.go @@ -10,14 +10,15 @@ import ( ) type options struct { - config *Config - initializers []func(config *Config, fs *embed.FS) error - extensions *extension.Registry - metrics *gmetric.Service - repository *repository.Service - statusHandler http.Handler - embedFs *embed.FS - configURL string + config *Config + initializers []func(config *Config, fs *embed.FS) error + extensions *extension.Registry + metrics *gmetric.Service + repository *repository.Service + statusHandler http.Handler + embedFs *embed.FS + configURL string + refreshDisabled bool } func newOptions(ctx context.Context, opts ...Option) (*options, error) { @@ -103,3 +104,9 @@ func WithConfigURL(configURL string) Option { o.configURL = configURL } } + +func WithRefreshDisabled(enabled bool) Option { + return func(o *options) { + o.refreshDisabled = enabled + } +} diff --git a/gateway/patch_basic_one_e2e_test.go b/gateway/patch_basic_one_e2e_test.go new file mode 100644 index 000000000..87d84b933 --- /dev/null +++ b/gateway/patch_basic_one_e2e_test.go @@ -0,0 +1,142 @@ +package gateway + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/service/operator" + readerpkg "github.com/viant/datly/service/reader" + "github.com/viant/datly/service/session" + "github.com/viant/datly/view/state" + "github.com/viant/datly/view/state/kind/locator" +) + +func TestGateway_PatchBasicOne_NoRefresh(t *testing.T) { + root := filepath.Clean(filepath.Join("..", "e2e", "v1", "autogen", "Datly", "config_8081.json")) + svc, err := New(context.Background(), + WithConfigURL(root), + WithRefreshDisabled(true), + ) + require.NoError(t, err) + t.Cleanup(func() { + _ = svc.Close() + ResetSingleton() + }) + + req := httptest.NewRequest(http.MethodPatch, "/v1/api/shape/dev/basic/foos", strings.NewReader(`{"ID":4,"Quantity":2500,"Name":"changed - foo 4"}`)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + svc.ServeHTTP(rec, req) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + require.Contains(t, rec.Body.String(), `"id":4`) + require.Contains(t, rec.Body.String(), `"quantity":2500`) + require.Contains(t, rec.Body.String(), `"name":"changed - foo 4"`) +} + +func TestGateway_PatchBasicOne_CurFoosValue(t *testing.T) { + root := filepath.Clean(filepath.Join("..", "e2e", "v1", "autogen", "Datly", "config_8081.json")) + svc, err := New(context.Background(), + WithConfigURL(root), + WithRefreshDisabled(true), + ) + require.NoError(t, err) + t.Cleanup(func() { + _ = svc.Close() + ResetSingleton() + }) + + component, err := svc.repository.Registry().Lookup(context.Background(), contract.NewPath(http.MethodPatch, "/v1/api/shape/dev/basic/foos")) + require.NoError(t, err) + resource := component.View.GetResource() + require.NotNil(t, resource) + curFoosView, err := resource.GetViews().Lookup("CurFoos") + require.NoError(t, err) + require.NotNil(t, curFoosView) + + plainDest := reflect.New(curFoosView.Schema.SliceType()).Interface() + err = readerpkg.New().ReadInto(context.Background(), plainDest, curFoosView) + require.NoError(t, err) + + req := httptest.NewRequest(http.MethodPatch, "/v1/api/shape/dev/basic/foos", strings.NewReader(`{"ID":4,"Quantity":2500,"Name":"changed - foo 4"}`)) + req.Header.Set("Content-Type", "application/json") + + unmarshal := component.UnmarshalFunc(req) + locatorOptions := append(component.LocatorOptions(req, nil, unmarshal)) + locatorOptions = append(locatorOptions, locator.WithLogger(nil)) + aSession := session.New(component.View, + session.WithComponent(component), + session.WithLocatorOptions(locatorOptions...), + session.WithRegistry(svc.repository.Registry()), + session.WithOperate(operator.New().Operate)) + + err = aSession.InitKinds(state.KindComponent, state.KindHeader, state.KindRequestBody, state.KindForm, state.KindQuery) + require.NoError(t, err) + err = aSession.Populate(context.Background()) + require.NoError(t, err) + + param, err := component.View.ParamByName("CurFoos") + require.NoError(t, err) + value, has, err := aSession.LookupValue(context.Background(), param, aSession.Indirect(true)) + require.NoError(t, err) + require.True(t, has) + require.NotNil(t, value) +} + +func TestGateway_PatchBasicOne_CurFoosReadInto_WithAndWithoutResourceState(t *testing.T) { + root := filepath.Clean(filepath.Join("..", "e2e", "v1", "autogen", "Datly", "config_8081.json")) + svc, err := New(context.Background(), + WithConfigURL(root), + WithRefreshDisabled(true), + ) + require.NoError(t, err) + t.Cleanup(func() { + _ = svc.Close() + ResetSingleton() + }) + + component, err := svc.repository.Registry().Lookup(context.Background(), contract.NewPath(http.MethodPatch, "/v1/api/shape/dev/basic/foos")) + require.NoError(t, err) + resource := component.View.GetResource() + require.NotNil(t, resource) + curFoosView, err := resource.GetViews().Lookup("CurFoos") + require.NoError(t, err) + require.NotNil(t, curFoosView) + + req := httptest.NewRequest(http.MethodPatch, "/v1/api/shape/dev/basic/foos", strings.NewReader(`{"ID":4,"Quantity":2500,"Name":"changed - foo 4"}`)) + req.Header.Set("Content-Type", "application/json") + + unmarshal := component.UnmarshalFunc(req) + locatorOptions := append(component.LocatorOptions(req, nil, unmarshal)) + locatorOptions = append(locatorOptions, locator.WithLogger(nil)) + aSession := session.New(component.View, + session.WithComponent(component), + session.WithLocatorOptions(locatorOptions...), + session.WithRegistry(svc.repository.Registry()), + session.WithOperate(operator.New().Operate)) + + err = aSession.InitKinds(state.KindComponent, state.KindHeader, state.KindRequestBody, state.KindForm, state.KindQuery) + require.NoError(t, err) + + err = aSession.SetViewState(context.Background(), curFoosView) + require.NoError(t, err) + sqlQuery, buildErr := readerpkg.NewBuilder().Build(context.Background(), + readerpkg.WithBuilderView(curFoosView), + readerpkg.WithBuilderStatelet(aSession.State().Lookup(curFoosView)), + ) + require.NoError(t, buildErr) + require.NotNil(t, sqlQuery) + t.Logf("curFoos sql=%s args=%#v", sqlQuery.SQL, sqlQuery.Args) + + stateDest := reflect.New(curFoosView.Schema.SliceType()).Interface() + err = readerpkg.New().ReadInto(context.Background(), stateDest, curFoosView, readerpkg.WithResourceState(aSession.State())) + require.Error(t, err) +} diff --git a/gateway/route_struct.go b/gateway/route_struct.go index 7e2bbe291..57e7c9d94 100644 --- a/gateway/route_struct.go +++ b/gateway/route_struct.go @@ -7,6 +7,7 @@ import ( "github.com/viant/xreflect" "net/http" "reflect" + "strings" ) func (r *Router) NewStructRoute(URL string, provider *repository.Provider) *Route { @@ -45,5 +46,10 @@ func (r *Router) generateGoStruct(component *repository.Component) (int, []byte) fieldTag, _ = xreflect.RemoveTag(fieldTag, "sql") *tag = fieldTag })) + structContent = legacyStructFormatting(structContent) return http.StatusOK, []byte(structContent) } + +func legacyStructFormatting(content string) string { + return strings.ReplaceAll(content, ` internal:"true"`, ` internal:"true"`) +} diff --git a/gateway/route_struct_test.go b/gateway/route_struct_test.go new file mode 100644 index 000000000..1d78e7991 --- /dev/null +++ b/gateway/route_struct_test.go @@ -0,0 +1,35 @@ +package gateway + +import ( + "net/http" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +func TestRouterGenerateGoStruct_PreservesLegacyInternalTagSpacing(t *testing.T) { + schemaType := reflect.StructOf([]reflect.StructField{ + { + Name: "VendorId", + Type: reflect.TypeOf((*int)(nil)), + Tag: reflect.StructTag(`sqlx:"VENDOR_ID" internal:"true"`), + }, + }) + schema := &state.Schema{Cardinality: state.Many} + schema.SetType(reflect.SliceOf(schemaType)) + + component := &repository.Component{ + View: &view.View{Schema: schema}, + } + + router := &Router{} + statusCode, content := router.generateGoStruct(component) + + require.Equal(t, http.StatusOK, statusCode) + require.True(t, strings.Contains(string(content), `sqlx:"VENDOR_ID" internal:"true"`), string(content)) +} diff --git a/gateway/service.go b/gateway/service.go index 794c2b2d7..5c055bf99 100644 --- a/gateway/service.go +++ b/gateway/service.go @@ -113,6 +113,7 @@ func New(ctx context.Context, opts ...Option) (*Service, error) { repository.WithFirebaseAuth(aConfig.Firebase), repository.WithDependencyURL(aConfig.DependencyURL), repository.WithRefreshFrequency(aConfig.SyncFrequency()), + repository.WithRefreshDisabled(options.refreshDisabled), repository.WithDispatcher(dispatcher.New), ) if err != nil { diff --git a/go.mod b/go.mod index 8781dfe24..1d47c9ced 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/viant/datly go 1.25.0 +replace github.com/viant/xdatly => ../xdatly + require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible github.com/aws/aws-lambda-go v1.31.0 @@ -53,7 +55,7 @@ require ( github.com/viant/mcp-protocol v0.11.0 github.com/viant/structology v0.8.0 github.com/viant/tagly v0.3.0 - github.com/viant/x v0.4.0 + github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 github.com/viant/xdatly/handler v0.0.0-20251208172928-dd34b7f09fd5 @@ -190,4 +192,3 @@ require ( modernc.org/strutil v1.1.3 // indirect modernc.org/token v1.0.0 // indirect ) -replace github.com/viant/x => /Users/awitas/go/src/github.com/viant/x diff --git a/go.sum b/go.sum index 04ca6ae0b..fb3ce7263 100644 --- a/go.sum +++ b/go.sum @@ -1210,10 +1210,8 @@ github.com/viant/toolbox v0.37.0 h1:+zwSdbQh6I6ZEyxokQJr+1gQKbLEw6erc+Av5dwKtLU= github.com/viant/toolbox v0.37.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/viant/velty v0.4.0 h1:eesQES/vCpcoPbM+gQLUBuLEL2sEO+A6s6lPpl8eKc4= github.com/viant/velty v0.4.0/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= -github.com/viant/x v0.4.0 h1:n2xuxQdw4lYtMdi59IAQEZHPioNT9InENGGbapyz+P4= -github.com/viant/x v0.4.0/go.mod h1:1TvsnpZFqI9dYVzIkaSYJyJ/UkfxW7fnk0YFafWXrPg= -github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a h1:7CLO2LjVnFgOwN0FL3Q4y5NrD7DpclS21AiW6tDLIc8= -github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= +github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef h1:KqWKMNloyzEg6nIn1pBK4CDEIcaRRhMrMUJr+k+xcPw= +github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef/go.mod h1:1TvsnpZFqI9dYVzIkaSYJyJ/UkfxW7fnk0YFafWXrPg= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUBDzc4rWacOYvAq/TKrTV0agvMF0gwm2gaoHI= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259/go.mod h1:fb8YgbVadk8X5ZLz49LWGzWmQlZd7Y/I5wE0ru44bIo= github.com/viant/xdatly/handler v0.0.0-20251208172928-dd34b7f09fd5 h1:CrT0HTlQul8FoGN0peylVczAOUEXKVqRAiB35ypRNHY= diff --git a/internal/translator/function/groupable.go b/internal/translator/function/groupable.go new file mode 100644 index 000000000..a97b075d3 --- /dev/null +++ b/internal/translator/function/groupable.go @@ -0,0 +1,37 @@ +package function + +import ( + "github.com/viant/datly/view" + "github.com/viant/sqlparser" +) + +type groupable struct{} + +func (c *groupable) Apply(args []string, column *sqlparser.Column, resource *view.Resource, aView *view.View) error { + values, err := convertArguments(c, args) + if err != nil { + return err + } + aView.Groupable = values[0].(bool) + return nil +} + +func (c *groupable) Name() string { + return "groupable" +} + +func (c *groupable) Description() string { + return "sets view.Groupable flag to enable dynamic group by rewriting for the view" +} + +func (c *groupable) Arguments() []*Argument { + return []*Argument{ + { + Name: "flag", + Description: "enable dynamic group by for the view", + Required: false, + Default: true, + DataType: "bool", + }, + } +} diff --git a/internal/translator/function/groupable_test.go b/internal/translator/function/groupable_test.go new file mode 100644 index 000000000..ef7b3fb2a --- /dev/null +++ b/internal/translator/function/groupable_test.go @@ -0,0 +1,38 @@ +package function + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" +) + +func TestGroupable_Apply(t *testing.T) { + useCases := []struct { + description string + args []string + expected bool + }{ + { + description: "defaults to true when flag omitted", + args: nil, + expected: true, + }, + { + description: "supports explicit false", + args: []string{"false"}, + expected: false, + }, + } + + for _, useCase := range useCases { + t.Run(useCase.description, func(t *testing.T) { + aView := &view.View{} + fn := &groupable{} + + err := fn.Apply(useCase.args, nil, nil, aView) + require.NoError(t, err) + require.Equal(t, useCase.expected, aView.Groupable) + }) + } +} diff --git a/internal/translator/view_selector_test.go b/internal/translator/view_selector_test.go new file mode 100644 index 000000000..4c8065a53 --- /dev/null +++ b/internal/translator/view_selector_test.go @@ -0,0 +1,58 @@ +package translator + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/internal/inference" + tparser "github.com/viant/datly/internal/translator/parser" + "github.com/viant/datly/view" +) + +func TestView_buildSelector_MergesDefaultConstraints(t *testing.T) { + namespace := &Viewlet{ + Name: "vendor", + Resource: &Resource{ + Declarations: &tparser.Declarations{ + QuerySelectors: map[string]inference.State{}, + }, + }, + } + + aView := &View{ + View: view.View{ + Name: "vendor", + Selector: &view.Config{ + Constraints: &view.Constraints{ + OrderBy: true, + OrderByColumn: map[string]string{ + "accountId": "ACCOUNT_ID", + }, + }, + }, + }, + } + + rule := &Rule{ + Root: "vendor", + Viewlets: Viewlets{ + registry: map[string]*Viewlet{ + "vendor": {Name: "vendor", View: aView}, + }, + keys: []string{"vendor"}, + }, + } + + aView.buildSelector(namespace, rule) + + require.NotNil(t, aView.Selector) + require.NotNil(t, aView.Selector.Constraints) + require.Equal(t, 25, aView.Selector.Limit) + require.True(t, aView.Selector.Constraints.Criteria) + require.True(t, aView.Selector.Constraints.Limit) + require.True(t, aView.Selector.Constraints.Offset) + require.True(t, aView.Selector.Constraints.Projection) + require.True(t, aView.Selector.Constraints.OrderBy) + require.Equal(t, "ACCOUNT_ID", aView.Selector.Constraints.OrderByColumn["accountId"]) + require.Equal(t, []string{"*"}, aView.Selector.Constraints.Filterable) +} diff --git a/internal/translator/viewlet_groupable_test.go b/internal/translator/viewlet_groupable_test.go new file mode 100644 index 000000000..c2079173c --- /dev/null +++ b/internal/translator/viewlet_groupable_test.go @@ -0,0 +1,60 @@ +package translator + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" +) + +func TestViewlet_discoverTables_GroupableColumnConfig(t *testing.T) { + ctx := context.Background() + dsn := filepath.Join(t.TempDir(), "viewlet_groupable.sqlite") + db, err := sql.Open("sqlite3", dsn) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + _, err = db.ExecContext(ctx, `CREATE TABLE sales (region_id TEXT, total_sales REAL, country_id TEXT)`) + require.NoError(t, err) + + useCases := []struct { + description string + sql string + expect map[string]bool + }{ + { + description: "flags groupable columns from ordinal group by", + sql: `SELECT region_id, SUM(total_sales) AS total_sales, country_id FROM sales GROUP BY 1, 3`, + expect: map[string]bool{ + "region_id": true, + "country_id": true, + }, + }, + { + description: "flags groupable columns from alias and name group by", + sql: `SELECT region_id AS region, SUM(total_sales) AS total_sales, country_id FROM sales GROUP BY region, country_id`, + expect: map[string]bool{ + "region": true, + "country_id": true, + }, + }, + } + + for _, useCase := range useCases { + t.Run(useCase.description, func(t *testing.T) { + viewlet := NewViewlet("sales", useCase.sql, nil, &Resource{}) + err := viewlet.discoverTables(ctx, db, useCase.sql) + require.NoError(t, err) + + actual := map[string]bool{} + for _, config := range viewlet.ColumnConfig { + require.NotNil(t, config.Groupable) + actual[config.Name] = *config.Groupable + } + require.Equal(t, useCase.expect, actual) + }) + } +} diff --git a/repository/option.go b/repository/option.go index 9c2b9b34b..8ac783c14 100644 --- a/repository/option.go +++ b/repository/option.go @@ -45,6 +45,7 @@ type Options struct { authConfig aconfig.Config shapePipeline bool legacyTypeContext bool + refreshDisabled bool } func (o *Options) UseColumn() bool { @@ -195,6 +196,14 @@ func WithRefreshFrequency(refreshFrequency time.Duration) Option { } } +// WithRefreshDisabled suppresses repository change polling and lazy hot-reload. +// Disabled by default to preserve existing behavior. +func WithRefreshDisabled(enabled bool) Option { + return func(o *Options) { + o.refreshDisabled = enabled + } +} + func WithResourceURL(URL string) Option { return func(o *Options) { o.resourceURL = URL diff --git a/repository/option_shape_test.go b/repository/option_shape_test.go index 11bf4ecba..ce232bcf2 100644 --- a/repository/option_shape_test.go +++ b/repository/option_shape_test.go @@ -27,3 +27,14 @@ func TestWithLegacyTypeContext(t *testing.T) { WithLegacyTypeContext(false)(opts) assert.False(t, opts.legacyTypeContext) } + +func TestWithRefreshDisabled(t *testing.T) { + opts := NewOptions(nil) + assert.False(t, opts.refreshDisabled) + + WithRefreshDisabled(true)(opts) + assert.True(t, opts.refreshDisabled) + + WithRefreshDisabled(false)(opts) + assert.False(t, opts.refreshDisabled) +} diff --git a/repository/path/service.go b/repository/path/service.go index 2ce530d9d..44118dbbd 100644 --- a/repository/path/service.go +++ b/repository/path/service.go @@ -240,8 +240,11 @@ func (s *Service) load(ctx context.Context) error { } func (s *Service) onModify(ctx context.Context, object storage.Object) error { - path := url.Path(object.URL()) - prev := s.lookupRouteBySourceURL(path) + sourceURL := object.URL() + prev := s.lookupRouteBySourceURL(sourceURL) + if prev == nil { + prev = s.lookupRouteBySourceURL(url.Path(sourceURL)) + } if prev != nil && prev.Version.HasChanged(object.ModTime()) { return nil } @@ -263,8 +266,11 @@ func (s *Service) onModify(ctx context.Context, object storage.Object) error { } func (s *Service) onDelete(ctx context.Context, object storage.Object) error { - path := url.Path(object.URL()) - prev := s.lookupRouteBySourceURL(path) + sourceURL := object.URL() + prev := s.lookupRouteBySourceURL(sourceURL) + if prev == nil { + prev = s.lookupRouteBySourceURL(url.Path(sourceURL)) + } if prev == nil { return nil } @@ -272,7 +278,7 @@ func (s *Service) onDelete(ctx context.Context, object storage.Object) error { prev.Version.Increase() // TODO delete works fine but after adding back rule file we get panic - //s.delete(prev, path) + //s.delete(prev, sourceURL) return nil } diff --git a/repository/path/service_test.go b/repository/path/service_test.go index ed1888b3a..d723ada37 100644 --- a/repository/path/service_test.go +++ b/repository/path/service_test.go @@ -4,6 +4,7 @@ import ( "context" _ "embed" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/viant/afs" "github.com/viant/afs/asset" "github.com/viant/afs/file" @@ -64,3 +65,29 @@ func TestNew(t *testing.T) { } } + +func TestService_onModify_DoesNotAppendDuplicateForTrackedURL(t *testing.T) { + location := "mem://localhost/test/routes_modify" + mgr, err := afs.Manager(location) + require.NoError(t, err) + err = asset.Create(mgr, location, []*asset.Resource{ + asset.New("dev/vendor.yml", file.DefaultFileOsMode, false, "", ruleVendor), + }) + require.NoError(t, err) + + service, err := New(context.Background(), afs.New(), location, time.Second) + require.NoError(t, err) + require.Len(t, service.Container.Items, 1) + + fs := afs.New() + object, err := fs.Object(context.Background(), "mem://localhost/test/routes_modify/dev/vendor.yml") + require.NoError(t, err) + + err = service.onModify(context.Background(), object) + require.NoError(t, err) + + require.Len(t, service.Container.Items, 1) + aPath := &contract.Path{URI: "/v1/api/dev/hauth/vendors/{vendorID}", Method: "GET"} + element := service.Lookup(aPath) + require.NotNil(t, element) +} diff --git a/repository/service.go b/repository/service.go index fdd95286c..f1de543ed 100644 --- a/repository/service.go +++ b/repository/service.go @@ -34,6 +34,7 @@ type ( auth *auth.Service plugins *plugin.Service refreshFrequency time.Duration + refreshDisabled bool options *Options } @@ -68,6 +69,9 @@ func (s *Service) Container() *path.Container { // SyncChanges checks if resource, plugin or components have changes // if so it would increase individual or all component/paths version number resulting in lazy reload func (s *Service) SyncChanges(ctx context.Context) (bool, error) { + if s == nil || s.refreshDisabled { + return false, nil + } now := time.Now() //fmt.Printf("[INFO] sync changes started\n") snap := &snapshot{} @@ -346,6 +350,7 @@ func New(ctx context.Context, opts ...Option) (*Service, error) { ret := &Service{ options: options, refreshFrequency: options.refreshFrequency, + refreshDisabled: options.refreshDisabled, resources: options.resources, extensions: options.extensions, } diff --git a/repository/service_refresh_test.go b/repository/service_refresh_test.go new file mode 100644 index 000000000..30fcc3789 --- /dev/null +++ b/repository/service_refresh_test.go @@ -0,0 +1,19 @@ +package repository + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestService_SyncChanges_RefreshDisabled(t *testing.T) { + service := &Service{ + refreshDisabled: true, + } + + changed, err := service.SyncChanges(context.Background()) + require.NoError(t, err) + assert.False(t, changed) +} diff --git a/repository/shape/compile/compiler.go b/repository/shape/compile/compiler.go index 4f5ef49ea..17118fe0c 100644 --- a/repository/shape/compile/compiler.go +++ b/repository/shape/compile/compiler.go @@ -133,7 +133,6 @@ func (c *DQLCompiler) assembleResult( result.Diagnostics = diags result.TypeContext = prepared.Pre.TypeCtx result.Directives = prepared.Pre.Directives - applyDefaultConnectorDirective(result) applyConstDirective(result) hints := extractViewHints(source.DQL) relationSQLSource := prepared.Pre.SQL @@ -142,6 +141,7 @@ func (c *DQLCompiler) assembleResult( } appendRelationViews(result, root, hints, relationSQLSource) appendDeclaredViews(source.DQL, result) + applyDefaultConnectorDirective(result) appendDeclaredStates(source.DQL, result) applyViewHints(result, hints) result.Diagnostics = append(result.Diagnostics, appendComponentTypesWithLayout(source, result, pathLayout)...) @@ -153,6 +153,8 @@ func (c *DQLCompiler) assembleResult( } applyInlineParamHints(source.DQL, result) applySourceParityEnrichmentWithLayout(result, source, pathLayout) + ensureDQLComponentRouteWithLayout(result, source, pathLayout) + applySummaryTypeSupport(result, source) if compileOptions.UseLinkedTypes == nil || *compileOptions.UseLinkedTypes { applyLinkedTypeSupport(result, source) } @@ -190,6 +192,7 @@ func (c *DQLCompiler) compileRoot(sourceName, sqlText string, statements dqlstmt mode = normalizeMixedMode(mode) unknownMode = normalizeUnknownNonReadMode(unknownMode) consts := map[string]string(nil) + groupableAliases := explicitGroupableAliases(extractViewHints(sqlText)) if directives != nil && len(directives.Const) > 0 { consts = directives.Const } @@ -228,7 +231,7 @@ func (c *DQLCompiler) compileRoot(sourceName, sqlText string, statements dqlstmt break } } - view, diags, err := pipeline.BuildReadWithConsts(sourceName, readSQL, consts) + view, diags, err := pipeline.BuildReadWithOptions(sourceName, readSQL, consts, groupableAliases) diags = append(diags, &dqlshape.Diagnostic{ Code: dqldiag.CodeDMLMixed, Severity: dqlshape.SeverityWarning, @@ -252,7 +255,28 @@ func (c *DQLCompiler) compileRoot(sourceName, sqlText string, statements dqlstmt } return view, diags, nil } - return pipeline.BuildReadWithConsts(sourceName, sqlText, consts) + return pipeline.BuildReadWithOptions(sourceName, sqlText, consts, groupableAliases) +} + +func explicitGroupableAliases(hints map[string]viewHint) map[string]bool { + if len(hints) == 0 { + return nil + } + result := map[string]bool{} + for alias, hint := range hints { + if hint.Groupable == nil || !*hint.Groupable { + continue + } + alias = strings.ToLower(strings.TrimSpace(alias)) + if alias == "" { + continue + } + result[alias] = true + } + if len(result) == 0 { + return nil + } + return result } func normalizeMixedMode(mode shape.CompileMixedMode) shape.CompileMixedMode { diff --git a/repository/shape/compile/compiler_test.go b/repository/shape/compile/compiler_test.go index d568b5586..b84cd75e3 100644 --- a/repository/shape/compile/compiler_test.go +++ b/repository/shape/compile/compiler_test.go @@ -138,6 +138,26 @@ SELECT id FROM ORDERS o assert.Equal(t, "analytics", planned.Views[0].Connector) } +func TestDQLCompiler_Compile_AppliesDefaultConnectorToDeclaredViews(t *testing.T) { + compiler := New() + dqlPath := filepath.Join("..", "..", "..", "e2e", "v1", "dql", "dev", "team", "user_team.dql") + dql, err := os.ReadFile(dqlPath) + require.NoError(t, err) + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "user_team", Path: dqlPath, DQL: string(dql)}) + require.NoError(t, err) + planned, ok := plan.ResultFrom(res) + require.True(t, ok) + require.GreaterOrEqual(t, len(planned.Views), 2) + connectors := map[string]string{} + for _, candidate := range planned.Views { + if candidate != nil { + connectors[candidate.Name] = candidate.Connector + } + } + assert.Equal(t, "dev", connectors["user_team"]) + assert.Equal(t, "dev", connectors["TeamStats"]) +} + func TestDQLCompiler_Compile_ColumnDiscoveryAutoForWildcard(t *testing.T) { compiler := New() res, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "SELECT * FROM ORDERS o"}) @@ -261,6 +281,23 @@ JOIN (SELECT * FROM PRODUCT t) products ON products.VENDOR_ID = vendor.ID` assert.Equal(t, `internal:"true"`, planned.ViewsByName["products"].Declaration.ColumnsConfig["VENDOR_ID"].Tag) } +func TestDQLCompiler_Compile_PopulatesComponentRouteFromDirective(t *testing.T) { + compiler := New() + dql := ` +#setting($_ = $route('/v1/api/shape/dev/vendors/{vendorID}', 'DELETE')) +#define($_ = $VendorID(path/vendorID)) +SELECT ID FROM VENDOR WHERE ID = $VendorID` + + res, err := compiler.Compile(context.Background(), &shape.Source{Name: "vendor_delete", DQL: dql}) + require.NoError(t, err) + planned, ok := plan.ResultFrom(res) + require.True(t, ok) + require.Len(t, planned.Components, 1) + assert.Equal(t, "DELETE", planned.Components[0].Method) + assert.Equal(t, "/v1/api/shape/dev/vendors/{vendorID}", planned.Components[0].RoutePath) + assert.Equal(t, planned.Views[0].Name, planned.Components[0].ViewName) +} + func TestDQLCompiler_Compile_DirectiveOnly_HasLineAndChar(t *testing.T) { compiler := New() _, err := compiler.Compile(context.Background(), &shape.Source{Name: "orders_report", DQL: "#package('x')"}) diff --git a/repository/shape/compile/component_route_shape.go b/repository/shape/compile/component_route_shape.go new file mode 100644 index 000000000..33ef9f6df --- /dev/null +++ b/repository/shape/compile/component_route_shape.go @@ -0,0 +1,59 @@ +package compile + +import ( + "strings" + + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" +) + +func ensureDQLComponentRouteWithLayout(result *plan.Result, source *shape.Source, layout compilePathLayout) { + if result == nil || len(result.Components) > 0 { + return + } + root := firstPlannedView(result.Views) + if root == nil { + return + } + + settings := extractRuleSettings(source, result.Directives) + method := httpMethod(settings) + uri := strings.TrimSpace(settings.URI) + if uri == "" { + namespace := "" + if source != nil && strings.TrimSpace(source.Path) != "" { + namespace, _ = dqlToRouteNamespaceWithLayout(source.Path, layout) + } + if namespace != "" { + uri = inferDefaultURI(namespace) + } + if uri == "" && source != nil { + uri = normalizeURI(source.Name) + } + } + if uri == "" { + return + } + + name := root.Name + if source != nil && strings.TrimSpace(source.Name) != "" { + name = strings.TrimSpace(source.Name) + } + result.Components = []*plan.ComponentRoute{{ + Name: name, + ViewName: strings.TrimSpace(root.Name), + RoutePath: normalizeURI(uri), + Method: method, + Connector: strings.TrimSpace(root.Connector), + SourceURL: strings.TrimSpace(root.SQLURI), + }} +} + +func firstPlannedView(views []*plan.View) *plan.View { + for _, item := range views { + if item != nil { + return item + } + } + return nil +} diff --git a/repository/shape/compile/hints.go b/repository/shape/compile/hints.go index e198ba0f5..6e83302d2 100644 --- a/repository/shape/compile/hints.go +++ b/repository/shape/compile/hints.go @@ -10,15 +10,18 @@ import ( ) type viewHint struct { - Connector string - AllowNulls *bool - NoLimit *bool - CacheRef string - Limit *int - Cardinality string - Dest string - TypeName string - Self *plan.SelfReference + Connector string + AllowNulls *bool + Groupable *bool + NoLimit *bool + CacheRef string + Limit *int + Cardinality string + Dest string + TypeName string + Self *plan.SelfReference + SelectorOrderBy *bool + SelectorOrderByNames map[string]string } func extractViewHints(dql string) map[string]viewHint { @@ -49,6 +52,35 @@ func extractViewHints(dql string) map[string]viewHint { value := true hint.AllowNulls = &value result[alias] = hint + case "groupable": + if len(call.args) != 1 { + continue + } + alias := normalizeHintAlias(call.args[0]) + if !isIdentifier(alias) { + continue + } + hint := result[alias] + value := true + hint.Groupable = &value + result[alias] = hint + case "allowed_order_by_columns": + if len(call.args) != 2 { + continue + } + alias := normalizeHintAlias(call.args[0]) + columns := strings.TrimSpace(unquote(strings.TrimSpace(call.args[1]))) + if !isIdentifier(alias) || columns == "" { + continue + } + hint := result[alias] + value := true + hint.SelectorOrderBy = &value + if hint.SelectorOrderByNames == nil { + hint.SelectorOrderByNames = map[string]string{} + } + appendAllowedOrderByColumns(hint.SelectorOrderByNames, columns) + result[alias] = hint case "set_limit": if len(call.args) != 2 { continue @@ -146,14 +178,16 @@ type hintCall struct { func scanHintCalls(input string) []hintCall { names := map[string]bool{ - "use_connector": true, - "allow_nulls": true, - "set_limit": true, - "set_cache": true, - "cardinality": true, - "self_ref": true, - "dest": true, - "type": true, + "use_connector": true, + "allow_nulls": true, + "groupable": true, + "allowed_order_by_columns": true, + "set_limit": true, + "set_cache": true, + "cardinality": true, + "self_ref": true, + "dest": true, + "type": true, } parsed, _ := decl.ScanCalls(input, decl.CallScanOptions{ AllowedNames: names, @@ -280,10 +314,24 @@ func applyViewHints(result *plan.Result, hints map[string]viewHint) { value := *hint.AllowNulls item.AllowNulls = &value } + if item.Groupable == nil && hint.Groupable != nil { + value := *hint.Groupable + item.Groupable = &value + } if item.SelectorNoLimit == nil && hint.NoLimit != nil { value := *hint.NoLimit item.SelectorNoLimit = &value } + if item.SelectorOrderBy == nil && hint.SelectorOrderBy != nil { + value := *hint.SelectorOrderBy + item.SelectorOrderBy = &value + } + if len(item.SelectorOrderByColumns) == 0 && len(hint.SelectorOrderByNames) > 0 { + item.SelectorOrderByColumns = map[string]string{} + for key, value := range hint.SelectorOrderByNames { + item.SelectorOrderByColumns[key] = value + } + } if item.SelectorLimit == nil && hint.Limit != nil { value := *hint.Limit item.SelectorLimit = &value @@ -316,6 +364,33 @@ func normalizeHintAlias(value string) string { return strings.ToLower(strings.TrimSpace(value)) } +func appendAllowedOrderByColumns(target map[string]string, columns string) { + for _, expression := range strings.Split(columns, ",") { + expression = strings.TrimSpace(expression) + if expression == "" { + continue + } + key := expression + value := expression + if strings.Contains(expression, ":") { + parts := strings.SplitN(expression, ":", 2) + key = strings.TrimSpace(parts[0]) + value = strings.TrimSpace(parts[1]) + } + if key == "" || value == "" { + continue + } + target[key] = value + lcKey := strings.ToLower(key) + if lcKey != key { + target[lcKey] = value + } + if index := strings.Index(key, "."); index != -1 && index+1 < len(key) { + target[key[index+1:]] = value + } + } +} + func lookupViewHint(hints map[string]viewHint, key string) (viewHint, bool) { key = normalizeHintAlias(key) if key == "" { diff --git a/repository/shape/compile/hints_strip.go b/repository/shape/compile/hints_strip.go index 5fd1847c5..67a1a0a5b 100644 --- a/repository/shape/compile/hints_strip.go +++ b/repository/shape/compile/hints_strip.go @@ -9,14 +9,16 @@ import ( ) var projectionHintCalls = map[string]bool{ - "useconnector": true, - "allownulls": true, - "setlimit": true, - "setcache": true, - "cardinality": true, - "selfref": true, - "dest": true, - "type": true, + "useconnector": true, + "allownulls": true, + "groupable": true, + "allowedorderbycolumns": true, + "setlimit": true, + "setcache": true, + "cardinality": true, + "selfref": true, + "dest": true, + "type": true, } // stripProjectionHintCalls removes hint-only projection functions (e.g. self_ref, dest) diff --git a/repository/shape/compile/hints_test.go b/repository/shape/compile/hints_test.go index 6ead9fde7..64710d44c 100644 --- a/repository/shape/compile/hints_test.go +++ b/repository/shape/compile/hints_test.go @@ -10,17 +10,34 @@ import ( ) func TestExtractViewHints_WithQuotedConnector(t *testing.T) { - dql := "SELECT use_connector(match, 'bq_sitemgmt_match'), use_connector(site, \"ci_ads\"), allow_nulls(match), set_limit(match, 0)" + dql := "SELECT use_connector(match, 'bq_sitemgmt_match'), use_connector(site, \"ci_ads\"), allow_nulls(match), groupable(match), set_limit(match, 0)" hints := extractViewHints(dql) require.Len(t, hints, 2) assert.Equal(t, "bq_sitemgmt_match", hints["match"].Connector) assert.Equal(t, "ci_ads", hints["site"].Connector) require.NotNil(t, hints["match"].AllowNulls) assert.True(t, *hints["match"].AllowNulls) + require.NotNil(t, hints["match"].Groupable) + assert.True(t, *hints["match"].Groupable) require.NotNil(t, hints["match"].NoLimit) assert.True(t, *hints["match"].NoLimit) } +func TestExtractViewHints_AllowedOrderByColumns(t *testing.T) { + dql := "SELECT allowed_order_by_columns(vendor, 'accountId:ACCOUNT_ID,vendor.userCreated:USER_CREATED,totalId:TOTAL_ID')" + hints := extractViewHints(dql) + require.Contains(t, hints, "vendor") + require.NotNil(t, hints["vendor"].SelectorOrderBy) + assert.True(t, *hints["vendor"].SelectorOrderBy) + assert.Equal(t, "ACCOUNT_ID", hints["vendor"].SelectorOrderByNames["accountId"]) + assert.Equal(t, "ACCOUNT_ID", hints["vendor"].SelectorOrderByNames["accountid"]) + assert.Equal(t, "USER_CREATED", hints["vendor"].SelectorOrderByNames["vendor.userCreated"]) + assert.Equal(t, "USER_CREATED", hints["vendor"].SelectorOrderByNames["vendor.usercreated"]) + assert.Equal(t, "USER_CREATED", hints["vendor"].SelectorOrderByNames["userCreated"]) + assert.Equal(t, "TOTAL_ID", hints["vendor"].SelectorOrderByNames["totalId"]) + assert.Equal(t, "TOTAL_ID", hints["vendor"].SelectorOrderByNames["totalid"]) +} + func TestExtractViewHints_MixedCaseAndUnquotedConnector(t *testing.T) { dql := "SELECT USE_CONNECTOR(match, ci_ads), Allow_Nulls(match), set_limit(match, -1)" hints := extractViewHints(dql) @@ -61,20 +78,30 @@ func TestApplyViewHints_Metadata(t *testing.T) { } applyViewHints(result, map[string]viewHint{ "match": { - Connector: "ci_ads", - AllowNulls: &trueValue, - NoLimit: &trueValue, - Cardinality: "one", - Dest: "match.go", - TypeName: "Match", + Connector: "ci_ads", + AllowNulls: &trueValue, + Groupable: &trueValue, + NoLimit: &trueValue, + Cardinality: "one", + Dest: "match.go", + TypeName: "Match", + SelectorOrderBy: &trueValue, + SelectorOrderByNames: map[string]string{ + "accountId": "ACCOUNT_ID", + }, }, }) require.Len(t, result.Views, 1) assert.Equal(t, "ci_ads", result.Views[0].Connector) require.NotNil(t, result.Views[0].AllowNulls) assert.True(t, *result.Views[0].AllowNulls) + require.NotNil(t, result.Views[0].Groupable) + assert.True(t, *result.Views[0].Groupable) require.NotNil(t, result.Views[0].SelectorNoLimit) assert.True(t, *result.Views[0].SelectorNoLimit) + require.NotNil(t, result.Views[0].SelectorOrderBy) + assert.True(t, *result.Views[0].SelectorOrderBy) + assert.Equal(t, "ACCOUNT_ID", result.Views[0].SelectorOrderByColumns["accountId"]) assert.Equal(t, "one", strings.ToLower(result.Views[0].Cardinality)) require.NotNil(t, result.Views[0].Declaration) assert.Equal(t, "match.go", result.Views[0].Declaration.Dest) @@ -102,10 +129,12 @@ func TestApplyViewHints_MetadataCaseInsensitiveAlias(t *testing.T) { } func TestStripProjectionHintCalls_RemovesSelfRefFromSQL(t *testing.T) { - sqlText := "SELECT user.* EXCEPT MGR_ID, self_ref(user, 'Team', 'ID', 'MGR_ID'), cardinality(user, 'one') FROM (SELECT t.* FROM USER t) user" + sqlText := "SELECT user.* EXCEPT MGR_ID, self_ref(user, 'Team', 'ID', 'MGR_ID'), cardinality(user, 'one'), groupable(user), allowed_order_by_columns(user, 'id:ID') FROM (SELECT t.* FROM USER t) user" actual := stripProjectionHintCalls(sqlText) assert.NotContains(t, strings.ToLower(actual), "self_ref(") assert.NotContains(t, strings.ToLower(actual), "cardinality(") + assert.NotContains(t, strings.ToLower(actual), "groupable(") + assert.NotContains(t, strings.ToLower(actual), "allowed_order_by_columns(") assert.Contains(t, strings.ToLower(actual), "user.* except mgr_id") } diff --git a/repository/shape/compile/pipeline/infer.go b/repository/shape/compile/pipeline/infer.go index 3d0e11e15..435b3e5d7 100644 --- a/repository/shape/compile/pipeline/infer.go +++ b/repository/shape/compile/pipeline/infer.go @@ -135,6 +135,11 @@ func InferProjectionType(queryNode *query.Select) (reflect.Type, reflect.Type, s if queryNode == nil || len(queryNode.List) == 0 || queryNode.List.IsStarExpr() { return reflect.TypeOf([]map[string]interface{}{}), reflect.TypeOf(map[string]interface{}{}), "many" } + for _, item := range queryNode.List { + if requiresDeferredProjectionType(sqlparser.Stringify(item)) { + return reflect.TypeOf([]map[string]interface{}{}), reflect.TypeOf(map[string]interface{}{}), "many" + } + } fields := make([]reflect.StructField, 0, len(queryNode.List)) used := map[string]int{} for index, item := range queryNode.List { @@ -152,7 +157,7 @@ func InferProjectionType(queryNode *query.Select) (reflect.Type, reflect.Type, s } used[fieldName]++ - typ := parseColumnType(column.Type) + typ := inferColumnType(sqlparser.Stringify(item), column.Type) veltyNames := []string{columnName} if fieldName != "" && fieldName != columnName { veltyNames = append(veltyNames, fieldName) @@ -160,13 +165,39 @@ func InferProjectionType(queryNode *query.Select) (reflect.Type, reflect.Type, s fields = append(fields, reflect.StructField{ Name: fieldName, Type: typ, - Tag: reflect.StructTag(fmt.Sprintf(`json:"%s,omitempty" sqlx:"name=%s" velty:"names=%s"`, strings.ToLower(fieldName), columnName, strings.Join(veltyNames, "|"))), + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s,omitempty" sqlx:"name=%s" velty:"names=%s"`, lowerCamel(fieldName), columnName, strings.Join(veltyNames, "|"))), }) } element := reflect.StructOf(fields) return reflect.SliceOf(element), element, "many" } +func requiresDeferredProjectionType(expression string) bool { + expression = strings.ToLower(strings.TrimSpace(expression)) + if expression == "" { + return false + } + if strings.Contains(expression, ".*") { + return true + } + if strings.Contains(expression, " except ") { + return true + } + if strings.HasPrefix(expression, "allow_nulls(") { + return true + } + return false +} + +func lowerCamel(value string) string { + if value == "" { + return "" + } + runes := []rune(value) + runes[0] = unicode.ToLower(runes[0]) + return string(runes) +} + func SanitizeName(value string) string { value = strings.TrimSpace(value) if value == "" { @@ -273,7 +304,7 @@ func parseColumnType(dataType string) reflect.Type { return reflect.TypeOf("") case "bool", "boolean": return reflect.TypeOf(false) - case "int", "int32", "smallint", "integer": + case "int", "int32", "smallint", "integer", "signed": return reflect.TypeOf(int(0)) case "int64", "bigint": return reflect.TypeOf(int64(0)) @@ -285,3 +316,54 @@ func parseColumnType(dataType string) reflect.Type { return reflect.TypeOf("") } } + +func inferColumnType(expression, dataType string) reflect.Type { + lower := strings.ToLower(strings.TrimSpace(expression)) + switch { + case isPureAggregateProjection(lower, "count("): + return reflect.TypeOf(int(0)) + case strings.Contains(lower, " as signed"), strings.Contains(lower, " as integer"), strings.Contains(lower, " as int)"), strings.Contains(lower, " as int "): + if isComputedNumericProjection(lower) { + return reflect.TypeOf((*int)(nil)) + } + return reflect.TypeOf(int(0)) + case strings.Contains(lower, " as bigint"): + if isComputedNumericProjection(lower) { + return reflect.TypeOf((*int64)(nil)) + } + return reflect.TypeOf(int64(0)) + case strings.Contains(lower, "sum("), strings.Contains(lower, "avg("): + return reflect.TypeOf(float64(0)) + default: + dataType = strings.TrimSpace(dataType) + return parseColumnType(dataType) + } +} + +func isPureAggregateProjection(expression string, aggregate string) bool { + idx := strings.Index(expression, aggregate) + if idx == -1 { + return false + } + return strings.TrimSpace(expression[:idx]) == "" +} + +func isComputedNumericProjection(expression string) bool { + expression = strings.ToLower(strings.TrimSpace(expression)) + switch { + case strings.Contains(expression, "count("): + return !isPureAggregateProjection(expression, "count(") + case strings.Contains(expression, "sum("): + return !isPureAggregateProjection(expression, "sum(") + case strings.Contains(expression, "avg("): + return !isPureAggregateProjection(expression, "avg(") + } + return strings.Contains(expression, " + ") || + strings.Contains(expression, " - ") || + strings.Contains(expression, " * ") || + strings.Contains(expression, " / ") || + strings.Contains(expression, "case ") || + strings.Contains(expression, "coalesce(") || + strings.Contains(expression, "nullif(") || + strings.Contains(expression, "cast(") +} diff --git a/repository/shape/compile/pipeline/infer_test.go b/repository/shape/compile/pipeline/infer_test.go index 7a2073eef..2ae5bb82b 100644 --- a/repository/shape/compile/pipeline/infer_test.go +++ b/repository/shape/compile/pipeline/infer_test.go @@ -60,4 +60,33 @@ func TestInferProjectionType_AddsVeltyNames(t *testing.T) { idField, ok := element.FieldByName("Id") assert.True(t, ok) assert.Equal(t, `names=ID|Id`, idField.Tag.Get("velty")) + assert.Equal(t, "isAuth,omitempty", field.Tag.Get("json")) + assert.Equal(t, "id,omitempty", idField.Tag.Get("json")) +} + +func TestInferProjectionType_InfersSummaryExpressionTypes(t *testing.T) { + queryNode, err := sqlparser.ParseQuery(`SELECT CAST(1 + (COUNT(1) / 25) AS SIGNED) AS PAGE_CNT, COUNT(1) AS CNT FROM PRODUCT`) + require.NoError(t, err) + _, element, _ := InferProjectionType(queryNode) + require.Equal(t, reflect.Struct, element.Kind()) + + pageCnt, ok := element.FieldByName("PageCnt") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf((*int)(nil)), pageCnt.Type) + assert.Equal(t, "pageCnt,omitempty", pageCnt.Tag.Get("json")) + + cnt, ok := element.FieldByName("Cnt") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf(int(0)), cnt.Type) +} + +func TestInferProjectionType_DefersWildcardAliasProjection(t *testing.T) { + queryNode, err := sqlparser.ParseQuery(`SELECT vendor.*, products.* EXCEPT VENDOR_ID, allow_nulls(products) FROM VENDOR vendor JOIN PRODUCT products ON products.VENDOR_ID = vendor.ID`) + require.NoError(t, err) + + fieldType, elementType, cardinality := InferProjectionType(queryNode) + + assert.Equal(t, reflect.TypeOf([]map[string]interface{}{}), fieldType) + assert.Equal(t, reflect.TypeOf(map[string]interface{}{}), elementType) + assert.Equal(t, "many", cardinality) } diff --git a/repository/shape/compile/pipeline/read.go b/repository/shape/compile/pipeline/read.go index 462d51d2a..89ee9cf4d 100644 --- a/repository/shape/compile/pipeline/read.go +++ b/repository/shape/compile/pipeline/read.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" + "github.com/viant/datly/internal/inference" dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/plan" "github.com/viant/sqlparser" @@ -19,10 +20,14 @@ import ( // It applies multiple parse strategies and gracefully degrades to a // loose (schema-less) view for template-driven SQL that cannot be fully parsed. func BuildRead(sourceName, sqlText string) (*plan.View, []*dqlshape.Diagnostic, error) { - return BuildReadWithConsts(sourceName, sqlText, nil) + return BuildReadWithOptions(sourceName, sqlText, nil, nil) } func BuildReadWithConsts(sourceName, sqlText string, consts map[string]string) (*plan.View, []*dqlshape.Diagnostic, error) { + return BuildReadWithOptions(sourceName, sqlText, consts, nil) +} + +func BuildReadWithOptions(sourceName, sqlText string, consts map[string]string, groupableAliases map[string]bool) (*plan.View, []*dqlshape.Diagnostic, error) { queryNode, parseDiag, parserSQL, err := resolveQueryNode(sqlText) // Template-driven SQL may legitimately fail strict parsing; treat as warning. @@ -91,10 +96,17 @@ func BuildReadWithConsts(sourceName, sqlText string, consts map[string]string) ( Relations: relations, } exceptByAlias := extractExceptColumnsByNamespace(queryNode) - if except := lookupExceptColumns(exceptByAlias, name); len(except) > 0 { - view.Declaration = &plan.ViewDeclaration{ColumnsConfig: except} + groupableByAlias := extractGroupableColumnsByNamespace(queryNode, name, groupableAliases) + rootConfig := mergeColumnConfigs( + lookupExceptColumns(exceptByAlias, name), + lookupColumnConfigs(groupableByAlias, name), + extractRootGroupedColumnConfigs(rootSQL, name, groupableAliases), + ) + if len(rootConfig) > 0 { + view.Declaration = &plan.ViewDeclaration{ColumnsConfig: rootConfig} } applyRelationExceptColumns(relations, exceptByAlias) + applyRelationGroupableColumns(relations, groupableByAlias) applyConstTables(view, consts) return view, diags, nil } @@ -289,25 +301,147 @@ func applyRelationExceptColumns(relations []*plan.Relation, exceptByAlias map[st } } +func applyRelationGroupableColumns(relations []*plan.Relation, groupableByAlias map[string]map[string]*plan.ViewColumnConfig) { + if len(relations) == 0 || len(groupableByAlias) == 0 { + return + } + for _, relation := range relations { + if relation == nil { + continue + } + relation.ColumnsConfig = mergeColumnConfigs(relation.ColumnsConfig, lookupColumnConfigs(groupableByAlias, relation.Ref)) + } +} + func lookupExceptColumns(exceptByAlias map[string]map[string]*plan.ViewColumnConfig, alias string) map[string]*plan.ViewColumnConfig { - if len(exceptByAlias) == 0 { + return lookupColumnConfigs(exceptByAlias, alias) +} + +func lookupColumnConfigs(byAlias map[string]map[string]*plan.ViewColumnConfig, alias string) map[string]*plan.ViewColumnConfig { + if len(byAlias) == 0 { return nil } alias = strings.ToLower(strings.TrimSpace(alias)) if alias == "" { return nil } - result := exceptByAlias[alias] + result := byAlias[alias] if len(result) == 0 { return nil } ret := make(map[string]*plan.ViewColumnConfig, len(result)) for key, cfg := range result { - ret[key] = cfg + if cfg == nil { + continue + } + cloned := *cfg + if cfg.Groupable != nil { + value := *cfg.Groupable + cloned.Groupable = &value + } + ret[key] = &cloned + } + if len(ret) == 0 { + return nil } return ret } +func mergeColumnConfigs(base map[string]*plan.ViewColumnConfig, overlays ...map[string]*plan.ViewColumnConfig) map[string]*plan.ViewColumnConfig { + var result map[string]*plan.ViewColumnConfig + if len(base) > 0 { + result = lookupColumnConfigs(map[string]map[string]*plan.ViewColumnConfig{"_": base}, "_") + } + for _, overlay := range overlays { + for name, cfg := range overlay { + name = strings.TrimSpace(name) + if name == "" || cfg == nil { + continue + } + if result == nil { + result = map[string]*plan.ViewColumnConfig{} + } + target := result[name] + if target == nil { + target = &plan.ViewColumnConfig{} + result[name] = target + } + if dataType := strings.TrimSpace(cfg.DataType); dataType != "" && target.DataType == "" { + target.DataType = dataType + } + if tag := strings.TrimSpace(cfg.Tag); tag != "" && target.Tag == "" { + target.Tag = tag + } + if target.Groupable == nil && cfg.Groupable != nil { + value := *cfg.Groupable + target.Groupable = &value + } + } + } + if len(result) == 0 { + return nil + } + return result +} + +func extractGroupableColumnsByNamespace(queryNode *query.Select, rootName string, enabledAliases map[string]bool) map[string]map[string]*plan.ViewColumnConfig { + if queryNode == nil || len(enabledAliases) == 0 { + return nil + } + columns := sqlparser.NewColumns(queryNode.List) + groupable := inference.GroupableColumns(queryNode, columns) + if len(groupable) == 0 { + return nil + } + rootName = strings.ToLower(strings.TrimSpace(rootName)) + result := map[string]map[string]*plan.ViewColumnConfig{} + for _, column := range columns { + if column == nil || !groupable[column.Identity()] { + continue + } + name := strings.TrimSpace(column.Identity()) + if name == "" { + continue + } + namespace := strings.ToLower(strings.TrimSpace(column.Namespace)) + if namespace == "" { + namespace = rootName + } + if namespace == "" || !enabledAliases[namespace] { + continue + } + columnsConfig := result[namespace] + if columnsConfig == nil { + columnsConfig = map[string]*plan.ViewColumnConfig{} + result[namespace] = columnsConfig + } + if columnsConfig[name] == nil { + value := true + columnsConfig[name] = &plan.ViewColumnConfig{Groupable: &value} + } + } + if len(result) == 0 { + return nil + } + return result +} + +func extractRootGroupedColumnConfigs(sqlText, rootName string, enabledAliases map[string]bool) map[string]*plan.ViewColumnConfig { + sqlText = strings.TrimSpace(sqlText) + if sqlText == "" { + return nil + } + rootName = strings.ToLower(strings.TrimSpace(rootName)) + if len(enabledAliases) == 0 || !enabledAliases[rootName] { + return nil + } + queryNode, _, _, err := resolveQueryNode(sqlText) + if err != nil || queryNode == nil { + return nil + } + return lookupColumnConfigs(extractGroupableColumnsByNamespace(queryNode, rootName, enabledAliases), rootName) +} + func extractExceptColumnsByNamespace(queryNode *query.Select) map[string]map[string]*plan.ViewColumnConfig { if queryNode == nil { return nil diff --git a/repository/shape/compile/pipeline/read_normalize.go b/repository/shape/compile/pipeline/read_normalize.go index 41b2dd8ce..826cdcc77 100644 --- a/repository/shape/compile/pipeline/read_normalize.go +++ b/repository/shape/compile/pipeline/read_normalize.go @@ -14,6 +14,11 @@ func normalizeParserSQL(sqlText string) string { return rewritePrivateShorthand(replaceTemplateTokens(sqlText)) } +// NormalizeParserSQL exposes the parser-safe SQL normalization used by read compilation. +func NormalizeParserSQL(sqlText string) string { + return normalizeParserSQL(sqlText) +} + func rewritePrivateShorthand(input string) string { var b strings.Builder b.Grow(len(input)) diff --git a/repository/shape/compile/pipeline/read_test.go b/repository/shape/compile/pipeline/read_test.go index 5fb54e710..92cb702df 100644 --- a/repository/shape/compile/pipeline/read_test.go +++ b/repository/shape/compile/pipeline/read_test.go @@ -144,6 +144,77 @@ JOIN (SELECT * FROM PRODUCT t) products ON products.VENDOR_ID = wrapper.ID` assert.Equal(t, `internal:"true"`, settingCfg["ID"].Tag) } +func TestBuildRead_GroupByDoesNotMarkColumnsWithoutExplicitGrouping(t *testing.T) { + sqlText := `SELECT t.REGION AS REGION, COUNT(*) AS TOTAL FROM SALES t GROUP BY REGION` + view, _, err := BuildRead("sales_report", sqlText) + require.NoError(t, err) + require.NotNil(t, view) + if view.Declaration != nil { + assert.Empty(t, view.Declaration.ColumnsConfig) + } +} + +func TestBuildRead_GroupByMarksRootGroupedColumnsWithExplicitGrouping(t *testing.T) { + sqlText := `SELECT t.REGION AS REGION, COUNT(*) AS TOTAL FROM SALES t GROUP BY REGION` + view, _, err := BuildReadWithOptions("sales_report", sqlText, nil, map[string]bool{"t": true}) + require.NoError(t, err) + require.NotNil(t, view) + require.NotNil(t, view.Declaration) + require.NotNil(t, view.Declaration.ColumnsConfig) + cfg, ok := view.Declaration.ColumnsConfig["REGION"] + require.True(t, ok) + require.NotNil(t, cfg) + require.NotNil(t, cfg.Groupable) + assert.True(t, *cfg.Groupable) + _, ok = view.Declaration.ColumnsConfig["TOTAL"] + assert.False(t, ok) +} + +func TestBuildRead_GroupByMarksRelationGroupedColumnsWithExplicitGrouping(t *testing.T) { + sqlText := `SELECT vendor.REGION AS REGION, products.CATEGORY AS CATEGORY, COUNT(*) AS TOTAL +FROM VENDOR vendor +JOIN PRODUCT products ON products.VENDOR_ID = vendor.ID +GROUP BY vendor.REGION, products.CATEGORY` + view, _, err := BuildReadWithOptions("vendor_products", sqlText, nil, map[string]bool{"vendor": true, "products": true}) + require.NoError(t, err) + require.NotNil(t, view) + require.NotNil(t, view.Declaration) + require.Contains(t, view.Declaration.ColumnsConfig, "REGION") + require.NotNil(t, view.Declaration.ColumnsConfig["REGION"].Groupable) + assert.True(t, *view.Declaration.ColumnsConfig["REGION"].Groupable) + require.Len(t, view.Relations, 1) + require.Contains(t, view.Relations[0].ColumnsConfig, "CATEGORY") + require.NotNil(t, view.Relations[0].ColumnsConfig["CATEGORY"].Groupable) + assert.True(t, *view.Relations[0].ColumnsConfig["CATEGORY"].Groupable) + _, ok := view.Relations[0].ColumnsConfig["TOTAL"] + assert.False(t, ok) +} + +func TestBuildRead_GroupByInRootSubqueryMarksGroupedColumnsWithExplicitGrouping(t *testing.T) { + sqlText := `SELECT vendor.* +FROM ( + SELECT ACCOUNT_ID, + USER_CREATED, + SUM(ID) AS TOTAL_ID, + MAX(ID) AS MAX_ID + FROM VENDOR t + GROUP BY 1, 2 +) vendor` + view, _, err := BuildReadWithOptions("vendors_grouping", sqlText, nil, map[string]bool{"vendor": true}) + require.NoError(t, err) + require.NotNil(t, view) + require.NotNil(t, view.Declaration) + require.NotNil(t, view.Declaration.ColumnsConfig) + require.Contains(t, view.Declaration.ColumnsConfig, "ACCOUNT_ID") + require.NotNil(t, view.Declaration.ColumnsConfig["ACCOUNT_ID"].Groupable) + assert.True(t, *view.Declaration.ColumnsConfig["ACCOUNT_ID"].Groupable) + require.Contains(t, view.Declaration.ColumnsConfig, "USER_CREATED") + require.NotNil(t, view.Declaration.ColumnsConfig["USER_CREATED"].Groupable) + assert.True(t, *view.Declaration.ColumnsConfig["USER_CREATED"].Groupable) + _, ok := view.Declaration.ColumnsConfig["TOTAL_ID"] + assert.False(t, ok) +} + func TestBuildRead_TemplateTableSelector_PreservesRelations(t *testing.T) { sqlText := `SELECT vendor.*, products.* FROM (SELECT * FROM ${Unsafe.Vendor} t WHERE t.ID IN ($criteria.AppendBinding($Unsafe.vendorIDs))) vendor diff --git a/repository/shape/compile/statedecl.go b/repository/shape/compile/statedecl.go index 409718e0c..7e1a5bde8 100644 --- a/repository/shape/compile/statedecl.go +++ b/repository/shape/compile/statedecl.go @@ -18,16 +18,13 @@ func appendDeclaredStates(rawDQL string, result *plan.Result) { if result == nil || strings.TrimSpace(rawDQL) == "" { return } - seen := map[string]bool{} + seen := map[string]*plan.State{} for _, block := range extractSetBlocks(rawDQL) { holder, kind, location, tail, tailOffset, ok := parseSetDeclarationBody(block.Body) if !ok { continue } key := declaredStateKey(holder, kind, location) - if seen[key] { - continue - } inName := location if kind == "view" || kind == "data_view" { if isAttachedSummaryState(result, holder) { @@ -59,13 +56,17 @@ func appendDeclaredStates(rawDQL string, result *plan.Result) { state.Required = &required } applyDeclaredStateOptions(state, tail, rawDQL, block.BodyOffset+tailOffset, &result.Diagnostics) + if existing := seen[key]; existing != nil { + mergeDeclaredState(existing, state) + continue + } result.States = append(result.States, state) - seen[key] = true + seen[key] = state } appendInferredPathStates(rawDQL, result, seen) } -func appendInferredPathStates(rawDQL string, result *plan.Result, seen map[string]bool) { +func appendInferredPathStates(rawDQL string, result *plan.Result, seen map[string]*plan.State) { if result == nil || strings.TrimSpace(rawDQL) == "" { return } @@ -75,7 +76,7 @@ func appendInferredPathStates(rawDQL string, result *plan.Result, seen map[strin } for _, name := range extractRoutePathParams(prepared.Directives.Route.URI) { key := declaredStateKey(name, string(st.KindPath), name) - if seen[key] { + if seen[key] != nil { continue } result.States = append(result.States, &plan.State{ @@ -88,7 +89,42 @@ func appendInferredPathStates(rawDQL string, result *plan.Result, seen map[strin }, }, }) - seen[key] = true + seen[key] = result.States[len(result.States)-1] + } +} + +func mergeDeclaredState(dst, src *plan.State) { + if dst == nil || src == nil { + return + } + dst.EmitOutput = dst.EmitOutput || src.EmitOutput + dst.Async = dst.Async || src.Async + if dst.QuerySelector == "" { + dst.QuerySelector = src.QuerySelector + } + if dst.OutputDataType == "" { + dst.OutputDataType = src.OutputDataType + } + if dst.Tag == "" { + dst.Tag = src.Tag + } + if dst.Required == nil { + dst.Required = src.Required + } + if dst.Cacheable == nil { + dst.Cacheable = src.Cacheable + } + if dst.Schema == nil && src.Schema != nil { + schema := *src.Schema + dst.Schema = &schema + } + if dst.Schema != nil && src.Schema != nil { + if dst.Schema.DataType == "" { + dst.Schema.DataType = src.Schema.DataType + } + if dst.Schema.Cardinality == "" { + dst.Schema.Cardinality = src.Schema.Cardinality + } } } @@ -284,6 +320,11 @@ func applyDeclaredStateOptions(state *plan.State, tail, dql string, baseOffset i continue } state.Async = true + case strings.EqualFold(name, "Output"): + if !expectStateArgs(state, name, args, 0, 0, dql, optionOffset, diags) { + continue + } + state.EmitOutput = true default: if state != nil && state.In != nil { kind := strings.ToLower(state.KindString()) diff --git a/repository/shape/compile/statedecl_test.go b/repository/shape/compile/statedecl_test.go index d8cd344fd..64d21ecb7 100644 --- a/repository/shape/compile/statedecl_test.go +++ b/repository/shape/compile/statedecl_test.go @@ -144,6 +144,37 @@ SELECT id FROM USERS u` assert.Contains(t, result.States[0].Tag, `anonymous:"true"`) } +func TestAppendDeclaredStates_OutputOptionMarksStateForOutput(t *testing.T) { + dql := ` +#set($_ = $Foos(body/).Output().Tag('anonymous:"true"')) +SELECT * FROM FOOS` + result := &plan.Result{} + + appendDeclaredStates(dql, result) + + require.Len(t, result.States, 1) + assert.Equal(t, "Foos", result.States[0].Name) + assert.Equal(t, "body", result.States[0].KindString()) + assert.True(t, result.States[0].EmitOutput) +} + +func TestAppendDeclaredStates_DuplicateDeclarationMergesOutputMarker(t *testing.T) { + dql := ` +#set($_ = $Foos(body/).Cardinality('One').Tag('anonymous:"true"')) +#set($_ = $Foos(body/).Output().Tag('anonymous:"true"')) +SELECT * FROM FOOS` + result := &plan.Result{} + + appendDeclaredStates(dql, result) + + require.Len(t, result.States, 1) + assert.Equal(t, "Foos", result.States[0].Name) + assert.Equal(t, "body", result.States[0].KindString()) + assert.True(t, result.States[0].EmitOutput) + require.NotNil(t, result.States[0].Schema) + assert.Equal(t, "One", string(result.States[0].Schema.Cardinality)) +} + func TestAppendDeclaredStates_InvalidOption_ReportsExactSpan(t *testing.T) { dql := ` #set($_ = $Auth(header/Authorization).Cacheable('x').UnknownFlag()) diff --git a/repository/shape/compile/type_support.go b/repository/shape/compile/type_support.go index d3faa9329..585ae1e00 100644 --- a/repository/shape/compile/type_support.go +++ b/repository/shape/compile/type_support.go @@ -7,10 +7,12 @@ import ( "os" "path/filepath" "reflect" + "strconv" "strings" "time" "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/compile/pipeline" "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/repository/shape/typectx" "github.com/viant/x" @@ -38,6 +40,9 @@ func applyLinkedTypeSupport(result *plan.Result, source *shape.Source) { if rType == nil { continue } + if isPlaceholderLinkedViewType(rType) { + continue + } item.ElementType = rType if strings.EqualFold(strings.TrimSpace(item.Cardinality), "many") { item.FieldType = reflect.SliceOf(rType) @@ -67,6 +72,257 @@ func applyLinkedTypeSupport(result *plan.Result, source *shape.Source) { } } +func isPlaceholderLinkedViewType(rType reflect.Type) bool { + rType = unwrapResolvedType(rType) + if rType == nil || rType.Kind() != reflect.Struct { + return false + } + hasScalars := false + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if !field.IsExported() { + continue + } + rawTag := string(field.Tag) + if strings.Contains(rawTag, `view:"`) || strings.Contains(rawTag, `on:"`) || strings.Contains(rawTag, `sqlx:"-"`) { + continue + } + hasScalars = true + if !isPlaceholderFieldName(field.Name, summaryTagName(field.Tag.Get("sqlx"))) { + return false + } + } + return hasScalars +} + +func isPlaceholderFieldName(fieldName, sqlxName string) bool { + return isPlaceholderName(fieldName) || isPlaceholderName(sqlxName) +} + +func isPlaceholderName(name string) bool { + name = strings.TrimSpace(name) + if name == "" { + return false + } + name = strings.TrimPrefix(strings.TrimPrefix(name, "name="), "*") + lower := strings.ToLower(strings.ReplaceAll(name, "_", "")) + if !strings.HasPrefix(lower, "col") || len(lower) == len("col") { + return false + } + _, err := strconv.Atoi(lower[len("col"):]) + return err == nil +} + +func applySummaryTypeSupport(result *plan.Result, source *shape.Source) { + if result == nil || source == nil { + return + } + registry := source.EnsureTypeRegistry() + if registry == nil { + return + } + resolver := typectx.NewResolver(registry, result.TypeContext) + existing := existingTypesByName(result.Types) + applySummaryTypeSupportWithResolver(result, source, resolver, registry, existing) +} + +func applySummaryTypeSupportWithResolver(result *plan.Result, source *shape.Source, resolver *typectx.Resolver, registry *x.Registry, existing map[string]bool) { + if result == nil || source == nil || registry == nil { + return + } + for _, item := range result.Views { + if item == nil { + continue + } + summaryName := strings.TrimSpace(item.SummaryName) + summarySQL := strings.TrimSpace(item.Summary) + if summaryName == "" || summarySQL == "" { + continue + } + typeName := summaryTypeName(summaryName) + if typeName == "" { + continue + } + queryNode, _, err := pipeline.ParseSelectWithDiagnostic(pipeline.NormalizeParserSQL(summarySQL)) + if err == nil && queryNode != nil { + _, elementType, _ := pipeline.InferProjectionType(queryNode) + elementType = unwrapResolvedType(elementType) + elementType = refineSummaryProjectionType(elementType, item, result.TypeContext, source) + if elementType != nil { + registerOpts := []x.Option{x.WithName(typeName), x.WithForceFlag()} + if ctx := result.TypeContext; ctx != nil { + if pkgPath := strings.TrimSpace(ctx.PackagePath); pkgPath != "" { + registerOpts = append(registerOpts, x.WithPkgPath(pkgPath)) + } + } + registry.Register(x.NewType(elementType, registerOpts...)) + appendResolvedType(result, elementType, typeName, existing, result.TypeContext) + continue + } + } + if key := resolveTypeKey(typeName, resolver, registry); key != "" { + if resolved := registry.Lookup(key); resolved != nil && resolved.Type != nil { + appendResolvedType(result, resolved.Type, typeName, existing, result.TypeContext) + continue + } + } + } +} + +func refineSummaryProjectionType(summaryType reflect.Type, item *plan.View, ctx *typectx.Context, source *shape.Source) reflect.Type { + summaryType = unwrapResolvedType(summaryType) + if summaryType == nil || summaryType.Kind() != reflect.Struct || item == nil { + return summaryType + } + ownerType := unwrapResolvedType(item.ElementType) + if ownerType == nil { + ownerType = unwrapResolvedType(item.FieldType) + } + if ownerType == nil || ownerType.Kind() != reflect.Struct { + ownerType = resolveSummaryOwnerType(item, ctx, source) + } + if ownerType == nil || ownerType.Kind() != reflect.Struct { + return summaryType + } + ownerFields := map[string]reflect.StructField{} + for i := 0; i < ownerType.NumField(); i++ { + field := ownerType.Field(i) + ownerFields[strings.ToUpper(strings.TrimSpace(field.Name))] = field + if sqlxName := summaryTagName(field.Tag.Get("sqlx")); sqlxName != "" { + ownerFields[strings.ToUpper(sqlxName)] = field + } + } + fields := make([]reflect.StructField, 0, summaryType.NumField()) + changed := false + for i := 0; i < summaryType.NumField(); i++ { + field := summaryType.Field(i) + if ownerField, ok := ownerFields[strings.ToUpper(summaryLookupName(field))]; ok && ownerField.Type != nil && ownerField.Type != field.Type { + field.Type = ownerField.Type + changed = true + } + fields = append(fields, field) + } + if !changed { + return summaryType + } + return reflect.StructOf(fields) +} + +func resolveSummaryOwnerType(item *plan.View, ctx *typectx.Context, source *shape.Source) reflect.Type { + if item == nil { + return nil + } + for _, candidate := range summaryOwnerTypeCandidates(item) { + if linked := lookupLinkedType(candidate, ctx, source); linked != nil { + linked = unwrapResolvedType(linked) + if linked != nil && linked.Kind() == reflect.Struct { + return linked + } + } + } + return nil +} + +func summaryOwnerTypeCandidates(item *plan.View) []string { + if item == nil { + return nil + } + result := make([]string, 0, 6) + seen := map[string]bool{} + appendCandidate := func(value string) { + value = strings.TrimSpace(value) + if value == "" || seen[value] { + return + } + seen[value] = true + result = append(result, value) + } + if item.Declaration != nil { + appendCandidate(item.Declaration.DataType) + appendCandidate(item.Declaration.Of) + } + appendCandidate(item.SchemaType) + name := toExportedTypeName(item.Name) + if name != "" { + appendCandidate(name + "View") + appendCandidate(name) + } + return result +} + +func summaryLookupName(field reflect.StructField) string { + if sqlxName := summaryTagName(field.Tag.Get("sqlx")); sqlxName != "" { + return sqlxName + } + return strings.TrimSpace(field.Name) +} + +func summaryTagName(tag string) string { + tag = strings.TrimSpace(tag) + if tag == "" { + return "" + } + if strings.HasPrefix(tag, "name=") { + tag = strings.TrimPrefix(tag, "name=") + } + if idx := strings.Index(tag, ","); idx != -1 { + tag = tag[:idx] + } + return strings.TrimSpace(tag) +} + +func appendResolvedType(result *plan.Result, rType reflect.Type, typeName string, existing map[string]bool, ctx *typectx.Context) { + rType = unwrapResolvedType(rType) + typeName = strings.TrimSpace(typeName) + if result == nil || rType == nil || typeName == "" { + return + } + key := strings.ToLower(typeName) + if existing[key] { + return + } + typeExpr, typePkg := summarySchemaTypeExpression(typeName, ctx) + result.Types = append(result.Types, &plan.Type{ + Name: typeName, + DataType: typeExpr, + Cardinality: string(planStateOne()), + Package: typePkg, + ModulePath: strings.TrimSpace(rType.PkgPath()), + }) + existing[key] = true +} + +func summaryTypeName(summaryName string) string { + summaryName = strings.TrimSpace(summaryName) + if summaryName == "" { + return "" + } + if strings.HasSuffix(summaryName, "View") { + return summaryName + } + return toExportedTypeName(summaryName) + "View" +} + +func summarySchemaTypeExpression(typeName string, ctx *typectx.Context) (string, string) { + typeName = strings.TrimSpace(typeName) + if typeName == "" { + return "", "" + } + if ctx != nil { + if pkgAlias := strings.TrimSpace(ctx.PackageName); pkgAlias != "" { + return "*" + pkgAlias + "." + typeName, pkgAlias + } + if pkgPath := strings.TrimSpace(ctx.PackagePath); pkgPath != "" { + return "*" + packageAlias(pkgPath, ctx) + "." + typeName, packageAlias(pkgPath, ctx) + } + } + return "*" + typeName, "" +} + +func planStateOne() string { + return "one" +} + func resolveViewType(item *plan.View, root bool, rootTypeKey string, resolver *typectx.Resolver, registry *x.Registry, ctx *typectx.Context, source *shape.Source) *x.Type { for _, candidate := range viewTypeCandidates(item, root, rootTypeKey) { if key := resolveTypeKey(candidate, resolver, registry); key != "" { diff --git a/repository/shape/compile/type_support_summary_test.go b/repository/shape/compile/type_support_summary_test.go new file mode 100644 index 000000000..158056e48 --- /dev/null +++ b/repository/shape/compile/type_support_summary_test.go @@ -0,0 +1,241 @@ +package compile + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/x" +) + +func TestApplyLinkedTypeSupport_RegistersSummaryTypes(t *testing.T) { + source := &shape.Source{ + TypeRegistry: x.NewRegistry(), + } + result := &plan.Result{ + TypeContext: &typectx.Context{ + PackageName: "meta_nested", + PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/meta_nested", + }, + Views: []*plan.View{ + { + Name: "vendor", + SummaryName: "Meta", + Summary: "SELECT COUNT(*) AS CNT, 1 AS PAGE_CNT FROM ($View.vendor.SQL) t", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + Cardinality: "many", + }, + { + Name: "products", + SummaryName: "ProductsMeta", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) t GROUP BY VENDOR_ID", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + Cardinality: "many", + }, + }, + } + + applySummaryTypeSupport(result, source) + applyLinkedTypeSupport(result, source) + + registry := source.EnsureTypeRegistry() + require.NotNil(t, registry.Lookup("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/meta_nested.MetaView")) + require.NotNil(t, registry.Lookup("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/meta_nested.ProductsMetaView")) + + var names []string + for _, item := range result.Types { + if item != nil { + names = append(names, item.Name) + } + } + assert.Contains(t, names, "MetaView") + assert.Contains(t, names, "ProductsMetaView") +} + +func TestApplyLinkedTypeSupport_SkipsPlaceholderLinkedViewTypes(t *testing.T) { + type placeholderVendorView struct { + Col1 string `sqlx:"name=col_1"` + Col2 string `sqlx:"name=col_2"` + } + + registry := x.NewRegistry() + registry.Register(x.NewType( + reflect.TypeOf(placeholderVendorView{}), + x.WithName("VendorView"), + x.WithPkgPath("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary"), + )) + + source := &shape.Source{TypeRegistry: registry} + result := &plan.Result{ + TypeContext: &typectx.Context{ + PackageName: "multi_summary", + PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary", + }, + Views: []*plan.View{ + { + Name: "vendor", + SchemaType: "*VendorView", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + Cardinality: "many", + }, + }, + } + + applyLinkedTypeSupport(result, source) + + assert.Equal(t, reflect.TypeOf(map[string]interface{}{}), result.Views[0].ElementType) + assert.Equal(t, reflect.TypeOf([]map[string]interface{}{}), result.Views[0].FieldType) +} + +func TestApplySummaryTypeSupport_RegistersSummaryTypesWithoutLinkedViews(t *testing.T) { + source := &shape.Source{ + TypeRegistry: x.NewRegistry(), + } + result := &plan.Result{ + TypeContext: &typectx.Context{ + PackageName: "meta_nested", + PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/meta_nested", + }, + Views: []*plan.View{ + { + Name: "vendor", + SummaryName: "Meta", + Summary: "SELECT COUNT(*) AS CNT, 1 AS PAGE_CNT FROM ($View.vendor.SQL) t", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + Cardinality: "many", + }, + { + Name: "products", + SummaryName: "ProductsMeta", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) t GROUP BY VENDOR_ID", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + Cardinality: "many", + }, + }, + } + + applySummaryTypeSupport(result, source) + + registry := source.EnsureTypeRegistry() + require.NotNil(t, registry.Lookup("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/meta_nested.MetaView")) + require.NotNil(t, registry.Lookup("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/meta_nested.ProductsMetaView")) +} + +func TestApplySummaryTypeSupport_PreservesOwnerColumnTypes(t *testing.T) { + type productView struct { + VendorId *int `sqlx:"VENDOR_ID"` + } + + source := &shape.Source{TypeRegistry: x.NewRegistry()} + result := &plan.Result{ + TypeContext: &typectx.Context{ + PackageName: "multi_summary", + PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary", + }, + Views: []*plan.View{ + { + Name: "products", + SummaryName: "ProductsMeta", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) t GROUP BY VENDOR_ID", + FieldType: reflect.TypeOf([]productView{}), + ElementType: reflect.TypeOf(productView{}), + Cardinality: "many", + }, + }, + } + + applySummaryTypeSupport(result, source) + + registry := source.EnsureTypeRegistry() + registered := registry.Lookup("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary.ProductsMetaView") + require.NotNil(t, registered) + require.NotNil(t, registered.Type) + field, ok := registered.Type.FieldByName("VendorId") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf((*int)(nil)), field.Type) +} + +func TestApplySummaryTypeSupport_InfersNullableComputedSummaryColumns(t *testing.T) { + source := &shape.Source{TypeRegistry: x.NewRegistry()} + result := &plan.Result{ + TypeContext: &typectx.Context{ + PackageName: "multi_summary", + PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary", + }, + Views: []*plan.View{ + { + Name: "vendor", + SummaryName: "Meta", + Summary: "SELECT CAST(1 + (COUNT(1) / 25) AS SIGNED) AS PAGE_CNT, COUNT(1) AS CNT FROM ($View.vendor.SQL) t", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + Cardinality: "many", + }, + }, + } + + applySummaryTypeSupport(result, source) + + registry := source.EnsureTypeRegistry() + registered := registry.Lookup("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary.MetaView") + require.NotNil(t, registered) + require.NotNil(t, registered.Type) + pageCnt, ok := registered.Type.FieldByName("PageCnt") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf((*int)(nil)), pageCnt.Type) + cnt, ok := registered.Type.FieldByName("Cnt") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf(int(0)), cnt.Type) +} + +func TestApplySummaryTypeSupport_OverridesStaleRegisteredSummaryType(t *testing.T) { + type staleProductsMetaView struct { + VendorId string `sqlx:"VENDOR_ID"` + } + type productView struct { + VendorId *int `sqlx:"VENDOR_ID"` + } + + registry := x.NewRegistry() + registry.Register(x.NewType( + reflect.TypeOf(staleProductsMetaView{}), + x.WithName("ProductsMetaView"), + x.WithPkgPath("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary"), + )) + + source := &shape.Source{TypeRegistry: registry} + result := &plan.Result{ + TypeContext: &typectx.Context{ + PackageName: "multi_summary", + PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary", + }, + Views: []*plan.View{ + { + Name: "products", + SummaryName: "ProductsMeta", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) t GROUP BY VENDOR_ID", + FieldType: reflect.TypeOf([]productView{}), + ElementType: reflect.TypeOf(productView{}), + Cardinality: "many", + }, + }, + } + + applySummaryTypeSupport(result, source) + + registered := registry.Lookup("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary.ProductsMetaView") + require.NotNil(t, registered) + require.NotNil(t, registered.Type) + field, ok := registered.Type.FieldByName("VendorId") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf((*int)(nil)), field.Type) +} diff --git a/repository/shape/compile/viewdecl.go b/repository/shape/compile/viewdecl.go index 44c139a79..e60fa78c6 100644 --- a/repository/shape/compile/viewdecl.go +++ b/repository/shape/compile/viewdecl.go @@ -52,8 +52,9 @@ type declaredPredicate struct { } type declaredColumnConfig struct { - DataType string - Tag string + DataType string + Tag string + Groupable *bool } const ( diff --git a/repository/shape/compile/viewdecl_append.go b/repository/shape/compile/viewdecl_append.go index 297f75349..0f91303fc 100644 --- a/repository/shape/compile/viewdecl_append.go +++ b/repository/shape/compile/viewdecl_append.go @@ -13,6 +13,7 @@ func appendDeclaredViews(rawDQL string, result *plan.Result) { if result == nil { return } + appendRootOutputViewDeclaration(rawDQL, result) declared, diags := extractDeclaredViews(rawDQL) if len(diags) > 0 { result.Diagnostics = append(result.Diagnostics, diags...) @@ -77,6 +78,116 @@ func appendDeclaredViews(rawDQL string, result *plan.Result) { } } +func appendRootOutputViewDeclaration(rawDQL string, result *plan.Result) { + if result == nil { + return + } + root := lookupRootView(result) + if root == nil { + return + } + for _, block := range extractSetBlocks(rawDQL) { + _, kind, location, tail, tailOffset, ok := parseSetDeclarationBody(block.Body) + if !ok || !strings.EqualFold(strings.TrimSpace(kind), "output") || !strings.EqualFold(strings.TrimSpace(location), "view") { + continue + } + view := &declaredView{} + applyDeclaredViewOptions(view, tail, rawDQL, block.BodyOffset+tailOffset, &result.Diagnostics) + mergeViewDeclaration(root, buildViewDeclaration(view)) + if view.Required && !view.CardinalitySet { + root.Cardinality = "one" + } + if view.Cardinality != "" { + root.Cardinality = view.Cardinality + } + } +} + +func mergeViewDeclaration(target *plan.View, declared *plan.ViewDeclaration) { + if target == nil || declared == nil { + return + } + if target.Declaration == nil { + target.Declaration = declared + return + } + dst := target.Declaration + if strings.TrimSpace(dst.Tag) == "" { + dst.Tag = declared.Tag + } + if strings.TrimSpace(dst.TypeName) == "" { + dst.TypeName = declared.TypeName + } + if strings.TrimSpace(dst.Dest) == "" { + dst.Dest = declared.Dest + } + if strings.TrimSpace(dst.Codec) == "" { + dst.Codec = declared.Codec + if len(dst.CodecArgs) == 0 { + dst.CodecArgs = append([]string{}, declared.CodecArgs...) + } + } + if strings.TrimSpace(dst.HandlerName) == "" { + dst.HandlerName = declared.HandlerName + if len(dst.HandlerArgs) == 0 { + dst.HandlerArgs = append([]string{}, declared.HandlerArgs...) + } + } + if dst.StatusCode == nil { + dst.StatusCode = declared.StatusCode + } + if strings.TrimSpace(dst.ErrorMessage) == "" { + dst.ErrorMessage = declared.ErrorMessage + } + if strings.TrimSpace(dst.QuerySelector) == "" { + dst.QuerySelector = declared.QuerySelector + } + if strings.TrimSpace(dst.CacheRef) == "" { + dst.CacheRef = declared.CacheRef + } + if dst.Limit == nil { + dst.Limit = declared.Limit + } + if dst.Cacheable == nil { + dst.Cacheable = declared.Cacheable + } + if strings.TrimSpace(dst.When) == "" { + dst.When = declared.When + } + if strings.TrimSpace(dst.Scope) == "" { + dst.Scope = declared.Scope + } + if strings.TrimSpace(dst.DataType) == "" { + dst.DataType = declared.DataType + } + if strings.TrimSpace(dst.Of) == "" { + dst.Of = declared.Of + } + if strings.TrimSpace(dst.Value) == "" { + dst.Value = declared.Value + } + dst.Async = dst.Async || declared.Async + dst.Output = dst.Output || declared.Output + if len(dst.Predicates) == 0 && len(declared.Predicates) > 0 { + dst.Predicates = append([]*plan.ViewPredicate{}, declared.Predicates...) + } + if len(declared.ColumnsConfig) > 0 { + if dst.ColumnsConfig == nil { + dst.ColumnsConfig = map[string]*plan.ViewColumnConfig{} + } + for name, cfg := range declared.ColumnsConfig { + if strings.TrimSpace(name) == "" || cfg == nil { + continue + } + dst.ColumnsConfig[name] = &plan.ViewColumnConfig{ + DataType: strings.TrimSpace(cfg.DataType), + Tag: strings.TrimSpace(cfg.Tag), + Groupable: cloneBoolPtr(cfg.Groupable), + } + } + } +} + func normalizeSummarySQLForParent(parent *plan.View, sqlText string) string { normalized := strings.TrimSpace(sqlText) if parent == nil || normalized == "" { @@ -242,8 +353,9 @@ func buildViewDeclaration(item *declaredView) *plan.ViewDeclaration { continue } ret.ColumnsConfig[name] = &plan.ViewColumnConfig{ - DataType: strings.TrimSpace(cfg.DataType), - Tag: strings.TrimSpace(cfg.Tag), + DataType: strings.TrimSpace(cfg.DataType), + Tag: strings.TrimSpace(cfg.Tag), + Groupable: cloneBoolPtr(cfg.Groupable), } } if len(ret.ColumnsConfig) == 0 { @@ -260,3 +372,11 @@ func buildViewDeclaration(item *declaredView) *plan.ViewDeclaration { } return ret } + +func cloneBoolPtr(value *bool) *bool { + if value == nil { + return nil + } + ret := *value + return &ret +} diff --git a/repository/shape/compile/viewdecl_options.go b/repository/shape/compile/viewdecl_options.go index 93c3aad29..19f74b471 100644 --- a/repository/shape/compile/viewdecl_options.go +++ b/repository/shape/compile/viewdecl_options.go @@ -260,6 +260,22 @@ func applyDeclaredViewOptions(view *declaredView, tail, dql string, offset int, } cfg := ensureDeclaredColumnConfig(view, columnName) cfg.Tag = tag + case strings.EqualFold(name, "WithColumnGroupable"), strings.EqualFold(name, "ColumnGroupable"): + if !expectArgs(view, name, args, 2, 2, dql, optionOffset, diags) { + continue + } + columnName := strings.TrimSpace(trimQuote(args[0])) + if columnName == "" { + appendOptionArgDiagnostic(view, name, "column name must be non-empty", dql, optionOffset, diags) + continue + } + groupable, err := strconv.ParseBool(strings.TrimSpace(trimQuote(args[1]))) + if err != nil { + appendOptionArgDiagnostic(view, name, fmt.Sprintf("invalid bool groupable %q", args[1]), dql, optionOffset, diags) + continue + } + cfg := ensureDeclaredColumnConfig(view, columnName) + cfg.Groupable = &groupable case strings.EqualFold(name, "Of"): if !expectArgs(view, name, args, 1, 1, dql, optionOffset, diags) { continue diff --git a/repository/shape/compile/viewdecl_test.go b/repository/shape/compile/viewdecl_test.go index a4cd80543..f7dff272b 100644 --- a/repository/shape/compile/viewdecl_test.go +++ b/repository/shape/compile/viewdecl_test.go @@ -72,7 +72,7 @@ func TestViewDecl_ApplyOptions_Extended(t *testing.T) { "WithStatusCode(422).WithErrorMessage('bad req').WithPredicate('ByID','id = ?', 101)." + "EnsurePredicate('Tenant','tenant_id = ?', 7).QuerySelector('qs').WithCache('c1').WithLimit(10)." + "Cacheable(true).When('x > 1').Scope('team').Type('OrderView').Dest('orders.go').WithType('[]Order')." + - "WithColumnType('Authorized','bool').WithColumnTag('Authorized','internal:\"true\"').Of('list').Value('abc').Async().Output()" + "WithColumnType('Authorized','bool').WithColumnTag('Authorized','internal:\"true\"').WithColumnGroupable('Authorized', true).Of('list').Value('abc').Async().Output()" applyDeclaredViewOptions(view, tail, "SELECT 1", 0, &diags) require.Empty(t, diags) @@ -107,6 +107,8 @@ func TestViewDecl_ApplyOptions_Extended(t *testing.T) { require.Contains(t, view.ColumnsConfig, "Authorized") assert.Equal(t, "bool", view.ColumnsConfig["Authorized"].DataType) assert.Equal(t, `internal:"true"`, view.ColumnsConfig["Authorized"].Tag) + require.NotNil(t, view.ColumnsConfig["Authorized"].Groupable) + assert.True(t, *view.ColumnsConfig["Authorized"].Groupable) assert.Equal(t, "list", view.Of) assert.Equal(t, "abc", view.Value) assert.True(t, view.Async) @@ -156,7 +158,7 @@ func TestViewDecl_AppendDeclaredViews_ExtendedDeclarationMetadata(t *testing.T) "WithStatusCode(409).WithErrorMessage('conflict').WithPredicate('ByID','id=?',1)." + "EnsurePredicate('Tenant','tenant=?',2).QuerySelector('items').WithCache('c1').WithLimit(5)." + "Cacheable(false).When('x').Scope('s').Type('OrderView').Dest('order.go').WithType('Order')." + - "WithColumnType('Authorized','bool').WithColumnTag('Authorized','internal:\"true\"').Of('o').Value('v').Async().Output() /* SELECT id FROM EXTRA e */)" + "WithColumnType('Authorized','bool').WithColumnTag('Authorized','internal:\"true\"').WithColumnGroupable('Authorized', true).Of('o').Value('v').Async().Output() /* SELECT id FROM EXTRA e */)" result := &plan.Result{ ViewsByName: map[string]*plan.View{}, ByPath: map[string]*plan.Field{}, @@ -193,6 +195,8 @@ func TestViewDecl_AppendDeclaredViews_ExtendedDeclarationMetadata(t *testing.T) require.Contains(t, target.Declaration.ColumnsConfig, "Authorized") assert.Equal(t, "bool", target.Declaration.ColumnsConfig["Authorized"].DataType) assert.Equal(t, `internal:"true"`, target.Declaration.ColumnsConfig["Authorized"].Tag) + require.NotNil(t, target.Declaration.ColumnsConfig["Authorized"].Groupable) + assert.True(t, *target.Declaration.ColumnsConfig["Authorized"].Groupable) assert.Equal(t, "o", target.Declaration.Of) assert.Equal(t, "v", target.Declaration.Value) assert.True(t, target.Declaration.Async) diff --git a/repository/shape/dql/preprocess/preprocess.go b/repository/shape/dql/preprocess/preprocess.go index 28d63f70e..01ab95682 100644 --- a/repository/shape/dql/preprocess/preprocess.go +++ b/repository/shape/dql/preprocess/preprocess.go @@ -136,6 +136,7 @@ func normalizeDirectives(input *dqlshape.Directives) *dqlshape.Directives { ret := &dqlshape.Directives{ Meta: strings.TrimSpace(input.Meta), DefaultConnector: strings.TrimSpace(input.DefaultConnector), + TemplateType: strings.TrimSpace(input.TemplateType), Dest: strings.TrimSpace(input.Dest), InputDest: strings.TrimSpace(input.InputDest), OutputDest: strings.TrimSpace(input.OutputDest), @@ -184,7 +185,7 @@ func normalizeDirectives(input *dqlshape.Directives) *dqlshape.Directives { ret.Const[k] = v } } - if ret.Meta == "" && ret.DefaultConnector == "" && + if ret.Meta == "" && ret.DefaultConnector == "" && ret.TemplateType == "" && ret.Dest == "" && ret.InputDest == "" && ret.OutputDest == "" && ret.RouterDest == "" && ret.InputType == "" && ret.OutputType == "" && ret.Cache == nil && ret.MCP == nil && ret.Route == nil && diff --git a/repository/shape/dql/preprocess/preprocess_test.go b/repository/shape/dql/preprocess/preprocess_test.go index 8ea4b3480..9a3433d84 100644 --- a/repository/shape/dql/preprocess/preprocess_test.go +++ b/repository/shape/dql/preprocess/preprocess_test.go @@ -145,6 +145,7 @@ func TestPrepare_SpecialDirectives(t *testing.T) { "#settings($_ = $format('tabular_json'))\n" + "#settings($_ = $date_format('2006-01-02'))\n" + "#settings($_ = $case_format('lc'))\n" + + "#settings($_ = $useTemplate('patch'))\n" + "SELECT id FROM ORDERS o" pre := Prepare(dql) require.NotNil(t, pre) @@ -170,6 +171,7 @@ func TestPrepare_SpecialDirectives(t *testing.T) { assert.Equal(t, "tabular", pre.Directives.Format) assert.Equal(t, "2006-01-02", pre.Directives.DateFormat) assert.Equal(t, "lc", pre.Directives.CaseFormat) + assert.Equal(t, "patch", pre.Directives.TemplateType) } func TestPrepare_InvalidDestDirectiveDiagnostic(t *testing.T) { diff --git a/repository/shape/dql/preprocess/settings_directives.go b/repository/shape/dql/preprocess/settings_directives.go index 53e1eb92b..b946d128d 100644 --- a/repository/shape/dql/preprocess/settings_directives.go +++ b/repository/shape/dql/preprocess/settings_directives.go @@ -13,26 +13,27 @@ import ( ) var ( - metaDirectiveName = map[string]bool{"meta": true} - connectorDirectiveName = map[string]bool{"connector": true} - cacheDirectiveName = map[string]bool{"cache": true} - mcpDirectiveName = map[string]bool{"mcp": true} - routeDirectiveName = map[string]bool{"route": true} - constDirectiveName = map[string]bool{"const": true} - marshalDirectiveName = map[string]bool{"marshal": true} - unmarshalDirectiveName = map[string]bool{"unmarshal": true} - formatDirectiveName = map[string]bool{"format": true} - dateFormatDirectiveName = map[string]bool{"date_format": true} - caseFormatDirectiveName = map[string]bool{"case_format": true} - destDirectiveName = map[string]bool{"dest": true} - inputDestDirectiveName = map[string]bool{"input_dest": true} - outputDestDirectiveName = map[string]bool{"output_dest": true} - routerDestDirectiveName = map[string]bool{"router_dest": true} - inputTypeDirectiveName = map[string]bool{"input_type": true} - outputTypeDirectiveName = map[string]bool{"output_type": true} - cacheProviderExpr = regexp.MustCompile(`(?i)\.withprovider\s*\(\s*['"]([^'"]+)['"]\s*\)`) - cacheLocationExpr = regexp.MustCompile(`(?i)\.withlocation\s*\(\s*['"]([^'"]+)['"]\s*\)`) - cacheTTLMsExpr = regexp.MustCompile(`(?i)\.withtimetolivems\s*\(\s*([0-9]+)\s*\)`) + metaDirectiveName = map[string]bool{"meta": true} + connectorDirectiveName = map[string]bool{"connector": true} + cacheDirectiveName = map[string]bool{"cache": true} + mcpDirectiveName = map[string]bool{"mcp": true} + routeDirectiveName = map[string]bool{"route": true} + constDirectiveName = map[string]bool{"const": true} + marshalDirectiveName = map[string]bool{"marshal": true} + unmarshalDirectiveName = map[string]bool{"unmarshal": true} + formatDirectiveName = map[string]bool{"format": true} + dateFormatDirectiveName = map[string]bool{"date_format": true} + caseFormatDirectiveName = map[string]bool{"case_format": true} + useTemplateDirectiveName = map[string]bool{"usetemplate": true} + destDirectiveName = map[string]bool{"dest": true} + inputDestDirectiveName = map[string]bool{"input_dest": true} + outputDestDirectiveName = map[string]bool{"output_dest": true} + routerDestDirectiveName = map[string]bool{"router_dest": true} + inputTypeDirectiveName = map[string]bool{"input_type": true} + outputTypeDirectiveName = map[string]bool{"output_type": true} + cacheProviderExpr = regexp.MustCompile(`(?i)\.withprovider\s*\(\s*['"]([^'"]+)['"]\s*\)`) + cacheLocationExpr = regexp.MustCompile(`(?i)\.withlocation\s*\(\s*['"]([^'"]+)['"]\s*\)`) + cacheTTLMsExpr = regexp.MustCompile(`(?i)\.withtimetolivems\s*\(\s*([0-9]+)\s*\)`) ) func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, directives *dqlshape.Directives) []*dqlshape.Diagnostic { @@ -193,6 +194,18 @@ func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, direct directives.CaseFormat = values[len(values)-1] } } + if strings.Contains(lower, "$usetemplate") { + calls, parseErrors := scanDollarCallsStrict(input, useTemplateDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirUnsupported, fullDQL, diagnosticOffset) + values := parseSingleArgQuotedDirectiveCalls(calls) + if len(values) == 0 { + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirUnsupported, "invalid $useTemplate directive", "expected: #settings($_ = $useTemplate('patch'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } + } else { + directives.TemplateType = values[len(values)-1] + } + } if strings.Contains(lower, "$dest") { calls, parseErrors := scanDollarCallsStrict(input, destDirectiveName) diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirDest, fullDQL, diagnosticOffset) diff --git a/repository/shape/dql/shape/model.go b/repository/shape/dql/shape/model.go index bbe53463f..2271d725e 100644 --- a/repository/shape/dql/shape/model.go +++ b/repository/shape/dql/shape/model.go @@ -42,6 +42,7 @@ type Diagnostic struct { type Directives struct { Meta string DefaultConnector string + TemplateType string Dest string InputDest string OutputDest string diff --git a/repository/shape/dql_engine_test.go b/repository/shape/dql_engine_test.go index 1b52a0e01..80ed16cf0 100644 --- a/repository/shape/dql_engine_test.go +++ b/repository/shape/dql_engine_test.go @@ -2,13 +2,24 @@ package shape_test import ( "context" + "fmt" + "os" + "path/filepath" + "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + marshalconfig "github.com/viant/datly/gateway/router/marshal/config" + marshaljson "github.com/viant/datly/gateway/router/marshal/json" shape "github.com/viant/datly/repository/shape" shapeCompile "github.com/viant/datly/repository/shape/compile" shapeLoad "github.com/viant/datly/repository/shape/load" + shapePlan "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/view" + "github.com/viant/datly/view/extension" + "github.com/viant/tagly/format/text" + "github.com/viant/xreflect" ) func TestEngine_LoadDQLViews(t *testing.T) { @@ -24,6 +35,20 @@ func TestEngine_LoadDQLViews(t *testing.T) { assert.Equal(t, "t", artifacts.Views[0].Name) } +func TestEngine_LoadDQLResource(t *testing.T) { + engine := shape.New( + shape.WithCompiler(shapeCompile.New()), + shape.WithLoader(shapeLoad.New()), + shape.WithName("/v1/api/reports/orders"), + ) + artifacts, err := engine.LoadDQLResource(context.Background(), "SELECT id FROM ORDERS t") + require.NoError(t, err) + require.NotNil(t, artifacts) + require.NotNil(t, artifacts.Resource) + require.Len(t, artifacts.Resource.Views, 1) + assert.Equal(t, "t", artifacts.Resource.Views[0].Name) +} + func TestEngine_LoadDQLComponent(t *testing.T) { engine := shape.New( shape.WithCompiler(shapeCompile.New()), @@ -84,3 +109,278 @@ SELECT 1 AS UserID` require.NotNil(t, component.Output[0].Schema) assert.Equal(t, "One", string(component.Output[0].Schema.Cardinality)) } + +type metaFormatOutput struct { + Meta *metaFormatMeta `json:"meta,omitempty"` + Data []metaFormatData `json:"data,omitempty"` + Status string `json:"status,omitempty"` +} + +type metaFormatMeta struct { + PageCnt *int `json:"pageCnt,omitempty"` + Cnt int `json:"cnt,omitempty"` +} + +type metaFormatData struct { + Id int `json:"id,omitempty"` + Name *string `json:"name,omitempty"` + AccountId *int `json:"accountId,omitempty"` + Products []*metaFormatProduct `json:"products,omitempty"` + ProductsMeta *metaFormatProductsMeta `json:"productsMeta,omitempty"` +} + +type metaFormatProduct struct { + Id int `json:"id,omitempty"` + Name *string `json:"name,omitempty"` + VendorId *int `json:"vendorId,omitempty"` +} + +type metaFormatProductsMeta struct { + VendorId *int `json:"vendorId,omitempty"` + PageCnt *int `json:"pageCnt,omitempty"` + TotalProducts int `json:"totalProducts,omitempty"` +} + +func TestMetaFormatLiveLikeOutput_Marshal(t *testing.T) { + name := "Acme" + id := 1 + pageCnt := 2 + output := &metaFormatOutput{ + Meta: &metaFormatMeta{PageCnt: &pageCnt, Cnt: 3}, + Data: []metaFormatData{ + { + Id: 1, + Name: &name, + AccountId: &id, + Products: []*metaFormatProduct{ + {Id: 10, Name: &name, VendorId: &id}, + }, + ProductsMeta: &metaFormatProductsMeta{VendorId: &id, PageCnt: &pageCnt, TotalProducts: 1}, + }, + }, + Status: "ok", + } + marshaller := marshaljson.New(&marshalconfig.IOConfig{CaseFormat: text.CaseFormatLowerCamel}) + _, err := marshaller.Marshal(output) + require.NoError(t, err) +} + +func TestDQLCompileLoad_MetaFormatPreservesSummariesWithoutLinkedTypes(t *testing.T) { + dqlPath := filepath.Join("..", "..", "e2e", "v1", "dql", "dev", "vendorsrv", "meta_format.dql") + dqlBytes, err := os.ReadFile(dqlPath) + require.NoError(t, err) + + source := &shape.Source{ + Name: "meta_format", + Path: dqlPath, + DQL: string(dqlBytes), + } + planResult, err := shapeCompile.New().Compile( + context.Background(), + source, + shape.WithLinkedTypes(false), + shape.WithTypeContextPackageDir(filepath.Join("e2e", "v1", "shape", "dev", "vendorsvc", "multi_summary")), + shape.WithTypeContextPackageName("multi_summary"), + ) + require.NoError(t, err) + registry := source.EnsureTypeRegistry() + require.NotNil(t, registry) + if lookup := registry.Lookup("ProductsView"); lookup != nil { + fmt.Printf("registry ProductsView: %T %v\n", lookup.Type, lookup.Type) + } else { + fmt.Printf("registry ProductsView: \n") + } + if lookup := registry.Lookup("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary.ProductsView"); lookup != nil { + fmt.Printf("registry fq ProductsView: %T %v\n", lookup.Type, lookup.Type) + } else { + fmt.Printf("registry fq ProductsView: \n") + } + planned, ok := shapePlan.ResultFrom(planResult) + require.True(t, ok) + foundPlannedProductsType := false + for _, item := range planned.Types { + if item != nil && item.Name == "ProductsView" { + foundPlannedProductsType = true + assert.NotEmpty(t, item.DataType) + } + } + var plannedProductsView *shapePlan.View + for _, item := range planned.Views { + if item != nil && item.Name == "products" { + plannedProductsView = item + break + } + } + require.NotNil(t, plannedProductsView) + require.NotNil(t, plannedProductsView.FieldType) + t.Logf("planned products fieldType=%v elementType=%v", plannedProductsView.FieldType, plannedProductsView.ElementType) + assert.False(t, foundPlannedProductsType) + registered := registry.Lookup("github.com/viant/datly/e2e/v1/shape/dev/vendorsvc/multi_summary.ProductsMetaView") + require.NotNil(t, registered) + require.NotNil(t, registered.Type) + registeredType := registered.Type + if registeredType.Kind() == reflect.Ptr { + registeredType = registeredType.Elem() + } + field, ok := registeredType.FieldByName("VendorId") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf((*int)(nil)), field.Type) + + resourceArtifacts, err := shapeLoad.New().LoadResource(context.Background(), planResult, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + require.NotNil(t, resourceArtifacts) + require.NotNil(t, resourceArtifacts.Resource) + + index := resourceArtifacts.Resource.Views.Index() + root, err := index.Lookup("vendor") + require.NoError(t, err) + require.NotNil(t, root) + t.Logf("root view: name=%s ref=%s schema=%v with=%d", root.Name, root.Ref, root.Schema != nil, len(root.With)) + for i, rel := range root.With { + if rel == nil || rel.Of == nil { + t.Logf("root relation[%d]: nil", i) + continue + } + relSchemaType := "" + if rel.Of.View.Schema != nil && rel.Of.View.Schema.Type() != nil { + relSchemaType = rel.Of.View.Schema.Type().String() + } + t.Logf("root relation[%d]: holder=%s name=%s ref=%s schema=%v schemaType=%s summary=%v", i, rel.Holder, rel.Of.View.Name, rel.Of.View.Ref, rel.Of.View.Schema != nil, relSchemaType, rel.Of.View.Template != nil && rel.Of.View.Template.Summary != nil) + } + require.NotNil(t, root.Template) + require.NotNil(t, root.Template.Summary) + require.NotNil(t, root.Template.Summary.Schema) + assert.Equal(t, "MetaView", root.Template.Summary.Schema.Name) + + child, err := index.Lookup("products") + require.NoError(t, err) + require.NotNil(t, child) + require.NotNil(t, child.Template) + require.NotNil(t, child.Template.Summary) + require.NotNil(t, child.Template.Summary.Schema) + assert.Equal(t, "ProductsMetaView", child.Template.Summary.Schema.Name) + childViewSummaryType := child.Template.Summary.Schema.Type() + require.NotNil(t, childViewSummaryType) + if childViewSummaryType.Kind() == reflect.Ptr { + childViewSummaryType = childViewSummaryType.Elem() + } + field, ok = childViewSummaryType.FieldByName("VendorId") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf((*int)(nil)), field.Type) + require.NotEmpty(t, root.With) + require.NotNil(t, root.With[0].Of) + require.NotNil(t, root.With[0].Of.View.Template) + require.NotNil(t, root.With[0].Of.View.Template.Summary) + childSummaryType := root.With[0].Of.View.Template.Summary.Schema.Type() + require.NotNil(t, childSummaryType) + if childSummaryType.Kind() == reflect.Ptr { + childSummaryType = childSummaryType.Elem() + } + field, ok = childSummaryType.FieldByName("VendorId") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf((*int)(nil)), field.Type) + + componentArtifact, err := shapeLoad.New().LoadComponent(context.Background(), planResult, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + component, ok := shapeLoad.ComponentFrom(componentArtifact) + require.True(t, ok) + foundProductsType := false + for _, item := range componentArtifact.Resource.Types { + if item != nil { + t.Logf("resource type: name=%s dataType=%s fields=%d package=%s module=%s", item.Name, item.DataType, len(item.Fields), item.Package, item.ModulePath) + } + if item == nil || item.Name != "ProductsView" { + continue + } + foundProductsType = true + require.NotEmpty(t, item.Fields) + break + } + require.True(t, foundProductsType) + typeRegistry, err := initTypeRegistryForResource(componentArtifact.Resource) + require.NoError(t, err) + + foundSummary := false + for _, param := range component.Output { + if param != nil && param.In != nil && param.In.Name == "summary" { + foundSummary = true + require.NotNil(t, param.Schema) + assert.Equal(t, "MetaView", param.Schema.Name) + } + } + assert.True(t, foundSummary) + + outputType, err := component.OutputReflectType("", typeRegistry.Lookup) + require.NoError(t, err) + require.NotNil(t, outputType) + + output := reflect.New(outputType).Elem() + dataField := output.FieldByName("Data") + require.True(t, dataField.IsValid()) + require.Equal(t, reflect.Slice, dataField.Kind()) + + rowType := dataField.Type().Elem() + fmt.Printf("output Data type: %T %v\n", dataField.Interface(), dataField.Type()) + rowValue := reflect.New(rowType) + if rowType.Kind() == reflect.Ptr { + rowValue = reflect.New(rowType.Elem()) + } + row := rowValue.Elem() + row.FieldByName("Id").SetInt(1) + + productsField := row.FieldByName("Products") + require.True(t, productsField.IsValid()) + productType := productsField.Type().Elem() + product := reflect.New(productType) + if productType.Kind() == reflect.Ptr { + product = reflect.New(productType.Elem()) + } + product.Elem().FieldByName("Id").SetInt(10) + if productType.Kind() == reflect.Ptr { + productsField.Set(reflect.Append(productsField, product)) + } else { + productsField.Set(reflect.Append(productsField, product.Elem())) + } + + data := reflect.MakeSlice(dataField.Type(), 0, 1) + if rowType.Kind() == reflect.Ptr { + data = reflect.Append(data, rowValue) + } else { + data = reflect.Append(data, row) + } + dataField.Set(data) + + marshaller := marshaljson.New(&marshalconfig.IOConfig{CaseFormat: text.CaseFormatLowerCamel}) + _, err = marshaller.Marshal(output.Addr().Interface()) + require.NoError(t, err) +} + +func initTypeRegistryForResource(resource *view.Resource) (*xreflect.Types, error) { + registry := extension.NewRegistry() + imports := view.Imports{} + for _, definition := range resource.Types { + if definition != nil && definition.ModulePath != "" { + imports.Add(definition.ModulePath) + if definition.Package != "" { + imports.AddWithAlias(definition.Package, definition.ModulePath) + } + } + } + for _, definition := range resource.Types { + if definition == nil { + continue + } + if err := definition.Init(context.Background(), registry.Types.Lookup, imports); err != nil { + return nil, err + } + if err := registry.Types.Register(definition.Name, xreflect.WithReflectType(definition.Type())); err != nil { + return nil, err + } + if definition.Package != "" { + if err := registry.Types.Register(definition.Name, xreflect.WithPackage(definition.Package), xreflect.WithReflectType(definition.Type())); err != nil { + return nil, err + } + } + } + return registry.Types, nil +} diff --git a/repository/shape/improvement.md b/repository/shape/improvement.md new file mode 100644 index 000000000..aceb6f67d --- /dev/null +++ b/repository/shape/improvement.md @@ -0,0 +1,267 @@ +# Shape Improvement Proposal + +This note captures the main internal improvements suggested by the translator-to-shape migration work. + +Scope: + +- Applies to `repository/shape` +- Focuses on `DQL -> shape -> IR` +- Uses migration findings from grouping, summary, selector, and generated patch routes + +## Goals + +- Make shape the authoritative semantic model for DQL and Go-derived routes. +- Reduce runtime/bootstrap recovery logic. +- Replace translator-era implicit behavior with explicit shape metadata. +- Keep load/materialization modular so views, components, and resources can be built independently. + +## 1. Promote `ComponentRoute` As A First-Class Shape Primitive + +Observed gap: + +- Route path, method, template strategy, and related component-level metadata were historically reconstructed outside shape. +- That encouraged direct `DQL -> IR` workarounds. + +Proposal: + +- Treat `ComponentRoute` as a first-class primitive produced by DQL compile. +- Carry at minimum: + - `Method` + - `RoutePath` + - `TemplateType` + - route-level connector/defaults + - route-level metadata/docs/cache/auth flags + +Target: + +- `DQL -> ComponentRoute` +- `DQL -> View` +- `shape/load -> component/resource IR` + +Benefit: + +- Transcribe, bootstrap, and future `AddRoute` APIs can consume route metadata from shape only. + +## 2. Make Template Strategy Explicit + +Observed gap: + +- Generated patch routes and translated exec routes were previously distinguished by inference. +- That was fragile and led to runtime fallbacks. + +Proposal: + +- Keep explicit DQL settings such as: + - `#setting($_ = $useTemplate('patch'))` +- Store the resolved value on shape route metadata as `TemplateType`. + +Recommended semantics: + +- `translate` or empty: preserve authored DQL/Velty behavior +- `patch`: synthesize mutable Velty from shape AST/metadata +- future values may include `post`, `put`, `upsert` + +Benefit: + +- Removes heuristic detection of generated mutable routes. + +## 3. Eliminate Runtime Type Recovery For Helper Parameters + +Observed gap: + +- Generated patch helpers such as `CurFoosId` and `CurFoos` needed runtime recovery of source type information. +- Local generator paths were more explicit than the early `v1` shape/transcribe path. + +Proposal: + +- Shape/transcribe should emit enough source/output type metadata so runtime codec initialization does not need to infer types from referenced params. +- Helper params should carry explicit source owner type and output type in shape/IR. + +Benefit: + +- Moves correctness back into shape. +- Reduces special handling in `view/state/parameter.go` and codec initialization. + +## 4. Keep View-Level And Column-Level Semantics Separate + +Observed gap: + +- Grouping and selector metadata were easy to blur across view and column layers. + +Proposal: + +- View-level metadata stays explicit: + - `Groupable` + - selector namespace + - selector constraints + - summary URI / summary behavior +- Column-level metadata is explicit or inferred independently: + - `ColumnConfig.Groupable` + - inferred grouped projections from `GROUP BY` + +Benefit: + +- Avoids deriving view semantics from column accidents. +- Keeps Go tags and DQL hints aligned. + +## 5. Add Dedicated Shape Primitives For Selector Holders + +Observed gap: + +- Flattening query selector fields into business input types makes Go shape contracts noisy and semantically wrong. + +Proposal: + +- Keep query-selector state as a separate shape concept. +- Support Go-derived contracts like: + - business input holder + - selector holder tagged with `querySelector:"viewAlias"` + +Target Go model: + +- `VendorInput` remains business input +- `ViewSelect` remains selector state +- shape merges both into component contract IR + +Benefit: + +- Aligns Go-derived shape with the DQL selector model. + +## 6. Make Summary A Real Shape Concept, Not A Side Effect + +Observed gap: + +- Summary handling drifted between tags, parent view attachment, and runtime conventions. +- Multi-level summaries exposed gaps in child summary attachment and typing. + +Proposal: + +- Represent summary explicitly in shape at any view level. +- Include: + - summary target view/ref + - summary URI/source + - parent attachment semantics + - summary output schema/type + +Benefit: + +- Root summaries and child summaries can be materialized consistently from shape. + +## 7. Add Recursive Mutable Generation For Nested Graphs + +Observed gap: + +- `patch_basic_one` and `patch_basic_many` became stable, but nested mutable graphs such as many-many flows need more general helper synthesis. + +Proposal: + +- Generalize mutable generation to recurse across relation graphs. +- Generate helper views, `IndexBy` maps, key propagation, and DML blocks per mutable node. + +Examples: + +- root collection patch +- nested child collection patch +- nested key propagation such as `FooId = parent.Id` + +Benefit: + +- Closes the remaining gap between local generate flows and shape-generated mutable routes. + +## 8. Introduce A Strong Shape Validation Stage + +Observed gap: + +- Some failures were discovered too late at bootstrap/runtime. + +Proposal: + +- Expand `datly validate` as the primary shape-only validation gate. +- Validate: + - DQL syntax and directives + - SQL asset existence + - route metadata completeness + - helper type completeness + - selector/summary/grouping consistency + - generated mutable prerequisites + +Benefit: + +- Detects incomplete shape before runtime. + +## 9. Add Deterministic Diagnostics Codes + +Observed gap: + +- Migration debugging spent too much time on ad hoc runtime errors. + +Proposal: + +- Extend shape diagnostics with stable codes across: + - route metadata + - selector metadata + - summary attachment + - mutable helper generation + - groupable inference + - type collisions + +Benefit: + +- Better tooling, tests, and compile-time failure handling. + +## 10. Keep Load Modular By Primitive + +Observed gap: + +- Some behavior was easier to validate once view/component/resource loading was separated. + +Proposal: + +- Continue building around primitive loaders: + - `LoadView` + - `LoadComponentRoute` + - `LoadComponent` + - `LoadResource` +- Keep both inputs supported: + - Go types + - DQL + +Benefit: + +- Allows future APIs such as `AddRoute` to stay thin. +- Makes unit coverage sharper and reduces cross-coupled runtime fixes. + +## 11. Reduce Direct Translator Dependence To Parity Specs And Fixtures + +Observed gap: + +- Migration often required checking local regression translator output to understand target semantics. + +Proposal: + +- Treat translator/local regression outputs as parity fixtures, not active implementation dependencies. +- Keep explicit parity docs and focused regression fixtures in shape tests. + +Benefit: + +- Shape remains independent while still preserving observable legacy behavior. + +## Suggested Priority + +1. `ComponentRoute` ownership in shape +2. Explicit `TemplateType` +3. Remove runtime helper type recovery by emitting complete helper metadata +4. Summary as explicit shape metadata +5. Recursive mutable generation +6. Broader `datly validate` coverage +7. Diagnostics standardization + +## Success Criteria + +The migration is structurally complete when: + +- Bootstrap and transcribe no longer need to reconstruct missing semantics from raw DQL. +- Generated patch routes are selected explicitly, not inferred heuristically. +- Summary, selector, and grouping behavior are fully representable in shape. +- Runtime does not need shape-recovery logic for helper/source types. +- Local translator outputs are matched by shape through tests, not through runtime workarounds. diff --git a/repository/shape/load/loader.go b/repository/shape/load/loader.go index 6cd3a585b..e6932e19e 100644 --- a/repository/shape/load/loader.go +++ b/repository/shape/load/loader.go @@ -3,10 +3,17 @@ package load import ( "context" "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path" "path/filepath" "reflect" + "regexp" "sort" "strings" + "time" "github.com/viant/datly/repository/shape" "github.com/viant/datly/repository/shape/compile/pipeline" @@ -21,6 +28,8 @@ import ( "github.com/viant/datly/view/state" "github.com/viant/datly/view/tags" "github.com/viant/sqlparser" + sqlxio "github.com/viant/sqlx/io" + "github.com/viant/tagly/format/text" "github.com/viant/xdatly/handler/response" ) @@ -43,7 +52,7 @@ func (l *Loader) LoadViews(ctx context.Context, planned *shape.PlanResult, opts opt(loadOptions) } } - pResult, resource, err := l.materialize(planned, loadOptions) + pResult, resource, err := l.materialize(ctx, planned, loadOptions) if err != nil { return nil, err } @@ -53,18 +62,36 @@ func (l *Loader) LoadViews(ctx context.Context, planned *shape.PlanResult, opts return &shape.ViewArtifacts{Resource: resource, Views: resource.Views}, nil } +// LoadResource implements shape.Loader. +func (l *Loader) LoadResource(ctx context.Context, planned *shape.PlanResult, opts ...shape.LoadOption) (*shape.ResourceArtifacts, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + loadOptions := &shape.LoadOptions{} + for _, opt := range opts { + if opt != nil { + opt(loadOptions) + } + } + _, resource, err := l.materialize(ctx, planned, loadOptions) + if err != nil { + return nil, err + } + return &shape.ResourceArtifacts{Resource: resource}, nil +} + // LoadComponent implements shape.Loader. func (l *Loader) LoadComponent(ctx context.Context, planned *shape.PlanResult, opts ...shape.LoadOption) (*shape.ComponentArtifact, error) { if err := ctx.Err(); err != nil { return nil, err } - loadOptions := &shape.LoadOptions{} + loadOptions := &shape.LoadOptions{UseTypeContextPackages: true} for _, opt := range opts { if opt != nil { opt(loadOptions) } } - pResult, resource, err := l.materialize(planned, loadOptions) + pResult, resource, err := l.materialize(ctx, planned, loadOptions) if err != nil { return nil, err } @@ -108,7 +135,7 @@ func allowsViewlessComponent(routes []*plan.ComponentRoute) bool { return false } -func (l *Loader) materialize(planned *shape.PlanResult, loadOptions *shape.LoadOptions) (*plan.Result, *view.Resource, error) { +func (l *Loader) materialize(ctx context.Context, planned *shape.PlanResult, loadOptions *shape.LoadOptions) (*plan.Result, *view.Resource, error) { if planned == nil || planned.Source == nil { return nil, nil, shape.ErrNilSource } @@ -130,10 +157,19 @@ func (l *Loader) materialize(planned *shape.PlanResult, loadOptions *shape.LoadO } resource.AddViews(aView) } + materializeConcreteViewSchemas(resource, planned.Source, pResult.TypeContext) + refineViewColumnConfigTypes(resource, planned.Source, pResult.TypeContext) + enrichConcreteViewColumns(resource) + assignViewSummarySchemas(resource, pResult, planned.Source) + enrichRelationLinkFields(pResult.Views) attachViewRelations(resource, pResult.Views) if err := enrichRelationHolderTypes(resource, pResult.Views); err != nil { return nil, nil, err } + refineSummarySchemas(resource) + applyVeltyAliasesToExecInputViews(resource, pResult) + materializeResourceTypes(resource, pResult.Views, planned.Source, pResult.TypeContext) + applyVeltyAliasesToExecInputViews(resource, pResult) rootView := rootResourceView(resource, pResult.Views) for _, item := range pResult.States { if item == nil { @@ -144,11 +180,14 @@ func (l *Loader) materialize(planned *shape.PlanResult, loadOptions *shape.LoadO continue } normalizeDerivedInputSchema(param, resource) + if rootView != nil { + inheritRootBodySchema(param, rootView) + } if rootView != nil { inheritRootOutputSchema(param, rootView) } - ensureMaterializedOutputSchema(param, rootView) - resource.AddParameters(param) + ensureMaterializedOutputSchema(param, rootView, planned.Source, pResult.TypeContext) + addResourceParameter(resource, param) } if err := shapevalidate.ValidateRelations(resource, resource.Views...); err != nil { return nil, nil, err @@ -166,7 +205,7 @@ func (l *Loader) materialize(planned *shape.PlanResult, loadOptions *shape.LoadO Cardinality: state.One, }, } - resource.AddParameters(constParam) + addResourceParameter(resource, constParam) } } bindTemplateParameters(resource) @@ -199,9 +238,11 @@ func buildComponent(source *shape.Source, pResult *plan.Result, resource *view.R component.TypeContext = cloneTypeContext(pResult.TypeContext) applyComponentRoutes(component, pResult.Components) applyViewMeta(component, pResult.Views) - applyStateBuckets(component, pResult.States, resource, loadOptions) - applyStateBuckets(component, synthesizeConstStates(pResult.Const), resource, loadOptions) - applyStateBuckets(component, synthesizeMissingRouteContractStates(component, pResult.Components), resource, loadOptions) + applyMutableRootMode(component, resource) + applyStateBuckets(component, pResult.States, resource, source, pResult.TypeContext, loadOptions) + applyStateBuckets(component, synthesizeConstStates(pResult.Const), resource, source, pResult.TypeContext, loadOptions) + applyStateBuckets(component, synthesizeMissingRouteContractStates(component, pResult.Components), resource, source, pResult.TypeContext, loadOptions) + synthesizeMutableExecHelpers(component, resource) component.Input = append(component.Input, synthesizePredicateStates(component.Input, component.Predicates)...) component.Directives = cloneDirectives(pResult.Directives) component.ColumnsDiscovery = pResult.ColumnsDiscovery @@ -209,6 +250,16 @@ func buildComponent(source *shape.Source, pResult *plan.Result, resource *view.R return component } +func addResourceParameter(resource *view.Resource, param *state.Parameter) { + if resource == nil || param == nil { + return + } + resource.AddParameters(param) + if named := resource.NamedParameters(); named != nil { + _ = named.Register(param) + } +} + func applyComponentRoutes(component *Component, routes []*plan.ComponentRoute) { if component == nil || len(routes) == 0 { return @@ -238,6 +289,22 @@ func applyComponentRoutes(component *Component, routes []*plan.ComponentRoute) { } } +func applyMutableRootMode(component *Component, resource *view.Resource) { + if component == nil || resource == nil { + return + } + if strings.EqualFold(strings.TrimSpace(component.Method), "GET") { + return + } + rootView := lookupNamedResourceView(resource, component.RootView) + if rootView == nil { + return + } + if rootView.Mode != view.ModeHandler { + rootView.Mode = view.ModeExec + } +} + func cloneComponentRoutes(routes []*plan.ComponentRoute) []*plan.ComponentRoute { if len(routes) == 0 { return nil @@ -256,6 +323,338 @@ func cloneComponentRoutes(routes []*plan.ComponentRoute) []*plan.ComponentRoute return result } +func synthesizeMutableExecHelpers(component *Component, resource *view.Resource) { + if component == nil || resource == nil { + return + } + if strings.EqualFold(strings.TrimSpace(component.Method), "GET") { + return + } + if templateType := strings.ToLower(strings.TrimSpace(componentTemplateType(component))); templateType != "" && templateType != "patch" { + return + } + rootView := lookupNamedResourceView(resource, component.RootView) + if rootView == nil || rootView.Schema == nil || rootView.Mode != view.ModeExec { + return + } + _ = view.WithTemplateParameterStateType(true)(rootView) + body := firstMutableBodyState(component.Input) + if body == nil || body.In == nil || body.Schema == nil { + return + } + bodyName := strings.TrimSpace(body.Name) + if bodyName == "" { + return + } + helperViewName := "Cur" + bodyName + if hasInputState(component.Input, helperViewName) { + return + } + keyFieldName, keyColumnName, keyType := mutableKeyDescriptor(rootView, body.Schema) + if keyFieldName == "" || keyColumnName == "" || keyType == nil { + return + } + componentDir := text.CaseFormatUpperCamel.Format(strings.TrimSpace(componentRootName(component, rootView, bodyName)), text.CaseFormatLowerUnderscore) + if componentDir == "" { + componentDir = text.CaseFormatUpperCamel.Format(bodyName, text.CaseFormatLowerUnderscore) + } + helperIDsName := helperViewName + keyFieldName + helperViewURI := path.Join(componentDir, text.CaseFormatUpperCamel.Format(helperViewName, text.CaseFormatLowerUnderscore)+".sql") + + valuesType := reflect.StructOf([]reflect.StructField{{ + Name: "Values", + Type: reflect.SliceOf(keyType), + Tag: reflect.StructTag(`json:",omitempty"`), + }}) + helperIDsSchema := state.NewSchema(reflect.PtrTo(valuesType)) + if helperIDsSchema != nil && strings.TrimSpace(helperIDsSchema.DataType) == "" { + helperIDsSchema.DataType = loaderSchemaTypeExpr(reflect.PtrTo(valuesType)) + } + helperIDsParam := &state.Parameter{ + Name: helperIDsName, + In: state.NewParameterLocation(bodyName), + Schema: helperIDsSchema.Clone(), + Output: &state.Codec{Name: "structql", Body: fmt.Sprintf(" SELECT ARRAY_AGG(%s) AS Values FROM `/` LIMIT 1", keyFieldName), Schema: helperIDsSchema.Clone()}, + PreserveSchema: true, + } + resource.Parameters.Append(helperIDsParam) + + helperSchema := rootView.Schema.Clone() + helperSchema.Cardinality = state.Many + helperViewParamSchema := helperSchema.Clone() + if helperViewParamSchema.Cardinality == "" { + helperViewParamSchema.Cardinality = state.Many + } + helperViewParam := &state.Parameter{ + Name: helperViewName, + In: state.NewViewLocation(helperViewName), + Tag: fmt.Sprintf(`view:"%s" sql:"uri=%s"`, helperViewName, helperViewURI), + Schema: helperViewParamSchema, + } + resource.Parameters.Append(helperViewParam) + bindViewTemplateParameters(rootView, []*state.Parameter{ + helperIDsParam, + helperViewParam, + }) + + helperView := view.NewView(helperViewName, "", view.WithSchema(helperSchema.Clone()), view.WithMode(view.ModeQuery)) + helperView.Table = rootView.Table + helperView.Connector = rootView.Connector + helperView.Columns = cloneViewColumns(rootView.Columns) + helperView.ColumnsConfig = cloneViewColumnsConfig(rootView.ColumnsConfig) + helperView.Selector = &view.Config{ + Namespace: strings.ToLower(truncateString(helperViewName, 2)), + Limit: 1000, + Constraints: &view.Constraints{ + Criteria: true, + Limit: true, + Offset: true, + Projection: true, + }, + } + helperView.Template = view.NewTemplate( + fmt.Sprintf("SELECT * FROM %s\nWHERE $criteria.In(%q, $Unsafe.%s.Values)", rootView.Table, keyColumnName, helperIDsName), + view.WithTemplateParameters(helperIDsParam), + view.WithTemplateUnsafeStateFromParameters(true), + view.WithTemplateDeclaredParametersOnly(true), + view.WithTemplateResourceParameterLookup(true), + ) + helperView.Template.SourceURL = helperViewURI + resource.AddViews(helperView) + component.Views = append(component.Views, helperViewName) + synthesizeMutableRootTemplate(component, rootView, body, keyFieldName, helperViewName) +} + +func componentTemplateType(component *Component) string { + if component == nil || component.Directives == nil { + return "" + } + return strings.TrimSpace(component.Directives.TemplateType) +} + +func synthesizeMutableRootTemplate(component *Component, rootView *view.View, body *plan.State, keyFieldName string, helperViewName string) { + if component == nil || rootView == nil || body == nil || body.Schema == nil { + return + } + method := strings.ToUpper(strings.TrimSpace(component.Method)) + switch method { + case "PATCH", "POST", "PUT": + default: + return + } + bodyName := strings.TrimSpace(body.Name) + if bodyName == "" || strings.TrimSpace(rootView.Table) == "" || strings.TrimSpace(keyFieldName) == "" { + return + } + if rootView.Template == nil { + rootView.Template = view.NewTemplate("", view.WithTemplateParameters()) + } + if rootView.TableBatches == nil { + rootView.TableBatches = map[string]bool{} + } + rootView.TableBatches[rootView.Table] = true + rootView.Template.Source = buildMutableRootTemplate(method, rootView.Table, bodyName, keyFieldName, helperViewName, body.Schema.Cardinality == state.Many) +} + +func buildMutableRootTemplate(method string, tableName string, bodyName string, keyFieldName string, helperViewName string, many bool) string { + var builder strings.Builder + if strings.ToUpper(strings.TrimSpace(method)) != "PUT" { + builder.WriteString(fmt.Sprintf("$sequencer.Allocate(%q, $Unsafe.%s, %q)\n\n", tableName, bodyName, keyFieldName)) + } + mapName := helperViewName + "By" + keyFieldName + builder.WriteString(fmt.Sprintf("#set($%s = $Unsafe.%s.IndexBy(%q))\n\n", mapName, helperViewName, keyFieldName)) + if many { + recordVar := "Rec" + bodyName + builder.WriteString(fmt.Sprintf("#foreach($%s in $Unsafe.%s)\n", recordVar, bodyName)) + builder.WriteString(fmt.Sprintf(" #if($%s.HasKey($%s.%s) == true)\n", mapName, recordVar, keyFieldName)) + builder.WriteString(fmt.Sprintf("$sql.Update($%s, %q);\n", recordVar, tableName)) + builder.WriteString(" #else\n") + builder.WriteString(fmt.Sprintf("$sql.Insert($%s, %q);\n", recordVar, tableName)) + builder.WriteString(" #end\n") + builder.WriteString("#end") + return builder.String() + } + builder.WriteString(fmt.Sprintf("#if($Unsafe.%s)\n", bodyName)) + builder.WriteString(fmt.Sprintf(" #if($%s.HasKey($Unsafe.%s.%s) == true)\n", mapName, bodyName, keyFieldName)) + builder.WriteString(fmt.Sprintf("$sql.Update($Unsafe.%s, %q);\n", bodyName, tableName)) + builder.WriteString(" #else\n") + builder.WriteString(fmt.Sprintf("$sql.Insert($Unsafe.%s, %q);\n", bodyName, tableName)) + builder.WriteString(" #end\n") + builder.WriteString("#end") + return builder.String() +} + +func loaderSchemaTypeExpr(rType reflect.Type) string { + if rType == nil { + return "" + } + switch rType.Kind() { + case reflect.Ptr: + return "*" + loaderSchemaTypeExpr(rType.Elem()) + case reflect.Slice: + return "[]" + loaderSchemaTypeExpr(rType.Elem()) + case reflect.Array: + return fmt.Sprintf("[%d]%s", rType.Len(), loaderSchemaTypeExpr(rType.Elem())) + case reflect.Map: + return "map[" + loaderSchemaTypeExpr(rType.Key()) + "]" + loaderSchemaTypeExpr(rType.Elem()) + default: + return rType.String() + } +} + +func firstMutableBodyState(states []*plan.State) *plan.State { + for _, item := range states { + if item == nil || item.In == nil || item.In.Kind != state.KindRequestBody { + continue + } + if !item.IsAnonymous() { + continue + } + return item + } + return nil +} + +func hasInputState(states []*plan.State, name string) bool { + for _, item := range states { + if item == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) { + return true + } + } + return false +} + +func componentRootName(component *Component, rootView *view.View, fallback string) string { + if rootView != nil && strings.TrimSpace(rootView.Name) != "" { + return rootView.Name + } + if component != nil && strings.TrimSpace(component.RootView) != "" { + return component.RootView + } + return fallback +} + +func mutableKeyDescriptor(rootView *view.View, schema *state.Schema) (string, string, reflect.Type) { + if fieldName, columnName, rType := mutableKeyFromType(schema); fieldName != "" && columnName != "" && rType != nil { + return fieldName, columnName, rType + } + if rootView == nil { + return "", "", nil + } + for _, column := range rootView.Columns { + if column == nil { + continue + } + dbColumn := strings.TrimSpace(column.DatabaseColumn) + if dbColumn == "" { + dbColumn = strings.TrimSpace(column.Name) + } + if !strings.EqualFold(dbColumn, "ID") { + continue + } + fieldName := strings.TrimSpace(column.FieldName()) + if fieldName == "" { + fieldName = "Id" + } + switch strings.ToLower(strings.TrimSpace(column.DataType)) { + case "int", "integer", "bigint", "smallint": + return fieldName, dbColumn, reflect.TypeOf(0) + } + } + return "", "", nil +} + +func mutableKeyFromType(schema *state.Schema) (string, string, reflect.Type) { + if schema == nil || schema.Type() == nil { + return "", "", nil + } + rType := schema.Type() + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return "", "", nil + } + if field, ok := rType.FieldByName("Id"); ok { + return "Id", "ID", derefType(field.Type) + } + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + sqlxTag := field.Tag.Get("sqlx") + if sqlxTag == "ID" || strings.Contains(sqlxTag, "name=ID") { + return field.Name, "ID", derefType(field.Type) + } + } + return "", "", nil +} + +func derefType(rType reflect.Type) reflect.Type { + for rType != nil && rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + return rType +} + +func truncateString(value string, max int) string { + value = strings.TrimSpace(value) + if max <= 0 || len(value) <= max { + return value + } + return value[:max] +} + +func cloneViewColumns(columns []*view.Column) []*view.Column { + if len(columns) == 0 { + return nil + } + result := make([]*view.Column, 0, len(columns)) + for _, item := range columns { + if item == nil { + continue + } + cloned := *item + result = append(result, &cloned) + } + return result +} + +func cloneViewColumnsConfig(columns map[string]*view.ColumnConfig) map[string]*view.ColumnConfig { + if len(columns) == 0 { + return nil + } + result := make(map[string]*view.ColumnConfig, len(columns)) + for key, item := range columns { + if item == nil { + continue + } + cloned := *item + result[key] = &cloned + } + return result +} + +func lookupNamedResourceView(resource *view.Resource, name string) *view.View { + if resource == nil { + return nil + } + if strings.TrimSpace(name) != "" { + for _, item := range resource.Views { + if item != nil && strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) { + return item + } + } + } + for _, item := range resource.Views { + if item != nil { + return item + } + } + return nil +} + func firstComponentRoute(routes []*plan.ComponentRoute) *plan.ComponentRoute { for _, item := range routes { if item != nil { @@ -580,7 +979,7 @@ func indexViewDeclaration(component *Component, viewName string, decl *plan.View // applyStateBuckets sorts plan states into the typed buckets on the component // (Input, Output, Meta, Async, Other) based on the state's location kind. -func applyStateBuckets(component *Component, states []*plan.State, resource *view.Resource, loadOptions *shape.LoadOptions) { +func applyStateBuckets(component *Component, states []*plan.State, resource *view.Resource, source *shape.Source, ctx *typectx.Context, loadOptions *shape.LoadOptions) { for _, item := range states { if item == nil { continue @@ -592,18 +991,35 @@ func applyStateBuckets(component *Component, states []*plan.State, resource *vie if loadOptions != nil && loadOptions.UseTypeContextPackages { inheritTypeContextSchemaPackage(&cloned.Parameter, component) } + if selector := strings.TrimSpace(cloned.QuerySelector); selector != "" { + if component.QuerySelectors == nil { + component.QuerySelectors = map[string][]string{} + } + component.QuerySelectors[selector] = append(component.QuerySelectors[selector], cloned.Name) + } normalizeDerivedInputSchema(&cloned.Parameter, resource) inheritRootBodySchema(&cloned.Parameter, rootResourceView(resource, nil)) + inheritRootOutputSchema(&cloned.Parameter, rootResourceView(resource, nil)) + ensureMaterializedOutputSchema(&cloned.Parameter, rootResourceView(resource, nil), source, ctx) kind := state.Kind(strings.ToLower(item.KindString())) inName := item.InName() if kind == "" && inName == "" { component.Other = append(component.Other, cloned) continue } + if cloned.EmitOutput && kind != state.KindOutput { + outputClone := clonePlanState(cloned) + if outputClone != nil { + component.Output = append(component.Output, outputClone) + } + } switch kind { case state.KindQuery, state.KindPath, state.KindHeader, state.KindRequestBody, state.KindView, state.KindComponent, state.KindConst, state.KindForm, state.KindCookie, state.KindRequest, "": + if kind == state.KindComponent { + normalizeDynamicComponentSchema(&cloned.Parameter) + } component.Input = append(component.Input, cloned) case state.KindOutput: component.Output = append(component.Output, cloned) @@ -617,6 +1033,15 @@ func applyStateBuckets(component *Component, states []*plan.State, resource *vie } } +func normalizeDynamicComponentSchema(param *state.Parameter) { + if param == nil || param.Schema == nil { + return + } + param.Schema.SetType(reflect.TypeOf((*interface{})(nil)).Elem()) + param.Schema.Package = "" + param.Schema.PackagePath = "" +} + func inheritTypeContextSchemaPackage(param *state.Parameter, component *Component) { if param == nil || param.Schema == nil || component == nil || component.TypeContext == nil { return @@ -628,20 +1053,24 @@ func inheritTypeContextSchemaPackage(param *state.Parameter, component *Componen return } typeName := strings.TrimSpace(shared.FirstNotEmpty(param.Schema.Name, param.Schema.DataType)) - if typeName == "" || strings.Contains(typeName, ".") { + if !shouldInheritTypeContextPackage(typeName) { return } if _, err := types.LookupType(nil, typeName); err == nil { return } - pkgPath := strings.TrimSpace(component.TypeContext.PackagePath) - if pkgPath == "" { - pkgPath = strings.TrimSpace(component.TypeContext.DefaultPackage) + if baseType := schemaBaseTypeName(typeName); baseType != typeName { + if _, err := types.LookupType(nil, baseType); err == nil { + return + } } + pkg, pkgPath := schemaTypeContextPackage(component.TypeContext) if pkgPath == "" { return } - param.Schema.Package = pkgPath + if pkg != "" { + param.Schema.Package = pkg + } param.Schema.PackagePath = pkgPath } @@ -656,23 +1085,103 @@ func inheritViewSchemaPackage(aView *view.View, ctx *typectx.Context) { return } typeName := strings.TrimSpace(shared.FirstNotEmpty(aView.Schema.Name, aView.Schema.DataType)) - if typeName == "" || strings.Contains(typeName, ".") { + if !shouldInheritTypeContextPackage(typeName) { return } if _, err := types.LookupType(nil, typeName); err == nil { return } - pkgPath := strings.TrimSpace(ctx.PackagePath) - if pkgPath == "" { - pkgPath = strings.TrimSpace(ctx.DefaultPackage) + if baseType := schemaBaseTypeName(typeName); baseType != typeName { + if _, err := types.LookupType(nil, baseType); err == nil { + return + } } + pkg, pkgPath := schemaTypeContextPackage(ctx) if pkgPath == "" { return } - aView.Schema.Package = pkgPath + if pkg != "" { + aView.Schema.Package = pkg + } aView.Schema.PackagePath = pkgPath } +func schemaTypeContextPackage(ctx *typectx.Context) (string, string) { + if ctx == nil { + return "", "" + } + pkg := strings.TrimSpace(ctx.PackageName) + pkgPath := strings.TrimSpace(ctx.PackagePath) + if pkgPath == "" { + pkgPath = strings.TrimSpace(ctx.DefaultPackage) + } + if pkg == "" && pkgPath != "" { + pkg = path.Base(pkgPath) + } + return pkg, pkgPath +} + +func shouldInheritTypeContextPackage(typeName string) bool { + baseType := schemaBaseTypeName(typeName) + if baseType == "" { + return false + } + if strings.Contains(baseType, ".") { + return false + } + if builtinSchemaTypes[baseType] { + return false + } + return true +} + +func schemaBaseTypeName(typeName string) string { + typeName = strings.TrimSpace(typeName) + for { + switch { + case strings.HasPrefix(typeName, "[]"): + typeName = strings.TrimSpace(typeName[2:]) + case strings.HasPrefix(typeName, "*"): + typeName = strings.TrimSpace(typeName[1:]) + default: + goto done + } + } +done: + if typeName == "" { + return "" + } + if strings.ContainsAny(typeName, " {}[](),") { + return "" + } + return typeName +} + +var builtinSchemaTypes = map[string]bool{ + "any": true, + "bool": true, + "byte": true, + "complex128": true, + "complex64": true, + "error": true, + "float32": true, + "float64": true, + "int": true, + "int16": true, + "int32": true, + "int64": true, + "int8": true, + "interface{}": true, + "rune": true, + "string": true, + "uint": true, + "uint16": true, + "uint32": true, + "uint64": true, + "uint8": true, + "uintptr": true, +} + func synthesizeMissingRouteContractStates(component *Component, routes []*plan.ComponentRoute) []*plan.State { if component == nil || len(routes) == 0 { return nil @@ -887,6 +1396,7 @@ func cloneDirectives(input *dqlshape.Directives) *dqlshape.Directives { ret := &dqlshape.Directives{ Meta: strings.TrimSpace(input.Meta), DefaultConnector: strings.TrimSpace(input.DefaultConnector), + TemplateType: strings.TrimSpace(input.TemplateType), Dest: strings.TrimSpace(input.Dest), InputDest: strings.TrimSpace(input.InputDest), OutputDest: strings.TrimSpace(input.OutputDest), @@ -927,7 +1437,7 @@ func cloneDirectives(input *dqlshape.Directives) *dqlshape.Directives { } } } - if ret.Meta == "" && ret.DefaultConnector == "" && + if ret.Meta == "" && ret.DefaultConnector == "" && ret.TemplateType == "" && ret.Dest == "" && ret.InputDest == "" && ret.OutputDest == "" && ret.RouterDest == "" && ret.InputType == "" && ret.OutputType == "" && ret.Cache == nil && ret.MCP == nil && ret.Route == nil && len(ret.Const) == 0 { @@ -984,6 +1494,9 @@ func materializeView(item *plan.View) (*view.View, error) { schema := newSchema(schemaType, item.Cardinality) opts := []view.Option{view.WithSchema(schema), view.WithMode(mode)} + if item.Groupable != nil { + opts = append(opts, view.WithGroupable(*item.Groupable)) + } if item.Connector != "" { opts = append(opts, view.WithConnectorRef(item.Connector)) @@ -991,19 +1504,20 @@ func materializeView(item *plan.View) (*view.View, error) { if item.SQL != "" || item.SQLURI != "" { tmpl := view.NewTemplate(item.SQL) tmpl.SourceURL = item.SQLURI - if strings.TrimSpace(item.Summary) != "" { - name := strings.TrimSpace(item.SummaryName) - if name == "" { - name = "Summary" - } - tmpl.Summary = &view.TemplateSummary{ - Name: name, - Source: item.Summary, - Kind: view.MetaKindRecord, - } - } opts = append(opts, view.WithTemplate(tmpl)) } + if strings.TrimSpace(item.Summary) != "" || strings.TrimSpace(item.SummaryURL) != "" { + name := strings.TrimSpace(item.SummaryName) + if name == "" { + name = "Summary" + } + opts = append(opts, view.WithSummary(&view.TemplateSummary{ + Name: name, + Source: item.Summary, + SourceURL: item.SummaryURL, + Kind: view.MetaKindRecord, + })) + } if item.CacheRef != "" { opts = append(opts, view.WithCache(&view.Cache{Reference: shared.Reference{Ref: item.CacheRef}})) } @@ -1045,20 +1559,60 @@ func materializeView(item *plan.View) (*view.View, error) { if tag := strings.TrimSpace(cfg.Tag); tag != "" { columnCfg.Tag = stringPtr(tag) } + if cfg.Groupable != nil { + columnCfg.Groupable = boolPtr(*cfg.Groupable) + } } } - if strings.TrimSpace(item.SelectorNamespace) != "" || item.SelectorNoLimit != nil || item.SelectorLimit != nil { + if strings.TrimSpace(item.SelectorNamespace) != "" || item.SelectorNoLimit != nil || item.SelectorLimit != nil || + item.SelectorCriteria != nil || item.SelectorProjection != nil || item.SelectorOrderBy != nil || + item.SelectorOffset != nil || item.SelectorPage != nil || len(item.SelectorFilterable) > 0 || + len(item.SelectorOrderByColumns) > 0 { if aView.Selector == nil { aView.Selector = &view.Config{} } + if aView.Selector.Constraints == nil { + aView.Selector.Constraints = &view.Constraints{} + } if strings.TrimSpace(item.SelectorNamespace) != "" { aView.Selector.Namespace = strings.TrimSpace(item.SelectorNamespace) } if item.SelectorNoLimit != nil { aView.Selector.NoLimit = *item.SelectorNoLimit + aView.Selector.Constraints.Limit = true } if item.SelectorLimit != nil { aView.Selector.Limit = *item.SelectorLimit + aView.Selector.Constraints.Limit = true + } + if item.SelectorCriteria != nil || item.SelectorProjection != nil || item.SelectorOrderBy != nil || + item.SelectorOffset != nil || item.SelectorPage != nil || len(item.SelectorFilterable) > 0 || + len(item.SelectorOrderByColumns) > 0 { + if item.SelectorCriteria != nil { + aView.Selector.Constraints.Criteria = *item.SelectorCriteria + } + if item.SelectorProjection != nil { + aView.Selector.Constraints.Projection = *item.SelectorProjection + } + if item.SelectorOrderBy != nil { + aView.Selector.Constraints.OrderBy = *item.SelectorOrderBy + } + if item.SelectorOffset != nil { + aView.Selector.Constraints.Offset = *item.SelectorOffset + } + if item.SelectorPage != nil { + value := *item.SelectorPage + aView.Selector.Constraints.Page = &value + } + if len(item.SelectorFilterable) > 0 { + aView.Selector.Constraints.Filterable = append([]string(nil), item.SelectorFilterable...) + } + if len(item.SelectorOrderByColumns) > 0 { + aView.Selector.Constraints.OrderByColumn = map[string]string{} + for key, value := range item.SelectorOrderByColumns { + aView.Selector.Constraints.OrderByColumn[key] = value + } + } } } if item.Self != nil { @@ -1084,44 +1638,1824 @@ func materializeView(item *plan.View) (*view.View, error) { aView.Columns = cols } } + if aView.Schema != nil && aView.Schema.Type() == nil { + if rowType := synthesizeViewSchemaType(aView); rowType != nil { + aView.Schema.SetType(rowType) + aView.Schema.EnsurePointer() + } + } return aView, nil } -func allowsDeferredSchema(item *plan.View, mode view.Mode) bool { - if item == nil { - return false +func assignViewSummarySchemas(resource *view.Resource, pResult *plan.Result, source *shape.Source) { + if resource == nil || pResult == nil { + return } - if mode != view.ModeQuery { - return false + index := resource.Views.Index() + for _, item := range pResult.Views { + if item == nil || strings.TrimSpace(item.Summary) == "" { + continue + } + aView, err := index.Lookup(item.Name) + if err != nil || aView == nil || aView.Template == nil || aView.Template.Summary == nil { + continue + } + if schema := aView.Template.Summary.Schema; schema != nil && (schema.Type() != nil || (strings.TrimSpace(schema.DataType) != "" && strings.TrimSpace(schema.DataType) != "?")) { + continue + } + summaryType := resolveSummarySchemaType(source, pResult.TypeContext, item.SummaryName) + if summaryType == nil { + summaryType = inferSummarySchemaType(item) + } + if summaryType == nil { + continue + } + aView.Template.Summary.Schema = materializedSummarySchema(summaryType, item.SummaryName, pResult.TypeContext) } - return strings.TrimSpace(item.Table) != "" || strings.TrimSpace(item.SQL) != "" || strings.TrimSpace(item.SQLURI) != "" } -func shouldDeferQuerySchemaType(rType reflect.Type, mode view.Mode) bool { - if rType == nil || mode != view.ModeQuery { +func inferSummarySchemaType(item *plan.View) reflect.Type { + if item == nil { + return nil + } + summarySQL := strings.TrimSpace(item.Summary) + if summarySQL == "" { + return nil + } + queryNode, _, err := pipeline.ParseSelectWithDiagnostic(pipeline.NormalizeParserSQL(summarySQL)) + if err != nil || queryNode == nil { + return nil + } + _, elementType, _ := pipeline.InferProjectionType(queryNode) + return unwrapSummarySchemaType(elementType) +} + +func unwrapSummarySchemaType(rType reflect.Type) reflect.Type { + for rType != nil { + switch rType.Kind() { + case reflect.Slice, reflect.Array, reflect.Ptr: + rType = rType.Elem() + default: + return rType + } + } + return nil +} + +func resolveSummarySchemaType(source *shape.Source, ctx *typectx.Context, summaryName string) reflect.Type { + summaryName = strings.TrimSpace(summaryName) + if summaryName == "" || source == nil { + return nil + } + registry := source.EnsureTypeRegistry() + if registry == nil { + return nil + } + candidates := []string{summaryName} + if !strings.HasSuffix(summaryName, "View") { + candidates = append([]string{summaryName + "View"}, candidates...) + } + resolver := typectx.NewResolver(registry, ctx) + for _, candidate := range candidates { + if lookup := registry.Lookup(candidate); lookup != nil && lookup.Type != nil { + return lookup.Type + } + if resolved, err := resolver.Resolve(candidate); err == nil && resolved != "" { + if lookup := registry.Lookup(resolved); lookup != nil && lookup.Type != nil { + return lookup.Type + } + } + } + return nil +} + +func resolveViewSchemaType(source *shape.Source, ctx *typectx.Context, aView *view.View, typeName string) reflect.Type { + candidates := []string{strings.TrimSpace(typeName)} + if aView != nil && aView.Schema != nil { + if name := strings.TrimSpace(aView.Schema.Name); name != "" { + candidates = append([]string{name}, candidates...) + } + } + seen := map[string]bool{} + for _, candidate := range candidates { + candidate = strings.TrimSpace(candidate) + if candidate == "" || seen[candidate] { + continue + } + seen[candidate] = true + if astType := resolveViewSchemaASTType(source, ctx, aView, candidate); astType != nil { + return astType + } + if source != nil { + registry := source.EnsureTypeRegistry() + if registry != nil { + resolver := typectx.NewResolver(registry, ctx) + if lookup := registry.Lookup(candidate); lookup != nil && lookup.Type != nil { + return lookup.Type + } + if resolved, err := resolver.Resolve(candidate); err == nil && resolved != "" { + if lookup := registry.Lookup(resolved); lookup != nil && lookup.Type != nil { + return lookup.Type + } + } + } + } + } + return nil +} + +func materializeConcreteViewSchemas(resource *view.Resource, source *shape.Source, ctx *typectx.Context) { + if resource == nil { + return + } + visited := map[*view.View]bool{} + for _, aView := range resource.Views { + applyConcreteViewSchemaType(aView, source, ctx, visited) + } +} + +func enrichConcreteViewColumns(resource *view.Resource) { + if resource == nil { + return + } + visited := map[*view.View]bool{} + for _, aView := range resource.Views { + enrichViewColumnsFromSchema(aView, visited) + } +} + +func enrichViewColumnsFromSchema(aView *view.View, visited map[*view.View]bool) { + if aView == nil || visited[aView] { + return + } + visited[aView] = true + appendMissingColumnsFromSchema(aView) + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + enrichViewColumnsFromSchema(&rel.Of.View, visited) + } +} + +func refineViewColumnConfigTypes(resource *view.Resource, source *shape.Source, ctx *typectx.Context) { + if resource == nil { + return + } + visited := map[*view.View]bool{} + for _, aView := range resource.Views { + refineViewColumnConfigType(aView, source, ctx, visited) + } +} + +func refineViewColumnConfigType(aView *view.View, source *shape.Source, ctx *typectx.Context, visited map[*view.View]bool) { + if aView == nil || visited[aView] { + return + } + visited[aView] = true + applyConfiguredColumnTypes(aView, source, ctx) + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + refineViewColumnConfigType(&rel.Of.View, source, ctx, visited) + } +} + +func applyConfiguredColumnTypes(aView *view.View, source *shape.Source, ctx *typectx.Context) { + if aView == nil || len(aView.ColumnsConfig) == 0 { + return + } + if aView.Schema != nil && aView.Schema.Type() != nil { + if refined := refineSchemaTypeByColumnConfig(aView.Schema.Type(), aView.ColumnsConfig, source, ctx); refined != nil && refined != aView.Schema.Type() { + aView.Schema.SetType(refined) + aView.Schema.EnsurePointer() + } + } + for _, column := range aView.Columns { + if column == nil { + continue + } + cfg := lookupColumnConfig(aView.ColumnsConfig, column.Name, column.DatabaseColumn, column.FieldName()) + if cfg == nil || strings.TrimSpace(valueOrEmpty(cfg.DataType)) == "" { + continue + } + if resolved := resolveColumnConfigType(strings.TrimSpace(*cfg.DataType), source, ctx); resolved != nil { + column.DataType = strings.TrimSpace(*cfg.DataType) + column.SetColumnType(resolved) + } + } +} + +func refineSchemaTypeByColumnConfig(rType reflect.Type, configs map[string]*view.ColumnConfig, source *shape.Source, ctx *typectx.Context) reflect.Type { + if rType == nil { + return nil + } + switch rType.Kind() { + case reflect.Ptr: + if refined := refineSchemaTypeByColumnConfig(rType.Elem(), configs, source, ctx); refined != nil && refined != rType.Elem() { + return reflect.PtrTo(refined) + } + return rType + case reflect.Slice: + if refined := refineSchemaTypeByColumnConfig(rType.Elem(), configs, source, ctx); refined != nil && refined != rType.Elem() { + return reflect.SliceOf(refined) + } + return rType + case reflect.Array: + if refined := refineSchemaTypeByColumnConfig(rType.Elem(), configs, source, ctx); refined != nil && refined != rType.Elem() { + return reflect.ArrayOf(rType.Len(), refined) + } + return rType + case reflect.Struct: + fields := make([]reflect.StructField, 0, rType.NumField()) + changed := false + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + cfg := lookupColumnConfig(configs, field.Name, summaryLookupName(field)) + if cfg != nil && strings.TrimSpace(valueOrEmpty(cfg.DataType)) != "" { + if resolved := resolveColumnConfigType(strings.TrimSpace(*cfg.DataType), source, ctx); resolved != nil && resolved != field.Type { + field.Type = resolved + changed = true + } + } + fields = append(fields, field) + } + if changed { + return reflect.StructOf(fields) + } + } + return rType +} + +func lookupColumnConfig(configs map[string]*view.ColumnConfig, names ...string) *view.ColumnConfig { + if len(configs) == 0 { + return nil + } + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" { + continue + } + if cfg := configs[name]; cfg != nil { + return cfg + } + for key, cfg := range configs { + if strings.EqualFold(strings.TrimSpace(key), name) { + return cfg + } + } + } + return nil +} + +func valueOrEmpty(value *string) string { + if value == nil { + return "" + } + return *value +} + +func resolveColumnConfigType(dataType string, source *shape.Source, ctx *typectx.Context) reflect.Type { + dataType = strings.TrimSpace(dataType) + if dataType == "" { + return nil + } + if resolved, err := types.LookupType(extension.Config.Types.Lookup, dataType); err == nil && resolved != nil { + return resolved + } + if source == nil { + return nil + } + registry := source.EnsureTypeRegistry() + if registry == nil { + return nil + } + resolver := typectx.NewResolver(registry, ctx) + if lookup := registry.Lookup(dataType); lookup != nil && lookup.Type != nil { + return lookup.Type + } + if resolved, err := resolver.Resolve(dataType); err == nil && resolved != "" { + if lookup := registry.Lookup(resolved); lookup != nil && lookup.Type != nil { + return lookup.Type + } + } + return nil +} + +func appendMissingColumnsFromSchema(aView *view.View) { + if aView == nil || aView.Schema == nil || aView.Schema.Type() == nil { + return + } + structType := types.EnsureStruct(aView.Schema.Type()) + if structType == nil || structType.Kind() != reflect.Struct { + return + } + ioColumns, err := sqlxio.StructColumns(structType, "sqlx") + if err != nil || len(ioColumns) == 0 { + return + } + type columnMeta struct { + dataType string + nullable bool + } + metadata := map[string]columnMeta{} + for _, ioColumn := range ioColumns { + if ioColumn == nil { + continue + } + meta := columnMeta{dataType: columnDataTypeFromScanType(ioColumn.ScanType())} + meta.nullable, _ = ioColumn.Nullable() + tagName := "" + if tag := ioColumn.Tag(); tag != nil { + tagName = strings.TrimSpace(tag.Name()) + } + for _, key := range []string{ + strings.ToUpper(strings.TrimSpace(ioColumn.Name())), + strings.ToUpper(tagName), + } { + if key != "" { + metadata[key] = meta + } + } + } + for _, column := range aView.Columns { + if column == nil || strings.TrimSpace(column.DataType) != "" { + continue + } + for _, key := range []string{ + strings.ToUpper(strings.TrimSpace(column.Name)), + strings.ToUpper(strings.TrimSpace(column.DatabaseColumn)), + strings.ToUpper(strings.TrimSpace(column.FieldName())), + } { + if meta, ok := metadata[key]; ok { + if meta.dataType != "" { + column.DataType = meta.dataType + } + column.Nullable = meta.nullable + break + } + } + } + existing := map[string]bool{} + for _, column := range aView.Columns { + if column == nil { + continue + } + for _, key := range []string{ + strings.ToUpper(strings.TrimSpace(column.Name)), + strings.ToUpper(strings.TrimSpace(column.DatabaseColumn)), + strings.ToUpper(strings.TrimSpace(column.FieldName())), + } { + if key != "" { + existing[key] = true + } + } + } + for _, ioColumn := range ioColumns { + name := strings.TrimSpace(ioColumn.Name()) + if name == "" || existing[strings.ToUpper(name)] { + continue + } + tagValue := "" + if tag := ioColumn.Tag(); tag != nil { + tagValue = tag.Raw + if tag.Ns != "" { + if strings.HasSuffix(tagValue, `"`) { + tagValue = strings.TrimRight(tagValue, `"`) + ",ns=" + tag.Ns + `"` + } else { + tagValue += `",ns=` + tag.Ns + `"` + } + } + } + nullable, _ := ioColumn.Nullable() + column := view.NewColumn(name, ioColumn.DatabaseTypeName(), ioColumn.ScanType(), nullable, view.WithColumnTag(tagValue)) + if stateTag, _ := tags.ParseStateTags(reflect.StructTag(column.Tag), nil); stateTag != nil { + if stateTag.Format != nil { + column.FormatTag = stateTag.Format + } + if codec := stateTag.Codec; codec != nil { + column.Codec = &state.Codec{Name: codec.Name, Args: codec.Arguments} + } + } + aView.Columns = append(aView.Columns, column) + existing[strings.ToUpper(name)] = true + if dbName := strings.ToUpper(strings.TrimSpace(column.DatabaseColumn)); dbName != "" { + existing[dbName] = true + } + } +} + +func columnDataTypeFromScanType(scanType reflect.Type) string { + if scanType == nil { + return "" + } + if schema := schemaFromReflectType(scanType); schema != nil { + return strings.TrimSpace(schema.DataType) + } + return strings.TrimSpace(scanType.String()) +} + +func applyConcreteViewSchemaType(aView *view.View, source *shape.Source, ctx *typectx.Context, visited map[*view.View]bool) { + if aView == nil || visited[aView] { + return + } + visited[aView] = true + if aView.Schema != nil { + if resolved := resolveViewSchemaType(source, ctx, aView, relationTypeName(aView)); resolved != nil { + if resolved.Kind() != reflect.Ptr { + resolved = reflect.PtrTo(resolved) + } + aView.Schema.SetType(resolved) + aView.Schema.EnsurePointer() + } + } + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + applyConcreteViewSchemaType(&rel.Of.View, source, ctx, visited) + } +} + +func resolveViewSchemaASTType(source *shape.Source, ctx *typectx.Context, aView *view.View, typeName string) reflect.Type { + pkgDir := resolveViewSchemaPackageDir(source, ctx, aView) + if pkgDir == "" { + return nil + } + return parseNamedStructType(pkgDir, typeName) +} + +func resolveViewSchemaPackageDir(source *shape.Source, ctx *typectx.Context, aView *view.View) string { + if aView != nil && aView.Schema != nil { + if pkgPath := strings.TrimSpace(firstNonEmpty(aView.Schema.ModulePath, aView.Schema.PackagePath)); pkgPath != "" { + if dir := resolveTypePackageDirFromSource(pkgPath, ctx, source); dir != "" { + return dir + } + } + } + if ctx == nil { + return "" + } + if dir := strings.TrimSpace(ctx.PackageDir); dir != "" { + resolvedDir := dir + if filepath.IsAbs(dir) { + if isUsablePackageDir(dir) { + return dir + } + resolvedDir = dir + } else if moduleRoot := nearestModuleRoot(source); moduleRoot != "" { + resolvedDir = filepath.Join(moduleRoot, filepath.FromSlash(dir)) + if isUsablePackageDir(resolvedDir) { + return resolvedDir + } + } + } + if pkgPath := strings.TrimSpace(firstNonEmpty(ctx.PackagePath, ctx.DefaultPackage)); pkgPath != "" { + return resolveTypePackageDirFromSource(pkgPath, ctx, source) + } + return "" +} + +func isUsablePackageDir(dir string) bool { + if strings.TrimSpace(dir) == "" { + return false + } + info, err := os.Stat(dir) + return err == nil && info.IsDir() +} + +func resolveTypePackageDirFromSource(pkgPath string, ctx *typectx.Context, source *shape.Source) string { + if pkgPath == "" { + return "" + } + moduleRoot := nearestModuleRoot(source) + if moduleRoot == "" { + if ctx != nil && strings.TrimSpace(ctx.PackagePath) == strings.TrimSpace(pkgPath) { + if dir := strings.TrimSpace(ctx.PackageDir); dir != "" { + if filepath.IsAbs(dir) { + return dir + } + } + } + return "" + } + modulePath := detectModulePath(moduleRoot) + if modulePath != "" { + if rel, ok := packagePathRelative(modulePath, pkgPath); ok { + if rel == "" { + return moduleRoot + } + return filepath.Join(moduleRoot, filepath.FromSlash(rel)) + } + } + if ctx != nil && strings.TrimSpace(ctx.PackagePath) == strings.TrimSpace(pkgPath) { + if dir := strings.TrimSpace(ctx.PackageDir); dir != "" { + if filepath.IsAbs(dir) { + return dir + } + return filepath.Join(moduleRoot, filepath.FromSlash(dir)) + } + } + return "" +} + +func detectModulePath(moduleRoot string) string { + if moduleRoot == "" { + return "" + } + data, err := os.ReadFile(filepath.Join(moduleRoot, "go.mod")) + if err != nil { + return "" + } + lines := strings.Split(string(data), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "module ") { + continue + } + return strings.TrimSpace(strings.TrimPrefix(line, "module ")) + } + return "" +} + +func packagePathRelative(modulePath, pkgPath string) (string, bool) { + modulePath = strings.TrimSpace(modulePath) + pkgPath = strings.TrimSpace(pkgPath) + if modulePath == "" || pkgPath == "" { + return "", false + } + if pkgPath == modulePath { + return "", true + } + if !strings.HasPrefix(pkgPath, modulePath+"/") { + return "", false + } + return strings.TrimPrefix(pkgPath, modulePath+"/"), true +} + +func nearestModuleRoot(source *shape.Source) string { + if source == nil || strings.TrimSpace(source.Path) == "" { + return "" + } + current := filepath.Dir(strings.TrimSpace(source.Path)) + for current != "" && current != string(filepath.Separator) && current != "." { + if _, err := os.Stat(filepath.Join(current, "go.mod")); err == nil { + return current + } + parent := filepath.Dir(current) + if parent == current { + break + } + current = parent + } + return "" +} + +func parseNamedStructType(pkgDir, typeName string) reflect.Type { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, pkgDir, nil, parser.ParseComments) + if err != nil || len(pkgs) == 0 { + return nil + } + specs := map[string]*ast.TypeSpec{} + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.TYPE { + continue + } + for _, spec := range gen.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok || typeSpec.Name == nil { + continue + } + specs[typeSpec.Name.Name] = typeSpec + } + } + } + } + cache := map[string]reflect.Type{} + inProgress := map[string]bool{} + var buildNamed func(name string) reflect.Type + var buildExpr func(expr ast.Expr) reflect.Type + + buildNamed = func(name string) reflect.Type { + if cached, ok := cache[name]; ok { + return cached + } + if inProgress[name] { + return reflect.TypeOf(new(interface{})).Elem() + } + spec := specs[name] + if spec == nil { + return nil + } + inProgress[name] = true + rType := buildExpr(spec.Type) + delete(inProgress, name) + if rType != nil { + cache[name] = rType + } + return rType + } + + buildExpr = func(expr ast.Expr) reflect.Type { + switch actual := expr.(type) { + case *ast.Ident: + switch actual.Name { + case "string": + return reflect.TypeOf("") + case "bool": + return reflect.TypeOf(true) + case "int": + return reflect.TypeOf(int(0)) + case "int8": + return reflect.TypeOf(int8(0)) + case "int16": + return reflect.TypeOf(int16(0)) + case "int32": + return reflect.TypeOf(int32(0)) + case "int64": + return reflect.TypeOf(int64(0)) + case "uint": + return reflect.TypeOf(uint(0)) + case "uint8": + return reflect.TypeOf(uint8(0)) + case "uint16": + return reflect.TypeOf(uint16(0)) + case "uint32": + return reflect.TypeOf(uint32(0)) + case "uint64": + return reflect.TypeOf(uint64(0)) + case "float32": + return reflect.TypeOf(float32(0)) + case "float64": + return reflect.TypeOf(float64(0)) + case "interface{}", "any": + return reflect.TypeOf(new(interface{})).Elem() + default: + return buildNamed(actual.Name) + } + case *ast.StarExpr: + if inner := buildExpr(actual.X); inner != nil { + return reflect.PtrTo(inner) + } + case *ast.ArrayType: + if actual.Len == nil { + if inner := buildExpr(actual.Elt); inner != nil { + return reflect.SliceOf(inner) + } + } + case *ast.MapType: + key := buildExpr(actual.Key) + value := buildExpr(actual.Value) + if key != nil && value != nil { + return reflect.MapOf(key, value) + } + case *ast.InterfaceType: + return reflect.TypeOf(new(interface{})).Elem() + case *ast.SelectorExpr: + if ident, ok := actual.X.(*ast.Ident); ok && actual.Sel != nil { + if ident.Name == "time" && actual.Sel.Name == "Time" { + return reflect.TypeOf(time.Time{}) + } + if resolved, err := types.LookupType(extension.Config.Types.Lookup, ident.Name+"."+actual.Sel.Name); err == nil && resolved != nil { + return resolved + } + } + case *ast.StructType: + fields := make([]reflect.StructField, 0, len(actual.Fields.List)) + seen := map[string]bool{} + for _, field := range actual.Fields.List { + if field == nil { + continue + } + fieldType := buildExpr(field.Type) + if fieldType == nil { + continue + } + tag := reflect.StructTag("") + if field.Tag != nil { + tag = reflect.StructTag(strings.Trim(field.Tag.Value, "`")) + } + if len(field.Names) == 0 { + if name := exportedEmbeddedFieldName(field.Type); name != "" { + if seen[name] { + continue + } + seen[name] = true + fields = append(fields, reflect.StructField{Name: name, Type: fieldType, Tag: tag, Anonymous: true}) + } + continue + } + for _, name := range field.Names { + if name == nil || !name.IsExported() { + continue + } + if seen[name.Name] { + continue + } + seen[name.Name] = true + fields = append(fields, reflect.StructField{Name: name.Name, Type: fieldType, Tag: tag}) + } + } + if len(fields) > 0 { + return reflect.StructOf(fields) + } + } + return nil + } + return buildNamed(typeName) +} + +func exportedEmbeddedFieldName(expr ast.Expr) string { + switch actual := expr.(type) { + case *ast.Ident: + if actual.IsExported() { + return actual.Name + } + case *ast.SelectorExpr: + if actual.Sel != nil && actual.Sel.IsExported() { + return actual.Sel.Name + } + case *ast.StarExpr: + return exportedEmbeddedFieldName(actual.X) + } + return "" +} + +func materializedSummarySchema(summaryType reflect.Type, summaryName string, ctx *typectx.Context) *state.Schema { + if summaryType == nil { + return nil + } + if summaryType.Kind() != reflect.Ptr { + summaryType = reflect.PtrTo(summaryType) + } + schema := state.NewSchema(summaryType) + typeName := strings.TrimSpace(summarySchemaName(summaryName)) + if typeName != "" { + schema.Name = typeName + if typeExpr, typePkg := summarySchemaTypeRef(typeName, ctx); typeExpr != "" { + schema.DataType = typeExpr + schema.Package = typePkg + if ctx != nil { + schema.PackagePath = strings.TrimSpace(ctx.PackagePath) + } + } + } + schema.EnsurePointer() + return schema +} + +func refineSummarySchemas(resource *view.Resource) { + if resource == nil { + return + } + visited := map[*view.View]bool{} + for _, aView := range resource.Views { + refineViewSummarySchemas(aView, visited) + } +} + +func materializeResourceTypes(resource *view.Resource, planned []*plan.View, source *shape.Source, ctx *typectx.Context) { + if resource == nil { + return + } + seen := map[string]bool{} + plannedByName := map[string]*plan.View{} + for _, item := range planned { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + plannedByName[strings.ToLower(strings.TrimSpace(item.Name))] = item + } + for _, item := range resource.Types { + if item == nil { + continue + } + name := strings.ToLower(strings.TrimSpace(item.Name)) + if name == "" { + continue + } + seen[name] = true + } + visited := map[*view.View]bool{} + for _, aView := range resource.Views { + collectViewTypes(aView, resource, seen, visited, plannedByName, source, ctx) + } +} + +func collectViewTypes(aView *view.View, resource *view.Resource, seen map[string]bool, visited map[*view.View]bool, plannedByName map[string]*plan.View, source *shape.Source, ctx *typectx.Context) { + if aView == nil || resource == nil || visited[aView] { + return + } + visited[aView] = true + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + collectViewTypes(&rel.Of.View, resource, seen, visited, plannedByName, source, ctx) + } + if aView.Template != nil && aView.Template.Summary != nil { + addSchemaTypeDefinition(resource, aView.Template.Summary.Schema, seen) + } + addViewTypeDefinition(resource, aView, seen, plannedByName, source, ctx) +} + +func addViewTypeDefinition(resource *view.Resource, aView *view.View, seen map[string]bool, plannedByName map[string]*plan.View, source *shape.Source, ctx *typectx.Context) { + if resource == nil || aView == nil { + return + } + baseName := strings.TrimSpace(aView.Ref) + if baseName == "" { + baseName = strings.TrimSpace(aView.Name) + } + typeName := state.SanitizeTypeName(baseName) + "View" + key := strings.ToLower(typeName) + if seen[key] { + return + } + def := &view.TypeDefinition{ + Name: typeName, + Package: viewSchemaPackage(aView), + ModulePath: viewSchemaModulePath(aView), + Ptr: viewSchemaPtr(aView), + } + fieldNames := map[string]bool{} + typedFields := collectTypedViewDefinitionFields(aView, plannedByName, source, ctx, typeName) + if len(typedFields) > 0 { + for _, field := range typedFields { + addTypeDefinitionField(def, fieldNames, field) + } + } else { + for _, column := range aView.Columns { + if field := typeDefinitionFieldFromColumn(aView, column); field != nil { + addTypeDefinitionField(def, fieldNames, field) + } + } + } + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + if field := typeDefinitionFieldFromRelation(rel); field != nil { + addTypeDefinitionField(def, fieldNames, field) + } + if field := typeDefinitionFieldFromRelationSummary(rel); field != nil { + addTypeDefinitionField(def, fieldNames, field) + } + } + if len(def.Fields) == 0 { + return + } + resource.Types = append(resource.Types, def) + seen[key] = true +} + +func collectTypedViewDefinitionFields(aView *view.View, plannedByName map[string]*plan.View, source *shape.Source, ctx *typectx.Context, typeName string) []*view.Field { + var result []*view.Field + seen := map[string]bool{} + appendFields := func(fields []*view.Field) { + for _, field := range fields { + if field == nil { + continue + } + name := strings.TrimSpace(field.Name) + if name == "" || seen[name] { + continue + } + seen[name] = true + result = append(result, field) + } + } + appendFields(typeDefinitionFieldsFromSchema(aView.Schema)) + appendFields(typeDefinitionFieldsFromReflectType(resolveViewSchemaType(source, ctx, aView, typeName))) + if len(result) == 0 { + appendFields(typeDefinitionFieldsFromPlannedView(plannedViewFor(aView, plannedByName))) + } + return result +} + +func addTypeDefinitionField(def *view.TypeDefinition, names map[string]bool, field *view.Field) { + if def == nil || field == nil { + return + } + name := strings.TrimSpace(field.Name) + if name == "" || names[name] { + return + } + names[name] = true + def.AddField(field) +} + +func plannedViewFor(aView *view.View, plannedByName map[string]*plan.View) *plan.View { + if aView == nil || len(plannedByName) == 0 { + return nil + } + for _, key := range []string{strings.TrimSpace(aView.Ref), strings.TrimSpace(aView.Name)} { + if key == "" { + continue + } + if item := plannedByName[strings.ToLower(key)]; item != nil { + return item + } + } + return nil +} + +func addSchemaTypeDefinition(resource *view.Resource, schema *state.Schema, seen map[string]bool) { + addSchemaTypeDefinitionWithName(resource, schema, strings.TrimSpace(summarySchemaName(schemaName(schema))), seen) +} + +func addSchemaTypeDefinitionWithName(resource *view.Resource, schema *state.Schema, typeName string, seen map[string]bool) { + if resource == nil || schema == nil { + return + } + name := strings.TrimSpace(typeName) + if name == "" { + return + } + key := strings.ToLower(name) + if seen[key] { + return + } + cloned := schema.Clone() + cloned.Name = name + if cloned.DataType == "" && cloned.Type() != nil { + cloned.DataType = cloned.TypeName() + } + resource.Types = append(resource.Types, &view.TypeDefinition{ + Name: name, + DataType: strings.TrimSpace(cloned.DataType), + Cardinality: cloned.Cardinality, + Package: strings.TrimSpace(cloned.Package), + ModulePath: firstNonEmpty(strings.TrimSpace(cloned.ModulePath), strings.TrimSpace(cloned.PackagePath)), + Schema: cloned, + }) + seen[key] = true +} + +func schemaName(schema *state.Schema) string { + if schema == nil { + return "" + } + return schema.Name +} + +func viewSchemaPackage(aView *view.View) string { + if aView == nil || aView.Schema == nil { + return "" + } + return strings.TrimSpace(aView.Schema.Package) +} + +func viewSchemaModulePath(aView *view.View) string { + if aView == nil || aView.Schema == nil { + return "" + } + return firstNonEmpty(strings.TrimSpace(aView.Schema.ModulePath), strings.TrimSpace(aView.Schema.PackagePath)) +} + +func viewSchemaPtr(aView *view.View) bool { + if aView == nil || aView.Schema == nil { + return false + } + if rType := aView.Schema.Type(); rType != nil { + if rType.Kind() == reflect.Slice { + rType = rType.Elem() + } + return rType.Kind() == reflect.Ptr + } + typeName := strings.TrimSpace(firstNonEmpty(aView.Schema.DataType, aView.Schema.Name)) + return strings.HasPrefix(typeName, "*") +} + +func typeDefinitionFieldFromColumn(aView *view.View, column *view.Column) *view.Field { + if column == nil { + return nil + } + fieldName := strings.TrimSpace(column.FieldName()) + if fieldName == "" && column.Field() != nil { + fieldName = strings.TrimSpace(column.Field().Name) + } + if fieldName == "" { + caseFormat := text.CaseFormatUpperCamel + if aView != nil && aView.CaseFormat != "" { + caseFormat = aView.CaseFormat + } + fieldName = state.StructFieldName(caseFormat, column.Name) + } + if fieldName == "" { + return nil + } + schema := columnFieldSchema(column) + if schema == nil { + return nil + } + return &view.Field{ + Name: fieldName, + Column: strings.TrimSpace(column.DatabaseColumn), + FromName: fieldName, + Schema: schema, + Tag: strings.TrimSpace(column.Tag), + Cardinality: schema.Cardinality, + } +} + +func columnFieldSchema(column *view.Column) *state.Schema { + if column == nil { + return nil + } + if rType := column.ColumnType(); rType != nil { + return schemaFromReflectType(rType) + } + if dataType := strings.TrimSpace(column.DataType); dataType != "" { + return &state.Schema{DataType: dataType, Cardinality: state.One} + } + return nil +} + +func typeDefinitionFieldFromRelation(rel *view.Relation) *view.Field { + if rel == nil || rel.Of == nil { + return nil + } + typeName := relationTypeName(&rel.Of.View) + if typeName == "" || strings.TrimSpace(rel.Holder) == "" { + return nil + } + schema := relationSchema(&rel.Of.View, typeName, rel.Cardinality) + return &view.Field{ + Name: strings.TrimSpace(rel.Holder), + Schema: schema, + Cardinality: rel.Cardinality, + } +} + +func typeDefinitionFieldFromRelationSummary(rel *view.Relation) *view.Field { + if rel == nil || rel.Of == nil || rel.Of.View.Template == nil || rel.Of.View.Template.Summary == nil || rel.Of.View.Template.Summary.Schema == nil { + return nil + } + name := strings.TrimSpace(rel.Of.View.Template.Summary.Name) + if name == "" { + return nil + } + schema := rel.Of.View.Template.Summary.Schema.Clone() + schema.EnsurePointer() + return &view.Field{ + Name: name, + Schema: schema, + Tag: `json:",omitempty" yaml:",omitempty" sqlx:"-"`, + } +} + +func relationTypeName(aView *view.View) string { + if aView == nil { + return "" + } + baseName := strings.TrimSpace(aView.Ref) + if baseName == "" { + baseName = strings.TrimSpace(aView.Name) + } + if baseName == "" { + return "" + } + return state.SanitizeTypeName(baseName) + "View" +} + +func relationSchema(aView *view.View, typeName string, cardinality state.Cardinality) *state.Schema { + schema := &state.Schema{ + Name: typeName, + DataType: "*" + typeName, + Cardinality: cardinality, + } + if aView != nil && aView.Schema != nil { + schema.Package = strings.TrimSpace(aView.Schema.Package) + schema.PackagePath = strings.TrimSpace(aView.Schema.PackagePath) + schema.ModulePath = firstNonEmpty(strings.TrimSpace(aView.Schema.ModulePath), strings.TrimSpace(aView.Schema.PackagePath)) + } + return schema +} + +func typeDefinitionFieldsFromSchema(schema *state.Schema) []*view.Field { + if schema == nil || schema.Type() == nil { + return nil + } + rType := schema.Type() + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return nil + } + result := make([]*view.Field, 0, rType.NumField()) + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if !field.IsExported() { + continue + } + result = append(result, &view.Field{ + Name: field.Name, + Schema: schemaFromReflectType(field.Type), + Tag: string(field.Tag), + FromName: field.Name, + Cardinality: state.One, + }) + } + return result +} + +func typeDefinitionFieldsFromPlannedView(item *plan.View) []*view.Field { + if item == nil { + return nil + } + return typeDefinitionFieldsFromReflectType(bestSchemaType(item)) +} + +func typeDefinitionFieldsFromReflectType(rType reflect.Type) []*view.Field { + if rType == nil { + return nil + } + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return nil + } + result := make([]*view.Field, 0, rType.NumField()) + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if !field.IsExported() { + continue + } + result = append(result, &view.Field{ + Name: field.Name, + Schema: schemaFromReflectType(field.Type), + Tag: string(field.Tag), + FromName: field.Name, + Cardinality: state.One, + }) + } + return result +} + +func schemaFromReflectType(rType reflect.Type) *state.Schema { + if rType == nil { + return nil + } + schema := state.NewSchema(rType) + if schema == nil { + return nil + } + if schema.Name == "" && schema.DataType == "" { + schema.DataType = rType.String() + if schema.Cardinality == "" { + schema.Cardinality = state.One + } + } + return schema +} + +func synthesizeViewSchemaType(aView *view.View) reflect.Type { + return synthesizeViewSchemaTypeWithOptions(aView, false) +} + +func synthesizeViewSchemaTypeWithOptions(aView *view.View, includeVelty bool) reflect.Type { + if aView == nil || len(aView.Columns) == 0 { + return nil + } + fields := make([]reflect.StructField, 0, len(aView.Columns)) + seen := map[string]bool{} + for _, column := range aView.Columns { + structField := viewStructFieldFromColumn(aView, column, includeVelty) + if structField == nil { + continue + } + if seen[structField.Name] { + continue + } + seen[structField.Name] = true + fields = append(fields, *structField) + } + if len(fields) == 0 { + return nil + } + return reflect.PtrTo(reflect.StructOf(fields)) +} + +func viewStructFieldFromColumn(aView *view.View, column *view.Column, includeVelty bool) *reflect.StructField { + if column == nil { + return nil + } + schema := columnFieldSchema(column) + if schema == nil || schema.Type() == nil { + return nil + } + fieldName := strings.TrimSpace(column.FieldName()) + if fieldName == "" && column.Field() != nil { + fieldName = strings.TrimSpace(column.Field().Name) + } + if fieldName == "" { + caseFormat := text.CaseFormatUpperCamel + if aView != nil && aView.CaseFormat != "" { + caseFormat = aView.CaseFormat + } + fieldName = state.StructFieldName(caseFormat, column.Name) + } + fieldName = strings.TrimSpace(fieldName) + if fieldName == "" { + return nil + } + tag := strings.TrimSpace(column.Tag) + sqlxTag := strings.TrimSpace(strings.TrimSpace(column.DatabaseColumn)) + if sqlxTag == "" { + sqlxTag = strings.TrimSpace(column.Name) + } + if sqlxTag != "" && !strings.Contains(tag, `sqlx:"`) { + if tag != "" { + tag += " " + } + tag += fmt.Sprintf(`sqlx:"%s"`, sqlxTag) + } + if includeVelty && !strings.Contains(tag, `velty:"`) { + veltyNames := []string{column.Name} + if fieldName != "" && fieldName != column.Name { + veltyNames = append(veltyNames, fieldName) + } + if tag != "" { + tag += " " + } + tag += fmt.Sprintf(`velty:"names=%s"`, strings.Join(veltyNames, "|")) + } + return &reflect.StructField{ + Name: fieldName, + Type: schema.Type(), + Tag: reflect.StructTag(tag), + } +} + +func applyVeltyAliasesToExecInputViews(resource *view.Resource, pResult *plan.Result) { + if resource == nil || pResult == nil || !planUsesVelty(resource, pResult) { + return + } + viewNames := map[string]bool{} + for _, item := range pResult.States { + if item == nil || item.In == nil || item.In.Kind != state.KindView { + continue + } + viewName := strings.TrimSpace(item.Name) + if name := strings.TrimSpace(item.In.Name); name != "" { + viewName = name + } + if viewName == "" { + continue + } + viewNames[strings.ToLower(viewName)] = true + } + if len(viewNames) == 0 { + return + } + for _, aView := range resource.Views { + if aView == nil || aView.Schema == nil { + continue + } + if !viewNames[strings.ToLower(strings.TrimSpace(aView.Name))] && + !viewNames[strings.ToLower(strings.TrimSpace(aView.Reference.Ref))] { + continue + } + applyVeltyAliasesToViewColumns(aView) + if !schemaNeedsVeltyAliases(aView.Schema.Type()) { + continue + } + if rebuilt := synthesizeViewSchemaTypeWithOptions(aView, true); rebuilt != nil { + aView.Schema.SetType(rebuilt) + aView.Schema.EnsurePointer() + continue + } + if rebuilt := ensureSchemaTypeVeltyAliases(aView.Schema.Type()); rebuilt != nil { + aView.Schema.SetType(rebuilt) + } + } +} + +func planUsesVelty(resource *view.Resource, pResult *plan.Result) bool { + if pResult == nil { + return false + } + for _, route := range pResult.Components { + if route == nil { + continue + } + method := strings.ToUpper(strings.TrimSpace(route.Method)) + if method != "" && method != "GET" && strings.TrimSpace(route.Handler) == "" { + return true + } + } + if resource != nil { + for _, aView := range resource.Views { + if aView != nil && aView.Mode == view.ModeExec { + return true + } + } + } + return false +} + +func schemaNeedsVeltyAliases(rType reflect.Type) bool { + if rType == nil { + return true + } + if rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return false + } + for i := 0; i < rType.NumField(); i++ { + if strings.TrimSpace(rType.Field(i).Tag.Get("velty")) != "" { + return false + } + } + return true +} + +func applyVeltyAliasesToViewColumns(aView *view.View) { + if aView == nil { + return + } + for _, column := range aView.Columns { + if column == nil || strings.Contains(column.Tag, `velty:"`) { + continue + } + fieldName := strings.TrimSpace(column.FieldName()) + if fieldName == "" && column.Field() != nil { + fieldName = strings.TrimSpace(column.Field().Name) + } + if fieldName == "" { + caseFormat := text.CaseFormatUpperCamel + if aView.CaseFormat != "" { + caseFormat = aView.CaseFormat + } + fieldName = state.StructFieldName(caseFormat, column.Name) + } + veltyNames := []string{column.Name} + if fieldName != "" && fieldName != column.Name { + veltyNames = append(veltyNames, fieldName) + } + tag := strings.TrimSpace(column.Tag) + if tag != "" { + tag += " " + } + tag += fmt.Sprintf(`velty:"names=%s"`, strings.Join(veltyNames, "|")) + column.Tag = strings.TrimSpace(tag) + } +} + +func ensureSchemaTypeVeltyAliases(rType reflect.Type) reflect.Type { + if rType == nil { + return nil + } + original := rType + isSlice := false + if rType.Kind() == reflect.Slice { + isSlice = true + rType = rType.Elem() + } + wasPtr := false + if rType.Kind() == reflect.Ptr { + wasPtr = true + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return nil + } + fields := make([]reflect.StructField, 0, rType.NumField()) + changed := false + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + tag := string(field.Tag) + if strings.TrimSpace(field.Tag.Get("velty")) == "" { + sqlxName := summaryTagName(field.Tag.Get("sqlx")) + if sqlxName == "" { + sqlxName = field.Name + } + veltyNames := []string{sqlxName} + if field.Name != "" && field.Name != sqlxName { + veltyNames = append(veltyNames, field.Name) + } + if strings.TrimSpace(tag) != "" { + tag += " " + } + tag += fmt.Sprintf(`velty:"names=%s"`, strings.Join(veltyNames, "|")) + changed = true + } + field.Tag = reflect.StructTag(strings.TrimSpace(tag)) + fields = append(fields, field) + } + if !changed { + return original + } + rebuilt := reflect.StructOf(fields) + if wasPtr { + rebuilt = reflect.PtrTo(rebuilt) + } + if isSlice { + rebuilt = reflect.SliceOf(rebuilt) + } + return rebuilt +} + +func refineViewSummarySchemas(aView *view.View, visited map[*view.View]bool) { + if aView == nil || visited[aView] { + return + } + visited[aView] = true + if aView.Template != nil && aView.Template.Summary != nil && aView.Template.Summary.Schema != nil { + if refined := refineSummarySchemaType(aView.Template.Summary.Schema, aView); refined != nil { + aView.Template.Summary.Schema = refined + } + } + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + refineViewSummarySchemas(&rel.Of.View, visited) + } +} + +func refineSummarySchemaType(summarySchema *state.Schema, ownerView *view.View) *state.Schema { + if summarySchema == nil || ownerView == nil { + return nil + } + summaryType := summarySchema.Type() + if summaryType == nil { + return nil + } + if summaryType.Kind() == reflect.Ptr { + summaryType = summaryType.Elem() + } + if summaryType.Kind() != reflect.Struct { + return nil + } + ownerFields := map[string]reflect.StructField{} + if ownerSchema := ownerView.Schema; ownerSchema != nil && ownerSchema.CompType() != nil { + ownerType := ownerSchema.CompType() + if ownerType.Kind() == reflect.Ptr { + ownerType = ownerType.Elem() + } + if ownerType.Kind() == reflect.Struct { + for i := 0; i < ownerType.NumField(); i++ { + field := ownerType.Field(i) + ownerFields[strings.ToUpper(strings.TrimSpace(field.Name))] = field + if sqlxName := summaryTagName(field.Tag.Get("sqlx")); sqlxName != "" { + ownerFields[strings.ToUpper(sqlxName)] = field + } + } + } + } + for _, column := range ownerView.Columns { + if column == nil { + continue + } + columnType := summaryColumnType(column) + if columnType == nil { + continue + } + fieldName := strings.TrimSpace(column.FieldName()) + if fieldName == "" { + fieldName = strings.TrimSpace(column.Name) + } + field := reflect.StructField{Name: fieldName, Type: columnType, Tag: reflect.StructTag(column.Tag)} + if key := strings.ToUpper(strings.TrimSpace(column.Name)); key != "" { + if _, ok := ownerFields[key]; ok { + continue + } + ownerFields[key] = field + } + if key := strings.ToUpper(strings.TrimSpace(column.DatabaseColumn)); key != "" { + if _, ok := ownerFields[key]; ok { + continue + } + ownerFields[key] = field + } + } + if len(ownerFields) == 0 { + return nil + } + fields := make([]reflect.StructField, 0, summaryType.NumField()) + changed := false + for i := 0; i < summaryType.NumField(); i++ { + field := summaryType.Field(i) + if ownerField, ok := ownerFields[strings.ToUpper(summaryLookupName(field))]; ok && ownerField.Type != nil && ownerField.Type != field.Type { + field.Type = ownerField.Type + changed = true + } + fields = append(fields, field) + } + if !changed { + return nil + } + refinedType := reflect.StructOf(fields) + refined := summarySchema.Clone() + refined.SetType(refinedType) + return refined +} + +func refreshInlineSummarySchemas(ctx context.Context, resource *view.Resource) { + if resource == nil { + return + } + visited := map[*view.View]bool{} + for _, aView := range resource.Views { + refreshViewInlineSummarySchema(ctx, resource, aView, visited) + } +} + +// RefineSummarySchemas reapplies summary schema refinement using current view schema/column metadata. +// This is useful after late column discovery updated view columns post-load. +func RefineSummarySchemas(resource *view.Resource) { + if resource == nil { + return + } + refineSummarySchemas(resource) +} + +func refreshViewInlineSummarySchema(ctx context.Context, resource *view.Resource, aView *view.View, visited map[*view.View]bool) { + if aView == nil || visited[aView] { + return + } + visited[aView] = true + if aView.GetResource() == nil { + aView.SetResource(resource) + } + if shouldRefreshInlineSummarySchema(aView) { + restore := suppressInlineTemplateURLs(aView.Template) + _ = aView.Template.Init(ctx, resource, aView) + restore() + } + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + if rel.Of.View.GetResource() == nil { + rel.Of.View.SetResource(resource) + } + refreshViewInlineSummarySchema(ctx, resource, &rel.Of.View, visited) + } +} + +func shouldRefreshInlineSummarySchema(aView *view.View) bool { + if aView == nil || aView.Template == nil || aView.Template.Summary == nil { + return false + } + if aView.Connector == nil || strings.TrimSpace(aView.Connector.Ref) == "" { + return false + } + if strings.TrimSpace(aView.Template.Summary.Source) == "" { + return false + } + if strings.TrimSpace(aView.Template.Source) == "" && strings.TrimSpace(aView.Template.SourceURL) == "" { + return false + } + return true +} + +func suppressInlineTemplateURLs(tmpl *view.Template) func() { + if tmpl == nil { + return func() {} + } + sourceURL := tmpl.SourceURL + summarySourceURL := "" + if strings.TrimSpace(tmpl.Source) != "" { + tmpl.SourceURL = "" + } + if tmpl.Summary != nil { + summarySourceURL = tmpl.Summary.SourceURL + if strings.TrimSpace(tmpl.Summary.Source) != "" { + tmpl.Summary.SourceURL = "" + } + } + return func() { + tmpl.SourceURL = sourceURL + if tmpl.Summary != nil { + tmpl.Summary.SourceURL = summarySourceURL + } + } +} + +func summaryColumnType(column *view.Column) reflect.Type { + if column == nil { + return nil + } + if rType := column.ColumnType(); rType != nil { + return rType + } + switch strings.ToLower(strings.TrimSpace(column.DataType)) { + case "int", "integer", "smallint", "signed", "int32": + if column.Nullable { + return reflect.TypeOf((*int)(nil)) + } + return reflect.TypeOf(int(0)) + case "int64", "bigint": + if column.Nullable { + return reflect.TypeOf((*int64)(nil)) + } + return reflect.TypeOf(int64(0)) + case "float", "float32", "real": + if column.Nullable { + return reflect.TypeOf((*float32)(nil)) + } + return reflect.TypeOf(float32(0)) + case "float64", "double", "numeric", "decimal": + if column.Nullable { + return reflect.TypeOf((*float64)(nil)) + } + return reflect.TypeOf(float64(0)) + case "bool", "boolean": + if column.Nullable { + return reflect.TypeOf((*bool)(nil)) + } + return reflect.TypeOf(false) + case "string", "text", "varchar", "char", "uuid", "json", "jsonb", "": + if column.Nullable { + return reflect.TypeOf((*string)(nil)) + } + return reflect.TypeOf("") + default: + return nil + } +} + +func summarySchemaName(summaryName string) string { + summaryName = strings.TrimSpace(summaryName) + if summaryName == "" { + return "" + } + if strings.HasSuffix(summaryName, "View") { + return summaryName + } + return exportedSummaryTypeName(summaryName) + "View" +} + +func summarySchemaTypeRef(typeName string, ctx *typectx.Context) (string, string) { + typeName = strings.TrimSpace(typeName) + if typeName == "" { + return "", "" + } + if ctx != nil { + if pkgAlias := strings.TrimSpace(ctx.PackageName); pkgAlias != "" { + return "*" + pkgAlias + "." + typeName, pkgAlias + } + if pkgPath := strings.TrimSpace(ctx.PackagePath); pkgPath != "" { + alias := summaryPackageAlias(pkgPath, ctx) + return "*" + alias + "." + typeName, alias + } + } + return "*" + typeName, "" +} + +func summaryPackageAlias(pkgPath string, ctx *typectx.Context) string { + pkgPath = strings.TrimSpace(pkgPath) + if pkgPath == "" { + return "" + } + if ctx != nil { + for _, item := range ctx.Imports { + if strings.TrimSpace(item.Package) != pkgPath { + continue + } + if alias := strings.TrimSpace(item.Alias); alias != "" { + return alias + } + } + if strings.TrimSpace(ctx.PackagePath) == pkgPath && strings.TrimSpace(ctx.PackageName) != "" { + return strings.TrimSpace(ctx.PackageName) + } + } + if index := strings.LastIndex(pkgPath, "/"); index != -1 && index+1 < len(pkgPath) { + return pkgPath[index+1:] + } + return pkgPath +} + +func exportedSummaryTypeName(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "" + } + parts := strings.FieldsFunc(name, func(r rune) bool { + return r == '_' || r == '-' || r == ' ' || r == '.' + }) + if len(parts) == 0 { + return "" + } + var b strings.Builder + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + b.WriteString(strings.ToUpper(part[:1])) + if len(part) > 1 { + b.WriteString(part[1:]) + } + } + return b.String() +} + +func summaryLookupName(field reflect.StructField) string { + if sqlxName := summaryTagName(field.Tag.Get("sqlx")); sqlxName != "" { + return sqlxName + } + return strings.TrimSpace(field.Name) +} + +func summaryTagName(tag string) string { + tag = strings.TrimSpace(tag) + if tag == "" { + return "" + } + if strings.HasPrefix(tag, "name=") { + tag = strings.TrimPrefix(tag, "name=") + } + if idx := strings.Index(tag, ","); idx != -1 { + tag = tag[:idx] + } + return strings.TrimSpace(tag) +} + +func allowsDeferredSchema(item *plan.View, mode view.Mode) bool { + if item == nil { + return false + } + if mode != view.ModeQuery { + return false + } + return strings.TrimSpace(item.Table) != "" || strings.TrimSpace(item.SQL) != "" || strings.TrimSpace(item.SQLURI) != "" +} + +func shouldDeferQuerySchemaType(rType reflect.Type, mode view.Mode) bool { + if rType == nil || mode != view.ModeQuery { return false } for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { rType = rType.Elem() } - return rType.Kind() == reflect.Map || rType.Kind() == reflect.Interface + if rType.Kind() == reflect.Map || rType.Kind() == reflect.Interface { + return true + } + if rType.Kind() == reflect.Struct { + if cols := inferColumnsFromType(rType); len(cols) > 0 && inferredColumnsArePlaceholders(cols) { + return true + } + } + return false } func bestSchemaType(item *plan.View) reflect.Type { if item.FieldType != nil { - return item.FieldType + return normalizeViewSchemaReflectType(item, item.FieldType) } if item.ElementType != nil { - return item.ElementType + return normalizeViewSchemaReflectType(item, item.ElementType) } return nil } +func normalizeViewSchemaReflectType(item *plan.View, rType reflect.Type) reflect.Type { + if item == nil || rType == nil { + return rType + } + schemaType := strings.TrimSpace(item.SchemaType) + if !strings.HasPrefix(schemaType, "*") { + return rType + } + if strings.EqualFold(strings.TrimSpace(item.Cardinality), string(state.Many)) { + if rType.Kind() == reflect.Slice { + elem := rType.Elem() + if elem.Kind() != reflect.Ptr { + return reflect.SliceOf(reflect.PtrTo(elem)) + } + } + return rType + } + if rType.Kind() != reflect.Ptr { + return reflect.PtrTo(rType) + } + return rType +} + func stringPtr(value string) *string { ret := value return &ret } +func boolPtr(value bool) *bool { + ret := value + return &ret +} + func toViewRelations(input []*plan.Relation) []*view.Relation { if len(input) == 0 { return nil @@ -1171,6 +3505,95 @@ func toViewLinks(input []*plan.RelationLink, parent bool) view.Links { return result } +func enrichRelationLinkFields(planned []*plan.View) { + if len(planned) == 0 { + return + } + byName := map[string]*plan.View{} + for _, item := range planned { + if item == nil || strings.TrimSpace(item.Name) == "" { + continue + } + byName[strings.ToLower(strings.TrimSpace(item.Name))] = item + } + for _, item := range planned { + if item == nil || len(item.Relations) == 0 { + continue + } + for _, rel := range item.Relations { + if rel == nil || len(rel.On) == 0 { + continue + } + parentPlan := item + if parentName := strings.TrimSpace(rel.Parent); parentName != "" { + if candidate, ok := byName[strings.ToLower(parentName)]; ok && candidate != nil { + parentPlan = candidate + } + } + refPlan := byName[strings.ToLower(strings.TrimSpace(rel.Ref))] + for _, link := range rel.On { + if link == nil { + continue + } + if link.ParentField == "" { + if field := fieldNameForColumn(parentPlan, link.ParentColumn); field != "" { + link.ParentField = field + link.ParentNamespace = "" + } + } + if link.RefField == "" { + if field := fieldNameForColumn(refPlan, link.RefColumn); field != "" { + link.RefField = field + link.RefNamespace = "" + } + } + } + } + } +} + +func fieldNameForColumn(item *plan.View, column string) string { + column = strings.TrimSpace(column) + if column == "" { + return "" + } + fallback := pipeline.ExportedName(column) + if item == nil { + return fallback + } + rType := bestSchemaType(item) + if rType == nil { + return fallback + } + for rType.Kind() == reflect.Ptr || rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + } + if rType.Kind() != reflect.Struct { + return fallback + } + normalizedColumn := normalizeRelationColumnName(column) + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if !field.IsExported() || shouldSkipInferredField(field) { + continue + } + candidate := sqlxColumnName(field) + if candidate == "" { + candidate = field.Name + } + if normalizeRelationColumnName(candidate) == normalizedColumn { + return field.Name + } + } + return fallback +} + +func normalizeRelationColumnName(name string) string { + name = strings.TrimSpace(strings.ToLower(name)) + name = strings.ReplaceAll(name, "_", "") + return name +} + func newSchema(rType reflect.Type, cardinality string) *state.Schema { if rType == nil { schema := &state.Schema{} @@ -1238,14 +3661,173 @@ func attachViewRelations(resource *view.Resource, planned []*plan.View) { if inferOneToOneRelation(parent, ref, relation) { relation.Cardinality = state.One } - relation.Of.View.Ref = ref.Name - relation.Of.View.Name = "" - relation.Of.View.Columns = ref.Columns + relation.Of.View = cloneRelationView(ref, relation.Of.View) parent.With = append(parent.With, relation) } } } +func cloneRelationView(ref *view.View, current view.View) view.View { + if ref == nil { + return current + } + cloned := *ref + cloned.Ref = ref.Name + cloned.Name = "" + if currentName := strings.TrimSpace(current.Name); currentName != "" && !strings.EqualFold(currentName, ref.Name) { + cloned.Name = current.Name + } + if ref.Schema != nil { + cloned.Schema = ref.Schema.Clone() + } + if ref.Template != nil { + templateCopy := *ref.Template + if ref.Template.Schema != nil { + templateCopy.Schema = ref.Template.Schema.Clone() + } + if strings.TrimSpace(templateCopy.Source) != "" { + templateCopy.SourceURL = "" + } + if ref.Template.Summary != nil { + summaryCopy := *ref.Template.Summary + if ref.Template.Summary.Schema != nil { + summaryCopy.Schema = ref.Template.Summary.Schema.Clone() + } + if strings.TrimSpace(summaryCopy.Source) != "" { + summaryCopy.SourceURL = "" + } + templateCopy.Summary = &summaryCopy + } + cloned.Template = &templateCopy + } + if cloned.Selector != nil && cloned.Selector.Limit > 0 { + if cloned.Batch == nil { + cloned.Batch = &view.Batch{} + } + if cloned.Batch.Size == 0 || cloned.Batch.Size > 1 { + cloned.Batch.Size = 1 + } + } + return cloned +} + +func hideRelationSummaryLinkFields(relation *view.Relation) { + if relation == nil || relation.Of == nil { + return + } + child := &relation.Of.View + if child.Template == nil || child.Template.Summary == nil || child.Template.Summary.Schema == nil { + return + } + hidden := map[string]bool{} + for _, link := range relation.Of.On { + if link == nil { + continue + } + if field := normalizeRelationColumnName(link.Field); field != "" { + hidden[field] = true + } + if column := normalizeRelationColumnName(link.Column); column != "" { + hidden[column] = true + } + } + if len(hidden) == 0 { + return + } + if refined := hideSummarySchemaFields(child.Template.Summary.Schema, hidden); refined != nil { + child.Template.Summary.Schema = refined + } +} + +func hideSummarySchemaFields(summarySchema *state.Schema, hidden map[string]bool) *state.Schema { + if summarySchema == nil || len(hidden) == 0 { + return nil + } + summaryType := summarySchema.Type() + if summaryType == nil { + return nil + } + isPtr := false + if summaryType.Kind() == reflect.Ptr { + isPtr = true + summaryType = summaryType.Elem() + } + if summaryType.Kind() != reflect.Struct { + return nil + } + fields := make([]reflect.StructField, 0, summaryType.NumField()) + changed := false + for i := 0; i < summaryType.NumField(); i++ { + field := summaryType.Field(i) + if shouldHideSummaryField(field, hidden) { + field.Tag = hideSummaryFieldTag(field.Tag) + changed = true + } + fields = append(fields, field) + } + if !changed { + return nil + } + refinedType := reflect.StructOf(fields) + if isPtr { + refinedType = reflect.PtrTo(refinedType) + } + refined := summarySchema.Clone() + refined.SetType(refinedType) + return refined +} + +func shouldHideSummaryField(field reflect.StructField, hidden map[string]bool) bool { + if len(hidden) == 0 { + return false + } + candidates := []string{ + normalizeRelationColumnName(field.Name), + normalizeRelationColumnName(summaryLookupName(field)), + } + for _, candidate := range candidates { + if candidate != "" && hidden[candidate] { + return true + } + } + return false +} + +var structTagPattern = regexp.MustCompile(`([A-Za-z0-9_]+):"([^"]*)"`) + +func hideSummaryFieldTag(tag reflect.StructTag) reflect.StructTag { + values := map[string]string{} + order := make([]string, 0, 4) + for _, match := range structTagPattern.FindAllStringSubmatch(string(tag), -1) { + key := strings.TrimSpace(match[1]) + if key == "" { + continue + } + if _, ok := values[key]; !ok { + order = append(order, key) + } + values[key] = match[2] + } + for _, item := range []struct { + key string + value string + }{ + {key: "internal", value: "true"}, + } { + if _, ok := values[item.key]; !ok { + order = append(order, item.key) + } + values[item.key] = item.value + } + parts := make([]string, 0, len(order)) + for _, key := range order { + if value, ok := values[key]; ok { + parts = append(parts, fmt.Sprintf(`%s:%q`, key, value)) + } + } + return reflect.StructTag(strings.Join(parts, " ")) +} + func enrichRelationHolderTypes(resource *view.Resource, planned []*plan.View) error { if resource == nil || len(planned) == 0 { return nil @@ -1286,9 +3868,21 @@ func enrichRelationHolderTypes(resource *view.Resource, planned []*plan.View) er continue } if parent.Schema.Cardinality == state.Many { + parentType := parent.Schema.Type() + if parentType != nil && parentType.Kind() == reflect.Slice { + elemType := parentType.Elem() + if elemType.Kind() == reflect.Ptr { + parent.Schema.SetType(reflect.SliceOf(reflect.PtrTo(augmented))) + continue + } + } parent.Schema.SetType(reflect.SliceOf(augmented)) continue } + if parentType := parent.Schema.Type(); parentType != nil && parentType.Kind() == reflect.Ptr { + parent.Schema.SetType(reflect.PtrTo(augmented)) + continue + } parent.Schema.SetType(augmented) } } @@ -1309,9 +3903,6 @@ func ensureRelationHolderFields(parentType reflect.Type, item *plan.View, byName if rel == nil || strings.TrimSpace(rel.Holder) == "" { continue } - if _, ok := parentType.FieldByName(rel.Holder); ok { - continue - } childName := strings.TrimSpace(rel.Ref) if childName == "" { continue @@ -1332,16 +3923,23 @@ func ensureRelationHolderFields(parentType reflect.Type, item *plan.View, byName if childType == nil { continue } - fieldType := relationHolderFieldType(childType, childPlannedCardinality(childName, byName)) - if fieldType == nil { - continue + if _, ok := parentType.FieldByName(rel.Holder); !ok && !fieldNameInSlice(fields, rel.Holder) { + fieldType := relationHolderFieldType(childType, childPlannedCardinality(childName, byName)) + if fieldType != nil { + fields = append(fields, reflect.StructField{ + Name: rel.Holder, + Type: fieldType, + Tag: reflect.StructTag(buildRelationHolderTag(rel, childView)), + }) + changed = true + } + } + if summaryField := relationSummaryField(childView); summaryField != nil { + if _, ok := parentType.FieldByName(summaryField.Name); !ok && !fieldNameInSlice(fields, summaryField.Name) { + fields = append(fields, *summaryField) + changed = true + } } - fields = append(fields, reflect.StructField{ - Name: rel.Holder, - Type: fieldType, - Tag: reflect.StructTag(buildRelationHolderTag(rel, childView)), - }) - changed = true } if !changed { return parentType, false, nil @@ -1349,6 +3947,37 @@ func ensureRelationHolderFields(parentType reflect.Type, item *plan.View, byName return reflect.StructOf(fields), true, nil } +func relationSummaryField(childView *view.View) *reflect.StructField { + if childView == nil || childView.Template == nil || childView.Template.Summary == nil || childView.Template.Summary.Schema == nil { + return nil + } + fieldType := childView.Template.Summary.Schema.Type() + if fieldType == nil { + return nil + } + fieldName := strings.TrimSpace(childView.Template.Summary.Name) + if fieldName == "" { + return nil + } + if fieldType.Kind() == reflect.Struct { + fieldType = reflect.PtrTo(fieldType) + } + return &reflect.StructField{ + Name: fieldName, + Type: fieldType, + Tag: reflect.StructTag(`json:",omitempty" yaml:",omitempty" sqlx:"-"`), + } +} + +func fieldNameInSlice(fields []reflect.StructField, name string) bool { + for _, field := range fields { + if field.Name == name { + return true + } + } + return false +} + func childPlannedCardinality(childName string, byName map[string]*plan.View) state.Cardinality { if childPlanned, ok := byName[strings.ToLower(childName)]; ok && childPlanned != nil { if strings.EqualFold(strings.TrimSpace(childPlanned.Cardinality), string(state.One)) { @@ -1502,6 +4131,15 @@ func bindViewTemplateParameters(aView *view.View, params []*state.Parameter) { return } if aView.Template != nil { + if aView.Template.DeclaredParametersOnly { + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + bindViewTemplateParameters(&rel.Of.View, params) + } + return + } seen := map[string]bool{} for _, item := range aView.Template.Parameters { if item != nil { @@ -1674,6 +4312,14 @@ func normalizeDerivedInputSchema(param *state.Parameter, resource *view.Resource if param.Schema.Type() == nil && aView.Schema.Type() != nil { param.Schema.SetType(aView.Schema.Type()) } + if resourceUsesVelty(resource) { + if rebuilt := ensureSchemaTypeVeltyAliases(param.Schema.Type()); rebuilt != nil { + param.Schema.SetType(rebuilt) + if aView.Schema != nil && schemaNeedsVeltyAliases(aView.Schema.Type()) { + aView.Schema.SetType(rebuilt) + } + } + } if param.Schema.Cardinality == "" { if required { param.Schema.Cardinality = state.One @@ -1683,6 +4329,18 @@ func normalizeDerivedInputSchema(param *state.Parameter, resource *view.Resource } } +func resourceUsesVelty(resource *view.Resource) bool { + if resource == nil { + return false + } + for _, aView := range resource.Views { + if aView != nil && aView.Mode == view.ModeExec { + return true + } + } + return false +} + func rootResourceView(resource *view.Resource, planned []*plan.View) *view.View { if resource == nil { return nil @@ -1736,9 +4394,38 @@ func inheritRootOutputSchema(param *state.Parameter, root *view.View) { if explicit.Cardinality != "" { schema.Cardinality = explicit.Cardinality } + if schema.Cardinality == state.One && schema.Type() != nil { + if normalized := collapseSchemaTypeToOne(schema.Type()); normalized != nil { + schema.SetType(normalized) + if strings.TrimSpace(schema.DataType) == "" || strings.HasPrefix(strings.TrimSpace(schema.DataType), "[]") { + schema.DataType = normalized.String() + } + } + } param.Schema = &schema } +func collapseSchemaTypeToOne(rType reflect.Type) reflect.Type { + if rType == nil { + return nil + } + switch rType.Kind() { + case reflect.Slice: + return rType.Elem() + case reflect.Ptr: + elem := collapseSchemaTypeToOne(rType.Elem()) + if elem == nil { + return nil + } + if elem.Kind() == reflect.Slice { + return elem + } + return reflect.PtrTo(elem) + default: + return rType + } +} + func inheritRootBodySchema(param *state.Parameter, root *view.View) { if param == nil || param.In == nil || root == nil || root.Schema == nil { return @@ -1749,13 +4436,6 @@ func inheritRootBodySchema(param *state.Parameter, root *view.View) { if !param.IsAnonymous() { return } - dataType := "" - if param.Schema != nil { - dataType = strings.TrimSpace(param.Schema.DataType) - } - if dataType != "" && dataType != "?" { - return - } if param.Schema == nil { param.Schema = &state.Schema{} } @@ -1779,25 +4459,41 @@ func inheritRootBodySchema(param *state.Parameter, root *view.View) { if explicit.Cardinality != "" { schema.Cardinality = explicit.Cardinality } + if schema.Type() == nil && root.Schema.Type() != nil { + schema.SetType(root.Schema.Type()) + } + if schema.Cardinality == state.One && schema.Type() != nil { + if normalized := collapseSchemaTypeToOne(schema.Type()); normalized != nil { + schema.SetType(normalized) + if strings.TrimSpace(schema.DataType) == "" || strings.HasPrefix(strings.TrimSpace(schema.DataType), "[]") { + schema.DataType = normalized.String() + } + } + } param.Schema = &schema } -func ensureMaterializedOutputSchema(param *state.Parameter, root *view.View) { +func ensureMaterializedOutputSchema(param *state.Parameter, root *view.View, source *shape.Source, ctx *typectx.Context) { if param == nil || param.In == nil { return } if param.In.Kind != state.KindOutput { return } - if param.Schema != nil && (param.Schema.Type() != nil || strings.TrimSpace(param.Schema.DataType) != "") { - return - } switch strings.ToLower(strings.TrimSpace(param.In.Name)) { case "status": + if param.Schema != nil && (param.Schema.Type() != nil || strings.TrimSpace(param.Schema.DataType) != "") { + return + } param.Schema = state.NewSchema(reflect.TypeOf(response.Status{})) case "summary": - if root != nil && root.Template != nil && root.Template.Summary != nil && root.Template.Summary.Schema != nil { + if (param.Schema == nil || (param.Schema.Type() == nil && strings.TrimSpace(param.Schema.DataType) == "" || strings.TrimSpace(param.Schema.DataType) == "?")) && root != nil && root.Template != nil && root.Template.Summary != nil && root.Template.Summary.Schema != nil { param.Schema = root.Template.Summary.Schema.Clone() } + if (param.Schema == nil || (param.Schema.Type() == nil && (strings.TrimSpace(param.Schema.DataType) == "" || strings.TrimSpace(param.Schema.DataType) == "?"))) && strings.TrimSpace(param.Name) != "" { + if summaryType := resolveSummarySchemaType(source, ctx, param.Name); summaryType != nil { + param.Schema = materializedSummarySchema(summaryType, param.Name, ctx) + } + } } } diff --git a/repository/shape/load/loader_dql_test.go b/repository/shape/load/loader_dql_test.go new file mode 100644 index 000000000..146420884 --- /dev/null +++ b/repository/shape/load/loader_dql_test.go @@ -0,0 +1,108 @@ +package load_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/shape" + shapeCompile "github.com/viant/datly/repository/shape/compile" + shapeLoad "github.com/viant/datly/repository/shape/load" + shapePlan "github.com/viant/datly/repository/shape/plan" +) + +func TestLoadComponent_DQLUserMetadataPreservesBitColumns(t *testing.T) { + dqlPath := filepath.Join("..", "..", "..", "e2e", "v1", "dql", "dev", "user", "user_metadata.dql") + dqlPath, err := filepath.Abs(dqlPath) + require.NoError(t, err) + data, err := os.ReadFile(dqlPath) + require.NoError(t, err) + + source := &shape.Source{ + Name: "user_metadata", + Path: dqlPath, + DQL: string(data), + } + planned, err := shapeCompile.New().Compile(context.Background(), source) + require.NoError(t, err) + actualPlan, ok := shapePlan.ResultFrom(planned) + require.True(t, ok) + require.NotNil(t, actualPlan.TypeContext) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/user/mysql_boolean", actualPlan.TypeContext.PackagePath) + t.Logf("typectx: dir=%q name=%q path=%q", actualPlan.TypeContext.PackageDir, actualPlan.TypeContext.PackageName, actualPlan.TypeContext.PackagePath) + + artifact, err := shapeLoad.New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + root, err := artifact.Resource.Views.Index().Lookup("user_metadata") + require.NoError(t, err) + require.NotNil(t, root) + require.NotNil(t, root.Schema) + require.NotNil(t, root.Schema.Type()) + t.Logf("schema type: %v", root.Schema.Type()) + + names := make([]string, 0, len(root.Columns)) + for _, column := range root.Columns { + if column == nil { + continue + } + names = append(names, column.Name) + } + assert.Contains(t, names, "IS_ENABLED") + assert.Contains(t, names, "IS_ACTIVATED") +} + +func TestLoadComponent_DQLVarsHonorsDeclaredColumnType(t *testing.T) { + dqlPath := filepath.Join("..", "..", "..", "e2e", "v1", "dql", "dev", "vendorsrv", "vars.dql") + dqlPath, err := filepath.Abs(dqlPath) + require.NoError(t, err) + data, err := os.ReadFile(dqlPath) + require.NoError(t, err) + + source := &shape.Source{ + Name: "vars", + Path: dqlPath, + DQL: string(data), + } + planned, err := shapeCompile.New().Compile(context.Background(), source) + require.NoError(t, err) + actualPlan, ok := shapePlan.ResultFrom(planned) + require.True(t, ok) + for _, item := range actualPlan.Views { + if item == nil || item.Name != "main" { + continue + } + if item.Declaration != nil && item.Declaration.ColumnsConfig != nil { + if cfg := item.Declaration.ColumnsConfig["Key3"]; cfg != nil { + t.Logf("planned Key3 dataType=%q", cfg.DataType) + } + } + } + + artifact, err := shapeLoad.New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + root, err := artifact.Resource.Views.Index().Lookup("main") + require.NoError(t, err) + require.NotNil(t, root) + require.NotNil(t, root.Schema) + require.NotNil(t, root.Schema.Type()) + if cfg := root.ColumnsConfig["Key3"]; cfg != nil && cfg.DataType != nil { + t.Logf("Key3 config dataType=%q", *cfg.DataType) + } + t.Logf("vars schema type: %v", root.Schema.Type()) + + var key3Type string + for _, column := range root.Columns { + if column == nil || column.Name != "Key3" { + continue + } + if column.ColumnType() != nil { + key3Type = column.ColumnType().String() + } else { + key3Type = column.DataType + } + } + assert.Equal(t, "bool", key3Type) +} diff --git a/repository/shape/load/loader_test.go b/repository/shape/load/loader_test.go index e5d5a3913..13a67059c 100644 --- a/repository/shape/load/loader_test.go +++ b/repository/shape/load/loader_test.go @@ -3,8 +3,10 @@ package load import ( "context" "embed" + "os" "path/filepath" "reflect" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -59,6 +61,30 @@ type vendorProductRow struct { VendorID int `sqlx:"name=VENDOR_ID"` } +type metaSummaryRow struct { + PageCnt int `sqlx:"name=PAGE_CNT"` + Cnt int `sqlx:"name=CNT"` +} + +type productsMetaSummaryRow struct { + VendorID int `sqlx:"name=VENDOR_ID"` + PageCnt int `sqlx:"name=PAGE_CNT"` + TotalProducts int `sqlx:"name=TOTAL_PRODUCTS"` +} + +type productsOwnerPointerRow struct { + VendorID *int `sqlx:"name=VENDOR_ID"` +} + +type vendorSummaryParentRow struct { + ID int `sqlx:"name=ID"` +} + +type vendorSummaryChildRow struct { + ID int `sqlx:"name=ID"` + VendorID int `sqlx:"name=VENDOR_ID"` +} + type fieldOnlyUserACLRow struct { UserID int `sqlx:"name=UserID"` IsReadOnly int `sqlx:"name=IsReadOnly"` @@ -123,6 +149,15 @@ type dynamicRouteSource struct { Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET"` } +type selectorHolderSource struct { + Rows []reportRow `view:"rows,table=REPORT" sql:"uri=testdata/report.sql"` + Route xdatly.Component[typedRouteInput, typedRouteOutput] `component:",path=/v1/api/dev/report,method=GET"` + ViewSelect struct { + Fields []string `parameter:"fields,kind=query,in=_fields,cacheable=false"` + Page int `parameter:"page,kind=query,in=_page,cacheable=false"` + } `querySelector:"rows"` +} + type routerOnlyInput struct { ID int `parameter:"id,kind=query,in=id"` } @@ -169,6 +204,37 @@ func TestLoader_LoadViews(t *testing.T) { require.NotNil(t, artifacts.Resource.EmbedFS()) } +func TestLoader_LoadResource(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "report"}, + Plan: &plan.Result{ + EmbedFS: &testFS, + Views: []*plan.View{ + { + Name: "rows", + Table: "REPORT", + Connector: "dev", + SQL: "SELECT ID, NAME FROM REPORT", + SQLURI: "testdata/report.sql", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + }, + }, + ViewsByName: map[string]*plan.View{"rows": {Name: "rows"}}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifacts, err := New().LoadResource(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifacts) + require.NotNil(t, artifacts.Resource) + require.Len(t, artifacts.Resource.Views, 1) + assert.Equal(t, "rows", artifacts.Resource.Views[0].Name) + assert.Equal(t, "REPORT", artifacts.Resource.Views[0].Table) +} + // stubPlanSpec is a non-plan-Result implementation of shape.PlanSpec used to // verify that LoadViews() returns an error when given an unexpected plan type. type stubPlanSpec struct{} @@ -185,27 +251,42 @@ func TestLoader_LoadViews_InvalidPlanType(t *testing.T) { func TestLoader_LoadViews_Metadata(t *testing.T) { noLimit := true allowNulls := true + groupable := true + criteria := true + projection := true + orderBy := true + offset := true planned := &shape.PlanResult{ Source: &shape.Source{Name: "meta"}, Plan: &plan.Result{ Views: []*plan.View{ { - Name: "items", - Table: "ITEMS", - Module: "platform/items", - AllowNulls: &allowNulls, - SelectorNamespace: "it", - SelectorNoLimit: &noLimit, - SchemaType: "*ItemView", - Cardinality: "many", - FieldType: reflect.TypeOf([]map[string]interface{}{}), - ElementType: reflect.TypeOf(map[string]interface{}{}), - SQL: "SELECT * FROM ITEMS", + Name: "items", + Table: "ITEMS", + Module: "platform/items", + AllowNulls: &allowNulls, + Groupable: &groupable, + SelectorNamespace: "it", + SelectorNoLimit: &noLimit, + SelectorCriteria: &criteria, + SelectorProjection: &projection, + SelectorOrderBy: &orderBy, + SelectorOffset: &offset, + SelectorFilterable: []string{"*"}, + SelectorOrderByColumns: map[string]string{ + "accountId": "ACCOUNT_ID", + }, + SchemaType: "*ItemView", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM ITEMS", Declaration: &plan.ViewDeclaration{ ColumnsConfig: map[string]*plan.ViewColumnConfig{ "AUTHORIZED": { - DataType: "bool", - Tag: `internal:"true"`, + DataType: "bool", + Tag: `internal:"true"`, + Groupable: &groupable, }, }, }, @@ -224,9 +305,18 @@ func TestLoader_LoadViews_Metadata(t *testing.T) { assert.Equal(t, "platform/items", actual.Module) require.NotNil(t, actual.AllowNulls) assert.True(t, *actual.AllowNulls) + assert.True(t, actual.Groupable) require.NotNil(t, actual.Selector) assert.Equal(t, "it", actual.Selector.Namespace) assert.True(t, actual.Selector.NoLimit) + require.NotNil(t, actual.Selector.Constraints) + assert.True(t, actual.Selector.Constraints.Limit) + assert.True(t, actual.Selector.Constraints.Criteria) + assert.True(t, actual.Selector.Constraints.Projection) + assert.True(t, actual.Selector.Constraints.OrderBy) + assert.True(t, actual.Selector.Constraints.Offset) + assert.Equal(t, []string{"*"}, actual.Selector.Constraints.Filterable) + assert.Equal(t, "ACCOUNT_ID", actual.Selector.Constraints.OrderByColumn["accountId"]) require.NotNil(t, actual.Schema) assert.Equal(t, "*ItemView", actual.Schema.DataType) require.NotNil(t, actual.ColumnsConfig) @@ -235,6 +325,55 @@ func TestLoader_LoadViews_Metadata(t *testing.T) { assert.Equal(t, "bool", *actual.ColumnsConfig["AUTHORIZED"].DataType) require.NotNil(t, actual.ColumnsConfig["AUTHORIZED"].Tag) assert.Equal(t, `internal:"true"`, *actual.ColumnsConfig["AUTHORIZED"].Tag) + require.NotNil(t, actual.ColumnsConfig["AUTHORIZED"].Groupable) + assert.True(t, *actual.ColumnsConfig["AUTHORIZED"].Groupable) +} + +func TestLoader_LoadViews_SelectorLimitEnablesConstraint(t *testing.T) { + limit := 2 + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "district"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "cities", + Table: "CITY", + SelectorLimit: &limit, + SelectorNamespace: "ci", + SchemaType: "*CitiesView", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM CITY", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + loader := New() + artifacts, err := loader.LoadViews(context.Background(), planned) + require.NoError(t, err) + require.Len(t, artifacts.Views, 1) + actual := artifacts.Views[0] + require.NotNil(t, actual.Selector) + require.NotNil(t, actual.Selector.Constraints) + assert.Equal(t, 2, actual.Selector.Limit) + assert.True(t, actual.Selector.Constraints.Limit) +} + +func TestCloneRelationView_SelectorLimitUsesSingleParentBatch(t *testing.T) { + ref, err := view.New("cities", "CITY") + require.NoError(t, err) + ref.Selector = &view.Config{ + Limit: 2, + Constraints: &view.Constraints{ + Limit: true, + }, + } + cloned := cloneRelationView(ref, view.View{}) + require.NotNil(t, cloned.Batch) + assert.Equal(t, 1, cloned.Batch.Size) } func TestLoader_LoadViews_InfersColumnsFromBestSchemaType(t *testing.T) { @@ -361,6 +500,7 @@ func TestLoader_LoadComponent(t *testing.T) { actualPlan.Directives = &dqlshape.Directives{ Meta: "docs/report.md", DefaultConnector: "analytics", + TemplateType: "patch", Dest: "all.go", InputDest: "input.go", OutputDest: "output.go", @@ -411,6 +551,7 @@ func TestLoader_LoadComponent(t *testing.T) { require.NotNil(t, component.Directives) assert.Equal(t, "docs/report.md", component.Directives.Meta) assert.Equal(t, "analytics", component.Directives.DefaultConnector) + assert.Equal(t, "patch", component.Directives.TemplateType) require.NotNil(t, component.Directives.Cache) assert.True(t, component.Directives.Cache.Enabled) assert.Equal(t, "5m", component.Directives.Cache.TTL) @@ -460,6 +601,7 @@ func TestLoader_LoadComponent_InheritsTypeContextPackageForNamedStateSchemas(t * Source: &shape.Source{Name: "patch_basic_one"}, Plan: &plan.Result{ TypeContext: &typectx.Context{ + PackageName: "patch_basic_one", DefaultPackage: "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", }, @@ -505,17 +647,99 @@ func TestLoader_LoadComponent_InheritsTypeContextPackageForNamedStateSchemas(t * require.True(t, ok) require.Len(t, component.Input, 1) require.Len(t, component.Output, 1) - assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", component.Input[0].Schema.Package) + assert.Equal(t, "patch_basic_one", component.Input[0].Schema.Package) assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", component.Input[0].Schema.PackagePath) - assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", component.Output[0].Schema.Package) + assert.Equal(t, "patch_basic_one", component.Output[0].Schema.Package) assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", component.Output[0].Schema.PackagePath) } +func TestLoader_LoadComponent_DoesNotInheritTypeContextPackageForPrimitiveStateSchemas(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "district_pagination"}, + Plan: &plan.Result{ + TypeContext: &typectx.Context{ + DefaultPackage: "github.com/viant/datly/e2e/v1/shape/dev/district/pagination", + PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/district/pagination", + }, + Views: []*plan.View{ + { + Name: "districts", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + Cardinality: string(state.Many), + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "IDs", + In: state.NewQueryLocation("IDs"), + Schema: &state.Schema{DataType: "[]int"}, + }, + }, + { + Parameter: state.Parameter{ + Name: "Page", + In: state.NewQueryLocation("page"), + Schema: &state.Schema{DataType: "int"}, + }, + }, + }, + }, + } + + artifact, err := New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.Input, 2) + assert.Empty(t, component.Input[0].Schema.Package) + assert.Empty(t, component.Input[0].Schema.PackagePath) + assert.Empty(t, component.Input[1].Schema.Package) + assert.Empty(t, component.Input[1].Schema.PackagePath) +} + +func TestLoader_LoadComponent_MaterializesAnonymousBodySchemaIntoResourceParameters(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "patch_basic_one"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "foos", + Holder: "Foos", + SchemaType: "*FoosView", + Cardinality: string(state.Many), + SQL: "SELECT * FROM FOOS", + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewBodyLocation(""), + Tag: `anonymous:"true"`, + }, + }, + }, + }, + } + + artifact, err := New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + param, err := artifact.Resource.LookupParameter("Foos") + require.NoError(t, err) + require.NotNil(t, param) + require.NotNil(t, param.Schema) + assert.Equal(t, "FoosView", param.Schema.Name) + assert.Equal(t, "*FoosView", param.Schema.DataType) +} + func TestLoader_LoadComponent_InheritsTypeContextPackageForNamedViewSchemas(t *testing.T) { planned := &shape.PlanResult{ Source: &shape.Source{Name: "patch_basic_one"}, Plan: &plan.Result{ TypeContext: &typectx.Context{ + PackageName: "patch_basic_one", DefaultPackage: "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", PackagePath: "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", }, @@ -539,7 +763,7 @@ func TestLoader_LoadComponent_InheritsTypeContextPackageForNamedViewSchemas(t *t require.NoError(t, err) require.NotNil(t, root) require.NotNil(t, root.Schema) - assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", root.Schema.Package) + assert.Equal(t, "patch_basic_one", root.Schema.Package) assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/events/patch_basic_one", root.Schema.PackagePath) } @@ -565,86 +789,370 @@ func TestLoader_LoadComponent_PreservesComponentHolderTypes(t *testing.T) { assert.Empty(t, component.ComponentRoutes[0].OutputName) } -func TestLoader_LoadComponent_PreservesDynamicComponentHolderTypes(t *testing.T) { - scanner := scan.New() - scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &dynamicRouteSource{ - Route: xdatly.Component[any, any]{ - Inout: dynamicRouteInput{}, - Output: dynamicRouteOutput{}, +func TestLoader_LoadComponent_SynthesizesMutableHelpersForPatchBodyRoute(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "patch_basic_one"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "foos", + Holder: "Foos", + SchemaType: "*FoosView", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + Cardinality: string(state.Many), + Table: "FOOS", + SQL: "SELECT * FROM FOOS", + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewBodyLocation(""), + Tag: `anonymous:"true"`, + Schema: &state.Schema{ + Name: "FoosView", + DataType: "*FoosView", + Cardinality: state.One, + }, + }, + EmitOutput: true, + }, + }, + Components: []*plan.ComponentRoute{ + { + Method: "PATCH", + RoutePath: "/v1/api/shape/dev/basic/foos", + ViewName: "FoosView", + }, + }, }, - }}) - require.NoError(t, err) - - planner := plan.New() - planned, err := planner.Plan(context.Background(), scanned) - require.NoError(t, err) + } - loader := New() - artifact, err := loader.LoadComponent(context.Background(), planned) + artifact, err := New().LoadComponent(context.Background(), planned) require.NoError(t, err) - component, ok := ComponentFrom(artifact) require.True(t, ok) - require.Len(t, component.ComponentRoutes, 1) - assert.Equal(t, reflect.TypeOf(dynamicRouteInput{}), component.ComponentRoutes[0].InputType) - assert.Equal(t, reflect.TypeOf(dynamicRouteOutput{}), component.ComponentRoutes[0].OutputType) -} -func TestLoader_LoadComponent_PreservesDynamicComponentHolderExplicitNames(t *testing.T) { - scanner := scan.New() - scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { - embeddedFS - Rows []reportRow `view:"rows,table=REPORT" sql:"uri=testdata/report.sql"` - Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` - }{}}) - require.NoError(t, err) - - planner := plan.New() - planned, err := planner.Plan(context.Background(), scanned) - require.NoError(t, err) + root := lookupNamedResourceView(artifact.Resource, component.RootView) + require.NotNil(t, root) + assert.Equal(t, view.ModeExec, root.Mode) + require.NotNil(t, root.Template) + assert.True(t, root.Template.UseParameterStateType) + require.NotNil(t, root.Template.Parameters.Lookup("CurFoosId")) + require.NotNil(t, root.Template.Parameters.Lookup("CurFoos")) + assert.Contains(t, root.Template.Source, `$sequencer.Allocate("FOOS", $Unsafe.Foos, "Id")`) + assert.Contains(t, root.Template.Source, `#if($CurFoosById.HasKey($Unsafe.Foos.Id) == true)`) + assert.Contains(t, root.Template.Source, `$sql.Update($Unsafe.Foos, "FOOS");`) + assert.Contains(t, root.Template.Source, `$sql.Insert($Unsafe.Foos, "FOOS");`) + assert.Equal(t, state.Many, root.Template.Parameters.Lookup("CurFoos").Schema.Cardinality) + + require.Nil(t, component.InputParameters().Lookup("CurFoosId")) + require.Nil(t, component.InputParameters().Lookup("CurFoos")) + require.NotNil(t, artifact.Resource.Parameters.Lookup("CurFoosId")) + require.NotNil(t, artifact.Resource.Parameters.Lookup("CurFoos")) + assert.Equal(t, "*struct { Values []int \"json:\\\",omitempty\\\"\" }", artifact.Resource.Parameters.Lookup("CurFoosId").Schema.DataType) + require.NotNil(t, artifact.Resource.Parameters.Lookup("CurFoosId").Output) + assert.Equal(t, "structql", artifact.Resource.Parameters.Lookup("CurFoosId").Output.Name) + assert.Contains(t, artifact.Resource.Parameters.Lookup("CurFoosId").Output.Body, "SELECT ARRAY_AGG(Id) AS Values") + assert.Equal(t, state.One, artifact.Resource.Parameters.Lookup("CurFoosId").Schema.Cardinality) + assert.Equal(t, state.One, artifact.Resource.Parameters.Lookup("CurFoosId").Output.Schema.Cardinality) + assert.Equal(t, state.Many, artifact.Resource.Parameters.Lookup("CurFoos").Schema.Cardinality) + require.Len(t, component.Output, 1) + require.Equal(t, "Foos", component.Output[0].Name) + require.Equal(t, state.KindRequestBody, component.Output[0].In.Kind) - loader := New() - artifact, err := loader.LoadComponent(context.Background(), planned) + curFoos, err := artifact.Resource.View("CurFoos") require.NoError(t, err) - - component, ok := ComponentFrom(artifact) - require.True(t, ok) - require.Len(t, component.ComponentRoutes, 1) - assert.Nil(t, component.ComponentRoutes[0].InputType) - assert.Nil(t, component.ComponentRoutes[0].OutputType) - assert.Equal(t, "ReportInput", component.ComponentRoutes[0].InputName) - assert.Equal(t, "ReportOutput", component.ComponentRoutes[0].OutputName) + require.NotNil(t, curFoos) + require.NotNil(t, curFoos.Template) + assert.Equal(t, "foos/cur_foos.sql", curFoos.Template.SourceURL) + require.True(t, curFoos.Template.UseParameterStateType) + require.True(t, curFoos.Template.DeclaredParametersOnly) + require.True(t, curFoos.Template.UseResourceParameterLookup) + require.NotNil(t, curFoos.Template.Parameters.Lookup("CurFoosId")) + require.Nil(t, curFoos.Template.Parameters.Lookup("Foos")) } -func TestLoader_LoadComponent_PreservesDynamicComponentHolderExplicitNamesFromRegistry(t *testing.T) { - registry := x.NewRegistry() - registry.Register(x.NewType(reflect.TypeOf(namedDynamicRouteInput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/load"), x.WithName("ReportInput"))) - registry.Register(x.NewType(reflect.TypeOf(namedDynamicRouteOutput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/load"), x.WithName("ReportOutput"))) - - scanner := scan.New() - scanned, err := scanner.Scan(context.Background(), &shape.Source{ - Struct: &struct { - Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` - }{}, - TypeRegistry: registry, - }) - require.NoError(t, err) +func TestLoader_LoadComponent_SynthesizesMutableHelpersForPatchManyBodyRoute(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "patch_basic_many"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "foos", + Holder: "Foos", + SchemaType: "*FoosView", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + Cardinality: string(state.Many), + Table: "FOOS", + SQL: "SELECT * FROM FOOS", + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewBodyLocation(""), + Tag: `anonymous:"true"`, + Schema: &state.Schema{ + Name: "FoosView", + DataType: "*FoosView", + Cardinality: state.Many, + }, + }, + EmitOutput: true, + }, + }, + Components: []*plan.ComponentRoute{ + { + Method: "PATCH", + RoutePath: "/v1/api/shape/dev/basic/foos-many", + ViewName: "FoosView", + }, + }, + }, + } - planner := plan.New() - planned, err := planner.Plan(context.Background(), scanned) + artifact, err := New().LoadComponent(context.Background(), planned) require.NoError(t, err) - loader := New() - artifact, err := loader.LoadComponent(context.Background(), planned) - require.NoError(t, err) + curFoosID := artifact.Resource.Parameters.Lookup("CurFoosId") + require.NotNil(t, curFoosID) + require.NotNil(t, curFoosID.Schema) + require.NotNil(t, curFoosID.Output) + require.NotNil(t, curFoosID.Output.Schema) + assert.Equal(t, state.One, curFoosID.Schema.Cardinality) + assert.Equal(t, state.One, curFoosID.Output.Schema.Cardinality) + assert.Equal(t, "*struct { Values []int \"json:\\\",omitempty\\\"\" }", curFoosID.Schema.DataType) + assert.Contains(t, curFoosID.Output.Body, "SELECT ARRAY_AGG(Id) AS Values") + root := lookupNamedResourceView(artifact.Resource, "foos") + require.NotNil(t, root) + require.NotNil(t, root.Template) + assert.Contains(t, root.Template.Source, `$sequencer.Allocate("FOOS", $Unsafe.Foos, "Id")`) + assert.Contains(t, root.Template.Source, `#foreach($RecFoos in $Unsafe.Foos)`) + assert.Contains(t, root.Template.Source, `#if($CurFoosById.HasKey($RecFoos.Id) == true)`) + assert.Contains(t, root.Template.Source, `$sql.Update($RecFoos, "FOOS");`) + assert.Contains(t, root.Template.Source, `$sql.Insert($RecFoos, "FOOS");`) + require.NotNil(t, root.TableBatches) + assert.True(t, root.TableBatches["FOOS"]) +} - component, ok := ComponentFrom(artifact) - require.True(t, ok) - require.Len(t, component.ComponentRoutes, 1) - assert.Equal(t, reflect.TypeOf(namedDynamicRouteInput{}), component.ComponentRoutes[0].InputType) - assert.Equal(t, reflect.TypeOf(namedDynamicRouteOutput{}), component.ComponentRoutes[0].OutputType) - assert.Equal(t, "ReportInput", component.ComponentRoutes[0].InputName) - assert.Equal(t, "ReportOutput", component.ComponentRoutes[0].OutputName) +func TestLoader_LoadComponent_DoesNotSynthesizeMutableHelpersForScalarBodyRoute(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "product_update"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "product_update", + Holder: "ProductUpdate", + SchemaType: "*ProductUpdateView", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + Cardinality: string(state.Many), + Table: "PRODUCT", + SQL: "UPDATE PRODUCT SET STATUS = $Status WHERE ID IN ($Ids)", + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Ids", + In: state.NewBodyLocation("Ids"), + Schema: &state.Schema{ + DataType: "int", + Cardinality: state.Many, + }, + }, + }, + { + Parameter: state.Parameter{ + Name: "Status", + In: state.NewBodyLocation("Status"), + Schema: &state.Schema{ + DataType: "int", + Cardinality: state.One, + }, + }, + }, + { + Parameter: state.Parameter{ + Name: "Records", + In: state.NewViewLocation("Records"), + Schema: &state.Schema{ + Name: "RecordsView", + DataType: "*RecordsView", + Cardinality: state.Many, + }, + }, + }, + }, + Components: []*plan.ComponentRoute{ + { + Method: "POST", + RoutePath: "/v1/api/shape/dev/auth/products", + ViewName: "ProductUpdateView", + }, + }, + }, + } + + artifact, err := New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + + root := lookupNamedResourceView(artifact.Resource, component.RootView) + require.NotNil(t, root) + require.NotNil(t, root.Template) + require.Nil(t, root.Template.Parameters.Lookup("CurIdsId")) + require.Nil(t, artifact.Resource.Parameters.Lookup("CurIdsId")) + require.Nil(t, artifact.Resource.Parameters.Lookup("CurIds")) +} + +func TestLoader_LoadComponent_UserMetadataPreservesBitColumnsFromSchemaType(t *testing.T) { + projectRoot := t.TempDir() + err := os.WriteFile(filepath.Join(projectRoot, "go.mod"), []byte("module github.com/acme/app\n\ngo 1.23.0\n"), 0o644) + require.NoError(t, err) + packageDir := filepath.Join(projectRoot, "shape", "dev", "user", "mysql_boolean") + err = os.MkdirAll(packageDir, 0o755) + require.NoError(t, err) + sourcePath := filepath.Join(projectRoot, "routes", "dev", "user_metadata.dql") + err = os.MkdirAll(filepath.Dir(sourcePath), 0o755) + require.NoError(t, err) + err = os.WriteFile(sourcePath, []byte("SELECT * FROM USER_METADATA"), 0o644) + require.NoError(t, err) + typeFile := `package mysql_boolean + +import "github.com/viant/sqlx/types" + +type UserMetadataView struct { + Id int ` + "`sqlx:\"ID\"`" + ` + UserId *int ` + "`sqlx:\"USER_ID\"`" + ` + IsEnabled *types.BitBool ` + "`sqlx:\"IS_ENABLED\"`" + ` + IsActivated *types.BitBool ` + "`sqlx:\"IS_ACTIVATED\"`" + ` +} +` + err = os.WriteFile(filepath.Join(packageDir, "user_metadata.go"), []byte(typeFile), 0o644) + require.NoError(t, err) + + artifact, err := New().LoadComponent(context.Background(), &shape.PlanResult{ + Source: &shape.Source{Name: "user_metadata", Path: sourcePath}, + Plan: &plan.Result{ + TypeContext: &typectx.Context{ + PackagePath: "github.com/acme/app/shape/dev/user/mysql_boolean", + }, + Views: []*plan.View{ + { + Name: "user_metadata", + Table: "USER_METADATA", + SchemaType: "*UserMetadataView", + Cardinality: string(state.Many), + SQL: "SELECT user_metadata.* FROM (SELECT * FROM USER_METADATA t) user_metadata", + }, + }, + }, + }) + require.NoError(t, err) + root, err := artifact.Resource.Views.Index().Lookup("user_metadata") + require.NoError(t, err) + require.NotNil(t, root) + require.NotNil(t, root.Schema) + require.NotNil(t, root.Schema.Type()) + + names := make([]string, 0, len(root.Columns)) + for _, column := range root.Columns { + if column == nil { + continue + } + names = append(names, column.Name) + } + assert.Contains(t, names, "IS_ENABLED") + assert.Contains(t, names, "IS_ACTIVATED") +} + +func TestLoader_LoadComponent_PreservesDynamicComponentHolderTypes(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &dynamicRouteSource{ + Route: xdatly.Component[any, any]{ + Inout: dynamicRouteInput{}, + Output: dynamicRouteOutput{}, + }, + }}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.ComponentRoutes, 1) + assert.Equal(t, reflect.TypeOf(dynamicRouteInput{}), component.ComponentRoutes[0].InputType) + assert.Equal(t, reflect.TypeOf(dynamicRouteOutput{}), component.ComponentRoutes[0].OutputType) +} + +func TestLoader_LoadComponent_PreservesDynamicComponentHolderExplicitNames(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { + embeddedFS + Rows []reportRow `view:"rows,table=REPORT" sql:"uri=testdata/report.sql"` + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` + }{}}) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.ComponentRoutes, 1) + assert.Nil(t, component.ComponentRoutes[0].InputType) + assert.Nil(t, component.ComponentRoutes[0].OutputType) + assert.Equal(t, "ReportInput", component.ComponentRoutes[0].InputName) + assert.Equal(t, "ReportOutput", component.ComponentRoutes[0].OutputName) +} + +func TestLoader_LoadComponent_PreservesDynamicComponentHolderExplicitNamesFromRegistry(t *testing.T) { + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(namedDynamicRouteInput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/load"), x.WithName("ReportInput"))) + registry.Register(x.NewType(reflect.TypeOf(namedDynamicRouteOutput{}), x.WithPkgPath("github.com/viant/datly/repository/shape/load"), x.WithName("ReportOutput"))) + + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{ + Struct: &struct { + Route xdatly.Component[any, any] `component:",path=/v1/api/dev/report,method=GET,input=ReportInput,output=ReportOutput"` + }{}, + TypeRegistry: registry, + }) + require.NoError(t, err) + + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.ComponentRoutes, 1) + assert.Equal(t, reflect.TypeOf(namedDynamicRouteInput{}), component.ComponentRoutes[0].InputType) + assert.Equal(t, reflect.TypeOf(namedDynamicRouteOutput{}), component.ComponentRoutes[0].OutputType) + assert.Equal(t, "ReportInput", component.ComponentRoutes[0].InputName) + assert.Equal(t, "ReportOutput", component.ComponentRoutes[0].OutputName) require.Len(t, component.Input, 1) assert.Equal(t, "name", component.Input[0].Name) require.Len(t, component.Output, 2) @@ -677,106 +1185,677 @@ func TestLoader_LoadComponent_RouterOnlySourceSynthesizesStatesAndViews(t *testi scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &routerOnlySource{}}) require.NoError(t, err) - planner := plan.New() - planned, err := planner.Plan(context.Background(), scanned) + planner := plan.New() + planned, err := planner.Plan(context.Background(), scanned) + require.NoError(t, err) + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + assert.Equal(t, "/v1/api/dev/router-only", component.URI) + assert.Equal(t, "GET", component.Method) + require.Len(t, component.Input, 1) + assert.Equal(t, "id", component.Input[0].Name) + require.Len(t, component.Output, 2) + require.Len(t, artifact.Resource.Views, 1) + assert.Equal(t, "rows", artifact.Resource.Views[0].Name) +} + +func TestLoader_LoadComponent_SynthesizesStatesFromRouteContractsWhenPlanStatesAreEmpty(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "team"}, + Plan: &plan.Result{ + Components: []*plan.ComponentRoute{ + { + FieldName: "Team", + Name: "Team", + RoutePath: "/v1/api/dev/team/{teamID}", + Method: "DELETE", + InputType: reflect.TypeOf(typedTeamRouteInput{}), + OutputType: reflect.TypeOf(typedTeamRouteOutput{}), + ViewName: "Team", + }, + }, + }, + } + + artifact, err := New().LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.Input, 1) + assert.Equal(t, "TeamID", component.Input[0].Name) + require.NotNil(t, component.Input[0].In) + assert.Equal(t, state.KindPath, component.Input[0].In.Kind) + assert.Equal(t, "teamID", component.Input[0].In.Name) +} + +func TestLoader_LoadComponent_CacheProviderDoesNotBindRootView(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "/v1/api/shape/dev/vendors/"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Path: "vendor", + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]any{}), + ElementType: reflect.TypeOf(map[string]any{}), + SQL: "SELECT * FROM VENDOR", + }, + }, + Directives: &dqlshape.Directives{ + Cache: &dqlshape.CacheDirective{ + Enabled: true, + Name: "aerospike", + Provider: "aerospike://127.0.0.1:3000/test", + Location: "${view.Name}", + TimeToLiveMs: 3600000, + }, + }, + }, + } + + loader := New() + artifact, err := loader.LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + require.Len(t, artifact.Resource.Views, 1) + require.NotEmpty(t, artifact.Resource.CacheProviders) + + root := artifact.Resource.Views[0] + assert.Nil(t, root.Cache) +} + +func TestLoader_LoadViews_DoesNotSeedPlaceholderColumnsFromLinkedType(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "districts"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "districts", + Table: "DISTRICT", + Cardinality: "many", + SchemaType: "*DistrictsView", + FieldType: reflect.TypeOf([]*placeholderDistrictRow{}), + ElementType: reflect.TypeOf(placeholderDistrictRow{}), + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifact, err := New().LoadViews(context.Background(), planned) + require.NoError(t, err) + require.Len(t, artifact.Views, 1) + assert.Empty(t, artifact.Views[0].Columns) +} + +func TestLoader_LoadViews_DefersMapBackedQuerySchemaType(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "cities"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "cities", + Table: "CITY", + Mode: string(view.ModeQuery), + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM CITY", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifact, err := New().LoadViews(context.Background(), planned) + require.NoError(t, err) + require.Len(t, artifact.Views, 1) + assert.Nil(t, artifact.Views[0].Schema.Type()) +} + +func TestLoader_LoadViews_DefersPlaceholderStructQuerySchemaType(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "districts"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "districts", + Table: "DISTRICT", + Mode: string(view.ModeQuery), + Cardinality: "many", + FieldType: reflect.TypeOf([]placeholderDistrictRow{}), + ElementType: reflect.TypeOf(placeholderDistrictRow{}), + SQL: "SELECT t.* FROM DISTRICT t", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifact, err := New().LoadViews(context.Background(), planned) + require.NoError(t, err) + require.Len(t, artifact.Views, 1) + assert.Nil(t, artifact.Views[0].Schema.Type()) +} + +func TestLoader_LoadComponent_DoesNotMaterializePlaceholderOutputViewSchema(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "districts"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "districts", + Table: "DISTRICT", + Mode: string(view.ModeQuery), + Cardinality: "many", + FieldType: reflect.TypeOf([]placeholderDistrictRow{}), + ElementType: reflect.TypeOf(placeholderDistrictRow{}), + SQL: "SELECT t.* FROM DISTRICT t", + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Data", + In: state.NewOutputLocation("view"), + }, + }, + }, + }, + } + + artifact, err := New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.Len(t, component.Output, 1) + require.NotNil(t, component.Output[0].Schema) + assert.Nil(t, component.Output[0].Schema.Type()) +} + +func TestLoader_LoadViews_PreservesChildSummaryGraph(t *testing.T) { + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "meta"}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM VENDOR", + Relations: []*plan.Relation{ + { + Name: "products", + Parent: "vendor", + Holder: "Products", + Ref: "products", + Table: "PRODUCT", + On: []*plan.RelationLink{ + {ParentColumn: "ID", RefColumn: "VENDOR_ID"}, + }, + }, + }, + }, + { + Name: "products", + Table: "PRODUCT", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM PRODUCT", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) PROD_META GROUP BY VENDOR_ID", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + loader := New() + artifacts, err := loader.LoadViews(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifacts) + require.Len(t, artifacts.Views, 2) + + index := artifacts.Resource.Views.Index() + products, err := index.Lookup("products") + require.NoError(t, err) + require.NotNil(t, products) + require.NotNil(t, products.Template) + require.NotNil(t, products.Template.Summary) + assert.Contains(t, products.Template.Summary.Source, "TOTAL_PRODUCTS") + assert.Contains(t, products.Template.Summary.Source, "$View.products.SQL") +} + +func TestLoader_LoadViews_AttachesChildSummaryFieldToParentSchema(t *testing.T) { + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(productsMetaSummaryRow{}), x.WithName("ProductsMetaView"))) + + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "meta", TypeRegistry: registry}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]vendorSummaryParentRow{}), + ElementType: reflect.TypeOf(vendorSummaryParentRow{}), + SQL: "SELECT * FROM VENDOR", + Relations: []*plan.Relation{ + { + Name: "products", + Parent: "vendor", + Holder: "Products", + Ref: "products", + Table: "PRODUCT", + On: []*plan.RelationLink{ + {ParentColumn: "ID", RefColumn: "VENDOR_ID"}, + }, + }, + }, + }, + { + Name: "products", + Table: "PRODUCT", + Cardinality: "many", + FieldType: reflect.TypeOf([]vendorSummaryChildRow{}), + ElementType: reflect.TypeOf(vendorSummaryChildRow{}), + SQL: "SELECT * FROM PRODUCT", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) PROD_META GROUP BY VENDOR_ID", + SummaryName: "ProductsMeta", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifacts, err := New().LoadViews(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifacts) + require.Len(t, artifacts.Views, 2) + + index := artifacts.Resource.Views.Index() + root, err := index.Lookup("vendor") + require.NoError(t, err) + require.NotNil(t, root) + require.NotNil(t, root.Schema) + compType := root.Schema.CompType() + require.NotNil(t, compType) + field, ok := compType.FieldByName("ProductsMeta") + require.True(t, ok) + assert.Equal(t, `json:",omitempty" yaml:",omitempty" sqlx:"-"`, string(field.Tag)) +} + +func TestLoader_LoadResource_AssignsSummarySchemas(t *testing.T) { + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(metaSummaryRow{}), x.WithName("MetaView"))) + registry.Register(x.NewType(reflect.TypeOf(productsMetaSummaryRow{}), x.WithName("ProductsMetaView"))) + + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "meta", TypeRegistry: registry}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM VENDOR", + Summary: "SELECT COUNT(*) AS CNT FROM ($View.NonWindowSQL) t", + SummaryName: "Meta", + Relations: []*plan.Relation{ + { + Name: "products", + Parent: "vendor", + Holder: "Products", + Ref: "products", + Table: "PRODUCT", + On: []*plan.RelationLink{ + {ParentColumn: "ID", RefColumn: "VENDOR_ID"}, + }, + }, + }, + }, + { + Name: "products", + Table: "PRODUCT", + Cardinality: "many", + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQL: "SELECT * FROM PRODUCT", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) PROD_META GROUP BY VENDOR_ID", + SummaryName: "ProductsMeta", + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Meta", + In: state.NewOutputLocation("summary"), + Schema: &state.Schema{DataType: "?"}, + }, + }, + }, + Components: []*plan.ComponentRoute{ + { + RoutePath: "/v1/api/dev/meta/vendors-nested", + Method: "GET", + ViewName: "vendor", + Name: "vendor", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifacts, err := New().LoadResource(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifacts) + require.NotNil(t, artifacts.Resource) + + index := artifacts.Resource.Views.Index() + root, err := index.Lookup("vendor") + require.NoError(t, err) + require.NotNil(t, root) + require.NotNil(t, root.Template) + require.NotNil(t, root.Template.Summary) + require.NotNil(t, root.Template.Summary.Schema) + assert.Equal(t, "*load.metaSummaryRow", root.Template.Summary.Schema.Type().String()) + + products, err := index.Lookup("products") + require.NoError(t, err) + require.NotNil(t, products) + require.NotNil(t, products.Template) + require.NotNil(t, products.Template.Summary) + require.NotNil(t, products.Template.Summary.Schema) + assert.Equal(t, "*load.productsMetaSummaryRow", products.Template.Summary.Schema.Type().String()) + productsSummaryType := products.Template.Summary.Schema.Type() + if productsSummaryType.Kind() == reflect.Ptr { + productsSummaryType = productsSummaryType.Elem() + } + productsSummaryField, ok := productsSummaryType.FieldByName("VendorID") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf(int(0)), productsSummaryField.Type) + + metaParam, err := artifacts.Resource.LookupParameter("Meta") + require.NoError(t, err) + require.NotNil(t, metaParam) + require.NotNil(t, metaParam.Schema) + require.NotNil(t, metaParam.Schema.Type()) + assert.Equal(t, "*load.metaSummaryRow", metaParam.Schema.Type().String()) + + componentArtifact, err := New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, componentArtifact) + component, ok := componentArtifact.Component.(*Component) + require.True(t, ok) + require.NotEmpty(t, component.Output) + require.NotNil(t, component.Output[0].Schema) + require.NotNil(t, component.Output[0].Schema.Type()) + assert.Equal(t, "*load.metaSummaryRow", component.Output[0].Schema.Type().String()) +} + +func TestRefineSummarySchemas_PrefersOwnerSchemaFieldTypeOverDiscoveredColumn(t *testing.T) { + resource := &view.Resource{ + Views: []*view.View{ + { + Name: "products", + Schema: &state.Schema{ + Name: "ProductsView", + DataType: "*ProductsView", + Cardinality: state.Many, + }, + Columns: []*view.Column{ + {Name: "VENDOR_ID", DatabaseColumn: "VENDOR_ID", DataType: "int"}, + }, + Template: &view.Template{ + Summary: &view.TemplateSummary{ + Name: "ProductsMeta", + Schema: &state.Schema{ + Name: "ProductsMetaView", + DataType: "*ProductsMetaView", + Cardinality: state.One, + }, + }, + }, + }, + }, + } + resource.Views[0].Schema.SetType(reflect.TypeOf([]productsOwnerPointerRow{})) + resource.Views[0].Template.Summary.Schema.SetType(reflect.TypeOf(productsMetaSummaryRow{})) + + RefineSummarySchemas(resource) + + summaryType := resource.Views[0].Template.Summary.Schema.Type() + require.NotNil(t, summaryType) + if summaryType.Kind() == reflect.Ptr { + summaryType = summaryType.Elem() + } + field, ok := summaryType.FieldByName("VendorID") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf((*int)(nil)), field.Type) +} + +func TestLoader_LoadResource_AttachesChildSummaryTemplateToRelationView(t *testing.T) { + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(metaSummaryRow{}), x.WithName("MetaView"))) + registry.Register(x.NewType(reflect.TypeOf(productsMetaSummaryRow{}), x.WithName("ProductsMetaView"))) + + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "meta", TypeRegistry: registry}, + Plan: &plan.Result{ + Views: []*plan.View{ + { + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]vendorSummaryParentRow{}), + ElementType: reflect.TypeOf(vendorSummaryParentRow{}), + SQL: "SELECT * FROM VENDOR", + Summary: "SELECT COUNT(*) AS CNT FROM ($View.NonWindowSQL) t", + SummaryName: "Meta", + Relations: []*plan.Relation{ + { + Name: "products", + Parent: "vendor", + Holder: "Products", + Ref: "products", + Table: "PRODUCT", + On: []*plan.RelationLink{ + {ParentColumn: "ID", RefColumn: "VENDOR_ID"}, + }, + }, + }, + }, + { + Name: "products", + Table: "PRODUCT", + Cardinality: "many", + FieldType: reflect.TypeOf([]vendorSummaryChildRow{}), + ElementType: reflect.TypeOf(vendorSummaryChildRow{}), + SQL: "SELECT * FROM PRODUCT", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) PROD_META GROUP BY VENDOR_ID", + SummaryName: "ProductsMeta", + }, + }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, + }, + } + + artifacts, err := New().LoadResource(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifacts) + root, err := artifacts.Resource.Views.Index().Lookup("vendor") require.NoError(t, err) + require.NotNil(t, root) + require.Len(t, root.With, 1) + require.NotNil(t, root.With[0].Of.Template) + require.NotNil(t, root.With[0].Of.Template.Summary) + require.NotNil(t, root.With[0].Of.Template.Summary.Schema) + assert.Equal(t, "ProductsMetaView", root.With[0].Of.Template.Summary.Schema.Name) - loader := New() - artifact, err := loader.LoadComponent(context.Background(), planned) + products, err := artifacts.Resource.Views.Index().Lookup("products") require.NoError(t, err) + require.NotNil(t, products) + require.NotNil(t, products.Template) + require.NotNil(t, products.Template.Summary) + require.NotNil(t, products.Template.Summary.Schema) - component, ok := ComponentFrom(artifact) + relationSummaryType := root.With[0].Of.Template.Summary.Schema.Type() + require.NotNil(t, relationSummaryType) + if relationSummaryType.Kind() == reflect.Ptr { + relationSummaryType = relationSummaryType.Elem() + } + relationField, ok := relationSummaryType.FieldByName("VendorID") require.True(t, ok) - assert.Equal(t, "/v1/api/dev/router-only", component.URI) - assert.Equal(t, "GET", component.Method) - require.Len(t, component.Input, 1) - assert.Equal(t, "id", component.Input[0].Name) - require.Len(t, component.Output, 2) - require.Len(t, artifact.Resource.Views, 1) - assert.Equal(t, "rows", artifact.Resource.Views[0].Name) + assert.NotEqual(t, "true", relationField.Tag.Get("internal")) + + standaloneSummaryType := products.Template.Summary.Schema.Type() + require.NotNil(t, standaloneSummaryType) + if standaloneSummaryType.Kind() == reflect.Ptr { + standaloneSummaryType = standaloneSummaryType.Elem() + } + standaloneField, ok := standaloneSummaryType.FieldByName("VendorID") + require.True(t, ok) + assert.NotEqual(t, "true", standaloneField.Tag.Get("internal")) + assert.Equal(t, standaloneSummaryType, relationSummaryType) } -func TestLoader_LoadComponent_SynthesizesStatesFromRouteContractsWhenPlanStatesAreEmpty(t *testing.T) { +func TestLoader_LoadResource_MaterializesNamedResourceTypes(t *testing.T) { + registry := x.NewRegistry() + registry.Register(x.NewType(reflect.TypeOf(metaSummaryRow{}), x.WithName("MetaView"))) + registry.Register(x.NewType(reflect.TypeOf(productsMetaSummaryRow{}), x.WithName("ProductsMetaView"))) + planned := &shape.PlanResult{ - Source: &shape.Source{Name: "team"}, + Source: &shape.Source{Name: "meta", TypeRegistry: registry}, Plan: &plan.Result{ - Components: []*plan.ComponentRoute{ + Views: []*plan.View{ { - FieldName: "Team", - Name: "Team", - RoutePath: "/v1/api/dev/team/{teamID}", - Method: "DELETE", - InputType: reflect.TypeOf(typedTeamRouteInput{}), - OutputType: reflect.TypeOf(typedTeamRouteOutput{}), - ViewName: "Team", + Name: "vendor", + Table: "VENDOR", + Cardinality: "many", + FieldType: reflect.TypeOf([]vendorSummaryParentRow{}), + ElementType: reflect.TypeOf(vendorSummaryParentRow{}), + SQL: "SELECT * FROM VENDOR", + Summary: "SELECT COUNT(*) AS CNT FROM ($View.NonWindowSQL) t", + SummaryName: "Meta", + Relations: []*plan.Relation{ + { + Name: "products", + Parent: "vendor", + Holder: "Products", + Ref: "products", + Table: "PRODUCT", + On: []*plan.RelationLink{ + {ParentColumn: "ID", RefColumn: "VENDOR_ID"}, + }, + }, + }, + }, + { + Name: "products", + Table: "PRODUCT", + Cardinality: "many", + FieldType: reflect.TypeOf([]vendorSummaryChildRow{}), + ElementType: reflect.TypeOf(vendorSummaryChildRow{}), + SQL: "SELECT * FROM PRODUCT", + Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) PROD_META GROUP BY VENDOR_ID", + SummaryName: "ProductsMeta", }, }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, }, } - artifact, err := New().LoadComponent(context.Background(), planned, shape.WithLoadTypeContextPackages(true)) + artifacts, err := New().LoadResource(context.Background(), planned) require.NoError(t, err) + require.NotNil(t, artifacts) + require.NotNil(t, artifacts.Resource) - component, ok := ComponentFrom(artifact) - require.True(t, ok) - require.Len(t, component.Input, 1) - assert.Equal(t, "TeamID", component.Input[0].Name) - require.NotNil(t, component.Input[0].In) - assert.Equal(t, state.KindPath, component.Input[0].In.Kind) - assert.Equal(t, "teamID", component.Input[0].In.Name) + var actual []string + for _, item := range artifacts.Resource.Types { + if item == nil { + continue + } + actual = append(actual, item.Name) + } + assert.ElementsMatch(t, []string{"VendorView", "MetaView", "ProductsView", "ProductsMetaView"}, actual) } -func TestLoader_LoadComponent_CacheProviderDoesNotBindRootView(t *testing.T) { +func TestLoader_LoadViews_PreservesSummarySourceURL(t *testing.T) { planned := &shape.PlanResult{ - Source: &shape.Source{Name: "/v1/api/shape/dev/vendors/"}, + Source: &shape.Source{Name: "meta"}, Plan: &plan.Result{ Views: []*plan.View{ { - Path: "vendor", Name: "vendor", Table: "VENDOR", Cardinality: "many", - FieldType: reflect.TypeOf([]map[string]any{}), - ElementType: reflect.TypeOf(map[string]any{}), - SQL: "SELECT * FROM VENDOR", - }, - }, - Directives: &dqlshape.Directives{ - Cache: &dqlshape.CacheDirective{ - Enabled: true, - Name: "aerospike", - Provider: "aerospike://127.0.0.1:3000/test", - Location: "${view.Name}", - TimeToLiveMs: 3600000, + FieldType: reflect.TypeOf([]map[string]interface{}{}), + ElementType: reflect.TypeOf(map[string]interface{}{}), + SQLURI: "vendor/vendor.sql", + SummaryURL: "vendor/vendor_summary.sql", + SummaryName: "Meta", }, }, + ViewsByName: map[string]*plan.View{}, + ByPath: map[string]*plan.Field{}, }, } - loader := New() - artifact, err := loader.LoadComponent(context.Background(), planned) + artifacts, err := New().LoadViews(context.Background(), planned) require.NoError(t, err) - require.NotNil(t, artifact) - require.NotNil(t, artifact.Resource) - require.Len(t, artifact.Resource.Views, 1) - require.NotEmpty(t, artifact.Resource.CacheProviders) - - root := artifact.Resource.Views[0] - assert.Nil(t, root.Cache) + require.NotNil(t, artifacts) + require.Len(t, artifacts.Views, 1) + require.NotNil(t, artifacts.Views[0].Template) + require.NotNil(t, artifacts.Views[0].Template.Summary) + assert.Equal(t, "vendor/vendor_summary.sql", artifacts.Views[0].Template.Summary.SourceURL) } -func TestLoader_LoadViews_DoesNotSeedPlaceholderColumnsFromLinkedType(t *testing.T) { +func TestLoader_LoadResource_TypedViewDefinitionsPreferSchemaFieldsOverColumns(t *testing.T) { + type foosViewHas struct { + Id bool + Name bool + Quantity bool + } + type foosView struct { + Id int `sqlx:"ID" velty:"names=ID|Id"` + Name *string `sqlx:"NAME" velty:"names=NAME|Name"` + Quantity *int `sqlx:"QUANTITY" velty:"names=QUANTITY|Quantity"` + Has *foosViewHas `setMarker:"true" format:"-" sqlx:"-" diff:"-" json:"-" typeName:"FoosViewHas"` + } + planned := &shape.PlanResult{ - Source: &shape.Source{Name: "districts"}, + Source: &shape.Source{Name: "patch_basic_one"}, Plan: &plan.Result{ Views: []*plan.View{ { - Name: "districts", - Table: "DISTRICT", + Name: "foos", + Table: "FOOS", Cardinality: "many", - SchemaType: "*DistrictsView", - FieldType: reflect.TypeOf([]*placeholderDistrictRow{}), - ElementType: reflect.TypeOf(placeholderDistrictRow{}), + FieldType: reflect.TypeOf([]foosView{}), + ElementType: reflect.TypeOf(foosView{}), + SQL: "SELECT * FROM FOOS", }, }, ViewsByName: map[string]*plan.View{}, @@ -784,25 +1863,60 @@ func TestLoader_LoadViews_DoesNotSeedPlaceholderColumnsFromLinkedType(t *testing }, } - artifact, err := New().LoadViews(context.Background(), planned) + artifacts, err := New().LoadResource(context.Background(), planned) require.NoError(t, err) - require.Len(t, artifact.Views, 1) - assert.Empty(t, artifact.Views[0].Columns) + require.NotNil(t, artifacts) + require.NotNil(t, artifacts.Resource) + + var actual []string + for _, item := range artifacts.Resource.Types { + if item == nil || item.Name != "FoosView" { + continue + } + for _, field := range item.Fields { + actual = append(actual, field.Name) + } + } + assert.Equal(t, []string{"Id", "Name", "Quantity", "Has"}, actual) } -func TestLoader_LoadViews_DefersMapBackedQuerySchemaType(t *testing.T) { +func TestLoader_LoadViews_InferRelationLinkFieldsFromSchemaTypes(t *testing.T) { planned := &shape.PlanResult{ - Source: &shape.Source{Name: "cities"}, + Source: &shape.Source{Name: "vendor"}, Plan: &plan.Result{ Views: []*plan.View{ { - Name: "cities", - Table: "CITY", - Mode: string(view.ModeQuery), + Name: "vendor", + Table: "VENDOR", Cardinality: "many", - FieldType: reflect.TypeOf([]map[string]interface{}{}), - ElementType: reflect.TypeOf(map[string]interface{}{}), - SQL: "SELECT * FROM CITY", + FieldType: reflect.TypeOf([]reportRow{}), + ElementType: reflect.TypeOf(reportRow{}), + SQL: "SELECT * FROM VENDOR", + Relations: []*plan.Relation{ + { + Name: "products", + Parent: "vendor", + Holder: "Products", + Ref: "products", + Table: "PRODUCT", + On: []*plan.RelationLink{ + { + ParentNamespace: "vendor", + ParentColumn: "ID", + RefNamespace: "products", + RefColumn: "VENDOR_ID", + }, + }, + }, + }, + }, + { + Name: "products", + Table: "PRODUCT", + Cardinality: "many", + FieldType: reflect.TypeOf([]vendorProductRow{}), + ElementType: reflect.TypeOf(vendorProductRow{}), + SQL: "SELECT * FROM PRODUCT", }, }, ViewsByName: map[string]*plan.View{}, @@ -810,15 +1924,27 @@ func TestLoader_LoadViews_DefersMapBackedQuerySchemaType(t *testing.T) { }, } - artifact, err := New().LoadViews(context.Background(), planned) + artifacts, err := New().LoadViews(context.Background(), planned) require.NoError(t, err) - require.Len(t, artifact.Views, 1) - assert.Nil(t, artifact.Views[0].Schema.Type()) + + root, err := artifacts.Resource.Views.Index().Lookup("vendor") + require.NoError(t, err) + require.NotNil(t, root) + require.Len(t, root.With, 1) + require.Len(t, root.With[0].On, 1) + require.Len(t, root.With[0].Of.On, 1) + + assert.Equal(t, "ID", root.With[0].On[0].Column) + assert.Equal(t, "ID", root.With[0].On[0].Field) + assert.Empty(t, root.With[0].On[0].Namespace) + assert.Equal(t, "VENDOR_ID", root.With[0].Of.On[0].Column) + assert.Equal(t, "VendorID", root.With[0].Of.On[0].Field) + assert.Empty(t, root.With[0].Of.On[0].Namespace) } -func TestLoader_LoadViews_PreservesChildSummaryGraph(t *testing.T) { +func TestLoader_LoadViews_FallsBackToColumnNamesForRelationLinkFields(t *testing.T) { planned := &shape.PlanResult{ - Source: &shape.Source{Name: "meta"}, + Source: &shape.Source{Name: "vendor"}, Plan: &plan.Result{ Views: []*plan.View{ { @@ -836,7 +1962,10 @@ func TestLoader_LoadViews_PreservesChildSummaryGraph(t *testing.T) { Ref: "products", Table: "PRODUCT", On: []*plan.RelationLink{ - {ParentColumn: "ID", RefColumn: "VENDOR_ID"}, + { + ParentColumn: "ID", + RefColumn: "VENDOR_ID", + }, }, }, }, @@ -848,7 +1977,6 @@ func TestLoader_LoadViews_PreservesChildSummaryGraph(t *testing.T) { FieldType: reflect.TypeOf([]map[string]interface{}{}), ElementType: reflect.TypeOf(map[string]interface{}{}), SQL: "SELECT * FROM PRODUCT", - Summary: "SELECT VENDOR_ID, COUNT(*) AS TOTAL_PRODUCTS FROM ($View.products.SQL) PROD_META GROUP BY VENDOR_ID", }, }, ViewsByName: map[string]*plan.View{}, @@ -856,20 +1984,18 @@ func TestLoader_LoadViews_PreservesChildSummaryGraph(t *testing.T) { }, } - loader := New() - artifacts, err := loader.LoadViews(context.Background(), planned) + artifacts, err := New().LoadViews(context.Background(), planned) require.NoError(t, err) - require.NotNil(t, artifacts) - require.Len(t, artifacts.Views, 2) - index := artifacts.Resource.Views.Index() - products, err := index.Lookup("products") + root, err := artifacts.Resource.Views.Index().Lookup("vendor") require.NoError(t, err) - require.NotNil(t, products) - require.NotNil(t, products.Template) - require.NotNil(t, products.Template.Summary) - assert.Contains(t, products.Template.Summary.Source, "TOTAL_PRODUCTS") - assert.Contains(t, products.Template.Summary.Source, "$View.products.SQL") + require.NotNil(t, root) + require.Len(t, root.With, 1) + require.Len(t, root.With[0].On, 1) + require.Len(t, root.With[0].Of.On, 1) + + assert.Equal(t, "Id", root.With[0].On[0].Field) + assert.Equal(t, "VendorId", root.With[0].Of.On[0].Field) } func TestLoader_LoadComponent_ConstDirectiveCreatesInternalConstParameter(t *testing.T) { @@ -1085,6 +2211,117 @@ func TestLoader_LoadComponent_RelationFieldsPreserved(t *testing.T) { assert.Equal(t, "ID", ref.Column) } +func TestLoader_LoadComponent_ExecViewInputPreservesVeltyAliases(t *testing.T) { + required := false + planned := &shape.PlanResult{ + Source: &shape.Source{Name: "user_team"}, + Plan: &plan.Result{ + Components: []*plan.ComponentRoute{ + { + Name: "user_team", + Method: "PUT", + Path: "/v1/api/shape/dev/teams", + ViewName: "user_team", + }, + }, + States: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "TeamIDs", + In: state.NewQueryLocation("TeamIDs"), + Required: &required, + Schema: &state.Schema{DataType: "[]int", Cardinality: state.Many}, + }, + }, + { + Parameter: state.Parameter{ + Name: "TeamStats", + In: state.NewViewLocation("TeamStats"), + Required: &required, + Schema: &state.Schema{Cardinality: state.Many}, + }, + }, + }, + Views: []*plan.View{ + { + Name: "user_team", + Table: "TEAM", + Cardinality: "many", + SQL: "UPDATE TEAM SET ACTIVE = false", + Mode: string(view.ModeExec), + FieldType: reflect.TypeOf([]struct { + Id int `sqlx:"ID"` + }{}), + ElementType: reflect.TypeOf(struct { + Id int `sqlx:"ID"` + }{}), + }, + { + Name: "TeamStats", + Table: "TEAM", + Cardinality: "many", + Mode: string(view.ModeQuery), + ColumnsDiscovery: true, + }, + }, + ByPath: map[string]*plan.Field{}, + ViewsByName: map[string]*plan.View{}, + }, + } + plannedResult, ok := plan.ResultFrom(planned) + require.True(t, ok) + require.Len(t, plannedResult.Views, 2) + plannedResult.Views[1].FieldType = reflect.TypeOf([]struct { + Id int `sqlx:"ID"` + TeamMembers int `sqlx:"TEAM_MEMBERS"` + Name *string `sqlx:"NAME"` + }{}) + plannedResult.Views[1].ElementType = plannedResult.Views[1].FieldType.Elem() + + artifact, err := New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + require.NotNil(t, artifact) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.NotNil(t, component) + + resourceView, err := artifact.Resource.View("TeamStats") + require.NoError(t, err) + require.NotNil(t, resourceView) + require.NotNil(t, resourceView.Schema) + resourceType := resourceView.Schema.Type() + require.NotNil(t, resourceType) + if resourceType.Kind() == reflect.Slice { + resourceType = resourceType.Elem() + } + if resourceType.Kind() == reflect.Ptr { + resourceType = resourceType.Elem() + } + require.Equal(t, reflect.Struct, resourceType.Kind()) + + var inputParam *plan.State + for _, item := range component.Input { + if item != nil && strings.EqualFold(item.Name, "TeamStats") { + inputParam = item + break + } + } + require.NotNil(t, inputParam) + require.NotNil(t, inputParam.Schema) + inputType := inputParam.Schema.Type() + require.NotNil(t, inputType) + if inputType.Kind() == reflect.Slice { + inputType = inputType.Elem() + } + if inputType.Kind() == reflect.Ptr { + inputType = inputType.Elem() + } + require.Equal(t, reflect.Struct, inputType.Kind()) + idField, ok := inputType.FieldByName("Id") + require.True(t, ok) + assert.Equal(t, "names=ID|Id", idField.Tag.Get("velty")) +} + func TestLoader_LoadComponent_AttachesRelationTreeByParent(t *testing.T) { planned := &shape.PlanResult{ Source: &shape.Source{Name: "/v1/api/tree"}, @@ -1489,6 +2726,8 @@ func TestLoader_LoadComponent_IncludesComponentStateInInput(t *testing.T) { require.Len(t, component.Input, 1) assert.Equal(t, state.KindComponent, component.Input[0].In.Kind) assert.Equal(t, "Auth", component.Input[0].Name) + require.NotNil(t, component.Input[0].Schema) + assert.Equal(t, reflect.TypeOf((*interface{})(nil)).Elem(), component.Input[0].Schema.Type()) } func TestLoader_LoadComponent_MaterializesOutputStatusSchema(t *testing.T) { @@ -1719,3 +2958,24 @@ func TestLoader_LoadComponent_AllowsViewlessComponentRoute(t *testing.T) { assert.Equal(t, "/v1/api/dev/team/{teamID}", component.URI) assert.Empty(t, artifact.Resource.Views) } + +func TestLoader_LoadComponent_QuerySelectorHolder(t *testing.T) { + scanned, err := scan.New().Scan(context.Background(), &shape.Source{Struct: &selectorHolderSource{}}) + require.NoError(t, err) + planned, err := plan.New().Plan(context.Background(), scanned) + require.NoError(t, err) + + artifact, err := New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + component, ok := ComponentFrom(artifact) + require.True(t, ok) + + require.Equal(t, []string{"fields", "page"}, component.QuerySelectors["rows"]) + fields := component.InputParameters().Lookup("fields") + require.NotNil(t, fields) + assert.Equal(t, state.KindQuery, fields.In.Kind) + assert.Equal(t, "_fields", fields.In.Name) + page := component.InputParameters().Lookup("page") + require.NotNil(t, page) + assert.Equal(t, "_page", page.In.Name) +} diff --git a/repository/shape/model.go b/repository/shape/model.go index d48c848ec..8f5f97335 100644 --- a/repository/shape/model.go +++ b/repository/shape/model.go @@ -89,6 +89,11 @@ type ViewArtifacts struct { Views view.Views } +// ResourceArtifacts is the runtime resource payload produced by Loader. +type ResourceArtifacts struct { + Resource *view.Resource +} + // ComponentArtifact is the runtime component payload produced by Loader. type ComponentArtifact struct { Resource *view.Resource diff --git a/repository/shape/parity_test.go b/repository/shape/parity_test.go index 8041328b1..4703d7d0d 100644 --- a/repository/shape/parity_test.go +++ b/repository/shape/parity_test.go @@ -75,6 +75,24 @@ func TestEngineParity_StructPipeline(t *testing.T) { assert.Equal(t, reflect.TypeOf(mv.Schema.CompType()), reflect.TypeOf(ev.Schema.CompType())) } +func TestEngineParity_LoadResource(t *testing.T) { + source := &paritySource{} + engine := shape.New( + shape.WithName("/v1/api/parity"), + shape.WithScanner(shapeScan.New()), + shape.WithPlanner(shapePlan.New()), + shape.WithLoader(shapeLoad.New()), + ) + + artifact, err := engine.LoadResource(context.Background(), source) + require.NoError(t, err) + require.NotNil(t, artifact) + require.NotNil(t, artifact.Resource) + require.Len(t, artifact.Resource.Views, 1) + assert.Equal(t, "rows", artifact.Resource.Views[0].Name) + assert.Equal(t, "REPORT", artifact.Resource.Views[0].Table) +} + func TestEngineParity_Component_SourceTagFieldJoin(t *testing.T) { source := &parityJoinSource{} scanner := shapeScan.New() diff --git a/repository/shape/plan/model.go b/repository/shape/plan/model.go index ae3b3879c..1aa77ba6e 100644 --- a/repository/shape/plan/model.go +++ b/repository/shape/plan/model.go @@ -82,17 +82,26 @@ type View struct { SQL string SQLURI string Summary string + SummaryURL string SummaryName string Relations []*Relation Holder string - AllowNulls *bool - SelectorNamespace string - SelectorLimit *int - SelectorNoLimit *bool - SchemaType string - ColumnsDiscovery bool - Self *SelfReference + AllowNulls *bool + Groupable *bool + SelectorNamespace string + SelectorLimit *int + SelectorNoLimit *bool + SelectorCriteria *bool + SelectorProjection *bool + SelectorOrderBy *bool + SelectorOffset *bool + SelectorPage *bool + SelectorFilterable []string + SelectorOrderByColumns map[string]string + SchemaType string + ColumnsDiscovery bool + Self *SelfReference Cardinality string ElementType reflect.Type @@ -136,8 +145,9 @@ type ViewPredicate struct { // ViewColumnConfig captures declaration-level per-column overrides. type ViewColumnConfig struct { - DataType string - Tag string + DataType string + Tag string + Groupable *bool } // Relation is normalized relation metadata extracted from DQL joins. @@ -177,6 +187,7 @@ type State struct { state.Parameter `yaml:",inline"` QuerySelector string OutputDataType string + EmitOutput bool } func (s *State) KindString() string { diff --git a/repository/shape/plan/planner.go b/repository/shape/plan/planner.go index f97139ac5..11852dcd4 100644 --- a/repository/shape/plan/planner.go +++ b/repository/shape/plan/planner.go @@ -90,6 +90,27 @@ func normalizeView(field *scan.Field) *View { result.Partitioner = tag.View.PartitionerType result.PartitionedConcurrency = tag.View.PartitionedConcurrency result.RelationalConcurrency = tag.View.RelationalConcurrency + result.Groupable = tag.View.Groupable + result.SelectorNamespace = strings.TrimSpace(tag.View.SelectorNamespace) + result.SelectorLimit = tag.View.Limit + if tag.View.Limit != nil { + noLimit := *tag.View.Limit == 0 + result.SelectorNoLimit = &noLimit + } + result.SelectorCriteria = tag.View.SelectorCriteria + result.SelectorProjection = tag.View.SelectorProjection + result.SelectorOrderBy = tag.View.SelectorOrderBy + result.SelectorOffset = tag.View.SelectorOffset + result.SelectorPage = tag.View.SelectorPage + if len(tag.View.SelectorFilterable) > 0 { + result.SelectorFilterable = append([]string(nil), tag.View.SelectorFilterable...) + } + if len(tag.View.SelectorOrderByColumns) > 0 { + result.SelectorOrderByColumns = map[string]string{} + for key, value := range tag.View.SelectorOrderByColumns { + result.SelectorOrderByColumns[key] = value + } + } if strings.TrimSpace(tag.View.CustomTag) != "" || strings.TrimSpace(field.ViewTypeName) != "" || strings.TrimSpace(field.ViewDest) != "" { result.Declaration = &ViewDeclaration{ Tag: strings.TrimSpace(tag.View.CustomTag), @@ -101,6 +122,11 @@ func normalizeView(field *scan.Field) *View { result.SQL = tag.SQL.SQL result.SQLURI = tag.SQL.URI result.Summary = tag.SummarySQL.SQL + if tag.View != nil && strings.TrimSpace(tag.View.SummaryURI) != "" { + result.SummaryURL = strings.TrimSpace(tag.View.SummaryURI) + } else { + result.SummaryURL = tag.SummarySQL.URI + } if len(tag.LinkOn) > 0 { result.Relations = append(result.Relations, relationFromTagLinks(field.Name, tag.LinkOn)) } @@ -184,13 +210,18 @@ func normalizeState(field *scan.Field) *State { Name: field.Name, In: &state.Location{}, }, + QuerySelector: strings.TrimSpace(field.QuerySelector), } - if field.StateTag == nil || field.StateTag.Parameter == nil { + if field.StateTag == nil { result.Schema = state.NewSchema(field.Type) return result } pTag := field.StateTag.Parameter + if pTag == nil { + result.Schema = state.NewSchema(field.Type) + return result + } result.Name = firstNonEmpty(pTag.Name, field.Name) result.In = &state.Location{ Kind: state.Kind(strings.ToLower(strings.TrimSpace(pTag.Kind))), @@ -205,7 +236,6 @@ func normalizeState(field *scan.Field) *State { result.URI = pTag.URI result.ErrorStatusCode = pTag.ErrorCode result.ErrorMessage = pTag.ErrorMessage - result.Schema = state.NewSchema(resolveStateType(result, field.Type)) if typeName := strings.TrimSpace(field.StateTag.TypeName); typeName != "" { applyStateTypeName(result.Schema, typeName) diff --git a/repository/shape/plan/planner_test.go b/repository/shape/plan/planner_test.go index 25fb8ee2c..cce058649 100644 --- a/repository/shape/plan/planner_test.go +++ b/repository/shape/plan/planner_test.go @@ -99,6 +99,19 @@ type codecStateSource struct { Run string `parameter:",kind=body,in=run" handler:"Exec"` } +type selectorHolderSource struct { + Route xdatly.Component[typedRouteInput, typedRouteOutput] `component:",path=/v1/api/dev/report,method=GET"` + ViewSelect struct { + Fields []string `parameter:"fields,kind=query,in=_fields"` + Page int `parameter:"page,kind=query,in=_page"` + } `querySelector:"rows"` +} + +type summaryViewSource struct { + embeddedFS + Rows []relationRow `view:"rows,table=REPORT,summaryURI=testdata/report_summary.sql" sql:"uri=testdata/report.sql"` +} + func TestPlanner_Plan(t *testing.T) { scanner := scan.New() scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportSource{}}) @@ -138,7 +151,6 @@ func TestPlanner_Plan(t *testing.T) { require.NotNil(t, stateByPath["id"]) assert.Equal(t, "query", stateByPath["id"].KindString()) assert.Equal(t, "id", stateByPath["id"].InName()) - require.Len(t, result.Components, 1) assert.Equal(t, "Route", result.Components[0].FieldName) assert.Equal(t, "/v1/api/dev/report", result.Components[0].RoutePath) @@ -146,6 +158,40 @@ func TestPlanner_Plan(t *testing.T) { assert.Equal(t, "dev", result.Components[0].Connector) } +func TestPlanner_Plan_QuerySelectorHolder(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &selectorHolderSource{}}) + require.NoError(t, err) + + planned, err := New().Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + + byName := map[string]*State{} + for _, item := range result.States { + byName[item.Name] = item + } + require.NotNil(t, byName["fields"]) + assert.Equal(t, "rows", byName["fields"].QuerySelector) + require.NotNil(t, byName["page"]) + assert.Equal(t, "rows", byName["page"].QuerySelector) +} + +func TestPlanner_Plan_ViewSummaryURI(t *testing.T) { + scanned, err := scan.New().Scan(context.Background(), &shape.Source{Struct: &summaryViewSource{}}) + require.NoError(t, err) + + planned, err := New().Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.Views, 1) + assert.Equal(t, "testdata/report_summary.sql", result.Views[0].SummaryURL) +} + func TestPlanner_Plan_LinkOnProducesStructuredRelations(t *testing.T) { scanner := scan.New() scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &relationSource{}}) diff --git a/repository/shape/plan/testdata/report_summary.sql b/repository/shape/plan/testdata/report_summary.sql new file mode 100644 index 000000000..3c601cdc2 --- /dev/null +++ b/repository/shape/plan/testdata/report_summary.sql @@ -0,0 +1 @@ +SELECT COUNT(*) AS CNT FROM REPORT diff --git a/repository/shape/scan/model.go b/repository/shape/scan/model.go index c750fd672..59e968a51 100644 --- a/repository/shape/scan/model.go +++ b/repository/shape/scan/model.go @@ -25,6 +25,7 @@ type Field struct { Name string Index []int Type reflect.Type + QuerySelector string ComponentInputType reflect.Type ComponentOutputType reflect.Type ComponentInputName string diff --git a/repository/shape/scan/scanner.go b/repository/shape/scan/scanner.go index f798f27a3..c4463380f 100644 --- a/repository/shape/scan/scanner.go +++ b/repository/shape/scan/scanner.go @@ -49,7 +49,7 @@ func (s *StructScanner) Scan(ctx context.Context, source *shape.Source, _ ...sha ByPath: map[string]*Field{}, } - if err = s.scanStruct(source, root, rootValue, "", nil, embedder, baseDir, result, map[reflect.Type]bool{}); err != nil { + if err = s.scanStruct(source, root, rootValue, "", nil, "", embedder, baseDir, result, map[reflect.Type]bool{}); err != nil { return nil, err } @@ -116,6 +116,7 @@ func (s *StructScanner) scanStruct( rootValue reflect.Value, prefix string, indexPrefix []int, + inheritedQuerySelector string, embedder *state.FSEmbedder, baseDir string, result *Result, @@ -136,12 +137,16 @@ func (s *StructScanner) scanStruct( combinedIndex := append(append([]int{}, indexPrefix...), field.Index...) descriptor := &Field{ - Path: path, - Name: field.Name, - Index: combinedIndex, - Type: field.Type, - Tag: field.Tag, - Anonymous: field.Anonymous, + Path: path, + Name: field.Name, + Index: combinedIndex, + Type: field.Type, + QuerySelector: inheritedQuerySelector, + Tag: field.Tag, + Anonymous: field.Anonymous, + } + if querySelector := tags.ParseQuerySelector(field.Tag.Get(tags.QuerySelectorTag)); querySelector != "" { + descriptor.QuerySelector = querySelector } fieldFS := parseFS(field.Tag, embedder.EmbedFS(), baseDir) @@ -195,7 +200,7 @@ func (s *StructScanner) scanStruct( nextType := nestedStructType(field.Type) if nextType != nil && shouldRecurseIntoField(field, descriptor, nextType) { - if err := s.scanStruct(source, nextType, rootValue, path, combinedIndex, embedder, baseDir, result, visited); err != nil { + if err := s.scanStruct(source, nextType, rootValue, path, combinedIndex, descriptor.QuerySelector, embedder, baseDir, result, visited); err != nil { return err } } @@ -230,6 +235,9 @@ func shouldRecurseIntoField(field reflect.StructField, descriptor *Field, nextTy // They still need recursive scanning so nested relation views are preserved. return true } + if descriptor != nil && strings.TrimSpace(descriptor.QuerySelector) != "" { + return true + } return false } @@ -248,14 +256,14 @@ func (s *StructScanner) scanComponentContracts( if contract.InputType != nil { embedder := state.NewFSEmbedder(nil) embedder.SetType(contract.InputType) - if err := s.scanStruct(source, contractInputRoot(contract.InputType), componentFieldValue(fieldValue, "Inout"), prefix+".Inout", nil, embedder, baseDir, result, visited); err != nil { + if err := s.scanStruct(source, contractInputRoot(contract.InputType), componentFieldValue(fieldValue, "Inout"), prefix+".Inout", nil, "", embedder, baseDir, result, visited); err != nil { return err } } if contract.OutputType != nil { embedder := state.NewFSEmbedder(nil) embedder.SetType(contract.OutputType) - if err := s.scanStruct(source, contractInputRoot(contract.OutputType), componentFieldValue(fieldValue, "Output"), prefix+".Output", nil, embedder, baseDir, result, visited); err != nil { + if err := s.scanStruct(source, contractInputRoot(contract.OutputType), componentFieldValue(fieldValue, "Output"), prefix+".Output", nil, "", embedder, baseDir, result, visited); err != nil { return err } } @@ -363,6 +371,8 @@ func parseShapeViewHints(tag reflect.StructTag) (string, string) { var typeName, dest string _ = values.MatchPairs(func(key, value string) error { switch strings.ToLower(strings.TrimSpace(key)) { + case "type": + typeName = strings.TrimSpace(value) case "typename": typeName = strings.TrimSpace(value) case "dest": diff --git a/repository/shape/scan/scanner_test.go b/repository/shape/scan/scanner_test.go index 42a53750e..c3c0f7fe6 100644 --- a/repository/shape/scan/scanner_test.go +++ b/repository/shape/scan/scanner_test.go @@ -64,6 +64,14 @@ type namedReportOutput struct { Data []reportRow `parameter:"data,kind=output,in=view"` } +type selectorHolderSource struct { + Route xdatly.Component[reportInput, reportOutput] `component:",path=/v1/api/dev/report,method=GET"` + ViewSelect struct { + Fields []string `parameter:"fields,kind=query,in=_fields"` + Page int `parameter:"page,kind=query,in=_page"` + } `querySelector:"rows"` +} + func TestStructScanner_Scan(t *testing.T) { scanner := New() result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportSource{}}) @@ -81,8 +89,8 @@ func TestStructScanner_Scan(t *testing.T) { require.True(t, rows.HasViewTag) require.NotNil(t, rows.ViewTag) assert.Equal(t, "rows", rows.ViewTag.View.Name) - assert.Equal(t, "ReportRow", rows.ViewTag.View.TypeName) - assert.Equal(t, "rows.go", rows.ViewTag.View.Dest) + assert.Equal(t, "ReportRow", rows.ViewTypeName) + assert.Equal(t, "rows.go", rows.ViewDest) assert.Contains(t, rows.ViewTag.SQL.SQL, "SELECT ID, NAME FROM REPORT") idField := descriptors.ByPath["ID"] @@ -127,6 +135,23 @@ func TestStructScanner_Scan_ComponentHolderTypes(t *testing.T) { assert.Empty(t, route.ComponentOutputName) } +func TestStructScanner_Scan_QuerySelectorHolder(t *testing.T) { + scanner := New() + result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &selectorHolderSource{}}) + require.NoError(t, err) + + descriptors, ok := DescriptorsFrom(result) + require.True(t, ok) + + fields := descriptors.ByPath["ViewSelect.Fields"] + require.NotNil(t, fields) + assert.Equal(t, "rows", fields.QuerySelector) + + page := descriptors.ByPath["ViewSelect.Page"] + require.NotNil(t, page) + assert.Equal(t, "rows", page.QuerySelector) +} + func TestStructScanner_Scan_DynamicComponentHolderTypes(t *testing.T) { scanner := New() result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { diff --git a/repository/shape/shape.go b/repository/shape/shape.go index 94e6f63c8..ef6f38042 100644 --- a/repository/shape/shape.go +++ b/repository/shape/shape.go @@ -21,6 +21,7 @@ type ( // Loader materializes runtime artifacts from normalized plan. Loader interface { LoadViews(ctx context.Context, plan *PlanResult, opts ...LoadOption) (*ViewArtifacts, error) + LoadResource(ctx context.Context, plan *PlanResult, opts ...LoadOption) (*ResourceArtifacts, error) LoadComponent(ctx context.Context, plan *PlanResult, opts ...LoadOption) (*ComponentArtifact, error) } @@ -106,11 +107,21 @@ func LoadComponent(ctx context.Context, src any, opts ...Option) (*ComponentArti return New(opts...).LoadComponent(ctx, src) } +// LoadResource is a package-level helper for struct source resource loading. +func LoadResource(ctx context.Context, src any, opts ...Option) (*ResourceArtifacts, error) { + return New(opts...).LoadResource(ctx, src) +} + // LoadDQLViews is a package-level helper for DQL source view loading. func LoadDQLViews(ctx context.Context, dql string, opts ...Option) (*ViewArtifacts, error) { return New(opts...).LoadDQLViews(ctx, dql) } +// LoadDQLResource is a package-level helper for DQL source resource loading. +func LoadDQLResource(ctx context.Context, dql string, opts ...Option) (*ResourceArtifacts, error) { + return New(opts...).LoadDQLResource(ctx, dql) +} + // LoadDQLComponent is a package-level helper for DQL source component loading. func LoadDQLComponent(ctx context.Context, dql string, opts ...Option) (*ComponentArtifact, error) { return New(opts...).LoadDQLComponent(ctx, dql) @@ -132,6 +143,22 @@ func (e *Engine) LoadViews(ctx context.Context, src any) (*ViewArtifacts, error) return e.options.Loader.LoadViews(ctx, plan) } +// LoadResource executes scan -> plan -> load for struct source. +func (e *Engine) LoadResource(ctx context.Context, src any) (*ResourceArtifacts, error) { + source, err := e.structSource(src) + if err != nil { + return nil, err + } + plan, err := e.scanAndPlan(ctx, source) + if err != nil { + return nil, err + } + if e.options.Loader == nil { + return nil, ErrLoaderNotConfigured + } + return e.options.Loader.LoadResource(ctx, plan) +} + // LoadComponent executes scan -> plan -> load for struct source. func (e *Engine) LoadComponent(ctx context.Context, src any) (*ComponentArtifact, error) { source, err := e.structSource(src) @@ -164,6 +191,22 @@ func (e *Engine) LoadDQLViews(ctx context.Context, dql string) (*ViewArtifacts, return e.options.Loader.LoadViews(ctx, plan) } +// LoadDQLResource executes compile -> load for DQL source. +func (e *Engine) LoadDQLResource(ctx context.Context, dql string) (*ResourceArtifacts, error) { + source, err := e.dqlSource(dql) + if err != nil { + return nil, err + } + plan, err := e.compile(ctx, source) + if err != nil { + return nil, err + } + if e.options.Loader == nil { + return nil, ErrLoaderNotConfigured + } + return e.options.Loader.LoadResource(ctx, plan) +} + // LoadDQLComponent executes compile -> load for DQL source. func (e *Engine) LoadDQLComponent(ctx context.Context, dql string) (*ComponentArtifact, error) { source, err := e.dqlSource(dql) diff --git a/repository/shape/xgen/codegen.go b/repository/shape/xgen/codegen.go index c0dd4fc77..5c5b858b2 100644 --- a/repository/shape/xgen/codegen.go +++ b/repository/shape/xgen/codegen.go @@ -2,6 +2,7 @@ package xgen import ( "bytes" + "context" "fmt" "go/ast" "go/format" @@ -65,6 +66,12 @@ type ComponentCodegenResult struct { Embeds map[string]string // SQL file name → SQL content } +type codegenSelectorHolder struct { + FieldName string + QuerySelector string + Type reflect.Type +} + // Generate produces the component Go source file. func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { if g.Component == nil { @@ -129,13 +136,17 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { lookupType := g.componentLookupType(packagePath) var inputType, outputType reflect.Type - if params := g.Component.InputParameters(); len(params) > 0 || strings.TrimSpace(g.Component.URI) != "" { - normalized := normalizeInputParametersForCodegen(params, g.Resource, g.Component.URI) + var selectorHolders []codegenSelectorHolder + inputParams := state.Parameters(nil) + if params := g.codegenInputParameters(); len(params) > 0 || strings.TrimSpace(g.Component.URI) != "" { + normalized := params + inputParams, selectorHolders = g.partitionInputParametersForCodegen(normalized, packagePath, lookupType) + normalizeBodyInputTypesForCodegen(inputParams, packagePath, lookupType) inputOpts := []state.ReflectOption{state.WithSetMarker(), state.WithTypeName(inputTypeName)} if g.componentUsesVelty() { inputOpts = append(inputOpts, state.WithVelty(true)) } - rt, err := normalized.ReflectType(packagePath, lookupType, inputOpts...) + rt, err := inputParams.ReflectType(packagePath, lookupType, inputOpts...) if err == nil && rt != nil { inputType = rt } @@ -146,6 +157,7 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { if !hasExplicitOutput { outputParams = g.defaultOutputParameters(componentName) } + g.syncOutputSummarySchemasForCodegen(outputParams) // Resolve wildcard output types to the view entity type g.resolveOutputWildcardTypes(outputParams, componentName) if len(outputParams) > 0 { @@ -166,6 +178,15 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { } inputHelpers := collectNamedHelperTypes(inputType, packagePath, shapeTypeNames) + selectorHelpers := []namedHelperType{} + selectorTypeImports := []string{} + for _, holder := range selectorHolders { + if holder.Type == nil { + continue + } + selectorHelpers = append(selectorHelpers, collectNamedHelperTypes(holder.Type, packagePath, shapeTypeNames)...) + selectorTypeImports = mergeImportPaths(selectorTypeImports, collectTypeImports(holder.Type, packagePath)) + } outputHelpers := collectNamedHelperTypes(outputType, packagePath, shapeTypeNames) mutableSupport := g.mutableSupport(inputType) emitResponseImport := g.outputUsesResponse(outputParams) || mutableSupport != nil @@ -218,6 +239,14 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { registryPackage, helper.TypeName, helper.TypeName)) registered[helper.TypeName] = true } + for _, helper := range selectorHelpers { + if helper.TypeName == "" || registered[helper.TypeName] { + continue + } + initBuilder.WriteString(fmt.Sprintf("\tcore.RegisterType(%q, %q, reflect.TypeOf(%s{}), checksum.GeneratedTime)\n", + registryPackage, helper.TypeName, helper.TypeName)) + registered[helper.TypeName] = true + } } initBuilder.WriteString("}\n\n") @@ -257,7 +286,10 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { outputBuilder.WriteString(helper.Decl) } if g.WithContract { - g.renderComponentHolder(&routerBuilder, componentName, inputTypeName, outputTypeName) + g.renderComponentHolder(&routerBuilder, componentName, inputTypeName, outputTypeName, selectorHolders) + for _, helper := range selectorHelpers { + routerBuilder.WriteString(helper.Decl) + } g.renderDefineComponent(&outputBuilder, componentName, inputTypeName, outputTypeName) } @@ -337,7 +369,9 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { viewImports, collectTypeImports(inputType, packagePath), collectTypeImports(outputType, packagePath), + selectorTypeImports, helperImports(inputHelpers), + helperImports(selectorHelpers), helperImports(outputHelpers), mutableOutputImports, ) @@ -354,6 +388,9 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { helperImports(outputHelpers), mutableOutputImports, ) + if routerFileName == "" || routerFileName == outputFileName { + outputImports = mergeImportPaths(outputImports, selectorTypeImports, helperImports(selectorHelpers)) + } if viewFileName == outputFileName { outputImports = mergeImportPaths(outputImports, viewImports) } @@ -403,7 +440,8 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { } } if len(routerParts) > 0 { - if writeErr = g.writeSectionFile(routerDest, packageName, g.buildRouterImports(), routerParts...); writeErr != nil { + routerImports := mergeImportPaths(g.buildRouterImports(), selectorTypeImports, helperImports(selectorHelpers)) + if writeErr = g.writeSectionFile(routerDest, packageName, routerImports, routerParts...); writeErr != nil { return nil, writeErr } appendGenerated(routerDest) @@ -481,6 +519,67 @@ func (g *ComponentCodegen) Generate() (*ComponentCodegenResult, error) { }, nil } +func normalizeBodyInputTypesForCodegen(params state.Parameters, pkgPath string, lookupType xreflect.LookupType) { + for _, param := range params { + if param == nil || param.In == nil || param.In.Kind != state.KindRequestBody || param.Schema == nil { + continue + } + if param.Schema.Cardinality != state.One { + continue + } + rType := param.Schema.Type() + if rType == nil { + if resolved, err := utypes.LookupType(lookupType, param.Schema.DataType, xreflect.WithPackage(param.Schema.Package)); err == nil && resolved != nil { + rType = resolved + } else if resolved, err := utypes.LookupType(lookupType, param.Schema.DataType, xreflect.WithPackage(pkgPath)); err == nil && resolved != nil { + rType = resolved + } + } + if rType != nil && rType.Kind() == reflect.Struct { + param.Schema.SetType(reflect.PtrTo(rType)) + } + } +} + +func (g *ComponentCodegen) refreshSummarySchemasForCodegen() { + if g == nil || g.Resource == nil { + return + } + visited := map[*view.View]bool{} + for _, aView := range g.Resource.Views { + g.refreshViewSummarySchemasForCodegen(context.Background(), aView, visited) + } +} + +func (g *ComponentCodegen) refreshViewSummarySchemasForCodegen(ctx context.Context, aView *view.View, visited map[*view.View]bool) { + if aView == nil || visited[aView] { + return + } + visited[aView] = true + if aView.Template != nil && aView.Template.Summary != nil { + _ = aView.Template.Init(ctx, g.Resource, aView) + } + for _, rel := range aView.With { + if rel == nil || rel.Of == nil { + continue + } + g.refreshViewSummarySchemasForCodegen(ctx, &rel.Of.View, visited) + } +} + +func (g *ComponentCodegen) syncOutputSummarySchemasForCodegen(params state.Parameters) { + root := g.rootResourceView() + if root == nil || root.Template == nil || root.Template.Summary == nil || root.Template.Summary.Schema == nil { + return + } + for _, param := range params { + if param == nil || param.In == nil || param.In.Name != "summary" { + continue + } + param.Schema = root.Template.Summary.Schema.Clone() + } +} + func normalizeInputParametersForCodegen(params state.Parameters, resource *view.Resource, uri string) state.Parameters { result := make(state.Parameters, 0, len(params)+4) seenPath := map[string]bool{} @@ -497,6 +596,9 @@ func normalizeInputParametersForCodegen(params state.Parameters, resource *view. cloned.Schema = schema if cloned.Schema != nil && stateResource != nil { _ = cloned.Schema.Init(stateResource) + if cloned.In != nil && cloned.In.Kind == state.KindRequestBody && cloned.Schema.Cardinality == state.One { + normalizeBodySchemaPointerForCodegen(cloned.Schema) + } } if cloned.Output != nil { output := *cloned.Output @@ -516,8 +618,8 @@ func normalizeInputParametersForCodegen(params state.Parameters, resource *view. if v := lookupInputView(resource, viewName); v != nil { cloned.Tag = mergeViewSQLTag(cloned.Tag, v) } - cloned.Tag = removeTagKeys(cloned.Tag, "typeName") } + cloned.Tag = ensureCodegenTypeNameTag(cloned.Tag, cloned.Schema) if in := cloned.In; in != nil && in.Kind == state.KindPath { key := strings.ToLower(strings.TrimSpace(in.Name)) if key == "" { @@ -548,6 +650,56 @@ func normalizeInputParametersForCodegen(params state.Parameters, resource *view. return result } +func (g *ComponentCodegen) codegenInputParameters() state.Parameters { + if g == nil || g.Component == nil { + return nil + } + params := cloneCodegenParameters(g.Component.InputParameters()) + params = g.mergeMutableTemplateInputParametersForCodegen(params) + return normalizeInputParametersForCodegen(params, g.Resource, g.Component.URI) +} + +func (g *ComponentCodegen) mergeMutableTemplateInputParametersForCodegen(params state.Parameters) state.Parameters { + if g == nil || !g.componentUsesVelty() { + return params + } + root := g.rootResourceView() + if root == nil || root.Template == nil || !root.Template.UseParameterStateType || len(root.Template.Parameters) == 0 { + return params + } + result := cloneCodegenParameters(params) + seen := map[string]bool{} + for _, item := range result { + if item == nil { + continue + } + seen[codegenParameterKey(item)] = true + } + for _, item := range root.Template.Parameters { + if item == nil { + continue + } + key := codegenParameterKey(item) + if seen[key] { + continue + } + cloned := *item + if item.Schema != nil { + cloned.Schema = item.Schema.Clone() + } + if item.Output != nil { + output := *item.Output + if item.Output.Schema != nil { + output.Schema = item.Output.Schema.Clone() + } + cloned.Output = &output + } + result = append(result, &cloned) + seen[key] = true + } + return result +} + func exportedCodegenParamName(name string) string { name = strings.TrimSpace(name) if name == "" { @@ -613,6 +765,94 @@ func cloneCodegenParameters(params state.Parameters) state.Parameters { return result } +func (g *ComponentCodegen) partitionInputParametersForCodegen(params state.Parameters, packagePath string, lookupType xreflect.LookupType) (state.Parameters, []codegenSelectorHolder) { + if len(params) == 0 { + return nil, nil + } + selectorByKey := map[string]string{} + if g != nil && g.Component != nil { + for _, item := range g.Component.Input { + if item == nil || strings.TrimSpace(item.QuerySelector) == "" { + continue + } + selectorByKey[codegenParameterKey(&item.Parameter)] = strings.TrimSpace(item.QuerySelector) + } + } + + business := make(state.Parameters, 0, len(params)) + grouped := map[string]state.Parameters{} + order := []string{} + for _, item := range params { + if item == nil { + continue + } + querySelector := selectorByKey[codegenParameterKey(item)] + if querySelector == "" { + business = append(business, item) + continue + } + if _, ok := grouped[querySelector]; !ok { + order = append(order, querySelector) + } + grouped[querySelector] = append(grouped[querySelector], item) + } + + if len(order) == 0 { + return business, nil + } + + holders := make([]codegenSelectorHolder, 0, len(order)) + usedNames := map[string]bool{} + for i, querySelector := range order { + group := grouped[querySelector] + holderType, err := group.ReflectType(packagePath, lookupType) + if err != nil || holderType == nil { + business = append(business, group...) + continue + } + holders = append(holders, codegenSelectorHolder{ + FieldName: selectorHolderFieldName(querySelector, i, len(order), usedNames), + QuerySelector: querySelector, + Type: holderType, + }) + } + return business, holders +} + +func codegenParameterKey(param *state.Parameter) string { + if param == nil { + return "" + } + kind := "" + inName := "" + if param.In != nil { + kind = strings.ToLower(strings.TrimSpace(string(param.In.Kind))) + inName = strings.ToLower(strings.TrimSpace(param.In.Name)) + } + return strings.ToLower(strings.TrimSpace(param.Name)) + "|" + kind + "|" + inName +} + +func selectorHolderFieldName(querySelector string, index, total int, used map[string]bool) string { + name := "ViewSelect" + if total > 1 { + base := toUpperCamel(querySelector) + if base != "" { + name = base + "Select" + } else { + name = fmt.Sprintf("ViewSelect%d", index+1) + } + } + candidate := name + if used == nil { + return candidate + } + for suffix := 2; used[candidate]; suffix++ { + candidate = fmt.Sprintf("%s%d", name, suffix) + } + used[candidate] = true + return candidate +} + func normalizeInputSchemaForCodegen(paramName string, in *state.Location, required bool, schema *state.Schema, resource *view.Resource) *state.Schema { var cloned state.Schema if schema != nil { @@ -660,14 +900,42 @@ func normalizeInputSchemaForCodegen(paramName string, in *state.Location, requir if kind != state.KindView && strings.TrimSpace(cloned.DataType) == "" { cloned.DataType = "string" } + if kind == state.KindRequestBody && cloned.Cardinality == state.One { + normalizeBodySchemaPointerForCodegen(&cloned) + } return &cloned } +func normalizeBodySchemaPointerForCodegen(schema *state.Schema) { + if schema == nil { + return + } + if rType := schema.Type(); rType != nil { + for rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + return + } + if rType.Kind() == reflect.Ptr { + return + } + if rType.Kind() == reflect.Struct { + schema.SetType(reflect.PtrTo(rType)) + } + } + dataType := strings.TrimSpace(schema.DataType) + if dataType == "" || strings.HasPrefix(dataType, "*") || strings.HasPrefix(dataType, "[]") { + return + } + if strings.HasPrefix(dataType, "struct {") || strings.HasPrefix(dataType, "interface{") || dataType == "string" || dataType == "int" || dataType == "bool" || dataType == "float64" { + return + } + schema.DataType = "*" + dataType +} + func exportedSchemaCopy(schema *state.Schema) state.Schema { if schema == nil { return state.Schema{} } - return state.Schema{ + result := state.Schema{ Package: schema.Package, PackagePath: schema.PackagePath, ModulePath: schema.ModulePath, @@ -676,6 +944,19 @@ func exportedSchemaCopy(schema *state.Schema) state.Schema { Cardinality: schema.Cardinality, Methods: append([]reflect.Method(nil), schema.Methods...), } + if rType := schema.Type(); rType != nil { + result.SetType(rType) + if schema.Package != "" { + result.Package = schema.Package + } + if schema.PackagePath != "" { + result.PackagePath = schema.PackagePath + } + if schema.ModulePath != "" { + result.ModulePath = schema.ModulePath + } + } + return result } func lookupViewSchemaForInput(resource *view.Resource, in *state.Location, paramName string) *state.Schema { @@ -734,28 +1015,114 @@ func normalizeViewLookupName(value string) string { } func mergeViewSQLTag(existing string, aView *view.View) string { - if aView == nil || aView.Template == nil { + tag := buildViewMetadataTag(aView, true, true) + if tag == nil { return existing } - viewName := strings.TrimSpace(aView.Name) - sourceURL := strings.TrimSpace(aView.Template.SourceURL) - if viewName == "" && sourceURL == "" { - return existing + return string(tag.UpdateTag(reflect.StructTag(existing))) +} + +func buildViewMetadataTag(aView *view.View, includeName bool, includeSQL bool) *viewtags.Tag { + if aView == nil { + return nil + } + result := &viewtags.Tag{} + tagView := &viewtags.View{} + if includeName { + tagView.Name = strings.TrimSpace(aView.Name) + } + if table := strings.TrimSpace(aView.Table); isStableTableName(table) { + tagView.Table = table + } + if aView.Template != nil && aView.Template.Summary != nil { + tagView.SummaryURI = strings.TrimSpace(aView.Template.Summary.SourceURL) + } + if aView.Groupable { + value := true + tagView.Groupable = &value } - updated := strings.TrimSpace(existing) - if viewName != "" && !strings.Contains(updated, `view:"`) { - if updated != "" { - updated += " " + if aView.Batch != nil && aView.Batch.Size > 0 && aView.Batch.Size != 10000 { + tagView.Batch = aView.Batch.Size + } + if aView.RelationalConcurrency != nil && aView.RelationalConcurrency.Number > 0 && aView.RelationalConcurrency.Number != 1 { + tagView.RelationalConcurrency = aView.RelationalConcurrency.Number + } + if aView.PublishParent { + tagView.PublishParent = true + } + if aView.Partitioned != nil { + tagView.PartitionerType = aView.Partitioned.DataType + tagView.PartitionedConcurrency = aView.Partitioned.Concurrency + } + if aView.MatchStrategy != "" && aView.MatchStrategy != view.ReadMatched { + tagView.Match = string(aView.MatchStrategy) + } + if aView.Cache != nil { + tagView.Cache = strings.TrimSpace(aView.Cache.Reference.Ref) + } + if aView.Connector != nil && aView.Connector.Ref != "" { + tagView.Connector = aView.Connector.Ref + } + if selector := aView.Selector; selector != nil { + if ns := strings.TrimSpace(selector.Namespace); ns != "" { + tagView.SelectorNamespace = ns + } + if selector.NoLimit || selector.Limit != 0 { + limit := selector.Limit + tagView.Limit = &limit } - updated += fmt.Sprintf(`view:"%s"`, viewName) + if constraints := selector.Constraints; constraints != nil { + if constraints.Criteria { + value := true + tagView.SelectorCriteria = &value + } + if constraints.Projection { + value := true + tagView.SelectorProjection = &value + } + if constraints.OrderBy { + value := true + tagView.SelectorOrderBy = &value + } + if constraints.Offset { + value := true + tagView.SelectorOffset = &value + } + if constraints.Page != nil { + value := *constraints.Page + tagView.SelectorPage = &value + } + if len(constraints.Filterable) > 0 { + tagView.SelectorFilterable = append([]string(nil), constraints.Filterable...) + } + if len(constraints.OrderByColumn) > 0 { + tagView.SelectorOrderByColumns = map[string]string{} + for key, value := range constraints.OrderByColumn { + tagView.SelectorOrderByColumns[key] = value + } + } + } + } + if aView.Tag != "" { + tagView.CustomTag = aView.Tag + } + if tagView.Name != "" || tagView.Table != "" || tagView.SummaryURI != "" || tagView.CustomTag != "" || tagView.Connector != "" || + tagView.Cache != "" || tagView.Limit != nil || tagView.Match != "" || tagView.Batch > 0 || + tagView.PublishParent || tagView.PartitionerType != "" || tagView.RelationalConcurrency > 0 || + tagView.Groupable != nil || tagView.SelectorNamespace != "" || tagView.SelectorCriteria != nil || + tagView.SelectorProjection != nil || tagView.SelectorOrderBy != nil || tagView.SelectorOffset != nil || + tagView.SelectorPage != nil || len(tagView.SelectorFilterable) > 0 || len(tagView.SelectorOrderByColumns) > 0 { + result.View = tagView } - if sourceURL != "" && !strings.Contains(updated, `sql:"`) { - if updated != "" { - updated += " " + if includeSQL && aView.Template != nil { + if sourceURL := strings.TrimSpace(aView.Template.SourceURL); sourceURL != "" { + result.SQL = viewtags.NewViewSQL("", sourceURL) } - updated += fmt.Sprintf(`sql:"uri=%s"`, sourceURL) } - return updated + if result.View == nil && result.SQL.URI == "" && result.SQL.SQL == "" { + return nil + } + return result } func removeTagKeys(tag string, keys ...string) string { @@ -771,6 +1138,22 @@ func removeTagKeys(tag string, keys ...string) string { return tag } +func ensureCodegenTypeNameTag(tag string, schema *state.Schema) string { + if schema == nil { + return strings.TrimSpace(tag) + } + typeName := strings.TrimSpace(schema.Name) + if typeName == "" { + return strings.TrimSpace(tag) + } + tag = removeTagKeys(tag, "typeName") + tag = strings.TrimSpace(tag) + if tag == "" { + return fmt.Sprintf(`typeName:"%s"`, typeName) + } + return tag + ` typeName:"` + typeName + `"` +} + func isDynamicTypeName(name string) bool { n := strings.TrimSpace(strings.ToLower(name)) n = strings.ReplaceAll(n, " ", "") @@ -810,6 +1193,17 @@ func (g *ComponentCodegen) componentLookupType(packagePath string) xreflect.Look } key := strings.ToLower(typeName) localTypes[key] = rType + if summary := summaryTemplateOf(aView); summary != nil && summary.Schema != nil { + if summaryType := summary.Schema.Type(); summaryType != nil { + summaryName := strings.TrimSpace(summary.Schema.Name) + if summaryName == "" { + summaryName = strings.TrimSpace(summary.Name) + } + if summaryName != "" { + localTypes[strings.ToLower(summaryName)] = summaryType + } + } + } } } return func(name string, opts ...xreflect.Option) (reflect.Type, error) { @@ -946,7 +1340,7 @@ func (g *ComponentCodegen) generateShapeFragment(projectDir, packageDir, package if g == nil || g.Resource == nil || len(g.Resource.Views) == 0 { return &shapeFragment{}, nil } - shapeDoc := resourceToCodegenDoc(g.Resource, g.TypeContext) + shapeDoc := resourceToShapeDocument(g.Resource, g.TypeContext) applyShapeDocViewTypeOverrides(shapeDoc.Root, g.Component) shapeCfg := &Config{ ProjectDir: projectDir, @@ -969,7 +1363,7 @@ func (g *ComponentCodegen) generateShapeFragment(projectDir, packageDir, package func (g *ComponentCodegen) renderSemanticShapeFragment(shapeCfg *Config, packagePath string) (*shapeFragment, error) { viewDescriptorsByName := map[string]viewDescriptor{} - shapeDoc := resourceToCodegenDoc(g.Resource, g.TypeContext) + shapeDoc := resourceToShapeDocument(g.Resource, g.TypeContext) for _, item := range extractViews(shapeDoc.Root) { viewDescriptorsByName[strings.ToLower(strings.TrimSpace(asString(item.name)))] = item } @@ -985,7 +1379,11 @@ func (g *ComponentCodegen) renderSemanticShapeFragment(shapeCfg *Config, package if typeName == "" || registered[typeName] { continue } - viewDecl, viewImports, err := g.renderSemanticViewDecl(shapeCfg, aView, packagePath) + mutable := false + if descriptor, ok := viewDescriptorsByName[strings.ToLower(strings.TrimSpace(aView.Name))]; ok { + mutable = descriptor.mutable + } + viewDecl, viewImports, err := g.renderSemanticViewDecl(shapeCfg, aView, packagePath, mutable) if err != nil { return nil, err } @@ -999,6 +1397,18 @@ func (g *ComponentCodegen) renderSemanticShapeFragment(shapeCfg *Config, package for _, imp := range viewImports { imports[imp] = true } + for _, summary := range g.summaryTypeDecls(aView, packagePath) { + if registered[summary.name] { + continue + } + registered[summary.name] = true + typeNames = append(typeNames, summary.name) + decls.WriteString(summary.decl) + decls.WriteString("\n") + for _, imp := range summary.imports { + imports[imp] = true + } + } if descriptor, ok := viewDescriptorsByName[strings.ToLower(strings.TrimSpace(aView.Name))]; ok && descriptor.mutable { structType := buildHasType(columnsFromView(aView)) @@ -1026,6 +1436,49 @@ func (g *ComponentCodegen) renderSemanticShapeFragment(shapeCfg *Config, package }, nil } +type emittedTypeDecl struct { + name string + decl string + imports []string +} + +func (g *ComponentCodegen) summaryTypeDecls(aView *view.View, currentPackage string) []emittedTypeDecl { + if aView == nil { + return nil + } + seen := map[string]bool{} + var result []emittedTypeDecl + appendSummary := func(summary *view.TemplateSummary) { + if summary == nil || summary.Schema == nil { + return + } + name := strings.TrimSpace(summary.Schema.Name) + rType := ensureCodegenStructType(summary.Schema.Type()) + if name == "" || rType == nil || seen[name] { + return + } + seen[name] = true + result = append(result, emittedTypeDecl{ + name: name, + decl: fmt.Sprintf("type %s struct {\n%s}\n\n", name, structFieldsSource(rType)), + imports: collectTypeImports(rType, currentPackage), + }) + } + appendSummary(summaryTemplateOf(aView)) + for _, rel := range aView.With { + child := g.semanticView(g.resolveRelationView(rel)) + appendSummary(summaryTemplateOf(child)) + } + return result +} + +func summaryTemplateOf(aView *view.View) *view.TemplateSummary { + if aView == nil || aView.Template == nil { + return nil + } + return aView.Template.Summary +} + func (g *ComponentCodegen) resourceViewTypeName(shapeCfg *Config, aView *view.View) string { if aView == nil { return "" @@ -1041,7 +1494,7 @@ func (g *ComponentCodegen) resourceViewTypeName(shapeCfg *Config, aView *view.Vi return viewTypeName(shapeCfg, descriptor) } -func (g *ComponentCodegen) renderSemanticViewDecl(shapeCfg *Config, aView *view.View, currentPackage string) (string, []string, error) { +func (g *ComponentCodegen) renderSemanticViewDecl(shapeCfg *Config, aView *view.View, currentPackage string, mutable bool) (string, []string, error) { aView = g.semanticView(aView) typeName := g.resourceViewTypeName(shapeCfg, aView) if typeName == "" { @@ -1099,6 +1552,10 @@ func (g *ComponentCodegen) renderSemanticViewDecl(shapeCfg *Config, aView *view. builder.WriteString(fmt.Sprintf("\t%s []interface{} `sqlx:\"-\"`\n", holder)) } } + if mutable { + hasTypeName := typeName + "Has" + builder.WriteString(fmt.Sprintf("\tHas *%s `setMarker:\"true\" format:\"-\" sqlx:\"-\" diff:\"-\" json:\"-\" typeName:\"%s\"`\n", hasTypeName, hasTypeName)) + } builder.WriteString("}\n\n") resultImports := make([]string, 0, len(imports)) for imp := range imports { @@ -1166,10 +1623,11 @@ func (g *ComponentCodegen) renderRelationField(shapeCfg *Config, parent *view.Vi } func (g *ComponentCodegen) renderRelationSummaryField(shapeCfg *Config, rel *view.Relation, currentPackage string) (string, []string) { - if rel == nil || rel.Of.Template == nil || rel.Of.Template.Summary == nil || rel.Of.Template.Summary.Schema == nil { + child := g.semanticView(g.resolveRelationView(rel)) + if child == nil || child.Template == nil || child.Template.Summary == nil || child.Template.Summary.Schema == nil { return "", nil } - meta := rel.Of.Template.Summary + meta := child.Template.Summary fieldName := state.StructFieldName(text.CaseFormatUpperCamel, meta.Name) if strings.TrimSpace(fieldName) == "" { return "", nil @@ -1213,14 +1671,22 @@ func (g *ComponentCodegen) relationTypeName(shapeCfg *Config, rel *view.Relation func (g *ComponentCodegen) columnFieldTag(aView *view.View, column *view.Column) string { tag := strings.TrimSpace(column.Tag) + cleaned, _ := xreflect.RemoveTag(tag, "velty") + tag = strings.TrimSpace(cleaned) + groupable := column.Groupable if aView != nil && aView.ColumnsConfig != nil { - if cfg := aView.ColumnsConfig[column.Name]; cfg != nil && cfg.Tag != nil { - configTag := strings.TrimSpace(strings.Trim(*cfg.Tag, ` `)) - if configTag != "" && !strings.Contains(tag, configTag) { - if tag != "" { - tag += " " + if cfg := aView.ColumnsConfig[column.Name]; cfg != nil { + if cfg.Groupable != nil { + groupable = *cfg.Groupable + } + if cfg.Tag != nil { + configTag := strings.TrimSpace(strings.Trim(*cfg.Tag, ` `)) + if configTag != "" && !strings.Contains(tag, configTag) { + if tag != "" { + tag += " " + } + tag += configTag } - tag += configTag } } } @@ -1230,6 +1696,12 @@ func (g *ComponentCodegen) columnFieldTag(aView *view.View, column *view.Column) } tag += `internal:"true"` } + if groupable && !strings.Contains(tag, `groupable:"`) { + if tag != "" { + tag += " " + } + tag += `groupable:"true"` + } sqlxValue := strings.TrimSpace(column.Name) if column.Codec != nil && strings.TrimSpace(column.DataType) != "" { sqlxValue += ",type=" + strings.TrimSpace(column.DataType) @@ -1266,27 +1738,51 @@ func (g *ComponentCodegen) viewUsesVelty(aView *view.View) bool { } func (g *ComponentCodegen) resourceViewUsesVelty(aView *view.View) bool { + if aView == nil || g == nil || g.Component == nil || !g.componentUsesVelty() || g.componentUsesHandler() { + return false + } if g.viewUsesVelty(aView) { return true } - if g == nil || aView == nil || !g.componentUsesVelty() || g.Component == nil { - return false - } - target := strings.TrimSpace(aView.Name) - if target == "" { - return false + matches := func(value string) bool { + value = strings.TrimSpace(value) + if value == "" { + return false + } + return strings.EqualFold(strings.TrimSpace(aView.Name), value) || + strings.EqualFold(strings.TrimSpace(aView.Reference.Ref), value) } for _, input := range g.Component.Input { if input == nil || input.In == nil || input.In.Kind != state.KindView { continue } - if strings.EqualFold(strings.TrimSpace(input.In.Name), target) { + if matches(input.In.Name) || matches(input.Name) { return true } } return false } +func (g *ComponentCodegen) componentUsesMutableHelpers() bool { + if g == nil || g.Component == nil || !g.componentUsesVelty() || g.componentUsesHandler() { + return false + } + hasBody := false + hasView := false + for _, input := range g.Component.Input { + if input == nil || input.In == nil { + continue + } + switch input.In.Kind { + case state.KindRequestBody: + hasBody = true + case state.KindView: + hasView = true + } + } + return hasBody && hasView +} + func (g *ComponentCodegen) componentUsesVelty() bool { if g == nil { return false @@ -1334,7 +1830,7 @@ func normalizeGeneratedTagOrder(tag string) string { return tag } ordered := make([]string, 0, 4) - for _, key := range []string{"sqlx", "internal", "velty", "json"} { + for _, key := range []string{"sqlx", "internal", "groupable", "velty", "json"} { value := reflect.StructTag(tag).Get(key) if value == "" { continue @@ -1352,9 +1848,12 @@ func normalizeGeneratedTagOrder(tag string) string { func (g *ComponentCodegen) relationFieldTag(parent *view.View, rel *view.Relation) string { child := g.semanticView(g.resolveRelationView(rel)) + if child == nil { + return "" + } tag := &viewtags.Tag{} - if table := strings.TrimSpace(child.Table); isStableTableName(table) { - tag.View = &viewtags.View{Table: table} + if metadata := buildViewMetadataTag(child, false, false); metadata != nil { + tag.View = metadata.View } if relTag := strings.TrimSpace(child.Tag); relTag != "" { if tag.View == nil { @@ -1362,37 +1861,6 @@ func (g *ComponentCodegen) relationFieldTag(parent *view.View, rel *view.Relatio } tag.View.CustomTag = relTag } - if child.Batch != nil && child.Batch.Size > 0 && child.Batch.Size != 10000 { - if tag.View == nil { - tag.View = &viewtags.View{} - } - tag.View.Batch = child.Batch.Size - } - if child.RelationalConcurrency != nil && child.RelationalConcurrency.Number > 0 && child.RelationalConcurrency.Number != 1 { - if tag.View == nil { - tag.View = &viewtags.View{} - } - tag.View.RelationalConcurrency = child.RelationalConcurrency.Number - } - if child.PublishParent { - if tag.View == nil { - tag.View = &viewtags.View{} - } - tag.View.PublishParent = true - } - if child.Partitioned != nil { - if tag.View == nil { - tag.View = &viewtags.View{} - } - tag.View.PartitionerType = child.Partitioned.DataType - tag.View.PartitionedConcurrency = child.Partitioned.Concurrency - } - if child.MatchStrategy != "" && child.MatchStrategy != view.ReadMatched { - if tag.View == nil { - tag.View = &viewtags.View{} - } - tag.View.Match = string(child.MatchStrategy) - } if parent != nil && parent.Cache != nil { if tag.View == nil { tag.View = &viewtags.View{} @@ -1572,9 +2040,38 @@ func (g *ComponentCodegen) mergeViewSemantics(dst, src *view.View) { dst.ColumnsConfig[key] = cfg } } - if (dst.Template == nil || strings.TrimSpace(dst.Template.SourceURL) == "") && src.Template != nil { + if dst.Template == nil && src.Template != nil { dst.Template = src.Template } + if dst.Template != nil && src.Template != nil { + if strings.TrimSpace(dst.Template.Source) == "" { + dst.Template.Source = src.Template.Source + } + if strings.TrimSpace(dst.Template.SourceURL) == "" { + dst.Template.SourceURL = src.Template.SourceURL + } + if src.Template.Summary != nil { + if dst.Template.Summary == nil { + dst.Template.Summary = src.Template.Summary + } else { + if strings.TrimSpace(dst.Template.Summary.Name) == "" { + dst.Template.Summary.Name = src.Template.Summary.Name + } + if dst.Template.Summary.Kind == "" { + dst.Template.Summary.Kind = src.Template.Summary.Kind + } + if strings.TrimSpace(dst.Template.Summary.Source) == "" { + dst.Template.Summary.Source = src.Template.Summary.Source + } + if strings.TrimSpace(dst.Template.Summary.SourceURL) == "" { + dst.Template.Summary.SourceURL = src.Template.Summary.SourceURL + } + if src.Template.Summary.Schema != nil && (dst.Template.Summary.Schema == nil || dst.Template.Summary.Schema.Type() == nil) { + dst.Template.Summary.Schema = src.Template.Summary.Schema + } + } + } + } if !isStableTableName(dst.Table) && isStableTableName(src.Table) { dst.Table = src.Table } @@ -1902,6 +2399,9 @@ func resourceViewNeedsRebuild(rType reflect.Type, columns []columnDescriptor, in if includeVelty && field.Tag.Get("sqlx") != "" && field.Tag.Get("sqlx") != "-" && field.Tag.Get("velty") == "" { return true } + if !includeVelty && field.Tag.Get("velty") != "" { + return true + } } return false } @@ -2093,6 +2593,7 @@ func extractTypeDeclsAndImports(source string) ([]string, string, error) { // } func (g *ComponentCodegen) renderOutputStruct(builder *strings.Builder, outputTypeName, viewTypeName, embedURI string, outputParams state.Parameters, outputType reflect.Type, mutableSupport *mutableComponentSupport) { rootView := g.Component.RootView + rootViewMetadata := g.rootResourceView() builder.WriteString(fmt.Sprintf("type %s struct {\n", outputTypeName)) @@ -2129,9 +2630,20 @@ func (g *ComponentCodegen) renderOutputStruct(builder *strings.Builder, outputTy if fieldName == "" || fieldName == "Output" { fieldName = "Data" } - tag := fmt.Sprintf(`parameter:",kind=output,in=view" view:"%s" sql:"uri=%s/%s.sql"`, - rootView, embedURI, rootView) - if p.Tag != "" && strings.Contains(p.Tag, "anonymous") { + tag := strings.TrimSpace(p.Tag) + if !strings.Contains(tag, `parameter:"`) { + tag = strings.TrimSpace(tag + ` parameter:",kind=output,in=view"`) + } + if !strings.Contains(tag, `view:"`) { + tag = strings.TrimSpace(tag + fmt.Sprintf(` view:"%s"`, rootView)) + } + if !strings.Contains(tag, `sql:"`) { + tag = strings.TrimSpace(tag + fmt.Sprintf(` sql:"uri=%s/%s.sql"`, embedURI, rootView)) + } + if rootViewMetadata != nil { + tag = mergeViewSQLTag(tag, rootViewMetadata) + } + if !strings.Contains(tag, `anonymous:"`) && p.Tag != "" && strings.Contains(p.Tag, "anonymous") { tag += ` anonymous:"true"` } builder.WriteString(fmt.Sprintf("\t%s %s%s `%s`\n", fieldName, typePrefix, viewTypeName, tag)) @@ -2163,6 +2675,7 @@ func (g *ComponentCodegen) renderOutputStruct(builder *strings.Builder, outputTy // schema to the view entity type. The legacy translator does this in updateParameterWithComponentOutputType. func (g *ComponentCodegen) resolveOutputWildcardTypes(params state.Parameters, componentName string) { viewType := componentName + "View" + rootView := g.rootResourceView() for _, p := range params { if p == nil || p.In == nil { continue @@ -2173,8 +2686,9 @@ func (g *ComponentCodegen) resolveOutputWildcardTypes(params state.Parameters, c if p.Schema == nil { p.Schema = &state.Schema{} } - // If schema type is wildcard or empty, resolve to the view type - if p.Schema.Name == "" || p.Schema.DataType == "" || p.Schema.DataType == "?" { + // Only view outputs default to the root view shape. Summary/status outputs + // must keep their own materialized schema types. + if p.In.Name == "view" && (p.Schema.Name == "" || p.Schema.DataType == "" || p.Schema.DataType == "?") { p.Schema.Name = viewType p.Schema.DataType = "*" + viewType if p.Schema.Cardinality == "" { @@ -2183,8 +2697,11 @@ func (g *ComponentCodegen) resolveOutputWildcardTypes(params state.Parameters, c } // Add view tag if missing if p.In.Name == "view" && !strings.Contains(p.Tag, "view:") { - rootView := g.Component.RootView - p.Tag += fmt.Sprintf(` view:"%s"`, rootView) + rootViewName := g.Component.RootView + p.Tag += fmt.Sprintf(` view:"%s"`, rootViewName) + } + if p.In.Name == "view" && rootView != nil { + p.Tag = mergeViewSQLTag(p.Tag, rootView) } } } @@ -2301,6 +2818,25 @@ func (g *ComponentCodegen) rootViewSourceURL() string { return strings.TrimSpace(g.Resource.Views[0].Template.SourceURL) } +func (g *ComponentCodegen) rootResourceView() *view.View { + if g == nil || g.Resource == nil { + return nil + } + rootView := "" + if g.Component != nil { + rootView = strings.TrimSpace(g.Component.RootView) + } + if rootView != "" { + if aView, _ := g.Resource.View(rootView); aView != nil { + return aView + } + } + if len(g.Resource.Views) == 0 { + return nil + } + return g.Resource.Views[0] +} + func (g *ComponentCodegen) rootSummarySourceURL() string { if g == nil || g.Resource == nil { return "" @@ -2693,7 +3229,7 @@ func (g *ComponentCodegen) buildRouterImports() []string { return []string{"github.com/viant/xdatly"} } -func (g *ComponentCodegen) renderComponentHolder(builder *strings.Builder, componentName, inputTypeName, outputTypeName string) { +func (g *ComponentCodegen) renderComponentHolder(builder *strings.Builder, componentName, inputTypeName, outputTypeName string, selectorHolders []codegenSelectorHolder) { method := strings.TrimSpace(g.Component.Method) if method == "" { method = "GET" @@ -2724,6 +3260,14 @@ func (g *ComponentCodegen) renderComponentHolder(builder *strings.Builder, compo tag += `"` builder.WriteString(fmt.Sprintf("type %sRouter struct {\n", componentName)) builder.WriteString(fmt.Sprintf("\t%s xdatly.Component[%s, %s] `%s`\n", componentName, inputTypeName, outputTypeName, tag)) + for _, holder := range selectorHolders { + if holder.Type == nil || strings.TrimSpace(holder.QuerySelector) == "" || strings.TrimSpace(holder.FieldName) == "" { + continue + } + builder.WriteString(fmt.Sprintf("\t%s struct {\n", holder.FieldName)) + builder.WriteString(indentSource(structFieldsSource(holder.Type), "\t\t")) + builder.WriteString(fmt.Sprintf("\t} `querySelector:%q`\n", holder.QuerySelector)) + } builder.WriteString("}\n\n") } @@ -2848,6 +3392,18 @@ func structFieldsSource(rType reflect.Type) string { return b.String() } +func indentSource(source, prefix string) string { + source = strings.TrimRight(source, "\n") + if source == "" { + return "" + } + lines := strings.Split(source, "\n") + for i, line := range lines { + lines[i] = prefix + line + } + return strings.Join(lines, "\n") + "\n" +} + func sourceFieldTypeExpr(field reflect.StructField) string { typeName := strings.TrimSpace(field.Tag.Get("typeName")) if typeName == "" { diff --git a/repository/shape/xgen/codegen_groupable_test.go b/repository/shape/xgen/codegen_groupable_test.go new file mode 100644 index 000000000..5ad9758b5 --- /dev/null +++ b/repository/shape/xgen/codegen_groupable_test.go @@ -0,0 +1,337 @@ +package xgen + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" + "github.com/viant/datly/repository/shape/load" + "github.com/viant/datly/repository/shape/plan" + "github.com/viant/datly/repository/shape/typectx" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +type codegenMetaView struct { + PageCnt *int +} + +type codegenProductsMetaView struct { + VendorId *int +} + +type staleCodegenProductsMetaView struct { + VendorId string +} + +type codegenProductsView struct { + VendorId *int +} + +type codegenVendorView struct { + ID int + Products []*codegenProductsView + ProductsMeta *codegenProductsMetaView +} + +func TestComponentCodegen_ColumnFieldTag_EmitsGroupableTag(t *testing.T) { + groupable := true + codegen := &ComponentCodegen{} + aView := &view.View{ + ColumnsConfig: map[string]*view.ColumnConfig{ + "REGION": {Name: "REGION", Groupable: &groupable}, + }, + } + column := &view.Column{Name: "REGION", DataType: "string"} + + tag := codegen.columnFieldTag(aView, column) + assert.Contains(t, tag, `groupable:"true"`) + assert.Contains(t, tag, `sqlx:"REGION"`) +} + +func TestComponentCodegen_GeneratesSelectorHolderOutsideBusinessInput(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "pkg", "dev", "vendor") + component := &load.Component{ + Name: "Vendor", + Method: "GET", + URI: "/v1/api/dev/vendors-grouping", + RootView: "Vendor", + Directives: &dqlshape.Directives{ + InputDest: "vendor_input.go", + OutputDest: "vendor_output.go", + RouterDest: "vendor_router.go", + }, + Input: []*plan.State{ + {Parameter: state.Parameter{Name: "VendorIDs", In: state.NewQueryLocation("vendorIDs"), Schema: state.NewSchema(reflect.TypeOf([]int{}))}}, + {Parameter: state.Parameter{Name: "Fields", In: state.NewQueryLocation("_fields"), Schema: state.NewSchema(reflect.TypeOf([]string{}))}, QuerySelector: "vendor"}, + {Parameter: state.Parameter{Name: "OrderBy", In: state.NewQueryLocation("_orderby"), Schema: state.NewSchema(reflect.TypeOf(""))}, QuerySelector: "vendor"}, + }, + } + resource := &view.Resource{ + Views: []*view.View{ + { + Name: "Vendor", + Groupable: true, + Selector: &view.Config{ + Constraints: &view.Constraints{ + OrderBy: true, + OrderByColumn: map[string]string{"accountId": "ACCOUNT_ID"}, + }, + }, + Template: &view.Template{SourceURL: "vendor/vendor.sql"}, + Columns: []*view.Column{{Name: "ACCOUNT_ID", DataType: "int"}}, + }, + }, + } + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "vendor", + PackagePath: "github.com/acme/project/pkg/dev/vendor", + } + + result, err := (&ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithContract: true, + }).Generate() + require.NoError(t, err) + + inputSource, err := os.ReadFile(result.InputFilePath) + require.NoError(t, err) + assert.Contains(t, string(inputSource), "type VendorInput struct {") + assert.Contains(t, string(inputSource), "VendorIDs []int") + assert.NotContains(t, string(inputSource), "Fields []string") + assert.NotContains(t, string(inputSource), "OrderBy string") + + routerSource, err := os.ReadFile(result.RouterFilePath) + require.NoError(t, err) + assert.Contains(t, string(routerSource), "ViewSelect struct {") + assert.Contains(t, string(routerSource), `querySelector:"vendor"`) + assert.Contains(t, string(routerSource), `Fields []string `+"`"+`parameter:"`) + assert.Contains(t, string(routerSource), `in=_fields`) + assert.Contains(t, string(routerSource), `OrderBy string `+"`"+`parameter:"`) + assert.Contains(t, string(routerSource), `in=_orderby`) + + outputSource, err := os.ReadFile(result.OutputFilePath) + require.NoError(t, err) + assert.Contains(t, string(outputSource), `view:"Vendor,groupable=true`) + assert.Contains(t, string(outputSource), `selectorOrderBy=true`) + assert.Contains(t, string(outputSource), `selectorOrderByColumns={accountId:ACCOUNT_ID}`) +} + +func TestComponentCodegen_GeneratesSummaryMetadata(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "pkg", "dev", "vendor") + vendorSchema := state.NewSchema(reflect.TypeOf(codegenVendorView{})) + vendorSchema.Name = "VendorView" + vendorSchema.DataType = "*VendorView" + productsSchema := state.NewSchema(reflect.TypeOf(codegenProductsView{})) + productsSchema.Name = "ProductsView" + productsSchema.DataType = "*ProductsView" + metaSchema := state.NewSchema(reflect.TypeOf(codegenMetaView{})) + metaSchema.Name = "MetaView" + metaSchema.DataType = "*MetaView" + productsMetaSchema := state.NewSchema(reflect.TypeOf(codegenProductsMetaView{})) + productsMetaSchema.Name = "ProductsMetaView" + productsMetaSchema.DataType = "*ProductsMetaView" + component := &load.Component{ + Name: "Vendor", + Method: "GET", + URI: "/v1/api/dev/meta/vendors-format", + RootView: "vendor", + Directives: &dqlshape.Directives{ + OutputDest: "vendor.go", + }, + Output: []*plan.State{ + { + Parameter: state.Parameter{ + Name: "Meta", + In: state.NewOutputLocation("summary"), + Schema: metaSchema, + }, + }, + { + Parameter: state.Parameter{ + Name: "Data", + In: state.NewOutputLocation("view"), + Schema: &state.Schema{ + Name: "VendorView", + DataType: "*VendorView", + Cardinality: state.Many, + }, + }, + }, + }, + } + resource := &view.Resource{ + Views: []*view.View{ + { + Name: "vendor", + Template: &view.Template{ + SourceURL: "vendor/vendor.sql", + Summary: &view.TemplateSummary{ + Name: "Meta", + SourceURL: "vendor/vendor_summary.sql", + Schema: metaSchema, + }, + }, + Schema: vendorSchema, + With: []*view.Relation{ + { + Holder: "Products", + Cardinality: state.Many, + Of: &view.ReferenceView{ + View: view.View{ + Name: "products", + Template: &view.Template{ + SourceURL: "vendor/products.sql", + Summary: &view.TemplateSummary{ + Name: "ProductsMeta", + SourceURL: "vendor/products_summary.sql", + Schema: productsMetaSchema, + }, + }, + Schema: productsSchema, + }, + On: []*view.Link{{Field: "VendorId", Column: "VENDOR_ID"}}, + }, + On: []*view.Link{{Field: "Id", Column: "ID"}}, + }, + }, + Columns: []*view.Column{{Name: "ID", DataType: "int"}}, + }, + { + Name: "products", + Template: &view.Template{ + SourceURL: "vendor/products.sql", + Summary: &view.TemplateSummary{ + Name: "ProductsMeta", + SourceURL: "vendor/products_summary.sql", + Schema: productsMetaSchema, + }, + }, + Schema: productsSchema, + Columns: []*view.Column{{Name: "VENDOR_ID", DataType: "int"}}, + }, + }, + } + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "vendor", + PackagePath: "github.com/acme/project/pkg/dev/vendor", + } + + result, err := (&ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithContract: true, + }).Generate() + require.NoError(t, err) + + outputSource, err := os.ReadFile(result.OutputFilePath) + require.NoError(t, err) + source := string(outputSource) + assert.Contains(t, source, `Meta MetaView`) + assert.Contains(t, source, `parameter:",kind=output,in=summary"`) + assert.Contains(t, source, `view:"vendor,summaryURI=vendor/vendor_summary.sql"`) + assert.Contains(t, source, `type MetaView struct {`) + assert.Contains(t, source, `ProductsMeta *ProductsMetaView`) + assert.Contains(t, source, `view:",summaryURI=vendor/products_summary.sql"`) + assert.Contains(t, source, `type ProductsMetaView struct {`) +} + +func TestComponentCodegen_PrefersStandaloneChildSummarySchemaOverStaleRelationCopy(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "pkg", "dev", "vendor") + vendorSchema := state.NewSchema(reflect.TypeOf(codegenVendorView{})) + vendorSchema.Name = "VendorView" + vendorSchema.DataType = "*VendorView" + productsSchema := state.NewSchema(reflect.TypeOf(codegenProductsView{})) + productsSchema.Name = "ProductsView" + productsSchema.DataType = "*ProductsView" + staleSummarySchema := state.NewSchema(reflect.TypeOf(staleCodegenProductsMetaView{})) + staleSummarySchema.Name = "ProductsMetaView" + staleSummarySchema.DataType = "*ProductsMetaView" + refinedSummarySchema := state.NewSchema(reflect.TypeOf(codegenProductsMetaView{})) + refinedSummarySchema.Name = "ProductsMetaView" + refinedSummarySchema.DataType = "*ProductsMetaView" + component := &load.Component{ + Name: "Vendor", + Method: "GET", + URI: "/v1/api/dev/meta/vendors-format", + RootView: "vendor", + } + resource := &view.Resource{ + Views: []*view.View{ + { + Name: "vendor", + Template: &view.Template{SourceURL: "vendor/vendor.sql"}, + Schema: vendorSchema, + With: []*view.Relation{ + { + Holder: "Products", + Cardinality: state.Many, + Of: &view.ReferenceView{ + View: view.View{ + Name: "products", + Template: &view.Template{ + SourceURL: "vendor/products.sql", + Summary: &view.TemplateSummary{ + Name: "ProductsMeta", + SourceURL: "vendor/products_summary.sql", + Schema: staleSummarySchema, + }, + }, + Schema: productsSchema, + }, + On: []*view.Link{{Field: "VendorId", Column: "VENDOR_ID"}}, + }, + On: []*view.Link{{Field: "Id", Column: "ID"}}, + }, + }, + }, + { + Name: "products", + Template: &view.Template{ + SourceURL: "vendor/products.sql", + Summary: &view.TemplateSummary{ + Name: "ProductsMeta", + SourceURL: "vendor/products_summary.sql", + Schema: refinedSummarySchema, + }, + }, + Schema: productsSchema, + }, + }, + } + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "vendor", + PackagePath: "github.com/acme/project/pkg/dev/vendor", + } + + result, err := (&ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithContract: true, + }).Generate() + require.NoError(t, err) + + outputSource, err := os.ReadFile(result.OutputFilePath) + require.NoError(t, err) + source := string(outputSource) + assert.Contains(t, source, `type ProductsMetaView struct {`) + assert.Contains(t, source, `VendorId *int`) + assert.NotContains(t, source, `VendorId string`) +} diff --git a/repository/shape/xgen/codegen_input_view_test.go b/repository/shape/xgen/codegen_input_view_test.go index fd1fe30f4..d492af469 100644 --- a/repository/shape/xgen/codegen_input_view_test.go +++ b/repository/shape/xgen/codegen_input_view_test.go @@ -46,6 +46,7 @@ func TestComponentCodegen_ViewInput_UsesResolvedViewType(t *testing.T) { Columns: []*view.Column{ {Name: "ID", DataType: "int"}, {Name: "STATUS", DataType: "int", Nullable: true}, + {Name: "IS_AUTH", DataType: "int", Nullable: true}, }, }, ) @@ -84,6 +85,9 @@ func TestComponentCodegen_ViewInput_UsesResolvedViewType(t *testing.T) { if !strings.Contains(generated, `Status *int `+"`"+`sqlx:"STATUS" velty:"names=STATUS|Status"`+"`") { t.Fatalf("expected exec view input helper type to retain velty aliases:\n%s", generated) } + if !strings.Contains(generated, `IsAuth *int `+"`"+`sqlx:"IS_AUTH" velty:"names=IS_AUTH|IsAuth"`+"`") { + t.Fatalf("expected exec view input helper type to retain SQL alias velty names:\n%s", generated) + } } func TestComponentCodegen_InputSynthesizesRoutePathParams(t *testing.T) { @@ -766,3 +770,101 @@ func TestComponentCodegen_ExecWithoutStatusDoesNotImportResponse(t *testing.T) { t.Fatalf("did not expect response import for empty exec output:\n%s", generated) } } + +func TestComponentCodegen_MutableView_EmbedsHasMarker(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "events", "patch_basic_one") + + component := &shapeload.Component{ + Method: "PATCH", + URI: "/v1/api/shape/dev/basic/foos", + RootView: "foos", + Input: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewBodyLocation(""), + Schema: &state.Schema{ + Name: "FoosView", + DataType: "*FoosView", + Cardinality: state.One, + }, + Tag: `anonymous:"true" typeName:"FoosView"`, + }, + }, + { + Parameter: state.Parameter{ + Name: "CurFoos", + In: state.NewViewLocation("CurFoos"), + Schema: &state.Schema{ + Name: "FoosView", + DataType: "*FoosView", + Cardinality: state.One, + }, + }, + }, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{ + Name: "foos", + Mode: view.ModeExec, + Template: &view.Template{ + Source: "#set($_ = $Foos(body/).Required())", + }, + Schema: &state.Schema{Name: "FoosView", Cardinality: state.Many}, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "NAME", DataType: "string", Nullable: true}, + {Name: "QUANTITY", DataType: "int", Nullable: true}, + }, + }, + &view.View{ + Name: "CurFoos", + Mode: view.ModeQuery, + Schema: &state.Schema{Name: "FoosView", Cardinality: state.Many}, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "NAME", DataType: "string", Nullable: true}, + {Name: "QUANTITY", DataType: "int", Nullable: true}, + }, + }, + ) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "patch_basic_one", + PackagePath: "github.com/acme/project/shape/dev/events/patch_basic_one", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + + if !strings.Contains(generated, "type FoosViewHas struct") { + t.Fatalf("expected mutable helper type declaration:\n%s", generated) + } + if !strings.Contains(generated, `Has *FoosViewHas `+"`"+`setMarker:"true" format:"-" sqlx:"-" diff:"-" json:"-" typeName:"FoosViewHas"`+"`") { + t.Fatalf("expected mutable view to embed Has marker:\n%s", generated) + } + if !strings.Contains(generated, `Foos *FoosView `+"`"+`parameter:",kind=body" typeName:"FoosView" anonymous:"true"`+"`") { + t.Fatalf("expected mutable body input to stay pointer typed:\n%s", generated) + } +} diff --git a/repository/shape/xgen/codegen_mutable_body_test.go b/repository/shape/xgen/codegen_mutable_body_test.go index 781c14114..787b37406 100644 --- a/repository/shape/xgen/codegen_mutable_body_test.go +++ b/repository/shape/xgen/codegen_mutable_body_test.go @@ -29,7 +29,7 @@ func TestComponentCodegen_BuildMutableVeltyBlock_PatchOne(t *testing.T) { actual := renderMutableBlock(t, codegen, inputType, support) for _, fragment := range []string{ `$sequencer.Allocate("FOOS", $Foos, "Id")`, - `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, `#if($Foos)`, `#if($CurFoosById.HasKey($Foos.Id) == true)`, `$sql.Update($Foos, "FOOS");`, @@ -46,7 +46,7 @@ func TestComponentCodegen_BuildMutableVeltyBlock_PatchMany(t *testing.T) { actual := renderMutableBlock(t, codegen, inputType, support) for _, fragment := range []string{ `$sequencer.Allocate("FOOS", $Foos, "Id")`, - `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, `#foreach($RecFoos in $Foos)`, `#if($CurFoosById.HasKey($RecFoos.Id) == true)`, `$sql.Update($RecFoos, "FOOS");`, @@ -68,7 +68,7 @@ func TestComponentCodegen_BuildMutableVeltyBlock_PutOne(t *testing.T) { t.Fatalf("did not expect insert branch in PUT body:\n%s", actual) } for _, fragment := range []string{ - `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, `#if($Foos)`, `#if($CurFoosById.HasKey($Foos.Id) == true)`, `$sql.Update($Foos, "FOOS");`, @@ -87,7 +87,7 @@ func TestComponentCodegen_BuildMutableVeltyBlock_PostMany(t *testing.T) { } for _, fragment := range []string{ `$sequencer.Allocate("FOOS", $Foos, "Id")`, - `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, `#foreach($RecFoos in $Foos)`, `$sql.Insert($RecFoos, "FOOS");`, } { @@ -161,14 +161,20 @@ func newMutableBodyFixture(method string, many bool) (*ComponentCodegen, reflect ViewFieldName: "CurFoos", MapFieldName: "CurFoosById", KeyFieldName: "Id", + KeyFieldType: "int", + TypeName: "Foos", ItemTypeExpr: "*xgen.mutableBodyFoos", + ItemIsPointer: true, }, { ViewParamName: "CurFoosPerformance", ViewFieldName: "CurFoosPerformance", MapFieldName: "CurFoosPerformanceById", KeyFieldName: "Id", + KeyFieldType: "int", + TypeName: "FoosPerformance", ItemTypeExpr: "*xgen.mutableBodyFoosPerformance", + ItemIsPointer: true, }, }, } diff --git a/repository/shape/xgen/codegen_mutable_helpers_test.go b/repository/shape/xgen/codegen_mutable_helpers_test.go index a044be72e..209d3182f 100644 --- a/repository/shape/xgen/codegen_mutable_helpers_test.go +++ b/repository/shape/xgen/codegen_mutable_helpers_test.go @@ -130,12 +130,15 @@ func TestComponentCodegen_MutableComponent_GeneratesPatchHelpers(t *testing.T) { } inputSource := mustReadCodegenFile(t, result.InputFilePath) - if !strings.Contains(inputSource, `CurFoosById map[int]*Foos`) { + if !strings.Contains(inputSource, `CurFoosById map[int]Foos`) { t.Fatalf("expected generated input to include indexed helper map:\n%s", inputSource) } + if !strings.Contains(inputSource, "CurFoosId *struct {") || !strings.Contains(inputSource, "Values []int") { + t.Fatalf("expected generated input to preserve helper ids struct type:\n%s", inputSource) + } initSource := mustReadCodegenFile(t, filepath.Join(packageDir, "input_init.go")) - if !strings.Contains(initSource, `i.CurFoosById = make(map[int]*Foos, len(i.CurFoos))`) { + if !strings.Contains(initSource, `i.CurFoosById = make(map[int]Foos, len(i.CurFoos))`) { t.Fatalf("expected generated init helper to allocate CurFoosById:\n%s", initSource) } if !strings.Contains(initSource, `if item.Id == nil {`) || !strings.Contains(initSource, `i.CurFoosById[*item.Id] = item`) { @@ -146,7 +149,7 @@ func TestComponentCodegen_MutableComponent_GeneratesPatchHelpers(t *testing.T) { if !strings.Contains(validateSource, `_, err := aValidator.Validate(ctx, value, append(options, validator.WithValidation(validation))...)`) { t.Fatalf("expected generated validate helper to call validator service:\n%s", validateSource) } - if !strings.Contains(validateSource, `case *Foos:`) || !strings.Contains(validateSource, `if actual.Id == nil {`) || !strings.Contains(validateSource, `_, ok := i.CurFoosById[*actual.Id]`) { + if !strings.Contains(validateSource, `case Foos:`) || !strings.Contains(validateSource, `if actual.Id == nil {`) || !strings.Contains(validateSource, `_, ok := i.CurFoosById[*actual.Id]`) { t.Fatalf("expected generated validate helper to use CurFoosById marker provider:\n%s", validateSource) } @@ -167,7 +170,8 @@ func TestComponentCodegen_MutableComponent_GeneratesPatchHelpers(t *testing.T) { veltySource := mustReadCodegenFile(t, result.VeltyFilePath) for _, fragment := range []string{ `$sequencer.Allocate("FOOS", $Foos, "Id")`, - `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, + `#set($_ = $CurFoos<[]Foos>(view/CurFoos) /*`, + `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, `$sql.Update($Foos, "FOOS");`, `$sql.Insert($Foos, "FOOS");`, } { @@ -225,7 +229,7 @@ func TestComponentCodegen_MutableComponent_DSQLParity_BasicOne(t *testing.T) { { Name: "CurFoos", Mode: view.ModeQuery, - Template: &view.Template{Source: "SELECT * FROM FOOS\nWHERE $criteria.In(\"ID\", $CurFoosId.Values)"}, + Template: &view.Template{Source: "SELECT * FROM FOOS\nWHERE $criteria.In(\"ID\", $Unsafe.CurFoosId.Values)"}, Schema: func() *state.Schema { s := state.NewSchema(reflect.TypeOf(&BasicFoos{})) s.Name, s.DataType, s.Cardinality = "Foos", "*Foos", state.Many @@ -280,7 +284,7 @@ func TestComponentCodegen_MutableComponent_DSQLParity_BasicMany(t *testing.T) { { Name: "CurFoos", Mode: view.ModeQuery, - Template: &view.Template{Source: "SELECT * FROM FOOS\nWHERE $criteria.In(\"ID\", $CurFoosId.Values)"}, + Template: &view.Template{Source: "SELECT * FROM FOOS\nWHERE $criteria.In(\"ID\", $Unsafe.CurFoosId.Values)"}, Schema: func() *state.Schema { s := state.NewSchema(reflect.TypeOf(&BasicFoos{})) s.Name, s.DataType, s.Cardinality = "Foos", "*Foos", state.Many @@ -335,7 +339,7 @@ func TestComponentCodegen_MutableComponent_DSQLParity_ManyMany(t *testing.T) { { Name: "CurFoos", Mode: view.ModeQuery, - Template: &view.Template{Source: "SELECT * FROM FOOS\nWHERE $criteria.In(\"ID\", $CurFoosId.Values)"}, + Template: &view.Template{Source: "SELECT * FROM FOOS\nWHERE $criteria.In(\"ID\", $Unsafe.CurFoosId.Values)"}, Schema: func() *state.Schema { s := state.NewSchema(reflect.TypeOf(&Foos{})) s.Name, s.DataType, s.Cardinality = "Foos", "*Foos", state.Many @@ -346,7 +350,7 @@ func TestComponentCodegen_MutableComponent_DSQLParity_ManyMany(t *testing.T) { { Name: "CurFoosPerformance", Mode: view.ModeQuery, - Template: &view.Template{Source: "SELECT * FROM FOOS_PERFORMANCE\nWHERE $criteria.In(\"ID\", $CurFoosFoosPerformanceId.Values)"}, + Template: &view.Template{Source: "SELECT * FROM FOOS_PERFORMANCE\nWHERE $criteria.In(\"ID\", $Unsafe.CurFoosFoosPerformanceId.Values)"}, Schema: func() *state.Schema { s := state.NewSchema(reflect.TypeOf(&FoosPerformance{})) s.Name, s.DataType, s.Cardinality = "FoosPerformance", "*FoosPerformance", state.Many @@ -505,3 +509,126 @@ func normalizeMutableSQL(value string) string { } return strings.Join(out, "\n") } + +func TestComponentCodegen_MutableComponent_MergesRootTemplateHelpersIntoInput(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "patch_basic_one") + + component := &shapeload.Component{ + Method: "PATCH", + URI: "/v1/api/shape/dev/basic/foos", + RootView: "Foos", + Input: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewBodyLocation(""), + Tag: `anonymous:"true"`, + Schema: &state.Schema{Name: "FoosView", DataType: "*FoosView", Cardinality: state.One}, + }, + }, + }, + Output: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Foos", + In: state.NewOutputLocation("body"), + Tag: `anonymous:"true"`, + Schema: &state.Schema{Name: "FoosView", DataType: "*FoosView", Cardinality: state.One}, + }, + }, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{ + Name: "Foos", + Mode: view.ModeExec, + Schema: &state.Schema{ + Name: "FoosView", + DataType: "*FoosView", + Cardinality: state.One, + }, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "NAME", DataType: "string", Nullable: true}, + {Name: "QUANTITY", DataType: "int", Nullable: true}, + }, + Template: &view.Template{ + UseParameterStateType: true, + Parameters: state.Parameters{ + { + Name: "Foos", + In: state.NewBodyLocation(""), + Tag: `anonymous:"true"`, + Schema: &state.Schema{Name: "FoosView", DataType: "*FoosView", Cardinality: state.One}, + }, + { + Name: "CurFoosId", + In: state.NewParameterLocation("Foos"), + Schema: state.NewSchema(reflect.TypeOf(&struct{ Values []int }{})), + Tag: `codec:"structql,uri=foos/cur_foos_id.sql"`, + }, + { + Name: "CurFoos", + In: state.NewViewLocation("CurFoos"), + Tag: `view:"CurFoos" sql:"uri=foos/cur_foos.sql"`, + Schema: &state.Schema{Name: "FoosView", DataType: "*FoosView", Cardinality: state.Many}, + }, + }, + }, + }, + &view.View{ + Name: "CurFoos", + Mode: view.ModeQuery, + Schema: &state.Schema{ + Name: "FoosView", + DataType: "*FoosView", + Cardinality: state.Many, + }, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "NAME", DataType: "string", Nullable: true}, + {Name: "QUANTITY", DataType: "int", Nullable: true}, + }, + }, + ) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "patch_basic_one", + PackagePath: "github.com/acme/project/shape/dev/patch_basic_one", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: true, + WithContract: true, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + + inputSource := mustReadCodegenFile(t, result.InputFilePath) + for _, fragment := range []string{ + `CurFoosId *struct {`, + `Values []int`, + `CurFoos `, + `CurFoosById map[int]`, + } { + if !strings.Contains(inputSource, fragment) { + t.Fatalf("expected generated input to include %q:\n%s", fragment, inputSource) + } + } + + initSource := mustReadCodegenFile(t, filepath.Join(packageDir, "input_init.go")) + if !strings.Contains(initSource, `i.CurFoosById = make(map[int]FoosView, len(i.CurFoos))`) { + t.Fatalf("expected generated init helper to index CurFoos:\n%s", initSource) + } +} diff --git a/repository/shape/xgen/mutable_body.go b/repository/shape/xgen/mutable_body.go index 31b5085c8..be1edba0c 100644 --- a/repository/shape/xgen/mutable_body.go +++ b/repository/shape/xgen/mutable_body.go @@ -3,6 +3,7 @@ package xgen import ( "path/filepath" "reflect" + "regexp" "sort" "strings" @@ -35,7 +36,9 @@ func (g *ComponentCodegen) renderMutableVeltyBody(inputType reflect.Type) (strin if err = block.Generate(builder); err != nil { return "", false, err } - return strings.TrimSpace(builder.String()) + "\n", true, nil + body := strings.TrimSpace(builder.String()) + body = g.normalizeMutableBodyReferences(body, support) + return body + "\n", true, nil } func (g *ComponentCodegen) renderMutableDSQL(inputType reflect.Type) (string, bool, error) { @@ -43,34 +46,7 @@ func (g *ComponentCodegen) renderMutableDSQL(inputType reflect.Type) (string, bo if err != nil || !ok { return "", ok, err } - support := g.mutableSupport(inputType) - if support == nil { - return "", false, nil - } - var builder strings.Builder - builder.WriteString("/* ") - builder.WriteString(g.mutableRouteOptionJSON()) - builder.WriteString(" */\n\n\n") - if imports := g.mutableTypeImports(support, inputType); len(imports) > 0 { - builder.WriteString("import (\n") - for _, item := range imports { - builder.WriteString("\t") - builder.WriteString(strconvQuote(item)) - builder.WriteString("\n") - } - builder.WriteString("\t)\n\n\n") - } - builder.WriteString(g.mutableBodyDeclaration(inputType, support)) - for _, helper := range g.mutableIDHelpers(support) { - builder.WriteString(g.mutableIDsDeclaration(helper)) - } - for _, helper := range g.mutableViewHelpers(support) { - builder.WriteString(g.mutableViewDeclaration(helper)) - } - builder.WriteString(g.mutableOutputDeclaration(inputType, support)) - builder.WriteString("\n\n") - builder.WriteString(body) - return builder.String(), true, nil + return strings.TrimSpace(body) + "\n", true, nil } func (g *ComponentCodegen) mutableTypeImports(support *mutableComponentSupport, inputType reflect.Type) []string { @@ -80,14 +56,14 @@ func (g *ComponentCodegen) mutableTypeImports(support *mutableComponentSupport, if typeName == "" { return } - pkg := strings.TrimSpace(g.PackageName) + pkg := strings.TrimSpace(g.PackagePath) if pkg == "" && g.TypeContext != nil { - pkg = strings.TrimSpace(g.TypeContext.PackageName) + pkg = strings.TrimSpace(g.TypeContext.PackagePath) } if pkg == "" { return } - items[pkg+"."+typeName] = struct{}{} + items[pkg] = struct{}{} } if bodyField, ok := inputType.FieldByName(support.BodyFieldName); ok { if itemType, _ := mutableBodyItemType(bodyField.Type); itemType != nil { @@ -132,19 +108,11 @@ func (g *ComponentCodegen) mutableBodyDeclaration(inputType reflect.Type, suppor if !ok { return "" } - itemType, many := mutableBodyItemType(bodyField.Type) - if itemType == nil { - return "" - } - typeName := strings.TrimSpace(support.BodyTypeName) - if typeName == "" { - typeName = itemType.Name() - } - typeExpr := typeName - if many { - typeExpr = "[]" + typeName + cardinality := "" + if g.mutableBodyMany(bodyField, support) { + cardinality = ".Cardinality('Many')" } - return "#set($_ = $" + support.BodyFieldName + "<" + typeExpr + ">(body/).WithTag('anonymous:\"true\"').Required())\n" + return "#set($_ = $" + support.BodyFieldName + "(body/)" + cardinality + ".WithTag('anonymous:\"true\"').Required())\n" } func (g *ComponentCodegen) mutableIDsDeclaration(helper mutableIndexHelper) string { @@ -157,19 +125,66 @@ func (g *ComponentCodegen) mutableIDsDeclaration(helper mutableIndexHelper) stri } func (g *ComponentCodegen) mutableViewDeclaration(helper mutableIndexHelper) string { - typeName := strings.TrimSpace(helper.TypeName) - if typeName == "" && helper.ItemStruct != nil { - typeName = strings.TrimSpace(helper.ItemStruct.Name()) - } - viewType := "[]*" + typeName - if typeName == "" || helper.ViewFieldName == "" { + viewName := strings.TrimSpace(helper.ViewFieldName) + if viewName == "" { return "" } sqlText := g.mutableDeclarationViewSQL(helper) if sqlText == "" { return "" } - return "\t#set($_ = $" + helper.ViewFieldName + "<" + viewType + ">(view/" + helper.ViewFieldName + ") /*\n" + sqlText + "\n*/\n)\n" + typeExpr := strings.TrimSpace(helper.ItemTypeExpr) + if typeExpr == "" { + return "" + } + if g.mutableHelperUsesMany(helper) && !strings.HasPrefix(typeExpr, "[]") { + typeExpr = "[]" + typeExpr + } + return "\t#set($_ = $" + viewName + "<" + typeExpr + ">(view/" + viewName + ") /*\n" + sqlText + "\n*/\n)\n" +} + +func (g *ComponentCodegen) mutableHelperUsesMany(helper mutableIndexHelper) bool { + if g == nil { + return true + } + lookup := func(params state.Parameters) (bool, bool) { + for _, input := range params { + if input == nil || input.In == nil || input.In.Kind != state.KindView { + continue + } + if !strings.EqualFold(strings.TrimSpace(input.Name), strings.TrimSpace(helper.ViewParamName)) { + continue + } + if input.Schema == nil { + return true, true + } + return input.Schema.Cardinality == state.Many, true + } + return false, false + } + if g.Component != nil { + componentInputs := make(state.Parameters, 0, len(g.Component.Input)) + for _, input := range g.Component.Input { + if input == nil { + continue + } + componentInputs = append(componentInputs, &input.Parameter) + } + if many, ok := lookup(componentInputs); ok { + return many + } + } + if root := g.rootResourceView(); root != nil && root.Template != nil { + if many, ok := lookup(root.Template.Parameters); ok { + return many + } + } + if g.Resource != nil { + if many, ok := lookup(g.Resource.Parameters); ok { + return many + } + } + return true } func (g *ComponentCodegen) mutableOutputDeclaration(inputType reflect.Type, support *mutableComponentSupport) string { @@ -177,22 +192,29 @@ func (g *ComponentCodegen) mutableOutputDeclaration(inputType reflect.Type, supp if !ok { return "" } - _, many := mutableBodyItemType(bodyField.Type) - typeExpr := "" - if many { - typeExpr = "[]" + cardinality := "" + if g.mutableBodyMany(bodyField, support) { + cardinality = ".Cardinality('Many')" } + tag := `anonymous:"true"` typeName := strings.TrimSpace(support.BodyTypeName) if typeName == "" { if itemType, _ := mutableBodyItemType(bodyField.Type); itemType != nil { typeName = itemType.Name() } } - tag := `anonymous:"true"` if typeName != "" { tag += ` typeName:"` + typeName + `"` } - return "#set($_ = $" + support.BodyFieldName + "<" + typeExpr + ">(body/).WithTag('" + tag + "').Required().Output())\n" + return "#set($_ = $" + support.BodyFieldName + "(body/)" + cardinality + ".WithTag('" + tag + "').Required().Output())\n" +} + +func (g *ComponentCodegen) mutableQualifiedTypeName(typeName string) string { + typeName = strings.TrimSpace(typeName) + if typeName == "" { + return "" + } + return typeName } func (g *ComponentCodegen) mutableIDSQL(helper mutableIndexHelper) string { @@ -209,22 +231,22 @@ func (g *ComponentCodegen) mutableIDSQL(helper mutableIndexHelper) string { func (g *ComponentCodegen) mutableViewSQL(helper mutableIndexHelper) string { if g == nil || g.Resource == nil { - return g.mutableFallbackViewSQL(helper) + return g.normalizeMutableViewSQL(helper, g.mutableFallbackViewSQL(helper)) } for _, aView := range g.Resource.Views { if aView == nil || !strings.EqualFold(strings.TrimSpace(aView.Name), strings.TrimSpace(helper.ViewParamName)) { continue } if aView.Template == nil { - return g.mutableFallbackViewSQL(helper) + return g.normalizeMutableViewSQL(helper, g.mutableFallbackViewSQL(helper)) } sqlText := strings.TrimSpace(aView.Template.Source) if sqlText != "" { - return sqlText + return g.normalizeMutableViewSQL(helper, sqlText) } - return g.mutableFallbackViewSQL(helper) + return g.normalizeMutableViewSQL(helper, g.mutableFallbackViewSQL(helper)) } - return g.mutableFallbackViewSQL(helper) + return g.normalizeMutableViewSQL(helper, g.mutableFallbackViewSQL(helper)) } func (g *ComponentCodegen) mutableDeclarationViewSQL(helper mutableIndexHelper) string { @@ -255,7 +277,48 @@ func (g *ComponentCodegen) mutableFallbackViewSQL(helper mutableIndexHelper) str if key == "" { key = "Id" } - return "SELECT * FROM " + tableName + "\nWHERE $criteria.In(\"" + key + "\", $" + idParam + ".Values)" + return "SELECT * FROM " + tableName + "\nWHERE $criteria.In(\"" + key + "\", $Unsafe." + idParam + ".Values)" +} + +func (g *ComponentCodegen) normalizeMutableViewSQL(helper mutableIndexHelper, sqlText string) string { + sqlText = strings.TrimSpace(sqlText) + if sqlText == "" { + return "" + } + idParam := strings.TrimSpace(g.mutableIDsParamName(helper)) + if idParam == "" { + return sqlText + } + legacy := "$" + idParam + ".Values" + normalized := "$Unsafe." + idParam + ".Values" + if strings.Contains(sqlText, legacy) && !strings.Contains(sqlText, normalized) { + sqlText = strings.ReplaceAll(sqlText, legacy, normalized) + } + return sqlText +} + +func (g *ComponentCodegen) normalizeMutableBodyReferences(body string, support *mutableComponentSupport) string { + body = strings.TrimSpace(body) + if body == "" || support == nil { + return body + } + names := []string{strings.TrimSpace(support.BodyFieldName)} + for _, helper := range support.Helpers { + if name := strings.TrimSpace(helper.ViewFieldName); name != "" { + names = append(names, name) + } + } + for _, name := range names { + if name == "" { + continue + } + pattern := regexp.MustCompile(`\$` + regexp.QuoteMeta(name) + `\b`) + body = pattern.ReplaceAllStringFunc(body, func(string) string { + return "$Unsafe." + name + }) + } + body = regexp.MustCompile(`#set\(\$([A-Za-z0-9_]+) =`).ReplaceAllString(body, `#set($$1 =`) + return body } func (g *ComponentCodegen) supportBodyFieldName(helper mutableIndexHelper) string { @@ -473,7 +536,7 @@ func (g *ComponentCodegen) mutableInputType() (reflect.Type, error) { if g == nil || g.Component == nil { return nil, nil } - params := normalizeInputParametersForCodegen(g.Component.InputParameters(), g.Resource, g.Component.URI) + params := g.codegenInputParameters() opts := []state.ReflectOption{state.WithSetMarker(), state.WithTypeName(g.inputTypeName(g.componentName()))} if g.componentUsesVelty() { opts = append(opts, state.WithVelty(true)) @@ -491,7 +554,7 @@ func (g *ComponentCodegen) buildMutableVeltyBlock(inputType reflect.Type, suppor if !ok { return nil, nil } - bodyItemType, bodyIsMany := mutableBodyItemType(bodyField.Type) + bodyItemType, _ := mutableBodyItemType(bodyField.Type) if bodyItemType == nil { return nil, nil } @@ -509,7 +572,7 @@ func (g *ComponentCodegen) buildMutableVeltyBlock(inputType reflect.Type, suppor for _, helper := range support.Helpers { block.Append(shapeast.NewAssign( - shapeast.NewIdent(helper.MapFieldName), + g.mutableHelperMapHolder(helper), shapeast.NewCallExpr(shapeast.NewIdent(helper.ViewFieldName), "IndexBy", shapeast.NewQuotedLiteral(helper.KeyFieldName)), )) } @@ -519,7 +582,7 @@ func (g *ComponentCodegen) buildMutableVeltyBlock(inputType reflect.Type, suppor rootHelper := support.rootHelper() bodyExpr := shapeast.NewIdent(support.BodyFieldName) - if bodyIsMany { + if g.mutableBodyMany(bodyField, support) { recordName := mutableRecordName(support.BodyFieldName) forEach := shapeast.NewForEach(shapeast.NewIdent(recordName), bodyExpr, shapeast.Block{}) g.appendMutableWriteLogic(&forEach.Body, shapeast.NewIdent(recordName), "", bodyItemType, bodyTable, support, rootHelper, bodyKeyField) @@ -527,12 +590,38 @@ func (g *ComponentCodegen) buildMutableVeltyBlock(inputType reflect.Type, suppor return block, nil } - condition := shapeast.NewCondition(bodyExpr, shapeast.Block{}, nil) - g.appendMutableWriteLogic(&condition.IFBlock, bodyExpr, "", bodyItemType, bodyTable, support, rootHelper, bodyKeyField) - block.Append(condition) + g.appendMutableWriteLogic(&block, bodyExpr, "", bodyItemType, bodyTable, support, rootHelper, bodyKeyField) return block, nil } +func (g *ComponentCodegen) mutableHelperMapHolder(helper mutableIndexHelper) shapeast.Expression { + return shapeast.NewIdent(helper.MapFieldName) +} + +func mutableItemExprIsPointer(itemTypeExpr string) bool { + itemTypeExpr = strings.TrimSpace(itemTypeExpr) + return strings.HasPrefix(itemTypeExpr, "*") || strings.HasPrefix(itemTypeExpr, "[]*") +} + +func (g *ComponentCodegen) mutableBodyMany(bodyField reflect.StructField, support *mutableComponentSupport) bool { + if support != nil && support.BodyMany { + return true + } + if g != nil && g.Component != nil { + for _, input := range g.Component.Input { + if input == nil || input.In == nil || input.In.Kind != state.KindRequestBody { + continue + } + if input.Schema != nil && input.Schema.Cardinality != "" { + return input.Schema.Cardinality == state.Many + } + break + } + } + _, many := mutableBodyItemType(bodyField.Type) + return many +} + func (g *ComponentCodegen) appendMutableWriteLogic(block *shapeast.Block, recordExpr *shapeast.Ident, logicalPath string, recordType reflect.Type, tableName string, support *mutableComponentSupport, rootHelper *mutableIndexHelper, keyField reflect.StructField) { method := strings.ToUpper(strings.TrimSpace(g.Component.Method)) hasCurrent := rootHelper != nil diff --git a/repository/shape/xgen/mutable_helpers.go b/repository/shape/xgen/mutable_helpers.go index c5ecd2f84..71f0e46ba 100644 --- a/repository/shape/xgen/mutable_helpers.go +++ b/repository/shape/xgen/mutable_helpers.go @@ -5,13 +5,13 @@ import ( "reflect" "strings" - "github.com/viant/datly/repository/shape/plan" "github.com/viant/datly/view/state" ) type mutableComponentSupport struct { BodyFieldName string BodyTypeName string + BodyMany bool Helpers []mutableIndexHelper } @@ -56,10 +56,11 @@ func (g *ComponentCodegen) mutableSupport(inputType reflect.Type) *mutableCompon if bodyTypeName := strings.TrimSpace(input.Schema.Name); bodyTypeName != "" { support.BodyTypeName = bodyTypeName } + support.BodyMany = input.Schema.Cardinality == state.Many } break } - for _, input := range g.Component.Input { + for _, input := range g.mutableHelperParametersForCodegen() { if input == nil || input.In == nil || input.In.Kind != state.KindView { continue } @@ -75,7 +76,7 @@ func (g *ComponentCodegen) mutableSupport(inputType reflect.Type) *mutableCompon return support } -func (g *ComponentCodegen) mutableIndexHelper(inputType reflect.Type, bodyFieldName string, param *plan.State) (mutableIndexHelper, bool) { +func (g *ComponentCodegen) mutableIndexHelper(inputType reflect.Type, bodyFieldName string, param *state.Parameter) (mutableIndexHelper, bool) { fieldName := exportedCodegenParamName(param.Name) if fieldName == "" { return mutableIndexHelper{}, false @@ -130,6 +131,21 @@ func (g *ComponentCodegen) mutableIndexHelper(inputType reflect.Type, bodyFieldN }, true } +func (g *ComponentCodegen) mutableHelperParametersForCodegen() []*state.Parameter { + params := g.codegenInputParameters() + if len(params) == 0 { + return nil + } + result := make([]*state.Parameter, 0, len(params)) + for _, item := range params { + if item == nil { + continue + } + result = append(result, item) + } + return result +} + func mutableRelationPath(inputType reflect.Type, itemType reflect.Type, bodyFieldName string) string { if inputType == nil || itemType == nil || bodyFieldName == "" { return "" diff --git a/service/executor/expand/data_unit.go b/service/executor/expand/data_unit.go index 9d2c22dad..7f0358b2e 100644 --- a/service/executor/expand/data_unit.go +++ b/service/executor/expand/data_unit.go @@ -3,7 +3,9 @@ package expand import ( "context" "fmt" + "os" "reflect" + "runtime/debug" "strings" "sync" @@ -64,6 +66,9 @@ func (c *DataUnit) Validate(dest interface{}, opts ...interface{}) (*validator.V } func (c *DataUnit) Allocate(tableName string, dest interface{}, selector string) (string, error) { + if os.Getenv("DATLY_DEBUG_MUTABLE") == "1" { + fmt.Printf("[MUTABLE DEBUG] Allocate table=%s selector=%s destType=%T dest=%#v\n", tableName, selector, dest, dest) + } db, err := c.MetaSource.Db() if err != nil { fmt.Printf("error occured while connecting to DB %v\n", err.Error()) @@ -208,6 +213,15 @@ func (c *DataUnit) FilterExecutables(statements []string, stopOnNonExec bool) [] } func (c *DataUnit) In(columnName string, args interface{}) (string, error) { + if os.Getenv("DATLY_DEBUG_DATAUNIT") == "1" { + defer func() { + if r := recover(); r != nil { + fmt.Printf("[DATAUNIT DEBUG] In panic column=%q argsType=%T args=%#v err=%v\n%s\n", columnName, args, args, r, debug.Stack()) + panic(r) + } + }() + fmt.Printf("[DATAUNIT DEBUG] In column=%q argsType=%T args=%#v\n", columnName, args, args) + } return c.in(columnName, args, true) } diff --git a/service/executor/expand/parent.go b/service/executor/expand/parent.go index 11ae1162f..97787429d 100644 --- a/service/executor/expand/parent.go +++ b/service/executor/expand/parent.go @@ -2,8 +2,10 @@ package expand import ( "database/sql" + "fmt" "github.com/viant/datly/utils/types" "github.com/viant/xunsafe" + "os" "reflect" "strings" ) @@ -247,10 +249,16 @@ func NotZeroOf(values ...int) int { } func (c *DataUnit) Insert(data interface{}, tableName string) (string, error) { + if os.Getenv("DATLY_DEBUG_MUTABLE") == "1" { + fmt.Printf("[MUTABLE DEBUG] Insert table=%s dataType=%T data=%#v\n", tableName, data, data) + } return c.Statements.InsertWithMarker(tableName, data), nil } func (c *DataUnit) Update(data interface{}, tableName string) (string, error) { + if os.Getenv("DATLY_DEBUG_MUTABLE") == "1" { + fmt.Printf("[MUTABLE DEBUG] Update table=%s dataType=%T data=%#v\n", tableName, data, data) + } return c.Statements.UpdateWithMarker(tableName, data), nil } diff --git a/service/operator/reader.go b/service/operator/reader.go index 801638e12..5a29d9eeb 100644 --- a/service/operator/reader.go +++ b/service/operator/reader.go @@ -3,6 +3,7 @@ package operator import ( "context" "net/http" + "os" "github.com/viant/datly/repository" "github.com/viant/datly/service/reader" @@ -23,6 +24,9 @@ func (s *Service) runQuery(ctx context.Context, component *repository.Component, defer func() { if r := recover(); r != nil { panicMsg := fmt.Sprintf("Panic occurred: %v, Stack trace: %v", r, string(debug.Stack())) + if os.Getenv("DATLY_DEBUG_OPERATOR") == "1" { + fmt.Printf("[OPERATOR DEBUG] %s\n", panicMsg) + } logger := aSession.Logger() if logger == nil { panic(panicMsg) diff --git a/service/reader/handler/handler.go b/service/reader/handler/handler.go index 6e83335e3..98a3e47d3 100644 --- a/service/reader/handler/handler.go +++ b/service/reader/handler/handler.go @@ -3,6 +3,7 @@ package handler import ( "context" "encoding/json" + "fmt" "github.com/viant/datly/gateway/router/status" _ "github.com/viant/datly/repository/locator/async" @@ -12,7 +13,9 @@ import ( _ "github.com/viant/datly/service/executor/handler/locator" "net/http" + "os" "reflect" + "runtime/debug" reader "github.com/viant/datly/service/reader" "github.com/viant/datly/service/session" @@ -96,6 +99,14 @@ func (h *Handler) Handle(ctx context.Context, aView *view.View, aSession *sessio } func (h *Handler) readData(ctx context.Context, aView *view.View, aState *session.Session, ret *Response, opts []reader.Option) error { + if os.Getenv("DATLY_DEBUG_HANDLER_READ") == "1" { + defer func() { + if r := recover(); r != nil { + fmt.Printf("[HANDLER READ DEBUG] panic view=%s err=%v\n%s\n", aView.Name, r, debug.Stack()) + panic(r) + } + }() + } destValue := reflect.New(aView.Schema.SliceType()) dest := destValue.Interface() aSession, err := reader.NewSession(dest, aView) diff --git a/service/reader/service.go b/service/reader/service.go index 1fda9fc9e..d362ec899 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -4,7 +4,9 @@ import ( "context" "database/sql" "fmt" + "os" "reflect" + "runtime/debug" "strings" "sync" "sync/atomic" @@ -35,6 +37,14 @@ type Service struct { // ReadInto reads Data into provided destination, * dDest` is required. It has to be a pointer to `interface{}` or pointer to slice of `T` or `*T` func (s *Service) ReadInto(ctx context.Context, dest interface{}, aView *view.View, opts ...Option) error { + if os.Getenv("DATLY_DEBUG_READER") == "1" { + defer func() { + if r := recover(); r != nil { + fmt.Printf("[READER DEBUG] panic view=%s dest=%T err=%v\n%s\n", aView.Name, dest, r, debug.Stack()) + panic(r) + } + }() + } session, err := NewSession(dest, aView, opts...) if err != nil { return err @@ -408,6 +418,9 @@ func (s *Service) BuildCriteria(ctx context.Context, value interface{}, options } func (s *Service) queryInBatches(ctx context.Context, session *Session, aView *view.View, collector *view.Collector, visitor view.VisitorFn, info *response.SQLExecutions, batchData *view.BatchData, selector *view.Statelet) error { + if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { + fmt.Printf("[QUERY DEBUG] queryInBatches view=%s selectorTemplateNil=%v batchValues=%d\n", aView.Name, selector == nil || selector.Template == nil, len(batchData.ValuesBatch)) + } wg := &sync.WaitGroup{} db, err := aView.Db() if err != nil { @@ -446,10 +459,19 @@ func (s *Service) queryObjects(ctx context.Context, session *Session, aView *vie return s.queryWithPartitions(ctx, session, aView, selector, batchData, db, collector, visitor, partitioned) } readData := 0 + if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { + fmt.Printf("[QUERY DEBUG] queryObjects view=%s schema=%v slice=%v collectorView=%s\n", aView.Name, aView.Schema.Type(), aView.Schema.SliceType(), collector.View().Name) + } parametrizedSQL, columnInMatcher, err := s.buildParametrizedSQL(ctx, aView, selector, batchData, collector, session, nil) if err != nil { + if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { + fmt.Printf("[QUERY DEBUG] buildParametrizedSQL error view=%s err=%v\n", aView.Name, err) + } return nil, err } + if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { + fmt.Printf("[QUERY DEBUG] builtSQL view=%s sql=%s args=%#v\n", aView.Name, parametrizedSQL.SQL, parametrizedSQL.Args) + } var parentProvider func(value interface{}) (interface{}, error) handler := func(row interface{}) error { @@ -514,6 +536,9 @@ func (s *Service) queryWithHandler(ctx context.Context, session *Session, aView stats, onDone := NewExecutionInfo(parametrizedSQL, cacheStats, collector) defer onDone() + if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { + fmt.Printf("[QUERY HANDLER] view=%s sql=%s args=%#v\n", aView.Name, parametrizedSQL.SQL, parametrizedSQL.Args) + } if session.DryRun { return []*response.SQLExecution{stats}, nil } @@ -543,7 +568,16 @@ BEGIN: } _ = stmt.Close() }() - err = reader.QueryAll(ctx, handler, parametrizedSQL.Args...) + debugHandler := handler + if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { + debugHandler = func(row interface{}) error { + fmt.Printf("[QUERY HANDLER] view=%s before unwrap row=%T readData=%d\n", aView.Name, row, *readData) + err := handler(row) + fmt.Printf("[QUERY HANDLER] view=%s after handler row=%T readData=%d err=%v\n", aView.Name, row, *readData, err) + return err + } + } + err = reader.QueryAll(ctx, debugHandler, parametrizedSQL.Args...) isInvalidConnection = err != nil && strings.Contains(err.Error(), "invalid connection") if isInvalidConnection && atomic.AddUint32(&retires, 1) < 3 { diff --git a/service/reader/sql.go b/service/reader/sql.go index 256cd26f3..d1c01485d 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -3,14 +3,19 @@ package reader import ( "context" "fmt" + "os" "strconv" "strings" + "github.com/viant/datly/internal/inference" "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/service/reader/metadata" "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/datly/view/keywords" + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/query" "github.com/viant/sqlx/io/read/cache" ) @@ -76,15 +81,21 @@ func (b *Builder) Build(ctx context.Context, opts ...BuilderOption) (*cache.Parm if len(state.Filters) > 0 { statelet.AppendFilters(state.Filters) } - if aView.Template.IsActualTemplate() && aView.ShouldTryDiscover() { - state.Expanded = metadata.EnrichWithDiscover(state.Expanded, true) - } sb := strings.Builder{} sb.WriteString(selectFragment) - if err = b.appendColumns(&sb, aView, statelet); err != nil { + projectedColumns, err := b.appendColumns(&sb, aView, statelet) + if err != nil { return nil, err } + if aView.Groupable { + if state.Expanded, err = b.rewriteGroupBy(state.Expanded, aView.Columns, projectedColumns); err != nil { + return nil, err + } + } + if aView.Template.IsActualTemplate() && aView.ShouldTryDiscover() { + state.Expanded = metadata.EnrichWithDiscover(state.Expanded, true) + } if err = b.appendRelationColumn(&sb, aView, statelet, relation); err != nil { return nil, err @@ -159,6 +170,9 @@ func (b *Builder) Build(ctx context.Context, opts ...BuilderOption) (*cache.Parm SQL: SQL, Args: placeholders, } + if os.Getenv("DATLY_DEBUG_SQL_BUILDER") == "1" { + fmt.Printf("[SQL BUILDER] view=%s sql=%s args=%#v state=%s\n", aView.Name, SQL, placeholders, state.Expanded) + } if exclude.ColumnsIn && relation != nil { parametrizedQuery.By = shared.FirstNotEmpty(relation.Of.On[0].Field, relation.Of.On[0].Column) @@ -173,20 +187,21 @@ func (b *Builder) Build(ctx context.Context, opts ...BuilderOption) (*cache.Parm return parametrizedQuery, err } -func (b *Builder) appendColumns(sb *strings.Builder, aView *view.View, selector *view.Statelet) error { +func (b *Builder) appendColumns(sb *strings.Builder, aView *view.View, selector *view.Statelet) ([]*view.Column, error) { if len(selector.Columns) == 0 { b.appendViewColumns(sb, aView) - return nil + return nil, nil } return b.appendSelectorColumns(sb, aView, selector) } -func (b *Builder) appendSelectorColumns(sb *strings.Builder, view *view.View, selector *view.Statelet) error { +func (b *Builder) appendSelectorColumns(sb *strings.Builder, aView *view.View, selector *view.Statelet) ([]*view.Column, error) { + result := make([]*view.Column, 0, len(selector.Columns)) for i, column := range selector.Columns { - viewColumn, ok := view.ColumnByName(column) + viewColumn, ok := aView.ColumnByName(column) if !ok { - return fmt.Errorf("not found column %v at view %v", column, view.Name) + return nil, fmt.Errorf("not found column %v at view %v", column, aView.Name) } if i != 0 { @@ -195,9 +210,10 @@ func (b *Builder) appendSelectorColumns(sb *strings.Builder, view *view.View, se sb.WriteString(" ") sb.WriteString(viewColumn.SqlExpression()) + result = append(result, viewColumn) } - return nil + return result, nil } func (b *Builder) viewAlias(view *view.View) string { @@ -225,6 +241,53 @@ func (b *Builder) appendViewColumns(sb *strings.Builder, view *view.View) { } } +func (b *Builder) rewriteGroupBy(SQL string, allColumns []*view.Column, projectedColumns []*view.Column) (string, error) { + if len(projectedColumns) == 0 { + return SQL, nil + } + + trimmed := strings.TrimSpace(SQL) + if trimmed == "" { + return SQL, nil + } + wrapped := strings.HasPrefix(trimmed, "(") && strings.HasSuffix(trimmed, ")") + querySQL := inference.TrimParenthesis(trimmed) + parsed, err := sqlparser.ParseQuery(querySQL) + if err != nil || parsed == nil { + return SQL, err + } + + positions := projectedGroupByPositions(allColumns, projectedColumns) + groupBy := make(query.List, 0, len(positions)) + for _, position := range positions { + groupBy = append(groupBy, query.NewItem(expr.NewIntLiteral(strconv.Itoa(position)))) + } + parsed.GroupBy = groupBy + + rewritten := sqlparser.Stringify(parsed) + if wrapped { + rewritten = "(" + rewritten + ")" + } + return rewritten, nil +} + +func projectedGroupByPositions(allColumns []*view.Column, projectedColumns []*view.Column) []int { + index := make(map[*view.Column]int, len(allColumns)) + for i, column := range allColumns { + index[column] = i + 1 + } + result := make([]int, 0, len(projectedColumns)) + for _, column := range projectedColumns { + if column == nil || !column.Groupable { + continue + } + if position, ok := index[column]; ok { + result = append(result, position) + } + } + return result +} + func (b *Builder) appendViewAlias(sb *strings.Builder, view *view.View) { if view.Alias == "" { return @@ -413,7 +476,7 @@ func (b *Builder) appendRelationColumn(sb *strings.Builder, aView *view.View, se } func (b *Builder) checkViewAndAppendRelColumn(sb *strings.Builder, aView *view.View, relation *view.Relation) error { - if _, ok := aView.ColumnByName(relation.Of.On[0].Column); ok { + if _, _, ok := b.lookupRelationColumn(aView, relation); ok { return nil } @@ -431,13 +494,16 @@ func (b *Builder) checkViewAndAppendRelColumn(sb *strings.Builder, aView *view.V } func (b *Builder) checkSelectorAndAppendRelColumn(sb *strings.Builder, aView *view.View, selector *view.Statelet, relation *view.Relation) error { - if relation == nil || selector.Has(relation.Of.On[0].Column) || aView.Template.IsActualTemplate() { + if relation == nil || aView.Template.IsActualTemplate() { + return nil + } + if b.selectorHasRelationColumn(selector, aView, relation) { return nil } sb.WriteString(separatorFragment) sb.WriteString(" ") - col, ok := aView.ColumnByName(relation.Of.On[0].Column) + col, _, ok := b.lookupRelationColumn(aView, relation) if !ok { sb.WriteString(relation.Of.On[0].Column) } else { @@ -447,6 +513,46 @@ func (b *Builder) checkSelectorAndAppendRelColumn(sb *strings.Builder, aView *vi return nil } +func (b *Builder) selectorHasRelationColumn(selector *view.Statelet, aView *view.View, relation *view.Relation) bool { + if selector == nil || relation == nil || relation.Of == nil || len(relation.Of.On) == 0 { + return false + } + link := relation.Of.On[0] + if selector.Has(link.Column) { + return true + } + if link.Field != "" && selector.Has(link.Field) { + return true + } + if column, _, ok := b.lookupRelationColumn(aView, relation); ok { + if selector.Has(column.Name) { + return true + } + if field := column.Field(); field != nil && selector.Has(field.Name) { + return true + } + } + return false +} + +func (b *Builder) lookupRelationColumn(aView *view.View, relation *view.Relation) (*view.Column, string, bool) { + if aView == nil || relation == nil || relation.Of == nil || len(relation.Of.On) == 0 { + return nil, "", false + } + link := relation.Of.On[0] + if link.Field != "" { + if column, ok := aView.ColumnByName(link.Field); ok { + return column, link.Field, true + } + } + if link.Column != "" { + if column, ok := aView.ColumnByName(link.Column); ok { + return column, link.Column, true + } + } + return nil, "", false +} + func actualLimit(aView *view.View, selector *view.Statelet) int { if selector.Limit != 0 { return selector.Limit diff --git a/service/session/reader.go b/service/session/reader.go index 2184634e1..94a91d528 100644 --- a/service/session/reader.go +++ b/service/session/reader.go @@ -2,11 +2,22 @@ package session import ( "context" + "fmt" reader "github.com/viant/datly/service/reader" "github.com/viant/datly/view" + "os" + "runtime/debug" ) func (s *Session) ReadInto(ctx context.Context, dest interface{}, aView *view.View) error { + if os.Getenv("DATLY_DEBUG_READINTO") == "1" { + defer func() { + if r := recover(); r != nil { + fmt.Printf("[READINTO DEBUG] panic view=%s dest=%T err=%v\n%s\n", aView.Name, dest, r, debug.Stack()) + panic(r) + } + }() + } if err := s.SetViewState(ctx, aView); err != nil { return err } diff --git a/service/session/state.go b/service/session/state.go index 5779250fc..9e182c61d 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "os" "reflect" "strings" "sync" @@ -219,6 +220,16 @@ func (s *Session) ViewOptions(aView *view.View, opts ...Option) *Options { var parameters state.NamedParameters if aView.Template != nil { parameters = aView.Template.Parameters.Index() + if aView.Template.UseResourceParameterLookup && aView.GetResource() != nil { + merged := state.NamedParameters{} + for k, v := range aView.GetResource().NamedParameters() { + merged[k] = v + } + for k, v := range parameters { + merged[k] = v + } + parameters = merged + } } viewOptions.kindLocator = s.kindLocator.With(s.viewLookupOptions(aView, parameters, viewOptions)...) @@ -539,6 +550,9 @@ func (s *Session) ensureValidValue(value interface{}, parameter *state.Parameter } return converted.Interface(), nil } + if wrapped, ok := wrapComponentResult(parameter, value, valueType, rawSrcType, rawDestType, destIsPtr); ok { + return wrapped, nil + } if options.shallReportNotAssignable() { fmt.Printf("parameter %v is not directly assignable from %s:(%s)\nsrc:%s \ndst:%s\n", parameter.Name, parameter.In.Kind, parameter.In.Name, valueType.String(), destType.String()) @@ -563,6 +577,43 @@ func (s *Session) ensureValidValue(value interface{}, parameter *state.Parameter return value, nil } +func wrapComponentResult(parameter *state.Parameter, value interface{}, valueType, rawSrcType, rawDestType reflect.Type, destIsPtr bool) (interface{}, bool) { + if parameter == nil || parameter.In == nil || parameter.In.Kind != state.KindComponent { + return nil, false + } + if rawSrcType.Kind() != reflect.Struct || rawDestType.Kind() != reflect.Struct { + return nil, false + } + field, ok := rawDestType.FieldByName("Data") + if !ok { + return nil, false + } + fieldType := field.Type + srcValue := reflect.ValueOf(value) + if valueType.Kind() == reflect.Ptr { + if srcValue.IsNil() { + return nil, true + } + } + if !valueType.AssignableTo(fieldType) { + if valueType.Kind() == reflect.Ptr && valueType.Elem().AssignableTo(fieldType) { + srcValue = srcValue.Elem() + } else if valueType.Kind() != reflect.Ptr && reflect.PointerTo(valueType).AssignableTo(fieldType) { + ptr := reflect.New(valueType) + ptr.Elem().Set(srcValue) + srcValue = ptr + } else { + return nil, false + } + } + target := reflect.New(rawDestType) + target.Elem().FieldByIndex(field.Index).Set(srcValue) + if destIsPtr { + return target.Interface(), true + } + return target.Elem().Interface(), true +} + func ensureAssignable(fieldName string, destFieldType reflect.Type, srcFieldType reflect.Type) bool { switch destFieldType.Kind() { case reflect.Slice: @@ -722,6 +773,15 @@ func (s *Session) lookupValue(ctx context.Context, parameter *state.Parameter, o func (s *Session) adjustAndCache(ctx context.Context, parameter *state.Parameter, opts *Options, has bool, value interface{}, cachable bool) (interface{}, bool, error) { var err error + if os.Getenv("DATLY_DEBUG_ADJUST") == "1" { + fmt.Printf("[ADJUST DEBUG][start] param=%s kind=%s has=%v value=%T outputType=%v schemaType=%v\n", + parameter.Name, parameter.In.Kind, has, value, parameter.OutputType(), func() reflect.Type { + if parameter.Schema == nil { + return nil + } + return parameter.Schema.Type() + }()) + } if !has && parameter.Value != nil { has = true value = parameter.Value @@ -732,6 +792,9 @@ func (s *Session) adjustAndCache(ctx context.Context, parameter *state.Parameter if value, err = s.adjustValue(parameter, value); err != nil { return nil, false, err } + if os.Getenv("DATLY_DEBUG_ADJUST") == "1" { + fmt.Printf("[ADJUST DEBUG][post-adjust] param=%s value=%T\n", parameter.Name, value) + } if parameter.Output != nil { // Defensive: ensure codec is initialized before Transform. if !parameter.Output.Initialized() { @@ -745,6 +808,9 @@ func (s *Session) adjustAndCache(ctx context.Context, parameter *state.Parameter return nil, false, fmt.Errorf("failed to transform %s with %s: %v, %w", parameter.Name, parameter.Output.Name, value, err) } value = transformed + if os.Getenv("DATLY_DEBUG_ADJUST") == "1" { + fmt.Printf("[ADJUST DEBUG][post-transform] param=%s value=%T\n", parameter.Name, value) + } } if has && err == nil && cachable { s.setValue(parameter, value) diff --git a/service/session/state_test.go b/service/session/state_test.go index 497d6aca6..8c4fac8e8 100644 --- a/service/session/state_test.go +++ b/service/session/state_test.go @@ -288,4 +288,36 @@ func TestSessionEnsureValidValue_Transitions(t *testing.T) { t.Fatalf("expected B=%d, got %d", *original.B, gotB.Elem().Int()) } }) + + t.Run("component_result_wraps_into_data_holder", func(t *testing.T) { + type componentRow struct { + IsReadOnly int + } + type componentHolder struct { + Data *componentRow + } + + value := &componentRow{IsReadOnly: 1} + parameter := &state.Parameter{ + Name: "Auth", + In: state.NewComponent("GET:/auth"), + Schema: state.NewSchema(reflect.TypeOf(componentHolder{})), + } + selector := newSelector(t, reflect.TypeOf(componentHolder{})) + sess := &Session{} + opts := NewOptions(WithReportNotAssignable(false)) + + got, err := sess.ensureValidValue(value, parameter, selector, opts) + if err != nil { + t.Fatalf("ensureValidValue error: %v", err) + } + + holder, ok := got.(componentHolder) + if !ok { + t.Fatalf("expected componentHolder, got %T", got) + } + if holder.Data == nil || holder.Data.IsReadOnly != 1 { + t.Fatalf("expected wrapped component result, got %#v", got) + } + }) } diff --git a/service/session/stater.go b/service/session/stater.go index 5b0b0d6ab..392a6d3a8 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "os" "reflect" "runtime/debug" @@ -46,6 +47,9 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt defer func() { if r := recover(); r != nil { panicMsg := fmt.Sprintf("Panic occurred: %v, Stack trace: %v", r, string(debug.Stack())) + if os.Getenv("DATLY_DEBUG_BIND") == "1" { + fmt.Printf("[BIND DEBUG] %s\n", panicMsg) + } logger := s.Logger() if logger == nil { panic(panicMsg) diff --git a/testutil/shapeparity/bridge.go b/testutil/shapeparity/bridge.go deleted file mode 100644 index 49e13d55e..000000000 --- a/testutil/shapeparity/bridge.go +++ /dev/null @@ -1,93 +0,0 @@ -package shapeparity - -import ( - "context" - "fmt" - "strings" - - "github.com/viant/afs" - "github.com/viant/afs/file" - "github.com/viant/afs/url" - "github.com/viant/datly/cmd/options" - "github.com/viant/datly/internal/translator" - "github.com/viant/datly/repository/shape/dql/sanitize" - dqlscan "github.com/viant/datly/repository/shape/dql/scan" -) - -// ScanDQL translates a DQL file through the legacy internal/translator pipeline -// and returns a scan.Result. This bridges internal/translator for parity tests -// without requiring repository/shape to depend on internal/*. -func ScanDQL(ctx context.Context, req *dqlscan.Request) (*dqlscan.Result, error) { - if req == nil || req.DQLURL == "" { - return nil, fmt.Errorf("dql scan: DQLURL was empty") - } - fs := afs.New() - sourceURL := req.DQLURL - project := inferProject(req.DQLURL) - translate := &options.Translate{} - translate.Rule.Project = project - translate.Rule.Source = []string{sourceURL} - translate.Rule.ModulePrefix = req.ModulePrefix - translate.Repository.RepositoryURL = req.Repository - translate.Repository.APIPrefix = req.APIPrefix - if len(req.Connectors) > 0 { - translate.Repository.Connectors = append(translate.Repository.Connectors, req.Connectors...) - } - if req.ConfigURL != "" { - translate.Repository.Configs.Append(req.ConfigURL) - } - if initErr := translate.Init(ctx); initErr != nil { - return nil, initErr - } - if req.ConfigURL == "" { - translate.Repository.Configs = nil - } - if translate.Rule.ModulePrefix == "" { - translate.Rule.ModulePrefix = "platform" - } - - svc := translator.New(translator.NewConfig(&translate.Repository), fs) - if initErr := svc.Init(ctx); initErr != nil { - return nil, initErr - } - if initErr := svc.InitSignature(ctx, &translate.Rule); initErr != nil { - return nil, initErr - } - dsql, loadErr := translate.Rule.LoadSource(ctx, fs, translate.Rule.SourceURL()) - if loadErr != nil { - return nil, loadErr - } - translate.Rule.NormalizeComponent(&dsql) - dsql = sanitize.SQL(dsql, sanitize.Options{Declared: sanitize.Declared(dsql)}) - top := &options.Options{Translate: translate} - if initErr := svc.Translate(ctx, &translate.Rule, dsql, top); initErr != nil { - return nil, initErr - } - ruleName := svc.Repository.RuleName(&translate.Rule) - targetSuffix := "/" + ruleName + ".yaml" - - scanner := dqlscan.New() - for _, item := range svc.Repository.Files { - if !strings.HasSuffix(item.URL, targetSuffix) { - continue - } - if strings.Contains(item.URL, "/.meta/") { - continue - } - return scanner.Result(ruleName, []byte(item.Content), dsql, req) - } - for _, item := range svc.Repository.Files { - if strings.HasSuffix(item.URL, targetSuffix) { - return scanner.Result(ruleName, []byte(item.Content), dsql, req) - } - } - return nil, fmt.Errorf("dql scan: generated YAML not found for %s", ruleName) -} - -func inferProject(dqlURL string) string { - base, _ := url.Split(dqlURL, file.Scheme) - if idx := strings.Index(base, "/dql/"); idx != -1 { - return base[:idx] - } - return base -} diff --git a/testutil/shapeparity/scan.go b/testutil/shapeparity/scan.go new file mode 100644 index 000000000..97acae814 --- /dev/null +++ b/testutil/shapeparity/scan.go @@ -0,0 +1,100 @@ +package shapeparity + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/viant/afs" + "github.com/viant/afs/file" + "github.com/viant/afs/url" + "github.com/viant/datly/cmd/command" + "github.com/viant/datly/cmd/options" + dqlscan "github.com/viant/datly/repository/shape/dql/scan" +) + +// ScanDQL runs the legacy translator into a temporary repository, then feeds the +// generated route YAML back through the shape scanner so parity tests can +// compare the legacy YAML contract against shape IR. +func ScanDQL(ctx context.Context, req *dqlscan.Request) (*dqlscan.Result, error) { + if req == nil { + return nil, fmt.Errorf("shape parity scan request was nil") + } + dqlURL := strings.TrimSpace(req.DQLURL) + if dqlURL == "" { + return nil, fmt.Errorf("shape parity scan request DQLURL was empty") + } + fs := afs.New() + dqlBytes, err := fs.DownloadWithURL(ctx, dqlURL) + if err != nil { + return nil, fmt.Errorf("failed to read DQL %s: %w", dqlURL, err) + } + tmpRepo, err := os.MkdirTemp("", "datly-shapeparity-*") + if err != nil { + return nil, fmt.Errorf("failed to create temp repository: %w", err) + } + defer os.RemoveAll(tmpRepo) + + projectRoot := inferProjectRoot(req, dqlURL) + modulePrefix := strings.Trim(strings.TrimSpace(req.ModulePrefix), "/") + apiPrefix := strings.TrimSpace(req.APIPrefix) + if apiPrefix == "" { + apiPrefix = "/v1/api" + } + repoOpts := options.Repository{ + RepositoryURL: tmpRepo, + ProjectURL: projectRoot, + APIPrefix: apiPrefix, + } + repoOpts.Connectors = append(repoOpts.Connectors, req.Connectors...) + if cfgURL := strings.TrimSpace(req.ConfigURL); cfgURL != "" { + repoOpts.Configs.Append(cfgURL) + } + opts := &options.Options{ + Translate: &options.Translate{ + Rule: options.Rule{ + Project: projectRoot, + ModulePrefix: modulePrefix, + Source: []string{dqlURL}, + ModuleLocation: func() string { + if req.Repository != "" { + return filepath.Join(req.Repository, "pkg") + } + return filepath.Join(projectRoot, "pkg") + }(), + Engine: options.EngineLegacy, + }, + Repository: repoOpts, + }, + } + if err = opts.Init(ctx); err != nil { + return nil, fmt.Errorf("failed to initialise legacy translate options: %w", err) + } + if err = command.New().Translate(ctx, opts); err != nil { + return nil, fmt.Errorf("failed to translate DQL %s with legacy pipeline: %w", dqlURL, err) + } + + ruleName := strings.TrimSuffix(filepath.Base(url.Path(dqlURL)), filepath.Ext(url.Path(dqlURL))) + routeYAMLURL := filepath.Join(tmpRepo, "Datly", "routes") + if modulePrefix != "" { + routeYAMLURL = filepath.Join(routeYAMLURL, filepath.FromSlash(modulePrefix)) + } + routeYAMLURL = filepath.Join(routeYAMLURL, ruleName+".yaml") + yamlBytes, err := fs.DownloadWithURL(ctx, routeYAMLURL) + if err != nil { + return nil, fmt.Errorf("failed to read generated route YAML %s: %w", routeYAMLURL, err) + } + return dqlscan.New().Result(ruleName, yamlBytes, string(dqlBytes), req) +} + +func inferProjectRoot(req *dqlscan.Request, dqlURL string) string { + if repositoryRoot := strings.TrimSpace(req.Repository); repositoryRoot != "" { + return filepath.Dir(filepath.Clean(repositoryRoot)) + } + if scheme := url.Scheme(dqlURL, file.Scheme); scheme != "" && scheme != file.Scheme { + return filepath.Dir(url.Path(dqlURL)) + } + return filepath.Dir(filepath.Clean(url.Path(dqlURL))) +} diff --git a/view/column.go b/view/column.go index 90bd0fad6..6f83f707e 100644 --- a/view/column.go +++ b/view/column.go @@ -10,6 +10,7 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/xreflect" "reflect" + "strconv" "strings" ) @@ -22,6 +23,7 @@ type ( Expression string `json:",omitempty"` Filterable bool `json:",omitempty"` + Groupable bool `json:",omitempty"` Nullable bool `json:",omitempty"` Default string `json:",omitempty"` FormatTag *format.Tag `json:",omitempty"` @@ -34,6 +36,7 @@ type ( field *reflect.StructField _initialized bool _fieldName string + _groupableSet bool } ColumnOption func(c *Column) ) @@ -79,6 +82,9 @@ func (c *Column) Init(resource state.Resource, caseFormat text.CaseFormat, allow if c.Name == "" { return fmt.Errorf("column name was empty") } + if err := c.initGroupable(); err != nil { + return err + } err := c.EnsureType(resource.LookupType()) if err != nil { return err @@ -100,6 +106,23 @@ func (c *Column) Init(resource state.Resource, caseFormat text.CaseFormat, allow return nil } +func (c *Column) initGroupable() error { + if c._groupableSet || c.Tag == "" { + return nil + } + value, ok := reflect.StructTag(c.Tag).Lookup("groupable") + if !ok { + return nil + } + groupable, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("invalid groupable tag for column %s: %w", c.Name, err) + } + c.Groupable = groupable + c._groupableSet = true + return nil +} + func (c *Column) EnsureType(lookupType xreflect.LookupType) error { if c.rType != nil && c.rType != xreflect.InterfaceType { return nil @@ -199,6 +222,10 @@ func (c *Column) ApplyConfig(config *ColumnConfig) { if config.Default != nil { c.Default = *config.Default } + if config.Groupable != nil { + c.Groupable = *config.Groupable + c._groupableSet = true + } c._initialized = false } @@ -237,6 +264,7 @@ type ( Codec *state.Codec `json:",omitempty"` DataType *string `json:",omitempty"` Required *bool `json:",omitempty"` + Groupable *bool `json:",omitempty"` Format *string `json:",omitempty"` Tag *string `json:",omitempty"` Default *string `json:",omitempty"` diff --git a/view/config_test.go b/view/config_test.go new file mode 100644 index 000000000..53f081717 --- /dev/null +++ b/view/config_test.go @@ -0,0 +1,9 @@ +package view + +import "testing" + +func TestQueryStateParameters_CriteriaParameterUsesCriteriaQuery(t *testing.T) { + if QueryStateParameters.CriteriaParameter == nil || QueryStateParameters.CriteriaParameter.In.Name != CriteriaQuery { + t.Fatalf("expected CriteriaParameter query name %q, got %#v", CriteriaQuery, QueryStateParameters.CriteriaParameter) + } +} diff --git a/view/groupable_test.go b/view/groupable_test.go new file mode 100644 index 000000000..0b2716713 --- /dev/null +++ b/view/groupable_test.go @@ -0,0 +1,46 @@ +package view + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/tagly/format/text" +) + +func TestColumn_Init_GroupableTag(t *testing.T) { + column := &Column{ + Name: "region", + DataType: "string", + Tag: `groupable:"true"`, + } + + err := column.Init(NewResources(EmptyResource(), &View{}), text.CaseFormatLowerUnderscore, true) + require.NoError(t, err) + require.True(t, column.Groupable) +} + +func TestView_IsGroupable(t *testing.T) { + groupable := &Column{Name: "region", Groupable: true} + metric := &Column{Name: "total"} + index := Columns{groupable, metric}.Index(text.CaseFormatLowerUnderscore) + index.RegisterWithName("Region", groupable) + + aView := &View{ + Columns: []*Column{groupable, metric}, + _columns: index, + } + + require.True(t, aView.IsGroupable("region")) + require.True(t, aView.IsGroupable("Region")) + require.False(t, aView.IsGroupable("total")) + require.False(t, aView.IsGroupable("missing")) +} + +func TestView_inherit_Groupable(t *testing.T) { + child := &View{} + parent := &View{Groupable: true} + + err := child.inherit(parent) + require.NoError(t, err) + require.True(t, child.Groupable) +} diff --git a/view/option.go b/view/option.go index fb776b94d..d630f7145 100644 --- a/view/option.go +++ b/view/option.go @@ -50,6 +50,56 @@ func WithColumns(columns Columns) Option { } } +func WithGroupable(groupable bool) Option { + return func(v *View) error { + v.Groupable = groupable + return nil + } +} + +func WithSummary(summary *TemplateSummary) Option { + return func(v *View) error { + v.EnsureTemplate() + v.Template.Summary = summary + return nil + } +} + +func WithSummaryURI(sourceURL string) Option { + return func(v *View) error { + v.EnsureTemplate() + if v.Template.Summary == nil { + v.Template.Summary = &TemplateSummary{} + } + v.Template.Summary.SourceURL = sourceURL + return nil + } +} + +func WithTemplateParameterStateType(enabled bool) Option { + return func(v *View) error { + v.EnsureTemplate() + v.Template.UseParameterStateType = enabled + return nil + } +} + +func WithDeclaredTemplateParametersOnly(enabled bool) Option { + return func(v *View) error { + v.EnsureTemplate() + v.Template.DeclaredParametersOnly = enabled + return nil + } +} + +func WithResourceParameterLookup(enabled bool) Option { + return func(v *View) error { + v.EnsureTemplate() + v.Template.UseResourceParameterLookup = enabled + return nil + } +} + // WithFS creates fs options func WithFS(fs *embed.FS) Option { return func(v *View) error { diff --git a/view/option_test.go b/view/option_test.go new file mode 100644 index 000000000..83d802dc4 --- /dev/null +++ b/view/option_test.go @@ -0,0 +1,83 @@ +package view + +import ( + "context" + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view/state" +) + +func TestWithSummary(t *testing.T) { + aView := NewView("vendor", "") + err := WithSummary(&TemplateSummary{ + Name: "Meta", + SourceURL: "vendor/vendor_summary.sql", + })(aView) + require.NoError(t, err) + require.NotNil(t, aView.Template) + require.NotNil(t, aView.Template.Summary) + require.Equal(t, "Meta", aView.Template.Summary.Name) + require.Equal(t, "vendor/vendor_summary.sql", aView.Template.Summary.SourceURL) +} + +func TestWithSummaryURI(t *testing.T) { + aView := NewView("vendor", "") + err := WithSummaryURI("vendor/vendor_summary.sql")(aView) + require.NoError(t, err) + require.NotNil(t, aView.Template) + require.NotNil(t, aView.Template.Summary) + require.Equal(t, "vendor/vendor_summary.sql", aView.Template.Summary.SourceURL) +} + +func TestWithTemplateParameterStateType(t *testing.T) { + type input struct { + Foos *struct { + ID int + } + } + + resource := EmptyResource() + aView := &View{ + Name: "foos", + Table: "FOOS", + Schema: state.NewSchema(reflect.TypeOf(&input{})), + _resource: resource, + } + aView.Template = NewTemplate( + `$CurFoosId.Values`, + WithTemplateParameters( + &state.Parameter{ + Name: "Foos", + In: state.NewBodyLocation(""), + Schema: state.NewSchema(reflect.TypeOf(&struct{ ID int }{})), + Tag: `anonymous:"true"`, + }, + &state.Parameter{ + Name: "CurFoosId", + In: state.NewParameterLocation("Foos"), + Schema: state.NewSchema(reflect.TypeOf(&struct{ Values []int }{})), + }, + ), + ) + require.NoError(t, WithTemplateParameterStateType(true)(aView)) + require.NoError(t, aView.Template.Init(context.Background(), resource, aView)) + require.NotNil(t, aView.Template.StateType()) + require.NotNil(t, aView.Template.StateType().Lookup("Foos")) + require.NotNil(t, aView.Template.StateType().Lookup("CurFoosId")) +} + +func TestWithDeclaredTemplateParametersOnly(t *testing.T) { + aView := NewView("vendor", "") + require.NoError(t, WithDeclaredTemplateParametersOnly(true)(aView)) + require.NotNil(t, aView.Template) + require.True(t, aView.Template.DeclaredParametersOnly) +} + +func TestWithResourceParameterLookup(t *testing.T) { + aView := NewView("vendor", "") + require.NoError(t, WithResourceParameterLookup(true)(aView)) + require.NotNil(t, aView.Template) + require.True(t, aView.Template.UseResourceParameterLookup) +} diff --git a/view/state/kind/locator/data.go b/view/state/kind/locator/data.go index db35876c5..ec5c6d135 100644 --- a/view/state/kind/locator/data.go +++ b/view/state/kind/locator/data.go @@ -6,6 +6,7 @@ import ( "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind" + "os" "reflect" ) @@ -18,19 +19,43 @@ func (p *DataView) Names() []string { return nil } -func (p *DataView) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { +func (p *DataView) Value(ctx context.Context, rType reflect.Type, name string) (interface{}, bool, error) { aView, ok := p.Views[name] if !ok { return nil, false, fmt.Errorf("failed to lookup view: %v", name) } + if os.Getenv("DATLY_DEBUG_VIEW_LOCATOR") == "1" { + fmt.Printf("[VIEW LOCATOR] name=%s schema=%v card=%s slice=%v\n", name, func() reflect.Type { + if aView.Schema == nil { + return nil + } + return aView.Schema.Type() + }(), func() state.Cardinality { + if aView.Schema == nil { + return "" + } + return aView.Schema.Cardinality + }(), func() reflect.Type { + if aView.Schema == nil { + return nil + } + return aView.Schema.SliceType() + }()) + } sliceValue := reflect.New(aView.Schema.SliceType()) destSlicePtr := sliceValue.Interface() err := p.ReadInto(ctx, destSlicePtr, aView) if err != nil { + if os.Getenv("DATLY_DEBUG_VIEW_LOCATOR") == "1" { + fmt.Printf("[VIEW LOCATOR] name=%s readIntoErr=%v dest=%T\n", name, err, destSlicePtr) + } return nil, false, err } + if os.Getenv("DATLY_DEBUG_VIEW_LOCATOR") == "1" { + fmt.Printf("[VIEW LOCATOR] name=%s len=%d dest=%T\n", name, sliceValue.Elem().Len(), destSlicePtr) + } - if aView.Schema.Cardinality == state.One { + if shouldReturnSingleValue(aView, rType) { switch sliceValue.Elem().Len() { case 0: return nil, true, nil @@ -43,6 +68,24 @@ func (p *DataView) Value(ctx context.Context, _ reflect.Type, name string) (inte return sliceValue.Elem().Interface(), true, err } +func shouldReturnSingleValue(aView *view.View, rType reflect.Type) bool { + if aView != nil && aView.Schema != nil && aView.Schema.Cardinality == state.One { + return true + } + if rType == nil { + return false + } + for rType.Kind() == reflect.Interface { + rType = rType.Elem() + } + switch rType.Kind() { + case reflect.Slice, reflect.Array, reflect.Map: + return false + default: + return true + } +} + func NewView(opts ...Option) (kind.Locator, error) { options := NewOptions(opts) if options.Views == nil { diff --git a/view/state/kind/locator/data_test.go b/view/state/kind/locator/data_test.go new file mode 100644 index 000000000..37ee43c14 --- /dev/null +++ b/view/state/kind/locator/data_test.go @@ -0,0 +1,62 @@ +package locator + +import ( + "context" + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +type dataViewLocatorRecord struct { + ID int +} + +func TestDataView_Value_UsesRequestedScalarTypeToUnwrapSingleResult(t *testing.T) { + aView := &view.View{ + Name: "CurFoos", + Schema: state.NewSchema(reflect.TypeOf([]*dataViewLocatorRecord{})), + } + aView.Schema.Cardinality = state.Many + locator := &DataView{ + Views: view.NamedViews{"CurFoos": aView}, + ReadInto: func(ctx context.Context, dest interface{}, aView *view.View) error { + target := dest.(*[]*dataViewLocatorRecord) + *target = append(*target, &dataViewLocatorRecord{ID: 7}) + return nil + }, + } + + value, ok, err := locator.Value(context.Background(), reflect.TypeOf(&dataViewLocatorRecord{}), "CurFoos") + require.NoError(t, err) + require.True(t, ok) + record, ok := value.(*dataViewLocatorRecord) + require.True(t, ok) + require.Equal(t, 7, record.ID) +} + +func TestDataView_Value_PreservesSliceForSliceTarget(t *testing.T) { + aView := &view.View{ + Name: "CurFoos", + Schema: state.NewSchema(reflect.TypeOf([]*dataViewLocatorRecord{})), + } + aView.Schema.Cardinality = state.Many + locator := &DataView{ + Views: view.NamedViews{"CurFoos": aView}, + ReadInto: func(ctx context.Context, dest interface{}, aView *view.View) error { + target := dest.(*[]*dataViewLocatorRecord) + *target = append(*target, &dataViewLocatorRecord{ID: 7}) + return nil + }, + } + + value, ok, err := locator.Value(context.Background(), reflect.TypeOf([]*dataViewLocatorRecord{}), "CurFoos") + require.NoError(t, err) + require.True(t, ok) + records, ok := value.([]*dataViewLocatorRecord) + require.True(t, ok) + require.Len(t, records, 1) + require.Equal(t, 7, records[0].ID) +} diff --git a/view/state/parameter.go b/view/state/parameter.go index a1ded9294..8bee637d6 100644 --- a/view/state/parameter.go +++ b/view/state/parameter.go @@ -53,6 +53,7 @@ type ( URI string `json:",omitempty" yaml:"URI"` Cacheable *bool `json:",omitempty" yaml:"Cacheable"` Async bool `json:",omitempty" yaml:"Async"` + PreserveSchema bool `json:",omitempty" yaml:"PreserveSchema"` isOutputType bool _timeLayout string _selector *structology.Selector @@ -546,6 +547,11 @@ func (p *Parameter) OutputType() reflect.Type { } func (p *Parameter) initParamBasedParameter(ctx context.Context, resource Resource) error { + if p.Schema != nil { + if p.Schema.Type() != nil { + return nil + } + } if p.Schema.Type() != nil { return nil } diff --git a/view/state/parameter_test.go b/view/state/parameter_test.go new file mode 100644 index 000000000..41f540646 --- /dev/null +++ b/view/state/parameter_test.go @@ -0,0 +1,97 @@ +package state + +import ( + "context" + "embed" + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/xdatly/codec" + "github.com/viant/xreflect" +) + +type parameterNamedPatchFoos struct { + ID int +} + +type testResource struct { + params map[string]*Parameter +} + +func (t *testResource) LookupParameter(name string) (*Parameter, error) { return t.params[name], nil } +func (t *testResource) AppendParameter(parameter *Parameter) {} +func (t *testResource) ViewSchema(context.Context, string) (*Schema, error) { + return nil, nil +} +func (t *testResource) ViewSchemaPointer(context.Context, string) (*Schema, error) { + return nil, nil +} +func (t *testResource) LookupType() xreflect.LookupType { return nil } +func (t *testResource) LoadText(context.Context, string) (string, error) { + return "", nil +} +func (t *testResource) Codecs() *codec.Registry { return nil } +func (t *testResource) CodecOptions() *codec.Options { return nil } +func (t *testResource) ExpandSubstitutes(text string) string { return text } +func (t *testResource) ReverseSubstitutes(text string) string { return text } +func (t *testResource) EmbedFS() *embed.FS { return nil } +func (t *testResource) SetFSEmbedder(*FSEmbedder) {} + +func TestParameter_initParamBasedParameter_ResolvesSourceSchemaEvenWithExplicitDataType(t *testing.T) { + resource := &testResource{ + params: map[string]*Parameter{ + "Foos": { + Name: "Foos", + In: NewBodyLocation(""), + Schema: NewSchema(reflect.TypeOf(&struct{ ID int }{})), + }, + }, + } + param := &Parameter{ + Name: "CurFoosId", + In: NewParameterLocation("Foos"), + Schema: &Schema{DataType: `*struct { Values []int "json:\",omitempty\"" }`}, + PreserveSchema: true, + } + + require.NoError(t, param.initParamBasedParameter(context.Background(), resource)) + require.NotNil(t, param.Schema) + require.Equal(t, reflect.TypeOf(&struct{ ID int }{}), param.Schema.Type()) +} + +func TestParameters_ReflectType_QualifiedNamedDataTypeResolves(t *testing.T) { + registry := xreflect.NewTypes() + require.NoError(t, registry.Register("FoosView", xreflect.WithPackage("patch_basic_one"), xreflect.WithReflectType(reflect.TypeOf(parameterNamedPatchFoos{})))) + + params := Parameters{ + &Parameter{ + Name: "Foos", + In: NewBodyLocation(""), + Schema: &Schema{Name: "FoosView", Package: "patch_basic_one", DataType: "*patch_basic_one.FoosView", Cardinality: One}, + }, + } + + rType, err := params.ReflectType("patch_basic_one", registry.Lookup) + require.NoError(t, err) + field, ok := rType.FieldByName("Foos") + require.True(t, ok) + require.Equal(t, reflect.TypeOf(¶meterNamedPatchFoos{}), field.Type) +} + +func TestParameter_buildTag_ParamDoesNotOverrideSourceDataType(t *testing.T) { + param := &Parameter{ + Name: "CurFoosId", + In: NewParameterLocation("Foos"), + Schema: &Schema{Name: "CurFoosId", DataType: `*struct { Values []int "json:\",omitempty\"" }`}, + Output: &Codec{ + Name: "structql", + Schema: &Schema{Name: "CurFoosId", DataType: `*struct { Values []int "json:\",omitempty\"" }`}, + }, + } + + tag := string(param.buildTag("CurFoosId")) + require.NotContains(t, tag, `dataType:"`) + require.Contains(t, tag, `kind=param`) + require.Contains(t, tag, `in=Foos`) +} diff --git a/view/state/parameters.go b/view/state/parameters.go index e464ce142..24235aab7 100644 --- a/view/state/parameters.go +++ b/view/state/parameters.go @@ -604,7 +604,9 @@ func (p *Parameter) buildTag(fieldName string) reflect.StructTag { } if p.Output != nil && p.Output.Schema != nil { if p.Output.Schema.TypeName() != p.Schema.TypeName() { - aTag.Parameter.DataType = p.Schema.TypeName() + if p.In == nil || p.In.Kind != KindParam { + aTag.Parameter.DataType = p.Schema.TypeName() + } } } if p.Handler != nil { diff --git a/view/state/type.go b/view/state/type.go index b9f3ee9af..262a83959 100644 --- a/view/state/type.go +++ b/view/state/type.go @@ -23,7 +23,6 @@ type ( Type struct { *Schema Parameters Parameters `json:",omitempty" yaml:"Parameters"` - Package string `json:",omitempty" yaml:",omitempty"` withMarker bool stateType *structology.StateType resource Resource @@ -136,15 +135,6 @@ func (t *Type) SetType(rType reflect.Type) { t.stateType = structology.NewStateType(rType) } -// PkgPath returns the effective package path for type generation. -// Uses the explicit Package field when set, otherwise falls back to the default. -func (t *Type) PkgPath() string { - if p := strings.TrimSpace(t.Package); p != "" { - return p - } - return pkgPath -} - func (t *Type) buildSchema(ctx context.Context, withMarker bool) (err error) { hasBodyParam := false for _, parameter := range t.Parameters { @@ -158,10 +148,9 @@ func (t *Type) buildSchema(ctx context.Context, withMarker bool) (err error) { if t.withBodyType && !hasBodyParam { t.withBodyType = hasBodyParam } - effectivePkgPath := t.PkgPath() var rType reflect.Type if t.withBodyType { - rType, err = t.Parameters.BuildBodyType(effectivePkgPath, t.resource.LookupType()) + rType, err = t.Parameters.BuildBodyType(pkgPath, t.resource.LookupType()) } else { var opts []ReflectOption if withMarker { @@ -170,7 +159,7 @@ func (t *Type) buildSchema(ctx context.Context, withMarker bool) (err error) { if t.Schema != nil && t.Schema.Name != "" { opts = append(opts, WithTypeName(t.Name)) } - rType, err = t.Parameters.ReflectType(effectivePkgPath, t.resource.LookupType(), opts...) + rType, err = t.Parameters.ReflectType(pkgPath, t.resource.LookupType(), opts...) } if err != nil { return err diff --git a/view/tags/query_selector.go b/view/tags/query_selector.go new file mode 100644 index 000000000..30c61f552 --- /dev/null +++ b/view/tags/query_selector.go @@ -0,0 +1,23 @@ +package tags + +import "strings" + +const QuerySelectorTag = "querySelector" + +// ParseQuerySelector returns the target view alias encoded in querySelector tag. +// Supported forms: +// +// querySelector:"vendor" +// querySelector:"view=vendor" +func ParseQuerySelector(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if key, mapped, ok := strings.Cut(value, "="); ok { + if strings.EqualFold(strings.TrimSpace(key), "view") { + return strings.TrimSpace(mapped) + } + } + return value +} diff --git a/view/tags/view.go b/view/tags/view.go index 3e96b4f5c..0002d3534 100644 --- a/view/tags/view.go +++ b/view/tags/view.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/viant/afs/storage" "github.com/viant/tagly/tags" + "sort" "strconv" "strings" ) @@ -19,6 +20,9 @@ type ( View struct { Name string Table string + SummaryURI string + TypeName string + Dest string CustomTag string Parameters []string //parameter references Connector string @@ -30,6 +34,15 @@ type ( PartitionerType string PartitionedConcurrency int RelationalConcurrency int + Groupable *bool + SelectorNamespace string + SelectorCriteria *bool + SelectorProjection *bool + SelectorOrderBy *bool + SelectorOffset *bool + SelectorPage *bool + SelectorFilterable []string + SelectorOrderByColumns map[string]string } ) @@ -60,6 +73,12 @@ func (t *Tag) updateView(key string, value string) error { tag.Limit = &limit case "table": tag.Table = strings.TrimSpace(value) + case "summaryuri": + tag.SummaryURI = strings.TrimSpace(value) + case "type": + tag.TypeName = strings.TrimSpace(value) + case "dest": + tag.Dest = strings.TrimSpace(value) case "connector": tag.Connector = strings.TrimSpace(value) case "partitioner": @@ -81,6 +100,24 @@ func (t *Tag) updateView(key string, value string) error { for _, parameter := range strings.Split(parameters, ",") { tag.Parameters = append(tag.Parameters, strings.TrimSpace(parameter)) } + case "groupable": + tag.Groupable = parseBoolPointer(value) + case "selectornamespace": + tag.SelectorNamespace = strings.TrimSpace(value) + case "selectorcriteria": + tag.SelectorCriteria = parseBoolPointer(value) + case "selectorprojection": + tag.SelectorProjection = parseBoolPointer(value) + case "selectororderby": + tag.SelectorOrderBy = parseBoolPointer(value) + case "selectoroffset": + tag.SelectorOffset = parseBoolPointer(value) + case "selectorpage": + tag.SelectorPage = parseBoolPointer(value) + case "selectorfilterable": + tag.SelectorFilterable = parseTagList(value) + case "selectororderbycolumns": + tag.SelectorOrderByColumns = parseTagMap(value) default: return fmt.Errorf("unsupported view tag option: '%s'", key) } @@ -105,6 +142,9 @@ func (v *View) Tag() *tags.Tag { appendNonEmpty(builder, "limit", strconv.Itoa(*v.Limit)) } appendNonEmpty(builder, "table", v.Table) + appendNonEmpty(builder, "summaryURI", v.SummaryURI) + appendNonEmpty(builder, "type", v.TypeName) + appendNonEmpty(builder, "dest", v.Dest) if v.Batch > 0 { appendNonEmpty(builder, "batch", strconv.Itoa(v.Batch)) } @@ -126,6 +166,28 @@ func (v *View) Tag() *tags.Tag { appendNonEmpty(builder, "concurrency", strconv.Itoa(v.PartitionedConcurrency)) } } + appendBool(builder, "groupable", v.Groupable) + appendNonEmpty(builder, "selectorNamespace", v.SelectorNamespace) + appendBool(builder, "selectorCriteria", v.SelectorCriteria) + appendBool(builder, "selectorProjection", v.SelectorProjection) + appendBool(builder, "selectorOrderBy", v.SelectorOrderBy) + appendBool(builder, "selectorOffset", v.SelectorOffset) + appendBool(builder, "selectorPage", v.SelectorPage) + if len(v.SelectorFilterable) > 0 { + appendNonEmpty(builder, "selectorFilterable", "{"+strings.Join(v.SelectorFilterable, ",")+"}") + } + if len(v.SelectorOrderByColumns) > 0 { + keys := make([]string, 0, len(v.SelectorOrderByColumns)) + for key := range v.SelectorOrderByColumns { + keys = append(keys, key) + } + sort.Strings(keys) + pairs := make([]string, 0, len(keys)) + for _, key := range keys { + pairs = append(pairs, key+":"+v.SelectorOrderByColumns[key]) + } + appendNonEmpty(builder, "selectorOrderByColumns", "{"+strings.Join(pairs, ",")+"}") + } return &tags.Tag{Name: ViewTag, Values: tags.Values(builder.String())} } @@ -138,3 +200,64 @@ func appendNonEmpty(builder *strings.Builder, key, value string) { builder.WriteString("=") builder.WriteString(value) } + +func appendBool(builder *strings.Builder, key string, value *bool) { + if value == nil { + return + } + appendNonEmpty(builder, key, strconv.FormatBool(*value)) +} + +func parseBoolPointer(value string) *bool { + v := true + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "true", "1": + v = true + case "false", "0": + v = false + } + return &v +} + +func parseTagList(value string) []string { + value = strings.Trim(value, "{}'\" ") + if value == "" { + return nil + } + items := strings.Split(value, ",") + ret := make([]string, 0, len(items)) + for _, item := range items { + item = strings.TrimSpace(item) + if item != "" { + ret = append(ret, item) + } + } + return ret +} + +func parseTagMap(value string) map[string]string { + value = strings.Trim(value, "{}'\" ") + if value == "" { + return nil + } + ret := map[string]string{} + for _, item := range strings.Split(value, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + key, mapped, ok := strings.Cut(item, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + mapped = strings.TrimSpace(mapped) + if key != "" && mapped != "" { + ret[key] = mapped + } + } + if len(ret) == 0 { + return nil + } + return ret +} diff --git a/view/tags/view_test.go b/view/tags/view_test.go index 1127cf66c..e84ded4c1 100644 --- a/view/tags/view_test.go +++ b/view/tags/view_test.go @@ -37,6 +37,29 @@ func TestTag_updateView(t *testing.T) { tag: `view:"foo,table=FOO,connector=dev,parameters={P1,P2}"`, expectView: &View{Name: "foo", Table: "FOO", Connector: "dev", Parameters: []string{"P1", "P2"}}, }, + { + description: "selector metadata view", + tag: `view:"foo,table=FOO,groupable=true,selectorNamespace=ve,selectorCriteria=true,selectorProjection=true,selectorOrderBy=true,selectorOffset=true,selectorFilterable={*},selectorOrderByColumns={accountId:ACCOUNT_ID,userCreated:USER_CREATED}"`, + expectView: &View{ + Name: "foo", + Table: "FOO", + Groupable: boolPtr(true), + SelectorNamespace: "ve", + SelectorCriteria: boolPtr(true), + SelectorProjection: boolPtr(true), + SelectorOrderBy: boolPtr(true), + SelectorOffset: boolPtr(true), + SelectorFilterable: []string{"*"}, + SelectorOrderByColumns: map[string]string{"accountId": "ACCOUNT_ID", "userCreated": "USER_CREATED"}, + }, + expectTag: "foo,table=FOO,groupable=true,selectorNamespace=ve,selectorCriteria=true,selectorProjection=true,selectorOrderBy=true,selectorOffset=true,selectorFilterable={*},selectorOrderByColumns={accountId:ACCOUNT_ID,userCreated:USER_CREATED}", + }, + { + description: "summary uri view", + tag: `view:"foo,table=FOO,summaryURI=testdata/foo_summary.sql"`, + expectView: &View{Name: "foo", Table: "FOO", SummaryURI: "testdata/foo_summary.sql"}, + expectTag: "foo,table=FOO,summaryURI=testdata/foo_summary.sql", + }, } for _, testCase := range testCases { @@ -55,3 +78,7 @@ func TestTag_updateView(t *testing.T) { assert.EqualValues(t, expectTag, string(actual.View.Tag().Values), testCase.description) } } + +func boolPtr(v bool) *bool { + return &v +} diff --git a/view/template.go b/view/template.go index ae4308723..abf549165 100644 --- a/view/template.go +++ b/view/template.go @@ -24,6 +24,15 @@ type ( Source string `json:",omitempty" yaml:"source,omitempty"` SourceURL string `json:",omitempty" yaml:"sourceURL,omitempty"` Schema *state.Schema `json:",omitempty" yaml:"schema,omitempty"` + // UseParameterStateType makes Velty compile against template parameters + // instead of the view schema when helper state exists outside the named IO type. + UseParameterStateType bool `json:",omitempty" yaml:"useParameterStateType,omitempty"` + // DeclaredParametersOnly prevents global resource parameter binding from + // appending undeclared parameters to this template. + DeclaredParametersOnly bool `json:",omitempty" yaml:"declaredParametersOnly,omitempty"` + // UseResourceParameterLookup allows param/state source lookup to resolve + // against resource parameters in addition to the declared template params. + UseResourceParameterLookup bool `json:",omitempty" yaml:"useResourceParameterLookup,omitempty"` stateType *structology.StateType @@ -92,7 +101,16 @@ func (t *Template) Init(ctx context.Context, resource *Resource, view *View) err if err = t.initTypes(ctx, resource); err != nil { return err } - if rType := t.Schema.Type(); rType != nil { + if t.UseParameterStateType && len(t.Parameters) > 0 { + rType, err := t.Parameters.ReflectType(t.Package(), resource.LookupType(), state.WithSetMarker()) + if err != nil { + return fmt.Errorf("failed to build template parameter state for %s: %w", t._view.Name, err) + } + if rType.Kind() == reflect.Struct { + rType = reflect.PtrTo(rType) + } + t.stateType = structology.NewStateType(rType) + } else if rType := t.Schema.Type(); rType != nil { t.stateType = structology.NewStateType(rType) } @@ -269,6 +287,30 @@ func WithTemplateParameters(parameters ...*state.Parameter) TemplateOption { } } +// WithTemplateUnsafeStateFromParameters configures template evaluation to derive +// the Velty Unsafe state from template parameters rather than the named view schema. +func WithTemplateUnsafeStateFromParameters(enabled bool) TemplateOption { + return func(t *Template) { + t.UseParameterStateType = enabled + } +} + +// WithTemplateDeclaredParametersOnly preserves only explicitly declared +// template parameters during later resource binding. +func WithTemplateDeclaredParametersOnly(enabled bool) TemplateOption { + return func(t *Template) { + t.DeclaredParametersOnly = enabled + } +} + +// WithTemplateResourceParameterLookup allows template parameter source lookup +// to resolve from resource parameters while keeping the declared template state minimal. +func WithTemplateResourceParameterLookup(enabled bool) TemplateOption { + return func(t *Template) { + t.UseResourceParameterLookup = enabled + } +} + // WithTemplateSchema returns with template schema func WithTemplateSchema(schema *state.Schema) TemplateOption { return func(t *Template) { diff --git a/view/view.go b/view/view.go index 6d744a74e..273742383 100644 --- a/view/view.go +++ b/view/view.go @@ -462,6 +462,9 @@ func (v *View) buildViewOptions(aViewType reflect.Type, tag *tags.Tag) ([]Option for _, name := range vTag.Parameters { parameters = append(parameters, state.NewRefParameter(name)) } + if vTag.SummaryURI != "" { + options = append(options, WithSummaryURI(vTag.SummaryURI)) + } } if SQL := tag.SQL; SQL.SQL != "" { tmpl := NewTemplate(string(SQL.SQL), WithTemplateParameters(parameters...)) From c9eee5c5ba4b71a01495e3b9596f57f4ce80733c Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 08:28:32 -0700 Subject: [PATCH 155/279] added dynamic grouping --- .../cases/010_grouping/vendors_grouping.sql | 3 +- .../regression/dev/.meta/vendor_meta.yaml | 137 ++++++++++++++++++ 2 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 e2e/local/regression/dev/.meta/vendor_meta.yaml diff --git a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql index 917eb3c74..71dee53fd 100644 --- a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql +++ b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql @@ -4,7 +4,7 @@ #set( $_ = $ID<[]int>(query/id)..WithPredicate(0, 'equal', 't', 'ID')) SELECT vendor.*, - groupable(vendor), + groupingEnabled(vendor), allowed_order_by_columns(vendor, 'accountId:ACCOUNT_ID,userCreated:USER_CREATED,totalId:TOTAL_ID,maxId:MAX_ID') FROM ( SELECT ACCOUNT_ID, @@ -16,4 +16,3 @@ FROM ( GROUP BY 1, 2 ) vendor - diff --git a/e2e/local/regression/dev/.meta/vendor_meta.yaml b/e2e/local/regression/dev/.meta/vendor_meta.yaml new file mode 100644 index 000000000..626be2a09 --- /dev/null +++ b/e2e/local/regression/dev/.meta/vendor_meta.yaml @@ -0,0 +1,137 @@ +items: + products: + - name: ID + datatype: int + tag: "" + expression: "" + filterable: false + nullable: false + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + - name: my_name + datatype: VARCHAR + tag: ' sqlx:"my_name"' + expression: "" + filterable: false + nullable: true + default: "" + formattag: null + codec: null + databasecolumn: my_name + indexedby: "" + - name: VENDOR_ID + datatype: int + tag: "" + expression: "" + filterable: false + nullable: true + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + vendor: + - name: ID + datatype: int + tag: "" + expression: "" + filterable: false + nullable: false + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + - name: NAME + datatype: varchar + tag: "" + expression: "" + filterable: false + nullable: true + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + - name: ACCOUNT_ID + datatype: int + tag: "" + expression: "" + filterable: false + nullable: true + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + - name: CREATED + datatype: datetime + tag: "" + expression: "" + filterable: false + nullable: true + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + - name: USER_CREATED + datatype: int + tag: "" + expression: "" + filterable: false + nullable: true + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + - name: UPDATED + datatype: datetime + tag: "" + expression: "" + filterable: false + nullable: true + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + - name: USER_UPDATED + datatype: int + tag: "" + expression: "" + filterable: false + nullable: true + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + vendor/DataSummary/Meta: + - name: PAGE_CNT + datatype: BIGINT + tag: source:"1 + (COUNT(1) / 1)" + expression: "" + filterable: false + nullable: true + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" + - name: CNT + datatype: BIGINT + tag: "" + expression: "" + filterable: false + nullable: false + default: "" + formattag: null + codec: null + databasecolumn: "" + indexedby: "" +sourceurl: "" From 2861f7b8f694ce7fa069f8ab70955b569a8118fe Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 08:29:12 -0700 Subject: [PATCH 156/279] added dynamic grouping --- e2e/local/regression/cases/010_grouping/vendors_grouping.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql index 71dee53fd..25760d7ab 100644 --- a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql +++ b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql @@ -4,7 +4,7 @@ #set( $_ = $ID<[]int>(query/id)..WithPredicate(0, 'equal', 't', 'ID')) SELECT vendor.*, - groupingEnabled(vendor), + grouping_enabled(vendor), allowed_order_by_columns(vendor, 'accountId:ACCOUNT_ID,userCreated:USER_CREATED,totalId:TOTAL_ID,maxId:MAX_ID') FROM ( SELECT ACCOUNT_ID, @@ -15,4 +15,3 @@ FROM ( WHERE t.ID IN ($vendorIDs) GROUP BY 1, 2 ) vendor - From d66bd818b61d7f4a4b98b393accbdf3d0f86adb5 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 08:55:46 -0700 Subject: [PATCH 157/279] added dynamic grouping --- e2e/local/regression/cases/010_grouping/vendors_grouping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql index 25760d7ab..e95ea4c2b 100644 --- a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql +++ b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql @@ -1,7 +1,7 @@ /* {"URI":"vendors-grouping/"} */ #set( $_ = $Data(output/view).Embed()) -#set( $_ = $ID<[]int>(query/id)..WithPredicate(0, 'equal', 't', 'ID')) +#set( $_ = $ID<[]int>(query/id).WithPredicate(0, 'equal', 't', 'ID')) SELECT vendor.*, grouping_enabled(vendor), From de7e44bd8827e78090edddb49a07c56bc18f80c1 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 08:56:24 -0700 Subject: [PATCH 158/279] added dynamic grouping --- cmd/command/transcribe_test.go | 18 ++++++------------ .../cases/001_relation_one_to_many/expect.json | 17 +++++++++-------- .../cases/001_relation_one_to_many/test.yaml | 3 ++- e2e/v1/cases/005_kind_uri_param/expect.json | 8 ++++---- e2e/v1/cases/005_kind_uri_param/test.yaml | 1 + .../cases/006_kind_header_params/expect.json | 8 ++++---- e2e/v1/cases/006_kind_header_params/test.yaml | 1 + e2e/v1/cases/007_kind_const/expect.json | 15 ++++++++------- e2e/v1/cases/007_kind_const/test.yaml | 3 ++- repository/shape/compile/hints.go | 3 ++- repository/shape/compile/hints_strip.go | 1 + repository/shape/compile/hints_test.go | 15 +++++++++++++++ repository/shape/load/loader.go | 9 ++++++++- repository/shape/load/loader_test.go | 8 ++++---- 14 files changed, 67 insertions(+), 43 deletions(-) diff --git a/cmd/command/transcribe_test.go b/cmd/command/transcribe_test.go index 73ac0786f..73d8b5d4a 100644 --- a/cmd/command/transcribe_test.go +++ b/cmd/command/transcribe_test.go @@ -433,17 +433,11 @@ func TestTranscribe_PatchBasicOneRouteYAMLPreservesNamedHelperParamTypes(t *test curFoosID := payload.Resource.Parameters.Lookup("CurFoosId") require.NotNil(t, curFoosID) require.NotNil(t, curFoosID.Schema) - require.Equal(t, "CurFoosId", curFoosID.Schema.DataType) - - var helperType *view.TypeDefinition - for _, item := range payload.Resource.Types { - if item != nil && item.Name == "CurFoosId" { - helperType = item - break - } - } - require.NotNil(t, helperType) - require.Equal(t, "struct { Values []int }", helperType.DataType) + require.Equal(t, "*patch_basic_one.FoosView", curFoosID.Schema.DataType) + + require.NotNil(t, curFoosID.Output) + require.NotNil(t, curFoosID.Output.Schema) + require.Equal(t, `*struct { Values []int "json:\",omitempty\"" }`, curFoosID.Output.Schema.DataType) curFoos := lookupNamedView(payload.Resource, "CurFoos") require.NotNil(t, curFoos) @@ -451,7 +445,7 @@ func TestTranscribe_PatchBasicOneRouteYAMLPreservesNamedHelperParamTypes(t *test curFoosParam := curFoos.Template.Parameters.Lookup("CurFoosId") require.NotNil(t, curFoosParam) require.NotNil(t, curFoosParam.Schema) - require.Equal(t, "CurFoosId", curFoosParam.Schema.DataType) + require.Equal(t, "*patch_basic_one.FoosView", curFoosParam.Schema.DataType) } func TestGenerateTranscribeTypes_MetaFormatPreservesChildSummaryType(t *testing.T) { diff --git a/e2e/v1/cases/001_relation_one_to_many/expect.json b/e2e/v1/cases/001_relation_one_to_many/expect.json index 1608ce562..e1ef36daa 100644 --- a/e2e/v1/cases/001_relation_one_to_many/expect.json +++ b/e2e/v1/cases/001_relation_one_to_many/expect.json @@ -3,7 +3,7 @@ "id": 1, "name": "Vendor 1", "accountId": 100, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "userCreated": 1, "updated": null, "userUpdated": null, @@ -11,7 +11,8 @@ { "id": 1, "name": "V1 Product 1", - "created": "2026-03-07T00:00:00Z", + "status": 2, + "created": "${created}", "userCreated": 1, "updated": null, "userUpdated": null @@ -20,7 +21,7 @@ "id": 2, "name": "V1 Product 2", "status": 1, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "userCreated": 1, "updated": null, "userUpdated": null @@ -31,7 +32,7 @@ "id": 2, "name": "Vendor 2", "accountId": 101, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "userCreated": 2, "updated": null, "userUpdated": null, @@ -40,7 +41,7 @@ "id": 3, "name": "V2 Product 1", "status": 1, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "userCreated": 2, "updated": null, "userUpdated": null @@ -49,7 +50,7 @@ "id": 4, "name": "V2 Product 2", "status": 1, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "userCreated": 2, "updated": null, "userUpdated": null @@ -58,7 +59,7 @@ "id": 5, "name": "V2 Product 3", "status": 1, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "userCreated": 2, "updated": null, "userUpdated": null @@ -69,7 +70,7 @@ "id": 3, "name": "Vendor 3", "accountId": 100, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "userCreated": 1, "updated": null, "userUpdated": null diff --git a/e2e/v1/cases/001_relation_one_to_many/test.yaml b/e2e/v1/cases/001_relation_one_to_many/test.yaml index f5f959d3b..2f186d10c 100644 --- a/e2e/v1/cases/001_relation_one_to_many/test.yaml +++ b/e2e/v1/cases/001_relation_one_to_many/test.yaml @@ -1,5 +1,6 @@ init: parentPath: $parent.path + created: $FormatTime('nowInUTC', 'yyyy-MM-ddT00:00:00Z') pipeline: test: @@ -9,7 +10,7 @@ pipeline: URL: http://127.0.0.1:8080/v1/api/shape/dev/vendors/ Expect: Code: 200 - JSONBody: $LoadJSON('${parentPath}/expect.json') + JSONBody: $LoadData('${parentPath}/expect.json') - Method: GET diff --git a/e2e/v1/cases/005_kind_uri_param/expect.json b/e2e/v1/cases/005_kind_uri_param/expect.json index 308b3c9a5..c286edbca 100644 --- a/e2e/v1/cases/005_kind_uri_param/expect.json +++ b/e2e/v1/cases/005_kind_uri_param/expect.json @@ -8,12 +8,12 @@ ], "vendor": { "accountId": 101, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 2, "name": "Vendor 2", "products": [ { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 3, "name": "V2 Product 1", "status": 1, @@ -22,7 +22,7 @@ "userUpdated": null }, { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 4, "name": "V2 Product 2", "status": 1, @@ -31,7 +31,7 @@ "userUpdated": null }, { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 5, "name": "V2 Product 3", "status": 1, diff --git a/e2e/v1/cases/005_kind_uri_param/test.yaml b/e2e/v1/cases/005_kind_uri_param/test.yaml index c7161bbf0..dda6ddff7 100644 --- a/e2e/v1/cases/005_kind_uri_param/test.yaml +++ b/e2e/v1/cases/005_kind_uri_param/test.yaml @@ -1,5 +1,6 @@ init: parentPath: $parent.path + created: $FormatTime('nowInUTC', 'yyyy-MM-ddT00:00:00Z') expect: $LoadData('${parentPath}/expect.json') pipeline: diff --git a/e2e/v1/cases/006_kind_header_params/expect.json b/e2e/v1/cases/006_kind_header_params/expect.json index d25967f48..34725db23 100644 --- a/e2e/v1/cases/006_kind_header_params/expect.json +++ b/e2e/v1/cases/006_kind_header_params/expect.json @@ -1,12 +1,12 @@ [ { "accountId": 101, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 2, "name": "Vendor 2", "products": [ { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 3, "name": "V2 Product 1", "status": 1, @@ -15,7 +15,7 @@ "userUpdated": null }, { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 4, "name": "V2 Product 2", "status": 1, @@ -24,7 +24,7 @@ "userUpdated": null }, { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 5, "name": "V2 Product 3", "status": 1, diff --git a/e2e/v1/cases/006_kind_header_params/test.yaml b/e2e/v1/cases/006_kind_header_params/test.yaml index 13ea703b1..5cee8ec46 100644 --- a/e2e/v1/cases/006_kind_header_params/test.yaml +++ b/e2e/v1/cases/006_kind_header_params/test.yaml @@ -1,5 +1,6 @@ init: parentPath: $parent.path + created: $FormatTime('nowInUTC', 'yyyy-MM-ddT00:00:00Z') expect: $LoadData('${parentPath}/expect.json') pipeline: test: diff --git a/e2e/v1/cases/007_kind_const/expect.json b/e2e/v1/cases/007_kind_const/expect.json index 8322bf6db..b137dff34 100644 --- a/e2e/v1/cases/007_kind_const/expect.json +++ b/e2e/v1/cases/007_kind_const/expect.json @@ -1,12 +1,13 @@ [ { "accountId": 100, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 1, "name": "Vendor 1", "products": [ { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", + "status": 2, "id": 1, "name": "V1 Product 1", "updated": null, @@ -15,7 +16,7 @@ "vendorId": 1 }, { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 2, "name": "V1 Product 2", "status": 1, @@ -31,12 +32,12 @@ }, { "accountId": 101, - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 2, "name": "Vendor 2", "products": [ { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 3, "name": "V2 Product 1", "status": 1, @@ -46,7 +47,7 @@ "vendorId": 2 }, { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 4, "name": "V2 Product 2", "status": 1, @@ -56,7 +57,7 @@ "vendorId": 2 }, { - "created": "2026-03-07T00:00:00Z", + "created": "${created}", "id": 5, "name": "V2 Product 3", "status": 1, diff --git a/e2e/v1/cases/007_kind_const/test.yaml b/e2e/v1/cases/007_kind_const/test.yaml index e538123f8..f476e3ed6 100644 --- a/e2e/v1/cases/007_kind_const/test.yaml +++ b/e2e/v1/cases/007_kind_const/test.yaml @@ -1,6 +1,7 @@ init: parentPath: $parent.path - expect: $LoadJSON('${parentPath}/expect.json') + created: $FormatTime('nowInUTC', 'yyyy-MM-ddT00:00:00Z') + expect: $LoadData('${parentPath}/expect.json') pipeline: printHello: diff --git a/repository/shape/compile/hints.go b/repository/shape/compile/hints.go index 6e83302d2..117209e13 100644 --- a/repository/shape/compile/hints.go +++ b/repository/shape/compile/hints.go @@ -52,7 +52,7 @@ func extractViewHints(dql string) map[string]viewHint { value := true hint.AllowNulls = &value result[alias] = hint - case "groupable": + case "groupable", "grouping_enabled": if len(call.args) != 1 { continue } @@ -181,6 +181,7 @@ func scanHintCalls(input string) []hintCall { "use_connector": true, "allow_nulls": true, "groupable": true, + "grouping_enabled": true, "allowed_order_by_columns": true, "set_limit": true, "set_cache": true, diff --git a/repository/shape/compile/hints_strip.go b/repository/shape/compile/hints_strip.go index 67a1a0a5b..080e78cab 100644 --- a/repository/shape/compile/hints_strip.go +++ b/repository/shape/compile/hints_strip.go @@ -12,6 +12,7 @@ var projectionHintCalls = map[string]bool{ "useconnector": true, "allownulls": true, "groupable": true, + "groupingenabled": true, "allowedorderbycolumns": true, "setlimit": true, "setcache": true, diff --git a/repository/shape/compile/hints_test.go b/repository/shape/compile/hints_test.go index 64710d44c..72f986770 100644 --- a/repository/shape/compile/hints_test.go +++ b/repository/shape/compile/hints_test.go @@ -23,6 +23,14 @@ func TestExtractViewHints_WithQuotedConnector(t *testing.T) { assert.True(t, *hints["match"].NoLimit) } +func TestExtractViewHints_GroupingEnabledAlias(t *testing.T) { + dql := "SELECT grouping_enabled(match), set_limit(match, 0)" + hints := extractViewHints(dql) + require.Contains(t, hints, "match") + require.NotNil(t, hints["match"].Groupable) + assert.True(t, *hints["match"].Groupable) +} + func TestExtractViewHints_AllowedOrderByColumns(t *testing.T) { dql := "SELECT allowed_order_by_columns(vendor, 'accountId:ACCOUNT_ID,vendor.userCreated:USER_CREATED,totalId:TOTAL_ID')" hints := extractViewHints(dql) @@ -138,6 +146,13 @@ func TestStripProjectionHintCalls_RemovesSelfRefFromSQL(t *testing.T) { assert.Contains(t, strings.ToLower(actual), "user.* except mgr_id") } +func TestStripProjectionHintCalls_RemovesGroupingEnabledAlias(t *testing.T) { + sqlText := "SELECT user.*, grouping_enabled(user) FROM (SELECT t.* FROM USER t) user" + actual := stripProjectionHintCalls(sqlText) + assert.NotContains(t, strings.ToLower(actual), "grouping_enabled(") + assert.Contains(t, strings.ToLower(actual), "user.*") +} + func TestAppendRelationViews_SQLSelection(t *testing.T) { testCases := []struct { name string diff --git a/repository/shape/load/loader.go b/repository/shape/load/loader.go index e6932e19e..f905d7a4a 100644 --- a/repository/shape/load/loader.go +++ b/repository/shape/load/loader.go @@ -370,10 +370,17 @@ func synthesizeMutableExecHelpers(component *Component, resource *view.Resource) if helperIDsSchema != nil && strings.TrimSpace(helperIDsSchema.DataType) == "" { helperIDsSchema.DataType = loaderSchemaTypeExpr(reflect.PtrTo(valuesType)) } + helperSourceSchema := body.Schema + if helperSourceSchema == nil && rootView.Schema != nil { + helperSourceSchema = rootView.Schema.Clone() + } + if helperSourceSchema != nil { + helperSourceSchema = helperSourceSchema.Clone() + } helperIDsParam := &state.Parameter{ Name: helperIDsName, In: state.NewParameterLocation(bodyName), - Schema: helperIDsSchema.Clone(), + Schema: helperSourceSchema, Output: &state.Codec{Name: "structql", Body: fmt.Sprintf(" SELECT ARRAY_AGG(%s) AS Values FROM `/` LIMIT 1", keyFieldName), Schema: helperIDsSchema.Clone()}, PreserveSchema: true, } diff --git a/repository/shape/load/loader_test.go b/repository/shape/load/loader_test.go index 13a67059c..6cde9b044 100644 --- a/repository/shape/load/loader_test.go +++ b/repository/shape/load/loader_test.go @@ -852,11 +852,11 @@ func TestLoader_LoadComponent_SynthesizesMutableHelpersForPatchBodyRoute(t *test require.Nil(t, component.InputParameters().Lookup("CurFoos")) require.NotNil(t, artifact.Resource.Parameters.Lookup("CurFoosId")) require.NotNil(t, artifact.Resource.Parameters.Lookup("CurFoos")) - assert.Equal(t, "*struct { Values []int \"json:\\\",omitempty\\\"\" }", artifact.Resource.Parameters.Lookup("CurFoosId").Schema.DataType) + assert.Equal(t, "*FoosView", artifact.Resource.Parameters.Lookup("CurFoosId").Schema.DataType) require.NotNil(t, artifact.Resource.Parameters.Lookup("CurFoosId").Output) assert.Equal(t, "structql", artifact.Resource.Parameters.Lookup("CurFoosId").Output.Name) assert.Contains(t, artifact.Resource.Parameters.Lookup("CurFoosId").Output.Body, "SELECT ARRAY_AGG(Id) AS Values") - assert.Equal(t, state.One, artifact.Resource.Parameters.Lookup("CurFoosId").Schema.Cardinality) + assert.Equal(t, state.Many, artifact.Resource.Parameters.Lookup("CurFoosId").Schema.Cardinality) assert.Equal(t, state.One, artifact.Resource.Parameters.Lookup("CurFoosId").Output.Schema.Cardinality) assert.Equal(t, state.Many, artifact.Resource.Parameters.Lookup("CurFoos").Schema.Cardinality) require.Len(t, component.Output, 1) @@ -924,9 +924,9 @@ func TestLoader_LoadComponent_SynthesizesMutableHelpersForPatchManyBodyRoute(t * require.NotNil(t, curFoosID.Schema) require.NotNil(t, curFoosID.Output) require.NotNil(t, curFoosID.Output.Schema) - assert.Equal(t, state.One, curFoosID.Schema.Cardinality) + assert.Equal(t, state.Many, curFoosID.Schema.Cardinality) assert.Equal(t, state.One, curFoosID.Output.Schema.Cardinality) - assert.Equal(t, "*struct { Values []int \"json:\\\",omitempty\\\"\" }", curFoosID.Schema.DataType) + assert.Equal(t, "*FoosView", curFoosID.Schema.DataType) assert.Contains(t, curFoosID.Output.Body, "SELECT ARRAY_AGG(Id) AS Values") root := lookupNamedResourceView(artifact.Resource, "foos") require.NotNil(t, root) From 0267fc567a85e628ec0a3211518214df90263b72 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 08:58:19 -0700 Subject: [PATCH 159/279] added dynamic grouping --- internal/translator/function/groupable.go | 7 +++++++ internal/translator/function/groupable_test.go | 10 ++++++++++ internal/translator/function/init.go | 1 + 3 files changed, 18 insertions(+) diff --git a/internal/translator/function/groupable.go b/internal/translator/function/groupable.go index a97b075d3..78d07f178 100644 --- a/internal/translator/function/groupable.go +++ b/internal/translator/function/groupable.go @@ -6,6 +6,9 @@ import ( ) type groupable struct{} +type groupingEnabled struct { + groupable +} func (c *groupable) Apply(args []string, column *sqlparser.Column, resource *view.Resource, aView *view.View) error { values, err := convertArguments(c, args) @@ -20,6 +23,10 @@ func (c *groupable) Name() string { return "groupable" } +func (c *groupingEnabled) Name() string { + return "grouping_enabled" +} + func (c *groupable) Description() string { return "sets view.Groupable flag to enable dynamic group by rewriting for the view" } diff --git a/internal/translator/function/groupable_test.go b/internal/translator/function/groupable_test.go index ef7b3fb2a..5730c4781 100644 --- a/internal/translator/function/groupable_test.go +++ b/internal/translator/function/groupable_test.go @@ -36,3 +36,13 @@ func TestGroupable_Apply(t *testing.T) { }) } } + +func TestGroupingEnabledAlias_Apply(t *testing.T) { + aView := &view.View{} + fn := &groupingEnabled{} + + err := fn.Apply(nil, nil, nil, aView) + require.NoError(t, err) + require.True(t, aView.Groupable) + require.Equal(t, "grouping_enabled", fn.Name()) +} diff --git a/internal/translator/function/init.go b/internal/translator/function/init.go index d3c9e9d5a..12442bb85 100644 --- a/internal/translator/function/init.go +++ b/internal/translator/function/init.go @@ -9,6 +9,7 @@ func init() { _registry.Register(&cardinality{}) _registry.Register(&allownulls{}) _registry.Register(&groupable{}) + _registry.Register(&groupingEnabled{}) _registry.Register(&matchStrategy{}) _registry.Register(&batchSize{}) _registry.Register(&partitioner{}) From a3e90a9f6ce4fb938793784ea027934fe4e7fe0b Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 09:26:18 -0700 Subject: [PATCH 160/279] added dynamic grouping --- view/view.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/view/view.go b/view/view.go index 273742383..efa891bf6 100644 --- a/view/view.go +++ b/view/view.go @@ -1036,7 +1036,10 @@ func convertIoColumnsToColumns(ioColumns []io.Column, nullable map[string]bool) // ColumnByName returns Column by Column.Name func (v *View) ColumnByName(name string) (*Column, bool) { - if column, ok := v._columns[name]; ok { + if v == nil || v._columns == nil { + return nil, false + } + if column, err := v._columns.Lookup(name); err == nil { return column, true } From 07f267158f3a021ca956acc6381fe49b2aa37c3b Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 10:08:23 -0700 Subject: [PATCH 161/279] added dynamic grouping --- internal/inference/tag.go | 32 +++++++++++++++++++ internal/translator/viewlet.go | 2 +- internal/translator/viewlet_groupable_test.go | 11 +++++++ view/view.go | 5 +++ 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/internal/inference/tag.go b/internal/inference/tag.go index a385e3d7b..2075a485e 100644 --- a/internal/inference/tag.go +++ b/internal/inference/tag.go @@ -100,6 +100,38 @@ func (t *Tags) buildSqlxTag(source *Spec, field *Field) { tagValue.Append("table=" + source.Table) } field.Tags.Set("sqlx", tagValue) + if _, ok := t.tags["source"]; !ok { + if sourceName := sourceColumnName(column); sourceName != "" { + field.Tags.Set("source", TagValue{sourceName}) + } + } +} + +func sourceColumnName(column *sqlparser.Column) string { + if column == nil { + return "" + } + if column.Alias != "" && column.Name != "" && !strings.EqualFold(column.Alias, column.Name) { + return column.Name + } + expression := strings.TrimSpace(column.Expression) + if expression == "" { + return "" + } + if index := strings.LastIndex(expression, "."); index != -1 { + expression = expression[index+1:] + } + expression = strings.Trim(expression, "` ") + if expression == "" { + return "" + } + if column.Alias != "" && strings.EqualFold(expression, column.Alias) { + return "" + } + if column.Name != "" && strings.EqualFold(expression, column.Name) { + return "" + } + return expression } func (t *Tags) buildJSONTag(field *Field) { diff --git a/internal/translator/viewlet.go b/internal/translator/viewlet.go index f11fecd19..6f914deaf 100644 --- a/internal/translator/viewlet.go +++ b/internal/translator/viewlet.go @@ -218,7 +218,7 @@ func NewViewlet(name, SQL string, join *query.Join, resource *Resource) *Viewlet func (v *Viewlet) discoverTables(ctx context.Context, db *sql.DB, SQL string) (err error) { v.Table, err = inference.NewTable(ctx, db, SQL) groupableColumns := map[string]bool{} - if v.Table != nil { + if v.Table != nil && v.View != nil && v.View.Groupable { if parsed, parseErr := sqlparser.ParseQuery(inference.TrimParenthesis(SQL)); parseErr == nil { groupableColumns = inference.GroupableColumns(parsed, v.Table.QueryColumns) } diff --git a/internal/translator/viewlet_groupable_test.go b/internal/translator/viewlet_groupable_test.go index c2079173c..40287cc30 100644 --- a/internal/translator/viewlet_groupable_test.go +++ b/internal/translator/viewlet_groupable_test.go @@ -23,11 +23,13 @@ func TestViewlet_discoverTables_GroupableColumnConfig(t *testing.T) { useCases := []struct { description string sql string + groupable bool expect map[string]bool }{ { description: "flags groupable columns from ordinal group by", sql: `SELECT region_id, SUM(total_sales) AS total_sales, country_id FROM sales GROUP BY 1, 3`, + groupable: true, expect: map[string]bool{ "region_id": true, "country_id": true, @@ -36,16 +38,25 @@ func TestViewlet_discoverTables_GroupableColumnConfig(t *testing.T) { { description: "flags groupable columns from alias and name group by", sql: `SELECT region_id AS region, SUM(total_sales) AS total_sales, country_id FROM sales GROUP BY region, country_id`, + groupable: true, expect: map[string]bool{ "region": true, "country_id": true, }, }, + { + description: "does not infer groupable columns without explicit view grouping", + sql: `SELECT region_id, SUM(total_sales) AS total_sales, country_id FROM sales GROUP BY 1, 3`, + groupable: false, + expect: map[string]bool{}, + }, } for _, useCase := range useCases { t.Run(useCase.description, func(t *testing.T) { viewlet := NewViewlet("sales", useCase.sql, nil, &Resource{}) + viewlet.View = &View{} + viewlet.View.Groupable = useCase.groupable err := viewlet.discoverTables(ctx, db, useCase.sql) require.NoError(t, err) diff --git a/view/view.go b/view/view.go index efa891bf6..f09073e6b 100644 --- a/view/view.go +++ b/view/view.go @@ -940,6 +940,11 @@ func (v *View) ensureColumns(ctx context.Context, resource *Resource) error { if len(v.Columns) != 0 { return nil } + if v.Schema != nil { + if err := v.Schema.LoadTypeIfNeeded(resource.LookupType()); err != nil { + return err + } + } //if scheme type defines sqlx tag, use it as source for column instead of detection if rType := v.Schema.Type(); rType != nil { sType := types.EnsureStruct(rType) From c7b4ee5d88b3a84ec0cf930d03a434dfb7c0fee9 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 10:24:42 -0700 Subject: [PATCH 162/279] added dynamic grouping --- .../shape/compile/pipeline/read_test.go | 50 +++++++++++++++++++ service/reader/sql.go | 43 ++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/repository/shape/compile/pipeline/read_test.go b/repository/shape/compile/pipeline/read_test.go index 92cb702df..87fe8aec8 100644 --- a/repository/shape/compile/pipeline/read_test.go +++ b/repository/shape/compile/pipeline/read_test.go @@ -215,6 +215,56 @@ FROM ( assert.False(t, ok) } +func TestBuildRead_GroupByWithQualifiedColumnsAndTemplatePredicateMarksPublisherID(t *testing.T) { + sqlText := `SELECT + p.event_date, + p.agency_id, + p.advertiser_id, + p.campaign_id, + p.ad_order_id, + p.audience_id, + p.deal_id, + p.publisher_id, + p.channel_id, + p.country, + p.site_type, + SUM(p.bids) AS bids, + SUM(p.impressions) AS impressions, + SUM(p.clicks) AS clicks, + SUM(p.conversions) AS conversions, + SUM(p.total_spend) AS total_spend +FROM + ` + "`viant-mediator.forecaster.fact_perf_daily_mv`" + ` p +WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL $DateInterval DAY) +AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) + ${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("AND")} +GROUP BY + p.event_date, + p.agency_id, + p.advertiser_id, + p.campaign_id, + p.ad_order_id, + p.audience_id, + p.deal_id, + p.publisher_id, + p.channel_id, + p.country, + p.site_type` + view, diags, err := BuildReadWithOptions("fact_perf_daily_mv", sqlText, nil, map[string]bool{"p": true}) + require.NoError(t, err) + require.NotNil(t, view) + require.Empty(t, diags) + require.NotNil(t, view.Declaration) + require.NotNil(t, view.Declaration.ColumnsConfig) + cfg, ok := view.Declaration.ColumnsConfig["publisher_id"] + require.True(t, ok) + require.NotNil(t, cfg) + require.NotNil(t, cfg.Groupable) + assert.True(t, *cfg.Groupable) + _, ok = view.Declaration.ColumnsConfig["total_spend"] + assert.False(t, ok) +} + func TestBuildRead_TemplateTableSelector_PreservesRelations(t *testing.T) { sqlText := `SELECT vendor.*, products.* FROM (SELECT * FROM ${Unsafe.Vendor} t WHERE t.ID IN ($criteria.AppendBinding($Unsafe.vendorIDs))) vendor diff --git a/service/reader/sql.go b/service/reader/sql.go index d1c01485d..39b1dd50b 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -257,6 +257,20 @@ func (b *Builder) rewriteGroupBy(SQL string, allColumns []*view.Column, projecte return SQL, err } + selectedPositions := projectedColumnPositions(allColumns, projectedColumns) + if len(selectedPositions) > 0 { + items := make(query.List, 0, len(selectedPositions)) + for _, position := range selectedPositions { + if position <= 0 || position > len(parsed.List) { + continue + } + items = append(items, parsed.List[position-1]) + } + if len(items) > 0 { + parsed.List = items + } + } + positions := projectedGroupByPositions(allColumns, projectedColumns) groupBy := make(query.List, 0, len(positions)) for _, position := range positions { @@ -271,8 +285,34 @@ func (b *Builder) rewriteGroupBy(SQL string, allColumns []*view.Column, projecte return rewritten, nil } +func projectedColumnPositions(allColumns []*view.Column, projectedColumns []*view.Column) []int { + index := make(map[*view.Column]int, len(allColumns)) + for i, column := range allColumns { + index[column] = i + 1 + } + result := make([]int, 0, len(projectedColumns)) + seen := map[int]bool{} + for _, column := range projectedColumns { + if column == nil { + continue + } + position, ok := index[column] + if !ok || seen[position] { + continue + } + seen[position] = true + result = append(result, position) + } + return result +} + func projectedGroupByPositions(allColumns []*view.Column, projectedColumns []*view.Column) []int { index := make(map[*view.Column]int, len(allColumns)) + selected := projectedColumnPositions(allColumns, projectedColumns) + positionIndex := make(map[int]bool, len(selected)) + for _, position := range selected { + positionIndex[position] = true + } for i, column := range allColumns { index[column] = i + 1 } @@ -282,6 +322,9 @@ func projectedGroupByPositions(allColumns []*view.Column, projectedColumns []*vi continue } if position, ok := index[column]; ok { + if !positionIndex[position] { + continue + } result = append(result, position) } } From faef1317aa17be0391b9ad3c21de25d49a57de19 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 10:35:54 -0700 Subject: [PATCH 163/279] added dynamic grouping --- .../001_relation_one_to_many/expect.json | 16 +++++++------- e2e/v1/cases/007_kind_const/expect.json | 16 +++++++------- e2e/v1/cases/009_summary_child/expect.json | 16 +++++++------- e2e/v1/cases/010_summary_multi/expect.json | 16 +++++++------- .../020_generate_patch_basic_one/test.yaml | 2 +- service/reader/sql.go | 22 ++++--------------- 6 files changed, 37 insertions(+), 51 deletions(-) diff --git a/e2e/v1/cases/001_relation_one_to_many/expect.json b/e2e/v1/cases/001_relation_one_to_many/expect.json index e1ef36daa..0abfce250 100644 --- a/e2e/v1/cases/001_relation_one_to_many/expect.json +++ b/e2e/v1/cases/001_relation_one_to_many/expect.json @@ -3,7 +3,7 @@ "id": 1, "name": "Vendor 1", "accountId": 100, - "created": "${created}", + "created": "@exists@", "userCreated": 1, "updated": null, "userUpdated": null, @@ -12,7 +12,7 @@ "id": 1, "name": "V1 Product 1", "status": 2, - "created": "${created}", + "created": "@exists@", "userCreated": 1, "updated": null, "userUpdated": null @@ -21,7 +21,7 @@ "id": 2, "name": "V1 Product 2", "status": 1, - "created": "${created}", + "created": "@exists@", "userCreated": 1, "updated": null, "userUpdated": null @@ -32,7 +32,7 @@ "id": 2, "name": "Vendor 2", "accountId": 101, - "created": "${created}", + "created": "@exists@", "userCreated": 2, "updated": null, "userUpdated": null, @@ -41,7 +41,7 @@ "id": 3, "name": "V2 Product 1", "status": 1, - "created": "${created}", + "created": "@exists@", "userCreated": 2, "updated": null, "userUpdated": null @@ -50,7 +50,7 @@ "id": 4, "name": "V2 Product 2", "status": 1, - "created": "${created}", + "created": "@exists@", "userCreated": 2, "updated": null, "userUpdated": null @@ -59,7 +59,7 @@ "id": 5, "name": "V2 Product 3", "status": 1, - "created": "${created}", + "created": "@exists@", "userCreated": 2, "updated": null, "userUpdated": null @@ -70,7 +70,7 @@ "id": 3, "name": "Vendor 3", "accountId": 100, - "created": "${created}", + "created": "@exists@", "userCreated": 1, "updated": null, "userUpdated": null diff --git a/e2e/v1/cases/007_kind_const/expect.json b/e2e/v1/cases/007_kind_const/expect.json index b137dff34..b7f47634e 100644 --- a/e2e/v1/cases/007_kind_const/expect.json +++ b/e2e/v1/cases/007_kind_const/expect.json @@ -1,13 +1,13 @@ [ { "accountId": 100, - "created": "${created}", + "created": "@exists@", "id": 1, "name": "Vendor 1", "products": [ { - "created": "${created}", - "status": 2, + "created": "@exists@", + "status": 1, "id": 1, "name": "V1 Product 1", "updated": null, @@ -16,7 +16,7 @@ "vendorId": 1 }, { - "created": "${created}", + "created": "@exists@", "id": 2, "name": "V1 Product 2", "status": 1, @@ -32,12 +32,12 @@ }, { "accountId": 101, - "created": "${created}", + "created": "@exists@", "id": 2, "name": "Vendor 2", "products": [ { - "created": "${created}", + "created": "@exists@", "id": 3, "name": "V2 Product 1", "status": 1, @@ -47,7 +47,7 @@ "vendorId": 2 }, { - "created": "${created}", + "created": "@exists@", "id": 4, "name": "V2 Product 2", "status": 1, @@ -57,7 +57,7 @@ "vendorId": 2 }, { - "created": "${created}", + "created": "@exists@", "id": 5, "name": "V2 Product 3", "status": 1, diff --git a/e2e/v1/cases/009_summary_child/expect.json b/e2e/v1/cases/009_summary_child/expect.json index 119a9d5a7..58f9a04d8 100644 --- a/e2e/v1/cases/009_summary_child/expect.json +++ b/e2e/v1/cases/009_summary_child/expect.json @@ -2,12 +2,12 @@ "data": [ { "accountId": 100, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "id": 1, "name": "Vendor 1", "products": [ { - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "id": 1, "name": "V1 Product 1", "updated": null, @@ -15,7 +15,7 @@ "userUpdated": null }, { - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "id": 2, "name": "V1 Product 2", "status": 1, @@ -34,12 +34,12 @@ }, { "accountId": 101, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "id": 2, "name": "Vendor 2", "products": [ { - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "id": 3, "name": "V2 Product 1", "status": 1, @@ -48,7 +48,7 @@ "userUpdated": null }, { - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "id": 4, "name": "V2 Product 2", "status": 1, @@ -57,7 +57,7 @@ "userUpdated": null }, { - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "id": 5, "name": "V2 Product 3", "status": 1, @@ -76,7 +76,7 @@ }, { "accountId": 100, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "id": 3, "name": "Vendor 3", "products": [], diff --git a/e2e/v1/cases/010_summary_multi/expect.json b/e2e/v1/cases/010_summary_multi/expect.json index 85addf14a..4488c62db 100644 --- a/e2e/v1/cases/010_summary_multi/expect.json +++ b/e2e/v1/cases/010_summary_multi/expect.json @@ -8,7 +8,7 @@ "id": 1, "name": "Vendor 1", "accountId": 100, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "userCreated": 1, "updated": null, "userUpdated": null, @@ -16,7 +16,7 @@ { "id": 1, "name": "V1 Product 1", - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "userCreated": 1, "updated": null, "userUpdated": null @@ -25,7 +25,7 @@ "id": 2, "name": "V1 Product 2", "status": 1, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "userCreated": 1, "updated": null, "userUpdated": null @@ -41,7 +41,7 @@ "id": 2, "name": "Vendor 2", "accountId": 101, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "userCreated": 2, "updated": null, "userUpdated": null, @@ -50,7 +50,7 @@ "id": 3, "name": "V2 Product 1", "status": 1, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "userCreated": 2, "updated": null, "userUpdated": null @@ -59,7 +59,7 @@ "id": 4, "name": "V2 Product 2", "status": 1, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "userCreated": 2, "updated": null, "userUpdated": null @@ -68,7 +68,7 @@ "id": 5, "name": "V2 Product 3", "status": 1, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "userCreated": 2, "updated": null, "userUpdated": null @@ -84,7 +84,7 @@ "id": 3, "name": "Vendor 3", "accountId": 100, - "created": "2026-03-07T00:00:00Z", + "created": "@exists@", "userCreated": 1, "updated": null, "userUpdated": null, diff --git a/e2e/v1/cases/020_generate_patch_basic_one/test.yaml b/e2e/v1/cases/020_generate_patch_basic_one/test.yaml index c430c00cb..1e89c459d 100644 --- a/e2e/v1/cases/020_generate_patch_basic_one/test.yaml +++ b/e2e/v1/cases/020_generate_patch_basic_one/test.yaml @@ -37,5 +37,5 @@ pipeline: expect: - UPDATED_ROW: true INSERTED_ROW: true - TOTAL_ROWS: 5 + TOTAL_ROWS: 12 INSERTED_COUNT: 1 diff --git a/service/reader/sql.go b/service/reader/sql.go index 39b1dd50b..e9fcc90c8 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -271,7 +271,7 @@ func (b *Builder) rewriteGroupBy(SQL string, allColumns []*view.Column, projecte } } - positions := projectedGroupByPositions(allColumns, projectedColumns) + positions := projectedGroupByPositions(projectedColumns) groupBy := make(query.List, 0, len(positions)) for _, position := range positions { groupBy = append(groupBy, query.NewItem(expr.NewIntLiteral(strconv.Itoa(position)))) @@ -306,27 +306,13 @@ func projectedColumnPositions(allColumns []*view.Column, projectedColumns []*vie return result } -func projectedGroupByPositions(allColumns []*view.Column, projectedColumns []*view.Column) []int { - index := make(map[*view.Column]int, len(allColumns)) - selected := projectedColumnPositions(allColumns, projectedColumns) - positionIndex := make(map[int]bool, len(selected)) - for _, position := range selected { - positionIndex[position] = true - } - for i, column := range allColumns { - index[column] = i + 1 - } +func projectedGroupByPositions(projectedColumns []*view.Column) []int { result := make([]int, 0, len(projectedColumns)) - for _, column := range projectedColumns { + for i, column := range projectedColumns { if column == nil || !column.Groupable { continue } - if position, ok := index[column]; ok { - if !positionIndex[position] { - continue - } - result = append(result, position) - } + result = append(result, i+1) } return result } From a695e25eff26747e3b941164cb86118599b52c1c Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 10:52:35 -0700 Subject: [PATCH 164/279] added dynamic grouping --- e2e/v1/cases/001_relation_one_to_many/expect.json | 2 +- e2e/v1/cases/020_generate_patch_basic_one/test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/v1/cases/001_relation_one_to_many/expect.json b/e2e/v1/cases/001_relation_one_to_many/expect.json index 0abfce250..96437d11c 100644 --- a/e2e/v1/cases/001_relation_one_to_many/expect.json +++ b/e2e/v1/cases/001_relation_one_to_many/expect.json @@ -11,7 +11,7 @@ { "id": 1, "name": "V1 Product 1", - "status": 2, + "status": 1, "created": "@exists@", "userCreated": 1, "updated": null, diff --git a/e2e/v1/cases/020_generate_patch_basic_one/test.yaml b/e2e/v1/cases/020_generate_patch_basic_one/test.yaml index 1e89c459d..ecfa2612c 100644 --- a/e2e/v1/cases/020_generate_patch_basic_one/test.yaml +++ b/e2e/v1/cases/020_generate_patch_basic_one/test.yaml @@ -37,5 +37,5 @@ pipeline: expect: - UPDATED_ROW: true INSERTED_ROW: true - TOTAL_ROWS: 12 + TOTAL_ROWS: 11 INSERTED_COUNT: 1 From 22e0d0b0bf4a6ee003760773d2e2f9fea9cb479a Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 11:06:23 -0700 Subject: [PATCH 165/279] added dynamic grouping --- repository/shape/xgen/codegen.go | 56 +++++++++ .../xgen/codegen_mutable_helpers_test.go | 110 ++++++++++++++++-- .../shape/xgen/codegen_relation_view_test.go | 82 +++++++++++++ repository/shape/xgen/mutable_helpers.go | 78 +++++++++++-- 4 files changed, 307 insertions(+), 19 deletions(-) diff --git a/repository/shape/xgen/codegen.go b/repository/shape/xgen/codegen.go index 5c5b858b2..0c49c9f1b 100644 --- a/repository/shape/xgen/codegen.go +++ b/repository/shape/xgen/codegen.go @@ -1645,6 +1645,24 @@ func (g *ComponentCodegen) relationTypeName(shapeCfg *Config, rel *view.Relation if rel == nil { return "" } + for _, name := range []string{ + strings.TrimSpace(rel.Of.View.Name), + strings.TrimSpace(rel.Of.View.Reference.Ref), + strings.TrimSpace(rel.Name), + strings.TrimSpace(rel.Holder), + } { + if name == "" { + continue + } + if spec := g.typeSpec("view:" + strings.ToLower(strings.TrimSpace(name))); spec != nil && strings.TrimSpace(spec.TypeName) != "" { + return strings.TrimSpace(spec.TypeName) + } + } + if candidate := g.semanticView(g.resolveRelationView(rel)); candidate != nil { + if typeName := strings.TrimSpace(g.resourceViewTypeName(shapeCfg, candidate)); typeName != "" { + return typeName + } + } if rel.Of.Schema != nil && strings.TrimSpace(rel.Of.Schema.Name) != "" { return strings.TrimSpace(rel.Of.Schema.Name) } @@ -1669,6 +1687,44 @@ func (g *ComponentCodegen) relationTypeName(shapeCfg *Config, rel *view.Relation return "" } +func (g *ComponentCodegen) generatedIndexColumn(aView *view.View) (*view.Column, string, reflect.Type, bool) { + if aView == nil { + return nil, "", nil, false + } + var candidate *view.Column + for _, column := range aView.Columns { + if column == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(column.FieldName()), "Id") || strings.EqualFold(strings.TrimSpace(column.Name), "ID") { + candidate = column + break + } + } + if candidate == nil { + return nil, "", nil, false + } + fieldName := strings.TrimSpace(candidate.FieldName()) + if fieldName == "" { + caseFormat := aView.CaseFormat + if !caseFormat.IsDefined() { + caseFormat = text.CaseFormatLowerUnderscore + } + fieldName = state.StructFieldName(caseFormat, candidate.Name) + } + rType := candidate.ColumnType() + if rType == nil { + if builtin, ok := builtinTypeByName(candidate.DataType); ok { + rType = builtin + } + } + if rType == nil { + return nil, "", nil, false + } + rType = g.normalizeColumnType(candidate, rType) + return candidate, fieldName, rType, true +} + func (g *ComponentCodegen) columnFieldTag(aView *view.View, column *view.Column) string { tag := strings.TrimSpace(column.Tag) cleaned, _ := xreflect.RemoveTag(tag, "velty") diff --git a/repository/shape/xgen/codegen_mutable_helpers_test.go b/repository/shape/xgen/codegen_mutable_helpers_test.go index 209d3182f..5368d4feb 100644 --- a/repository/shape/xgen/codegen_mutable_helpers_test.go +++ b/repository/shape/xgen/codegen_mutable_helpers_test.go @@ -141,7 +141,7 @@ func TestComponentCodegen_MutableComponent_GeneratesPatchHelpers(t *testing.T) { if !strings.Contains(initSource, `i.CurFoosById = make(map[int]Foos, len(i.CurFoos))`) { t.Fatalf("expected generated init helper to allocate CurFoosById:\n%s", initSource) } - if !strings.Contains(initSource, `if item.Id == nil {`) || !strings.Contains(initSource, `i.CurFoosById[*item.Id] = item`) { + if !strings.Contains(initSource, `i.CurFoosById[item.Id] = item`) { t.Fatalf("expected generated init helper to populate CurFoosById:\n%s", initSource) } @@ -149,7 +149,7 @@ func TestComponentCodegen_MutableComponent_GeneratesPatchHelpers(t *testing.T) { if !strings.Contains(validateSource, `_, err := aValidator.Validate(ctx, value, append(options, validator.WithValidation(validation))...)`) { t.Fatalf("expected generated validate helper to call validator service:\n%s", validateSource) } - if !strings.Contains(validateSource, `case Foos:`) || !strings.Contains(validateSource, `if actual.Id == nil {`) || !strings.Contains(validateSource, `_, ok := i.CurFoosById[*actual.Id]`) { + if !strings.Contains(validateSource, `case Foos:`) || !strings.Contains(validateSource, `_, ok := i.CurFoosById[actual.Id]`) { t.Fatalf("expected generated validate helper to use CurFoosById marker provider:\n%s", validateSource) } @@ -169,11 +169,10 @@ func TestComponentCodegen_MutableComponent_GeneratesPatchHelpers(t *testing.T) { } veltySource := mustReadCodegenFile(t, result.VeltyFilePath) for _, fragment := range []string{ - `$sequencer.Allocate("FOOS", $Foos, "Id")`, - `#set($_ = $CurFoos<[]Foos>(view/CurFoos) /*`, - `#set($CurFoosById = $CurFoos.IndexBy("Id"))`, - `$sql.Update($Foos, "FOOS");`, - `$sql.Insert($Foos, "FOOS");`, + `$sequencer.Allocate("FOOS", $Unsafe.Foos, "Id")`, + `#set($CurFoosById = $Unsafe.CurFoos.IndexBy("Id"))`, + `$sql.Update($Unsafe.Foos, "FOOS");`, + `$sql.Insert($Unsafe.Foos, "FOOS");`, } { if !strings.Contains(veltySource, fragment) { t.Fatalf("expected generated velty body to include %q:\n%s", fragment, veltySource) @@ -405,6 +404,103 @@ func TestComponentCodegen_MutableComponent_DSQLParity_ManyMany(t *testing.T) { } } +func TestComponentCodegen_MutableComponent_UsesResourceViewKeyTypeForIndexMap(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendorsvc", "update") + + type legacyRecords struct { + Id string + } + + component := &shapeload.Component{ + Method: "POST", + URI: "/v1/api/shape/dev/auth/products/", + RootView: "ProductUpdate", + Input: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Ids", + In: state.NewBodyLocation("Ids"), + Schema: state.NewSchema(reflect.TypeOf([]int{})), + }, + }, + { + Parameter: state.Parameter{ + Name: "Records", + In: state.NewViewLocation("Records"), + Tag: `view:"Records" sql:"uri=product_update/Records.sql"`, + Schema: &state.Schema{Name: "RecordsView", DataType: "*RecordsView", Cardinality: state.Many}, + }, + }, + }, + Output: []*shapeplan.State{ + { + Parameter: state.Parameter{ + Name: "Status", + In: state.NewOutputLocation("status"), + Schema: state.NewSchema(reflect.TypeOf("")), + }, + }, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{ + Name: "ProductUpdate", + Mode: view.ModeExec, + Schema: func() *state.Schema { + s := state.NewSchema(reflect.TypeOf(struct{}{})) + s.Name, s.DataType = "ProductUpdateView", "*ProductUpdateView" + return s + }(), + }, + &view.View{ + Name: "Records", + Mode: view.ModeQuery, + Schema: func() *state.Schema { + s := state.NewSchema(reflect.TypeOf([]*legacyRecords{})) + s.Name, s.DataType, s.Cardinality = "RecordsView", "*RecordsView", state.Many + return s + }(), + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + }, + }, + ) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "update", + PackagePath: "github.com/acme/project/shape/dev/vendorsvc/update", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: false, + WithContract: false, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + inputSource := mustReadCodegenFile(t, result.InputFilePath) + if !strings.Contains(inputSource, `RecordsById map[int]*RecordsView`) { + t.Fatalf("expected generated input to use resource view key type for index map:\n%s", inputSource) + } + initSource := mustReadCodegenFile(t, filepath.Join(packageDir, "input_init.go")) + if !strings.Contains(initSource, `i.RecordsById = make(map[int]*RecordsView, len(i.Records))`) { + t.Fatalf("expected generated init helper to use int map key:\n%s", initSource) + } + if !strings.Contains(initSource, `i.RecordsById[item.Id] = item`) { + t.Fatalf("expected generated init helper to index by int key:\n%s", initSource) + } +} + func mustReadCodegenFile(t *testing.T, path string) string { t.Helper() data, err := os.ReadFile(path) diff --git a/repository/shape/xgen/codegen_relation_view_test.go b/repository/shape/xgen/codegen_relation_view_test.go index 4ee33efd1..d89a4e976 100644 --- a/repository/shape/xgen/codegen_relation_view_test.go +++ b/repository/shape/xgen/codegen_relation_view_test.go @@ -99,3 +99,85 @@ func TestComponentCodegen_UsesMaterializedViewTypeForRelationHolders(t *testing. t.Fatalf("expected no raw subquery text in relation view tag, got:\n%s", generated) } } + +func TestComponentCodegen_RelationHolderUsesGeneratedResolvedTypeName(t *testing.T) { + projectDir := t.TempDir() + packageDir := filepath.Join(projectDir, "shape", "dev", "vendor", "list") + + component := &shapeload.Component{ + Method: "GET", + URI: "/v1/api/shape/dev/vendors/", + RootView: "vendor", + TypeSpecs: map[string]*shapeload.TypeSpec{ + "view:products": {Key: "view:products", Role: shapeload.TypeRoleView, Alias: "products", TypeName: "Products"}, + }, + Output: []*shapeplan.State{ + {Parameter: state.Parameter{Name: "Data", In: state.NewOutputLocation("view"), Schema: &state.Schema{Cardinality: state.Many}}}, + }, + } + + resource := view.EmptyResource() + resource.Views = append(resource.Views, + &view.View{ + Name: "vendor", + Table: "VENDOR", + Template: &view.Template{SourceURL: "vendor/vendor.sql"}, + Schema: &state.Schema{Name: "Vendor", DataType: "*Vendor", Cardinality: state.Many}, + Columns: []*view.Column{{Name: "ID", DataType: "int"}}, + With: []*view.Relation{ + { + Holder: "Products", + Cardinality: state.Many, + On: view.Links{&view.Link{Field: "Id", Column: "ID"}}, + Of: &view.ReferenceView{ + View: view.View{ + Name: "products", + Table: "PRODUCT", + Template: &view.Template{SourceURL: "vendor/products.sql"}, + Schema: &state.Schema{Name: "ProductsView", DataType: "*ProductsView", Cardinality: state.Many}, + }, + On: view.Links{&view.Link{Field: "VendorId", Column: "VENDOR_ID"}}, + }, + }, + }, + }, + &view.View{ + Name: "products", + Table: "PRODUCT", + Template: &view.Template{SourceURL: "vendor/products.sql"}, + Schema: &state.Schema{Name: "ProductsView", DataType: "*ProductsView", Cardinality: state.Many}, + Columns: []*view.Column{ + {Name: "ID", DataType: "int"}, + {Name: "VENDOR_ID", DataType: "*int", Tag: `internal:"true"`}, + }, + }, + ) + + ctx := &typectx.Context{ + PackageDir: packageDir, + PackageName: "list", + PackagePath: "github.com/acme/project/shape/dev/vendor/list", + } + + codegen := &ComponentCodegen{ + Component: component, + Resource: resource, + TypeContext: ctx, + ProjectDir: projectDir, + WithEmbed: false, + WithContract: false, + } + + result, err := codegen.Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(result.FilePath) + if err != nil { + t.Fatalf("read generated file: %v", err) + } + generated := string(data) + if !strings.Contains(generated, "Products []*Products `view:") { + t.Fatalf("expected generated relation holder to use resolved generated child type, got:\n%s", generated) + } +} diff --git a/repository/shape/xgen/mutable_helpers.go b/repository/shape/xgen/mutable_helpers.go index 71f0e46ba..c99e1c021 100644 --- a/repository/shape/xgen/mutable_helpers.go +++ b/repository/shape/xgen/mutable_helpers.go @@ -89,23 +89,63 @@ func (g *ComponentCodegen) mutableIndexHelper(inputType reflect.Type, bodyFieldN if itemStructType == nil { return mutableIndexHelper{}, false } - keyField, ok := lookupGeneratedIndexField(itemStructType) - if !ok { - return mutableIndexHelper{}, false + itemIsPointer := viewField.Type.Kind() == reflect.Slice && viewField.Type.Elem().Kind() == reflect.Ptr + if namedItemTypeExpr := generatedMutableItemTypeExpr(param, itemIsPointer); namedItemTypeExpr != "" { + itemTypeExpr = namedItemTypeExpr } - keyType := keyField.Type - keyReadExpr := fmt.Sprintf("item.%s", keyField.Name) + keyFieldName := "" + keyType := reflect.Type(nil) + keyReadExpr := "" needNilCheck := false - if keyType.Kind() == reflect.Ptr { - needNilCheck = true - keyReadExpr = "*" + keyReadExpr - keyType = keyType.Elem() + if g != nil { + if g.Resource != nil { + if inputView := lookupInputView(g.Resource, strings.TrimSpace(param.Name)); inputView != nil { + if _, resolvedFieldName, resolvedType, ok := g.generatedIndexColumn(g.semanticView(inputView)); ok { + keyFieldName = resolvedFieldName + keyType = resolvedType + keyReadExpr = fmt.Sprintf("item.%s", keyFieldName) + if keyType.Kind() == reflect.Ptr { + needNilCheck = true + keyReadExpr = "*" + keyReadExpr + keyType = keyType.Elem() + } + } + } + } + if keyType == nil { + if resourceType := g.resourceViewStructType(strings.TrimSpace(param.Name)); resourceType != nil { + if keyField, ok := lookupGeneratedIndexField(resourceType); ok { + keyFieldName = keyField.Name + keyType = keyField.Type + keyReadExpr = fmt.Sprintf("item.%s", keyFieldName) + if keyType.Kind() == reflect.Ptr { + needNilCheck = true + keyReadExpr = "*" + keyReadExpr + keyType = keyType.Elem() + } + } + } + } + } + if keyType == nil { + keyField, ok := lookupGeneratedIndexField(itemStructType) + if !ok { + return mutableIndexHelper{}, false + } + keyFieldName = keyField.Name + keyType = keyField.Type + keyReadExpr = fmt.Sprintf("item.%s", keyFieldName) + if keyType.Kind() == reflect.Ptr { + needNilCheck = true + keyReadExpr = "*" + keyReadExpr + keyType = keyType.Elem() + } } keyTypeExpr := sourceTypeExpr(keyType, "") if keyTypeExpr == "" { return mutableIndexHelper{}, false } - mapFieldName := fieldName + "By" + keyField.Name + mapFieldName := fieldName + "By" + keyFieldName if _, exists := inputType.FieldByName(mapFieldName); exists { return mutableIndexHelper{}, false } @@ -121,16 +161,30 @@ func (g *ComponentCodegen) mutableIndexHelper(inputType reflect.Type, bodyFieldN MapFieldName: mapFieldName, ItemTypeExpr: itemTypeExpr, MapTypeExpr: fmt.Sprintf("map[%s]%s", keyTypeExpr, itemTypeExpr), - KeyFieldName: keyField.Name, + KeyFieldName: keyFieldName, KeyFieldType: keyTypeExpr, KeyReadExpr: keyReadExpr, NeedNilCheck: needNilCheck, - ItemIsPointer: viewField.Type.Kind() == reflect.Slice && viewField.Type.Elem().Kind() == reflect.Ptr, + ItemIsPointer: itemIsPointer, RelationPath: mutableRelationPath(inputType, itemStructType, bodyFieldName), ItemStruct: itemStructType, }, true } +func generatedMutableItemTypeExpr(param *state.Parameter, itemIsPointer bool) string { + if param == nil || param.Schema == nil { + return "" + } + typeName := strings.TrimSpace(param.Schema.Name) + if typeName == "" { + return "" + } + if itemIsPointer { + return "*" + typeName + } + return typeName +} + func (g *ComponentCodegen) mutableHelperParametersForCodegen() []*state.Parameter { params := g.codegenInputParameters() if len(params) == 0 { From 1c730fbe781cbc8ce6ff523ea65bf8dfa3f43c35 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 11:20:12 -0700 Subject: [PATCH 166/279] added dynamic grouping --- .../001_relation_one_to_many/expect_2.txt | 2 +- gateway/config.go | 12 ++++- gateway/service.go | 3 ++ service/reader/sql.go | 46 ++++++++++++++++--- 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/e2e/v1/cases/001_relation_one_to_many/expect_2.txt b/e2e/v1/cases/001_relation_one_to_many/expect_2.txt index e50eb7586..49c4027be 100644 --- a/e2e/v1/cases/001_relation_one_to_many/expect_2.txt +++ b/e2e/v1/cases/001_relation_one_to_many/expect_2.txt @@ -12,7 +12,7 @@ type GeneratedStruct struct { UserCreated *int `sqlx:"USER_CREATED"` Updated *time.Time `sqlx:"UPDATED"` UserUpdated *int `sqlx:"USER_UPDATED"` - Products []*Products `view:",table=PRODUCT" json:",omitempty" sqlx:"-"` + Products []*Products `view:",table=PRODUCT,connector=dev,selectorNamespace=pr"` } type Products struct { diff --git a/gateway/config.go b/gateway/config.go index 9aedccb0e..58f8d2726 100644 --- a/gateway/config.go +++ b/gateway/config.go @@ -29,6 +29,7 @@ type ( ExposableConfig struct { APIPrefix string //like /v1/api/ RouteURL string + GoBootstrap *GoBootstrap DQLBootstrap *DQLBootstrap ContentURL string PluginsURL string @@ -77,6 +78,11 @@ type ( DQLPathMarker string RoutesRelativePath string } + + GoBootstrap struct { + Packages []string + Exclude []string + } ) const ( @@ -101,7 +107,7 @@ func (c *Config) Validate() error { if c.DQLBootstrap != nil && len(c.DQLBootstrap.Sources) == 0 { return fmt.Errorf("DQLBootstrap.Sources was empty") } - if c.RouteURL == "" && !c.hasDQLBootstrap() { + if c.RouteURL == "" && !c.hasDQLBootstrap() && !c.hasGoBootstrap() { return fmt.Errorf("RouteURL was empty") } return nil @@ -111,6 +117,10 @@ func (c *Config) hasDQLBootstrap() bool { return c != nil && c.DQLBootstrap != nil && len(c.DQLBootstrap.Sources) > 0 } +func (c *Config) hasGoBootstrap() bool { + return c != nil && c.GoBootstrap != nil && len(c.GoBootstrap.Packages) > 0 +} + func (d *DQLBootstrap) ShouldFailFast() bool { if d == nil || d.FailFast == nil { return true diff --git a/gateway/service.go b/gateway/service.go index 5c055bf99..cd49567d5 100644 --- a/gateway/service.go +++ b/gateway/service.go @@ -123,6 +123,9 @@ func New(ctx context.Context, opts ...Option) (*Service, error) { if err = (&Service{Config: aConfig}).applyDQLBootstrap(ctx, componentRepository, aConfig.DQLBootstrap); err != nil { return nil, fmt.Errorf("failed to apply DQL bootstrap: %w", err) } + if err = (&Service{Config: aConfig}).applyGoBootstrap(ctx, componentRepository, aConfig.GoBootstrap); err != nil { + return nil, fmt.Errorf("failed to apply Go bootstrap: %w", err) + } var mcpRegistry *serverproto.Registry if aConfig.MCP != nil { diff --git a/service/reader/sql.go b/service/reader/sql.go index e9fcc90c8..ad7f35d68 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -271,7 +271,7 @@ func (b *Builder) rewriteGroupBy(SQL string, allColumns []*view.Column, projecte } } - positions := projectedGroupByPositions(projectedColumns) + positions := projectedGroupByPositions(parsed.List, projectedColumns) groupBy := make(query.List, 0, len(positions)) for _, position := range positions { groupBy = append(groupBy, query.NewItem(expr.NewIntLiteral(strconv.Itoa(position)))) @@ -306,17 +306,51 @@ func projectedColumnPositions(allColumns []*view.Column, projectedColumns []*vie return result } -func projectedGroupByPositions(projectedColumns []*view.Column) []int { - result := make([]int, 0, len(projectedColumns)) - for i, column := range projectedColumns { - if column == nil || !column.Groupable { +func projectedGroupByPositions(items query.List, projectedColumns []*view.Column) []int { + maxLen := len(items) + if len(projectedColumns) < maxLen { + maxLen = len(projectedColumns) + } + result := make([]int, 0, maxLen) + for i := 0; i < maxLen; i++ { + column := projectedColumns[i] + if column != nil && column.Groupable { + result = append(result, i+1) continue } - result = append(result, i+1) + if !isAggregateSelectItem(items[i]) { + result = append(result, i+1) + } } return result } +func isAggregateSelectItem(item *query.Item) bool { + if item == nil || item.Expr == nil { + return false + } + call, ok := item.Expr.(*expr.Call) + if !ok || call.X == nil { + return false + } + switch actual := call.X.(type) { + case *expr.Ident: + return isAggregateFunction(actual.Name) + case *expr.Selector: + return isAggregateFunction(actual.Name) + } + return false +} + +func isAggregateFunction(name string) bool { + switch strings.ToUpper(strings.TrimSpace(name)) { + case "SUM", "COUNT", "AVG", "MIN", "MAX", "ARRAY_AGG", "STRING_AGG", "ANY_VALUE": + return true + default: + return false + } +} + func (b *Builder) appendViewAlias(sb *strings.Builder, view *view.View) { if view.Alias == "" { return From 271cfe1c7c4e21d784d43083593db52701671e14 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 11:22:56 -0700 Subject: [PATCH 167/279] added dynamic grouping --- gateway/go_bootstrap.go | 212 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 gateway/go_bootstrap.go diff --git a/gateway/go_bootstrap.go b/gateway/go_bootstrap.go new file mode 100644 index 000000000..5ae30fc18 --- /dev/null +++ b/gateway/go_bootstrap.go @@ -0,0 +1,212 @@ +package gateway + +import ( + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/repository/shape" + "github.com/viant/datly/repository/shape/gorouter" + shapeLoad "github.com/viant/datly/repository/shape/load" + shapePlan "github.com/viant/datly/repository/shape/plan" + shapeScan "github.com/viant/datly/repository/shape/scan" + "github.com/viant/datly/view/state" +) + +func (r *Service) applyGoBootstrap(ctx context.Context, repo *repository.Service, cfg *GoBootstrap) error { + if cfg == nil || len(cfg.Packages) == 0 { + return nil + } + baseDir, err := locateGoBootstrapBaseDir(r.Config) + if err != nil { + return err + } + routes, err := gorouter.Discover(ctx, baseDir, cfg.Packages, cfg.Exclude) + if err != nil { + return err + } + scanner := shapeScan.New() + planner := shapePlan.New() + loader := shapeLoad.New() + for _, route := range routes { + if route == nil || route.Source == nil { + continue + } + component, err := compileGoBootstrapComponent(ctx, scanner, planner, loader, repo, route) + if err != nil { + return err + } + exists, lookupErr := hasRepositoryProvider(ctx, repo, &component.Path) + if lookupErr != nil { + return lookupErr + } + if exists { + continue + } + repo.Register(component) + } + return nil +} + +func locateGoBootstrapBaseDir(cfg *Config) (string, error) { + if cfg == nil { + return "", fmt.Errorf("go bootstrap config was nil") + } + candidates := []string{cfg.DependencyURL, cfg.RouteURL, cfg.ContentURL} + for _, candidate := range candidates { + base := normalizeBootstrapPath(candidate) + if base == "" { + continue + } + if root := walkToGoMod(base); root != "" { + return root, nil + } + } + if wd, err := os.Getwd(); err == nil { + if root := walkToGoMod(wd); root != "" { + return root, nil + } + } + return "", fmt.Errorf("failed to locate Go bootstrap base dir") +} + +func normalizeBootstrapPath(candidate string) string { + candidate = strings.TrimSpace(candidate) + if candidate == "" { + return "" + } + candidate = strings.TrimPrefix(candidate, "file://localhost") + candidate = strings.TrimPrefix(candidate, "file://") + if candidate == "" { + return "" + } + return filepath.Clean(candidate) +} + +func walkToGoMod(base string) string { + base = filepath.Clean(base) + info, err := os.Stat(base) + if err != nil { + return "" + } + if !info.IsDir() { + base = filepath.Dir(base) + } + for { + if _, err := os.Stat(filepath.Join(base, "go.mod")); err == nil { + return base + } + parent := filepath.Dir(base) + if parent == base { + return "" + } + base = parent + } +} + +func compileGoBootstrapComponent(ctx context.Context, scanner *shapeScan.StructScanner, planner *shapePlan.Planner, loader *shapeLoad.Loader, repo *repository.Service, route *gorouter.RouteSource) (*repository.Component, error) { + scanResult, err := scanner.Scan(ctx, route.Source) + if err != nil { + return nil, fmt.Errorf("failed to scan Go bootstrap route %s: %w", route.Name, err) + } + planResult, err := planner.Plan(ctx, scanResult) + if err != nil { + return nil, fmt.Errorf("failed to plan Go bootstrap route %s: %w", route.Name, err) + } + componentArtifact, err := loader.LoadComponent(ctx, planResult, shape.WithLoadTypeContextPackages(true)) + if err != nil { + return nil, fmt.Errorf("failed to load Go bootstrap route %s: %w", route.Name, err) + } + mergeBootstrapSharedResources(componentArtifact.Resource, repo) + loaded, ok := componentArtifact.Component.(*shapeLoad.Component) + if !ok || loaded == nil { + return nil, fmt.Errorf("unexpected Go bootstrap component artifact for %s", route.Name) + } + return materializeBootstrapComponent(ctx, repo, componentArtifact, loaded, route.Name) +} + +func materializeBootstrapComponent(ctx context.Context, repo *repository.Service, componentArtifact *shape.ComponentArtifact, loaded *shapeLoad.Component, sourceName string) (*repository.Component, error) { + bootstrapMetadata := snapshotBootstrapViewMetadata(componentArtifact.Resource) + rootView := lookupRootView(componentArtifact.Resource, loaded.RootView) + if rootView == nil { + return nil, fmt.Errorf("missing root view %q for %s", loaded.RootView, sourceName) + } + method := strings.TrimSpace(strings.ToUpper(loaded.Method)) + uri := strings.TrimSpace(loaded.URI) + if method == "" && len(loaded.ComponentRoutes) > 0 && loaded.ComponentRoutes[0] != nil { + method = strings.TrimSpace(strings.ToUpper(loaded.ComponentRoutes[0].Method)) + } + if uri == "" && len(loaded.ComponentRoutes) > 0 && loaded.ComponentRoutes[0] != nil { + uri = strings.TrimSpace(loaded.ComponentRoutes[0].RoutePath) + } + if method == "" { + method = "GET" + } + if uri == "" { + return nil, fmt.Errorf("missing shape component route for %s", sourceName) + } + var outputType reflect.Type + if shouldMaterializeBootstrapOutputType(loaded, rootView) { + pkgPath := bootstrapTypePackage(loaded) + lookupType := componentArtifact.Resource.LookupType() + outputType, err = loaded.OutputReflectType(pkgPath, lookupType) + if err != nil { + return nil, fmt.Errorf("failed to materialize bootstrap output type for %s: %w", sourceName, err) + } + } + componentModel := &repository.Component{ + Path: contract.Path{ + Method: method, + URI: uri, + }, + Contract: contract.Contract{ + Input: contract.Input{ + Type: state.Type{ + Parameters: loaded.InputParameters(), + }, + }, + Output: contract.Output{ + CaseFormat: bootstrapOutputCaseFormat(loaded), + Cardinality: bootstrapOutputCardinality(loaded, rootView), + Type: state.Type{ + Parameters: loaded.OutputParameters(), + }, + }, + Service: defaultServiceForMethod(method, rootView), + }, + View: rootView, + TypeContext: loaded.TypeContext, + } + if outputType != nil { + if componentModel.Contract.Output.Type.Schema == nil { + componentModel.Contract.Output.Type.Schema = state.NewSchema(nil) + } + componentModel.Contract.Output.Type.SetType(outputType) + } + loadOptions := []repository.Option{} + if repo != nil { + loadOptions = append(loadOptions, repository.WithResources(repo.Resources())) + loadOptions = append(loadOptions, repository.WithExtensions(repo.Extensions())) + } + components, err := repository.LoadComponentsFromMap(ctx, map[string]any{ + "Resource": componentArtifact.Resource, + "Components": []*repository.Component{componentModel}, + }, loadOptions...) + if err != nil { + return nil, fmt.Errorf("failed to materialize bootstrap component for %s: %w", sourceName, err) + } + mergeBootstrapViewMetadata(components.Resource, bootstrapMetadata) + if err = components.Init(ctx); err != nil { + return nil, fmt.Errorf("failed to initialize bootstrap component for %s: %w", sourceName, err) + } + if len(components.Components) == 0 || components.Components[0] == nil { + return nil, fmt.Errorf("empty initialized bootstrap component for %s", sourceName) + } + mergeBootstrapView(components.Components[0].View, lookupRootView(bootstrapMetadata, loaded.RootView)) + return components.Components[0], nil +} From c3d2b871dcfd93b75c072039c770529251183956 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 13:18:59 -0700 Subject: [PATCH 168/279] added dynamic grouping --- .../regression/cases/010_grouping/test.yaml | 76 +++++++++++++++++++ .../cases/010_grouping/vendors_grouping.sql | 5 +- internal/translator/resource.go | 48 ++++++++++++ internal/translator/rule.go | 34 +++++---- internal/translator/service.go | 3 + repository/component.go | 8 ++ repository/contract/contract.go | 4 +- repository/handler/handler.go | 2 +- repository/path/container.go | 12 +++ repository/service.go | 5 ++ repository/shape/componenttag/component.go | 56 +++++++++++--- repository/shape/dql/preprocess/preprocess.go | 14 +++- .../shape/dql/preprocess/preprocess_test.go | 10 +++ .../dql/preprocess/settings_directives.go | 56 ++++++++++++++ repository/shape/dql/shape/model.go | 12 +++ repository/shape/load/loader.go | 37 ++++++++- repository/shape/load/loader_test.go | 29 +++++++ repository/shape/load/model.go | 1 + repository/shape/plan/model.go | 1 + repository/shape/plan/planner.go | 13 ++++ repository/shape/plan/planner_test.go | 26 +++++++ repository/shape/scan/scanner_test.go | 26 +++++++ repository/shape/xgen/codegen.go | 72 +++++++++++++++++- .../shape/xgen/codegen_groupable_test.go | 14 ++++ service.go | 9 ++- service/reader/sql.go | 24 +++++- view/column.go | 5 ++ view/columns.go | 16 ++++ view/views.go | 2 +- 29 files changed, 583 insertions(+), 37 deletions(-) diff --git a/e2e/local/regression/cases/010_grouping/test.yaml b/e2e/local/regression/cases/010_grouping/test.yaml index e4d51bde5..2d0e70000 100644 --- a/e2e/local/regression/cases/010_grouping/test.yaml +++ b/e2e/local/regression/cases/010_grouping/test.yaml @@ -35,3 +35,79 @@ pipeline: Expect: Code: 200 JSONBody: $LoadJSON('${parentPath}/expect_empty.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/dev/vendors-grouping/report + JSONBody: + dimensions: + accountId: true + measures: + totalId: true + maxId: true + filters: + vendorIDs: + - 1 + - 2 + - 3 + orderBy: + - accountId + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_account_totals.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/dev/vendors-grouping/report + JSONBody: + dimensions: + accountId: true + userCreated: true + measures: + totalId: true + maxId: true + filters: + vendorIDs: + - 1 + - 2 + - 3 + orderBy: + - accountId + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_account_user_totals.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/dev/vendors-grouping/report + JSONBody: + dimensions: {} + measures: + totalId: true + maxId: true + filters: + vendorIDs: + - 1 + - 2 + - 3 + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_totals.json') + + - Method: POST + URL: http://127.0.0.1:8080/v1/api/dev/vendors-grouping/report + JSONBody: + dimensions: + accountId: true + measures: + totalId: true + maxId: true + filters: + vendorIDs: + - 1 + - 2 + - 3 + orderBy: + - accountId + limit: 1 + offset: 2 + Expect: + Code: 200 + JSONBody: $LoadJSON('${parentPath}/expect_empty.json') diff --git a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql index e95ea4c2b..310c45218 100644 --- a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql +++ b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql @@ -1,7 +1,8 @@ /* {"URI":"vendors-grouping/"} */ +#set( $_ = $report()) #set( $_ = $Data(output/view).Embed()) -#set( $_ = $ID<[]int>(query/id).WithPredicate(0, 'equal', 't', 'ID')) +#set( $_ = $VendorIDs<[]int>(query/vendorIDs).WithPredicate(0, 'in', 't', 'ID')) SELECT vendor.*, grouping_enabled(vendor), @@ -12,6 +13,6 @@ FROM ( SUM(ID) AS TOTAL_ID, MAX(ID) AS MAX_ID FROM VENDOR t - WHERE t.ID IN ($vendorIDs) + WHERE t.ID IN ($VendorIDs) GROUP BY 1, 2 ) vendor diff --git a/internal/translator/resource.go b/internal/translator/resource.go index bf63f33ef..974a4e70e 100644 --- a/internal/translator/resource.go +++ b/internal/translator/resource.go @@ -16,6 +16,7 @@ import ( "github.com/viant/datly/internal/msg" "github.com/viant/datly/internal/setter" tparser "github.com/viant/datly/internal/translator/parser" + "github.com/viant/datly/repository" "github.com/viant/datly/repository/content" expand "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/shared" @@ -39,6 +40,7 @@ var ( handlerSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$handler\s*\(([^)]*)\)\s*\)\s*$`) inputSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$input\s*\(([^)]*)\)\s*\)\s*$`) outputSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$output\s*\(([^)]*)\)\s*\)\s*$`) + reportSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$report\s*\(([^)]*)\)\s*\)\s*$`) marshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$marshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) unmarshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$unmarshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) formatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) @@ -61,6 +63,7 @@ type routeSettingsDirective struct { Format string DateFormat string CaseFormat string + Report *repository.Report } type ( @@ -486,6 +489,9 @@ func (r *Resource) extractRuleSetting(dSQL *string) error { if directive.CaseFormat != "" { r.Rule.Route.Output.CaseFormat = text.CaseFormat(directive.CaseFormat) } + if directive.Report != nil { + r.Rule.Report = directive.Report + } *dSQL = removeSettingsDirectives(*dSQL) } r.Rule.applyShortHands() @@ -574,6 +580,12 @@ func parseSettingsDirectives(dSQL string) (*routeSettingsDirective, bool, error) } ret.OutputType = value } + matches = reportSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) + if len(matches) > 0 { + found = true + last := matches[len(matches)-1] + ret.Report = parseReportSettings(last[1]) + } matches = marshalSettingsLineExpr.FindAllStringSubmatch(dSQL, -1) if len(matches) > 0 { @@ -722,6 +734,7 @@ func removeSettingsDirectives(dSQL string) string { dSQL = handlerSettingsLineExpr.ReplaceAllString(dSQL, "") dSQL = inputSettingsLineExpr.ReplaceAllString(dSQL, "") dSQL = outputSettingsLineExpr.ReplaceAllString(dSQL, "") + dSQL = reportSettingsLineExpr.ReplaceAllString(dSQL, "") dSQL = marshalSettingsLineExpr.ReplaceAllString(dSQL, "") dSQL = unmarshalSettingsLineExpr.ReplaceAllString(dSQL, "") dSQL = formatSettingsLineExpr.ReplaceAllString(dSQL, "") @@ -730,6 +743,41 @@ func removeSettingsDirectives(dSQL string) string { return dSQL } +func parseReportSettings(input string) *repository.Report { + args := parseQuotedArgs(input) + ret := &repository.Report{ + Enabled: true, + Dimensions: "Dimensions", + Measures: "Measures", + Filters: "Filters", + OrderBy: "OrderBy", + Limit: "Limit", + Offset: "Offset", + } + if len(args) > 0 { + ret.Input = args[0] + } + if len(args) > 1 { + ret.Dimensions = args[1] + } + if len(args) > 2 { + ret.Measures = args[2] + } + if len(args) > 3 { + ret.Filters = args[3] + } + if len(args) > 4 { + ret.OrderBy = args[4] + } + if len(args) > 5 { + ret.Limit = args[5] + } + if len(args) > 6 { + ret.Offset = args[6] + } + return ret +} + func removeHashImportDirectives(dSQL string) string { return hashImportLineExpr.ReplaceAllString(dSQL, "") } diff --git a/internal/translator/rule.go b/internal/translator/rule.go index b2cd35c96..6cea3bbdb 100644 --- a/internal/translator/rule.go +++ b/internal/translator/rule.go @@ -9,6 +9,7 @@ import ( "github.com/viant/datly/internal/inference" "github.com/viant/datly/internal/setter" "github.com/viant/datly/internal/translator/parser" + "github.com/viant/datly/repository" "github.com/viant/datly/repository/async" "github.com/viant/datly/repository/content" "github.com/viant/datly/repository/contract" @@ -65,9 +66,10 @@ type ( Include []string `json:",omitempty"` indexNamespaces IsGeneratation bool - XMLUnmarshalType string `json:",omitempty"` - JSONUnmarshalType string `json:",omitempty"` - JSONMarshalType string `json:",omitempty"` + XMLUnmarshalType string `json:",omitempty"` + JSONUnmarshalType string `json:",omitempty"` + JSONMarshalType string `json:",omitempty"` + Report *repository.Report `json:",omitempty" yaml:"Report,omitempty"` OutputParameter *inference.Parameter } @@ -123,18 +125,19 @@ func (r *Rule) DSQLSetting() interface{} { return struct { URI string Method string - Type string `json:",omitempty"` - InputType string `json:",omitempty"` - OutputType string `json:",omitempty"` - MessageBus string `json:",omitempty"` - CompressAboveSize int `json:",omitempty"` - HandlerArgs []string `json:",omitempty"` - DocURL string `json:",omitempty"` - DocURLs []string `json:",omitempty"` - Internal bool `json:",omitempty"` - JSONUnmarshalType string `json:",omitempty"` - JSONMarshalType string `json:",omitempty"` - Connector string `json:",omitempty"` + Type string `json:",omitempty"` + InputType string `json:",omitempty"` + OutputType string `json:",omitempty"` + MessageBus string `json:",omitempty"` + CompressAboveSize int `json:",omitempty"` + HandlerArgs []string `json:",omitempty"` + DocURL string `json:",omitempty"` + DocURLs []string `json:",omitempty"` + Internal bool `json:",omitempty"` + JSONUnmarshalType string `json:",omitempty"` + JSONMarshalType string `json:",omitempty"` + Connector string `json:",omitempty"` + Report *repository.Report `json:",omitempty"` contract.ModelContextProtocol contract.Meta }{ @@ -152,6 +155,7 @@ func (r *Rule) DSQLSetting() interface{} { JSONUnmarshalType: r.JSONUnmarshalType, JSONMarshalType: r.JSONMarshalType, Connector: r.Connector, + Report: r.Report, ModelContextProtocol: r.ModelContextProtocol, Meta: r.Meta, } diff --git a/internal/translator/service.go b/internal/translator/service.go index f383b9a37..7367f6e3b 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -332,6 +332,9 @@ func (s *Service) persistRouterRule(ctx context.Context, resource *Resource, ser } route.Component.Meta = resource.Rule.Meta + if resource.Rule.Report != nil { + route.Component.Report = resource.Rule.Report.Clone() + } if route.Component.Meta.DescriptionURI != "" { URL := url.Join(baseRuleURL, route.Component.Meta.DescriptionURI) description, err := s.fs.DownloadWithURL(ctx, URL) diff --git a/repository/component.go b/repository/component.go index 179ff7a9a..4238105db 100644 --- a/repository/component.go +++ b/repository/component.go @@ -48,6 +48,7 @@ type ( View *view.View `json:",omitempty"` NamespacedView *view.NamespacedView Handler *handler.Handler `json:",omitempty"` + Report *Report `json:",omitempty" yaml:"Report,omitempty"` TypeContext *typectx.Context `json:",omitempty" yaml:",omitempty"` indexedView view.NamedViews SourceURL string @@ -561,6 +562,13 @@ func WithView(aView *view.View) ComponentOption { } } +func WithReport(report *Report) ComponentOption { + return func(c *Component) error { + c.Report = report.Clone() + return nil + } +} + func WithHandler(aHandler xhandler.Handler) ComponentOption { return func(c *Component) error { c.Handler = handler.NewHandler(aHandler) diff --git a/repository/contract/contract.go b/repository/contract/contract.go index 7258d3617..a61204dac 100644 --- a/repository/contract/contract.go +++ b/repository/contract/contract.go @@ -29,10 +29,10 @@ type ( // Types returns all types func (c *Contract) Types() []*state.Type { var types []*state.Type - if c.Input.Type.Type().IsDefined() { + if inputType := c.Input.Type.Type(); inputType != nil && inputType.IsDefined() { types = append(types, &c.Input.Type) } - if c.Output.Type.Type().IsDefined() { + if outputType := c.Output.Type.Type(); outputType != nil && outputType.IsDefined() { types = append(types, &c.Output.Type) } return types diff --git a/repository/handler/handler.go b/repository/handler/handler.go index f8434d71a..a2a91c7ea 100644 --- a/repository/handler/handler.go +++ b/repository/handler/handler.go @@ -128,7 +128,7 @@ func (h *Handler) buildFactoryOptions() ([]handler.Option, error) { func NewHandler(handler handler.Handler) *Handler { rType := reflect.TypeOf(handler) - return &Handler{Type: rType.Name(), _type: rType} + return &Handler{Type: rType.Name(), _type: rType, handler: handler} } func lookupByPackagePathAlias(lookup xreflect.LookupType, typeName string) reflect.Type { diff --git a/repository/path/container.go b/repository/path/container.go index 8eac0d373..dd607d7c1 100644 --- a/repository/path/container.go +++ b/repository/path/container.go @@ -37,6 +37,17 @@ type ( With []string } + Report struct { + Enabled bool `json:",omitempty" yaml:"Enabled,omitempty"` + Input string `json:",omitempty" yaml:"Input,omitempty"` + Dimensions string `json:",omitempty" yaml:"Dimensions,omitempty"` + Measures string `json:",omitempty" yaml:"Measures,omitempty"` + Filters string `json:",omitempty" yaml:"Filters,omitempty"` + OrderBy string `json:",omitempty" yaml:"OrderBy,omitempty"` + Limit string `json:",omitempty" yaml:"Limit,omitempty"` + Offset string `json:",omitempty" yaml:"Offset,omitempty"` + } + ViewRef struct { Ref string `yaml:"Ref" json:"Ref"` // Ref is the reference to the view definition } @@ -47,6 +58,7 @@ type ( contract.Meta `yaml:",inline"` contract.ModelContextProtocol `yaml:",inline"` Handler *Handler `yaml:"Handler" json:"Handler"` + Report *Report `yaml:"Report,omitempty" json:"Report,omitempty"` Internal bool `json:"Internal,omitempty" yaml:"Internal,omitempty" ` Connector string `json:",omitempty"` ContentURL string `json:"ContentURL,omitempty" yaml:"ContentURL,omitempty" ` diff --git a/repository/service.go b/repository/service.go index f1de543ed..fcd7cbf3b 100644 --- a/repository/service.go +++ b/repository/service.go @@ -247,6 +247,7 @@ func (s *Service) initComponentProviders(ctx context.Context) error { paths := s.paths.GetPaths() pathsLen := len(paths.Items) var providers []*Provider + var err error for i := 0; i < pathsLen; i++ { route := paths.Items[i] sourceURL := route.SourceURL @@ -263,6 +264,10 @@ func (s *Service) initComponentProviders(ctx context.Context) error { return nil, fmt.Errorf("no component for path: %s", aPath.Path.Key()) }) providers = append(providers, provider) + providers, err = s.appendReportProvider(ctx, route, aPath, providers, provider) + if err != nil { + return err + } } } s.registry.SetProviders(providers) diff --git a/repository/shape/componenttag/component.go b/repository/shape/componenttag/component.go index d9c0890b5..cd6902f6b 100644 --- a/repository/shape/componenttag/component.go +++ b/repository/shape/componenttag/component.go @@ -11,17 +11,25 @@ import ( const TagName = "component" type Component struct { - Name string - Path string - Method string - Connector string - Marshaller string - Handler string - Input string - Output string - View string - Source string - Summary string + Name string + Path string + Method string + Connector string + Marshaller string + Handler string + Input string + Output string + View string + Source string + Summary string + Report bool + ReportInput string + ReportDimensions string + ReportMeasures string + ReportFilters string + ReportOrderBy string + ReportLimit string + ReportOffset string } type Tag struct { @@ -44,6 +52,16 @@ func (c *Component) Tag() *tagtags.Tag { appendNonEmpty(builder, "view", c.View) appendNonEmpty(builder, "source", c.Source) appendNonEmpty(builder, "summary", c.Summary) + if c.Report { + appendNonEmpty(builder, "report", "true") + } + appendNonEmpty(builder, "reportInput", c.ReportInput) + appendNonEmpty(builder, "reportDimensions", c.ReportDimensions) + appendNonEmpty(builder, "reportMeasures", c.ReportMeasures) + appendNonEmpty(builder, "reportFilters", c.ReportFilters) + appendNonEmpty(builder, "reportOrderBy", c.ReportOrderBy) + appendNonEmpty(builder, "reportLimit", c.ReportLimit) + appendNonEmpty(builder, "reportOffset", c.ReportOffset) return &tagtags.Tag{Name: TagName, Values: tagtags.Values(builder.String())} } @@ -78,6 +96,22 @@ func Parse(tag reflect.StructTag) (*Tag, error) { component.Source = strings.TrimSpace(value) case "summary": component.Summary = strings.TrimSpace(value) + case "report": + component.Report = strings.EqualFold(strings.TrimSpace(value), "true") + case "reportinput": + component.ReportInput = strings.TrimSpace(value) + case "reportdimensions": + component.ReportDimensions = strings.TrimSpace(value) + case "reportmeasures": + component.ReportMeasures = strings.TrimSpace(value) + case "reportfilters": + component.ReportFilters = strings.TrimSpace(value) + case "reportorderby": + component.ReportOrderBy = strings.TrimSpace(value) + case "reportlimit": + component.ReportLimit = strings.TrimSpace(value) + case "reportoffset": + component.ReportOffset = strings.TrimSpace(value) default: return fmt.Errorf("unsupported component tag option: '%s'", key) } diff --git a/repository/shape/dql/preprocess/preprocess.go b/repository/shape/dql/preprocess/preprocess.go index 01ab95682..d932a1039 100644 --- a/repository/shape/dql/preprocess/preprocess.go +++ b/repository/shape/dql/preprocess/preprocess.go @@ -179,6 +179,18 @@ func normalizeDirectives(input *dqlshape.Directives) *dqlshape.Directives { Methods: normalizedMethods, } } + if input.Report != nil { + ret.Report = &dqlshape.ReportDirective{ + Enabled: input.Report.Enabled, + Input: strings.TrimSpace(input.Report.Input), + Dimensions: strings.TrimSpace(input.Report.Dimensions), + Measures: strings.TrimSpace(input.Report.Measures), + Filters: strings.TrimSpace(input.Report.Filters), + OrderBy: strings.TrimSpace(input.Report.OrderBy), + Limit: strings.TrimSpace(input.Report.Limit), + Offset: strings.TrimSpace(input.Report.Offset), + } + } if len(input.Const) > 0 { ret.Const = make(map[string]string, len(input.Const)) for k, v := range input.Const { @@ -188,7 +200,7 @@ func normalizeDirectives(input *dqlshape.Directives) *dqlshape.Directives { if ret.Meta == "" && ret.DefaultConnector == "" && ret.TemplateType == "" && ret.Dest == "" && ret.InputDest == "" && ret.OutputDest == "" && ret.RouterDest == "" && ret.InputType == "" && ret.OutputType == "" && - ret.Cache == nil && ret.MCP == nil && ret.Route == nil && + ret.Cache == nil && ret.MCP == nil && ret.Route == nil && ret.Report == nil && ret.JSONMarshalType == "" && ret.JSONUnmarshalType == "" && ret.XMLUnmarshalType == "" && ret.Format == "" && ret.DateFormat == "" && ret.CaseFormat == "" && len(ret.Const) == 0 { return nil diff --git a/repository/shape/dql/preprocess/preprocess_test.go b/repository/shape/dql/preprocess/preprocess_test.go index 9a3433d84..29e06bcb4 100644 --- a/repository/shape/dql/preprocess/preprocess_test.go +++ b/repository/shape/dql/preprocess/preprocess_test.go @@ -131,6 +131,7 @@ func TestPrepare_InvalidMultilineImportDiagnostic(t *testing.T) { func TestPrepare_SpecialDirectives(t *testing.T) { dql := "#settings($_ = $meta('docs/orders.md'))\n" + "#setting($_ = $connector('analytics'))\n" + + "#setting($_ = $report('OrderReportInput','Dims','Metrics','Predicates','Sort','Take','Skip'))\n" + "#setting($_ = $dest('vendor.go'))\n" + "#setting($_ = $input_dest('vendor_input.go'))\n" + "#setting($_ = $output_dest('vendor_output.go'))\n" + @@ -152,6 +153,15 @@ func TestPrepare_SpecialDirectives(t *testing.T) { require.NotNil(t, pre.Directives) assert.Equal(t, "docs/orders.md", pre.Directives.Meta) assert.Equal(t, "analytics", pre.Directives.DefaultConnector) + require.NotNil(t, pre.Directives.Report) + assert.True(t, pre.Directives.Report.Enabled) + assert.Equal(t, "OrderReportInput", pre.Directives.Report.Input) + assert.Equal(t, "Dims", pre.Directives.Report.Dimensions) + assert.Equal(t, "Metrics", pre.Directives.Report.Measures) + assert.Equal(t, "Predicates", pre.Directives.Report.Filters) + assert.Equal(t, "Sort", pre.Directives.Report.OrderBy) + assert.Equal(t, "Take", pre.Directives.Report.Limit) + assert.Equal(t, "Skip", pre.Directives.Report.Offset) assert.Equal(t, "vendor.go", pre.Directives.Dest) assert.Equal(t, "vendor_input.go", pre.Directives.InputDest) assert.Equal(t, "vendor_output.go", pre.Directives.OutputDest) diff --git a/repository/shape/dql/preprocess/settings_directives.go b/repository/shape/dql/preprocess/settings_directives.go index b946d128d..4327a7780 100644 --- a/repository/shape/dql/preprocess/settings_directives.go +++ b/repository/shape/dql/preprocess/settings_directives.go @@ -18,6 +18,7 @@ var ( cacheDirectiveName = map[string]bool{"cache": true} mcpDirectiveName = map[string]bool{"mcp": true} routeDirectiveName = map[string]bool{"route": true} + reportDirectiveName = map[string]bool{"report": true} constDirectiveName = map[string]bool{"const": true} marshalDirectiveName = map[string]bool{"marshal": true} unmarshalDirectiveName = map[string]bool{"unmarshal": true} @@ -111,6 +112,18 @@ func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, direct directives.Route = values[len(values)-1] } } + if strings.Contains(lower, "$report") { + calls, parseErrors := scanDollarCallsStrict(input, reportDirectiveName) + diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirRoute, fullDQL, diagnosticOffset) + values := parseReportDirectiveCalls(calls) + if len(values) == 0 { + if len(calls) > 0 { + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirRoute, "invalid $report directive", "expected: #settings($_ = $report()) or #settings($_ = $report('InputType','Dimensions','Measures','Filters','OrderBy','Limit','Offset'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + } + } else { + directives.Report = values[len(values)-1] + } + } if strings.Contains(lower, "$const") { calls, parseErrors := scanDollarCallsStrict(input, constDirectiveName) diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirConst, fullDQL, diagnosticOffset) @@ -548,6 +561,49 @@ func parseRouteDirectiveCalls(calls []directiveCall) []*dqlshape.RouteDirective return result } +func parseReportDirectiveCalls(calls []directiveCall) []*dqlshape.ReportDirective { + result := make([]*dqlshape.ReportDirective, 0, len(calls)) + for _, call := range calls { + args := make([]string, 0, len(call.args)) + valid := true + for _, raw := range call.args { + value, ok := parseQuotedLiteral(raw) + if !ok { + valid = false + break + } + args = append(args, strings.TrimSpace(value)) + } + if !valid { + continue + } + directive := &dqlshape.ReportDirective{Enabled: true} + if len(args) > 0 { + directive.Input = strings.TrimSpace(args[0]) + } + if len(args) > 1 { + directive.Dimensions = strings.TrimSpace(args[1]) + } + if len(args) > 2 { + directive.Measures = strings.TrimSpace(args[2]) + } + if len(args) > 3 { + directive.Filters = strings.TrimSpace(args[3]) + } + if len(args) > 4 { + directive.OrderBy = strings.TrimSpace(args[4]) + } + if len(args) > 5 { + directive.Limit = strings.TrimSpace(args[5]) + } + if len(args) > 6 { + directive.Offset = strings.TrimSpace(args[6]) + } + result = append(result, directive) + } + return result +} + func normalizeHTTPMethods(input []string) ([]string, bool) { if len(input) == 0 { return nil, true diff --git a/repository/shape/dql/shape/model.go b/repository/shape/dql/shape/model.go index 2271d725e..5b35af53a 100644 --- a/repository/shape/dql/shape/model.go +++ b/repository/shape/dql/shape/model.go @@ -52,6 +52,7 @@ type Directives struct { Cache *CacheDirective MCP *MCPDirective Route *RouteDirective + Report *ReportDirective Const map[string]string JSONMarshalType string JSONUnmarshalType string @@ -81,6 +82,17 @@ type RouteDirective struct { Methods []string } +type ReportDirective struct { + Enabled bool + Input string + Dimensions string + Measures string + Filters string + OrderBy string + Limit string + Offset string +} + type Route struct { Name string URI string diff --git a/repository/shape/load/loader.go b/repository/shape/load/loader.go index f905d7a4a..d5013ecb5 100644 --- a/repository/shape/load/loader.go +++ b/repository/shape/load/loader.go @@ -245,6 +245,29 @@ func buildComponent(source *shape.Source, pResult *plan.Result, resource *view.R synthesizeMutableExecHelpers(component, resource) component.Input = append(component.Input, synthesizePredicateStates(component.Input, component.Predicates)...) component.Directives = cloneDirectives(pResult.Directives) + if primary := firstComponentRoute(pResult.Components); primary != nil && primary.Report != nil { + component.Report = &dqlshape.ReportDirective{ + Enabled: primary.Report.Enabled, + Input: strings.TrimSpace(primary.Report.Input), + Dimensions: strings.TrimSpace(primary.Report.Dimensions), + Measures: strings.TrimSpace(primary.Report.Measures), + Filters: strings.TrimSpace(primary.Report.Filters), + OrderBy: strings.TrimSpace(primary.Report.OrderBy), + Limit: strings.TrimSpace(primary.Report.Limit), + Offset: strings.TrimSpace(primary.Report.Offset), + } + } else if component.Directives != nil && component.Directives.Report != nil { + component.Report = &dqlshape.ReportDirective{ + Enabled: component.Directives.Report.Enabled, + Input: strings.TrimSpace(component.Directives.Report.Input), + Dimensions: strings.TrimSpace(component.Directives.Report.Dimensions), + Measures: strings.TrimSpace(component.Directives.Report.Measures), + Filters: strings.TrimSpace(component.Directives.Report.Filters), + OrderBy: strings.TrimSpace(component.Directives.Report.OrderBy), + Limit: strings.TrimSpace(component.Directives.Report.Limit), + Offset: strings.TrimSpace(component.Directives.Report.Offset), + } + } component.ColumnsDiscovery = pResult.ColumnsDiscovery component.TypeSpecs = resolveTypeSpecs(pResult) return component @@ -1444,10 +1467,22 @@ func cloneDirectives(input *dqlshape.Directives) *dqlshape.Directives { } } } + if input.Report != nil { + ret.Report = &dqlshape.ReportDirective{ + Enabled: input.Report.Enabled, + Input: strings.TrimSpace(input.Report.Input), + Dimensions: strings.TrimSpace(input.Report.Dimensions), + Measures: strings.TrimSpace(input.Report.Measures), + Filters: strings.TrimSpace(input.Report.Filters), + OrderBy: strings.TrimSpace(input.Report.OrderBy), + Limit: strings.TrimSpace(input.Report.Limit), + Offset: strings.TrimSpace(input.Report.Offset), + } + } if ret.Meta == "" && ret.DefaultConnector == "" && ret.TemplateType == "" && ret.Dest == "" && ret.InputDest == "" && ret.OutputDest == "" && ret.RouterDest == "" && ret.InputType == "" && ret.OutputType == "" && - ret.Cache == nil && ret.MCP == nil && ret.Route == nil && len(ret.Const) == 0 { + ret.Cache == nil && ret.MCP == nil && ret.Route == nil && ret.Report == nil && len(ret.Const) == 0 { return nil } return ret diff --git a/repository/shape/load/loader_test.go b/repository/shape/load/loader_test.go index 6cde9b044..cfdc2e2ac 100644 --- a/repository/shape/load/loader_test.go +++ b/repository/shape/load/loader_test.go @@ -126,6 +126,12 @@ type typedRouteSource struct { Route xdatly.Component[typedRouteInput, typedRouteOutput] `component:",path=/v1/api/dev/report,method=GET"` } +type reportEnabledLoadSource struct { + embeddedFS + Rows []reportRow `view:"rows,table=REPORT" sql:"uri=testdata/report.sql"` + Route xdatly.Component[typedRouteInput, typedRouteOutput] `component:",path=/v1/api/dev/report,method=GET,report=true,reportInput=NamedReportInput,reportDimensions=Dims,reportMeasures=Metrics,reportFilters=Predicates,reportOrderBy=Sort,reportLimit=Take,reportOffset=Skip"` +} + type dynamicRouteInput struct { Name string } @@ -204,6 +210,29 @@ func TestLoader_LoadViews(t *testing.T) { require.NotNil(t, artifacts.Resource.EmbedFS()) } +func TestLoader_LoadComponent_PreservesReportConfig(t *testing.T) { + scanned, err := scan.New().Scan(context.Background(), &shape.Source{Struct: &reportEnabledLoadSource{}}) + require.NoError(t, err) + + planned, err := plan.New().Plan(context.Background(), scanned) + require.NoError(t, err) + + artifact, err := New().LoadComponent(context.Background(), planned) + require.NoError(t, err) + + component, ok := ComponentFrom(artifact) + require.True(t, ok) + require.NotNil(t, component.Report) + assert.True(t, component.Report.Enabled) + assert.Equal(t, "NamedReportInput", component.Report.Input) + assert.Equal(t, "Dims", component.Report.Dimensions) + assert.Equal(t, "Metrics", component.Report.Measures) + assert.Equal(t, "Predicates", component.Report.Filters) + assert.Equal(t, "Sort", component.Report.OrderBy) + assert.Equal(t, "Take", component.Report.Limit) + assert.Equal(t, "Skip", component.Report.Offset) +} + func TestLoader_LoadResource(t *testing.T) { planned := &shape.PlanResult{ Source: &shape.Source{Name: "report"}, diff --git a/repository/shape/load/model.go b/repository/shape/load/model.go index 9666bdffa..f091ee336 100644 --- a/repository/shape/load/model.go +++ b/repository/shape/load/model.go @@ -28,6 +28,7 @@ type Component struct { Predicates map[string][]*plan.ViewPredicate TypeContext *typectx.Context Directives *dqlshape.Directives + Report *dqlshape.ReportDirective ColumnsDiscovery bool TypeSpecs map[string]*TypeSpec diff --git a/repository/shape/plan/model.go b/repository/shape/plan/model.go index 1aa77ba6e..38eaca97b 100644 --- a/repository/shape/plan/model.go +++ b/repository/shape/plan/model.go @@ -46,6 +46,7 @@ type ComponentRoute struct { Connector string Marshaller string Handler string + Report *dqlshape.ReportDirective } // Type is normalized type metadata collected during compile. diff --git a/repository/shape/plan/planner.go b/repository/shape/plan/planner.go index 11852dcd4..2340c185c 100644 --- a/repository/shape/plan/planner.go +++ b/repository/shape/plan/planner.go @@ -10,6 +10,7 @@ import ( metakeys "github.com/viant/datly/repository/locator/meta/keys" outputkeys "github.com/viant/datly/repository/locator/output/keys" "github.com/viant/datly/repository/shape" + dqlshape "github.com/viant/datly/repository/shape/dql/shape" "github.com/viant/datly/repository/shape/scan" "github.com/viant/datly/view/state" ) @@ -303,6 +304,18 @@ func normalizeComponent(field *scan.Field) *ComponentRoute { result.ViewName = strings.TrimSpace(tag.View) result.SourceURL = strings.TrimSpace(tag.Source) result.SummaryURL = strings.TrimSpace(tag.Summary) + if tag.Report || tag.ReportInput != "" { + result.Report = &dqlshape.ReportDirective{ + Enabled: tag.Report, + Input: strings.TrimSpace(tag.ReportInput), + Dimensions: strings.TrimSpace(tag.ReportDimensions), + Measures: strings.TrimSpace(tag.ReportMeasures), + Filters: strings.TrimSpace(tag.ReportFilters), + OrderBy: strings.TrimSpace(tag.ReportOrderBy), + Limit: strings.TrimSpace(tag.ReportLimit), + Offset: strings.TrimSpace(tag.ReportOffset), + } + } } return result } diff --git a/repository/shape/plan/planner_test.go b/repository/shape/plan/planner_test.go index cce058649..483498636 100644 --- a/repository/shape/plan/planner_test.go +++ b/repository/shape/plan/planner_test.go @@ -70,6 +70,10 @@ type typedRouteSource struct { Route xdatly.Component[typedRouteInput, typedRouteOutput] `component:",path=/v1/api/dev/report,method=GET"` } +type reportRouteSource struct { + Route xdatly.Component[typedRouteInput, typedRouteOutput] `component:",path=/v1/api/dev/report,method=GET,report=true,reportInput=NamedReportInput,reportDimensions=Dims,reportMeasures=Metrics,reportFilters=Predicates,reportOrderBy=Sort,reportLimit=Take,reportOffset=Skip"` +} + type dynamicRouteInput struct { Name string } @@ -233,6 +237,28 @@ func TestPlanner_Plan_ComponentHolderTypes(t *testing.T) { assert.Empty(t, result.Components[0].OutputName) } +func TestPlanner_Plan_ComponentReportTags(t *testing.T) { + scanner := scan.New() + scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportRouteSource{}}) + require.NoError(t, err) + + planned, err := New().Plan(context.Background(), scanned) + require.NoError(t, err) + + result, ok := ResultFrom(planned) + require.True(t, ok) + require.Len(t, result.Components, 1) + require.NotNil(t, result.Components[0].Report) + assert.True(t, result.Components[0].Report.Enabled) + assert.Equal(t, "NamedReportInput", result.Components[0].Report.Input) + assert.Equal(t, "Dims", result.Components[0].Report.Dimensions) + assert.Equal(t, "Metrics", result.Components[0].Report.Measures) + assert.Equal(t, "Predicates", result.Components[0].Report.Filters) + assert.Equal(t, "Sort", result.Components[0].Report.OrderBy) + assert.Equal(t, "Take", result.Components[0].Report.Limit) + assert.Equal(t, "Skip", result.Components[0].Report.Offset) +} + func TestPlanner_Plan_DynamicComponentHolderTypes(t *testing.T) { scanner := scan.New() scanned, err := scanner.Scan(context.Background(), &shape.Source{Struct: &struct { diff --git a/repository/shape/scan/scanner_test.go b/repository/shape/scan/scanner_test.go index c3c0f7fe6..fc0bbda71 100644 --- a/repository/shape/scan/scanner_test.go +++ b/repository/shape/scan/scanner_test.go @@ -48,6 +48,10 @@ type typedComponentSource struct { Route xdatly.Component[reportInput, reportOutput] `component:",path=/v1/api/dev/report,method=GET"` } +type reportEnabledSource struct { + Route xdatly.Component[reportInput, reportOutput] `component:",path=/v1/api/dev/report,method=GET,report=true,reportInput=NamedReportInput,reportDimensions=Dims,reportMeasures=Metrics,reportFilters=Predicates,reportOrderBy=Sort,reportLimit=Take,reportOffset=Skip"` +} + type dynamicReportInput struct { Name string } @@ -135,6 +139,28 @@ func TestStructScanner_Scan_ComponentHolderTypes(t *testing.T) { assert.Empty(t, route.ComponentOutputName) } +func TestStructScanner_Scan_ComponentReportTags(t *testing.T) { + scanner := New() + result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &reportEnabledSource{}}) + require.NoError(t, err) + + descriptors, ok := DescriptorsFrom(result) + require.True(t, ok) + require.Len(t, descriptors.ComponentFields, 1) + route := descriptors.ComponentFields[0] + require.NotNil(t, route) + require.NotNil(t, route.ComponentTag) + require.NotNil(t, route.ComponentTag.Component) + assert.True(t, route.ComponentTag.Component.Report) + assert.Equal(t, "NamedReportInput", route.ComponentTag.Component.ReportInput) + assert.Equal(t, "Dims", route.ComponentTag.Component.ReportDimensions) + assert.Equal(t, "Metrics", route.ComponentTag.Component.ReportMeasures) + assert.Equal(t, "Predicates", route.ComponentTag.Component.ReportFilters) + assert.Equal(t, "Sort", route.ComponentTag.Component.ReportOrderBy) + assert.Equal(t, "Take", route.ComponentTag.Component.ReportLimit) + assert.Equal(t, "Skip", route.ComponentTag.Component.ReportOffset) +} + func TestStructScanner_Scan_QuerySelectorHolder(t *testing.T) { scanner := New() result, err := scanner.Scan(context.Background(), &shape.Source{Struct: &selectorHolderSource{}}) diff --git a/repository/shape/xgen/codegen.go b/repository/shape/xgen/codegen.go index 0c49c9f1b..7734557b1 100644 --- a/repository/shape/xgen/codegen.go +++ b/repository/shape/xgen/codegen.go @@ -3313,6 +3313,9 @@ func (g *ComponentCodegen) renderComponentHolder(builder *strings.Builder, compo if summaryURL := strings.TrimSpace(g.rootSummarySourceURL()); summaryURL != "" { tag += fmt.Sprintf(`,summary=%s`, summaryURL) } + if reportTag := g.reportComponentTag(); reportTag != "" { + tag += reportTag + } tag += `"` builder.WriteString(fmt.Sprintf("type %sRouter struct {\n", componentName)) builder.WriteString(fmt.Sprintf("\t%s xdatly.Component[%s, %s] `%s`\n", componentName, inputTypeName, outputTypeName, tag)) @@ -3354,7 +3357,14 @@ func (g *ComponentCodegen) renderDefineComponent(builder *strings.Builder, compo if connectorRef != "" { builder.WriteString(fmt.Sprintf(`, view.WithConnectorRef(%q)`, connectorRef)) } - builder.WriteString("))\n\n") + builder.WriteString(")") + builder.WriteString(")") + if reportOption := g.reportComponentOption(); reportOption != "" { + builder.WriteString(",\n") + builder.WriteString("\t\t") + builder.WriteString(reportOption) + } + builder.WriteString(")\n\n") builder.WriteString("\tif err != nil {\n") builder.WriteString(fmt.Sprintf("\t\treturn fmt.Errorf(\"failed to create %s component: %%w\", err)\n", componentName)) builder.WriteString("\t}\n") @@ -3365,6 +3375,66 @@ func (g *ComponentCodegen) renderDefineComponent(builder *strings.Builder, compo builder.WriteString("}\n\n") } +func (g *ComponentCodegen) reportComponentTag() string { + if g.Component == nil || g.Component.Report == nil || !g.Component.Report.Enabled { + return "" + } + report := g.Component.Report + tag := ",report=true" + if value := strings.TrimSpace(report.Input); value != "" { + tag += fmt.Sprintf(",reportInput=%s", value) + } + if value := strings.TrimSpace(report.Dimensions); value != "" { + tag += fmt.Sprintf(",reportDimensions=%s", value) + } + if value := strings.TrimSpace(report.Measures); value != "" { + tag += fmt.Sprintf(",reportMeasures=%s", value) + } + if value := strings.TrimSpace(report.Filters); value != "" { + tag += fmt.Sprintf(",reportFilters=%s", value) + } + if value := strings.TrimSpace(report.OrderBy); value != "" { + tag += fmt.Sprintf(",reportOrderBy=%s", value) + } + if value := strings.TrimSpace(report.Limit); value != "" { + tag += fmt.Sprintf(",reportLimit=%s", value) + } + if value := strings.TrimSpace(report.Offset); value != "" { + tag += fmt.Sprintf(",reportOffset=%s", value) + } + return tag +} + +func (g *ComponentCodegen) reportComponentOption() string { + if g.Component == nil || g.Component.Report == nil || !g.Component.Report.Enabled { + return "" + } + report := g.Component.Report + parts := []string{"Enabled: true"} + if value := strings.TrimSpace(report.Input); value != "" { + parts = append(parts, fmt.Sprintf("Input: %q", value)) + } + if value := strings.TrimSpace(report.Dimensions); value != "" { + parts = append(parts, fmt.Sprintf("Dimensions: %q", value)) + } + if value := strings.TrimSpace(report.Measures); value != "" { + parts = append(parts, fmt.Sprintf("Measures: %q", value)) + } + if value := strings.TrimSpace(report.Filters); value != "" { + parts = append(parts, fmt.Sprintf("Filters: %q", value)) + } + if value := strings.TrimSpace(report.OrderBy); value != "" { + parts = append(parts, fmt.Sprintf("OrderBy: %q", value)) + } + if value := strings.TrimSpace(report.Limit); value != "" { + parts = append(parts, fmt.Sprintf("Limit: %q", value)) + } + if value := strings.TrimSpace(report.Offset); value != "" { + parts = append(parts, fmt.Sprintf("Offset: %q", value)) + } + return fmt.Sprintf("repository.WithReport(&repository.Report{%s})", strings.Join(parts, ", ")) +} + func (g *ComponentCodegen) rootConnectorRef() string { if g.Resource == nil { return "" diff --git a/repository/shape/xgen/codegen_groupable_test.go b/repository/shape/xgen/codegen_groupable_test.go index 5ad9758b5..083dfda9c 100644 --- a/repository/shape/xgen/codegen_groupable_test.go +++ b/repository/shape/xgen/codegen_groupable_test.go @@ -61,6 +61,16 @@ func TestComponentCodegen_GeneratesSelectorHolderOutsideBusinessInput(t *testing Method: "GET", URI: "/v1/api/dev/vendors-grouping", RootView: "Vendor", + Report: &dqlshape.ReportDirective{ + Enabled: true, + Input: "VendorReportInput", + Dimensions: "Dims", + Measures: "Metrics", + Filters: "Predicates", + OrderBy: "Sort", + Limit: "Take", + Offset: "Skip", + }, Directives: &dqlshape.Directives{ InputDest: "vendor_input.go", OutputDest: "vendor_output.go", @@ -114,6 +124,9 @@ func TestComponentCodegen_GeneratesSelectorHolderOutsideBusinessInput(t *testing require.NoError(t, err) assert.Contains(t, string(routerSource), "ViewSelect struct {") assert.Contains(t, string(routerSource), `querySelector:"vendor"`) + assert.Contains(t, string(routerSource), `report=true`) + assert.Contains(t, string(routerSource), `reportInput=VendorReportInput`) + assert.Contains(t, string(routerSource), `reportDimensions=Dims`) assert.Contains(t, string(routerSource), `Fields []string `+"`"+`parameter:"`) assert.Contains(t, string(routerSource), `in=_fields`) assert.Contains(t, string(routerSource), `OrderBy string `+"`"+`parameter:"`) @@ -121,6 +134,7 @@ func TestComponentCodegen_GeneratesSelectorHolderOutsideBusinessInput(t *testing outputSource, err := os.ReadFile(result.OutputFilePath) require.NoError(t, err) + assert.Contains(t, string(outputSource), `repository.WithReport(&repository.Report{Enabled: true, Input: "VendorReportInput", Dimensions: "Dims", Measures: "Metrics", Filters: "Predicates", OrderBy: "Sort", Limit: "Take", Offset: "Skip"})`) assert.Contains(t, string(outputSource), `view:"Vendor,groupable=true`) assert.Contains(t, string(outputSource), `selectorOrderBy=true`) assert.Contains(t, string(outputSource), `selectorOrderByColumns={accountId:ACCOUNT_ID}`) diff --git a/service.go b/service.go index 3f7e9b28b..2c984ddf3 100644 --- a/service.go +++ b/service.go @@ -587,7 +587,14 @@ func (s *Service) AddComponent(ctx context.Context, component *repository.Compon return err } - s.repository.Register(components.Components...) + registerComponents := append([]*repository.Component{}, components.Components...) + if reportComponent, err := repository.BuildReportComponent(s.repository.Registry().Dispatcher(), components.Components[0]); err != nil { + return err + } else if reportComponent != nil { + registerComponents = append(registerComponents, reportComponent) + } + + s.repository.Register(registerComponents...) return nil } diff --git a/service/reader/sql.go b/service/reader/sql.go index ad7f35d68..02568c72b 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -209,13 +209,35 @@ func (b *Builder) appendSelectorColumns(sb *strings.Builder, aView *view.View, s } sb.WriteString(" ") - sb.WriteString(viewColumn.SqlExpression()) + if aView.Groupable { + sb.WriteString(groupedProjectionExpression(viewColumn)) + } else { + sb.WriteString(viewColumn.SqlExpression()) + } result = append(result, viewColumn) } return result, nil } +func groupedProjectionExpression(column *view.Column) string { + if column == nil { + return "" + } + expr := column.Name + if defaultValue := columnDefaultValue(column); defaultValue != "" { + return "COALESCE(" + expr + "," + defaultValue + ") AS " + column.Name + } + return expr +} + +func columnDefaultValue(column *view.Column) string { + if column == nil { + return "" + } + return column.DefaultValue() +} + func (b *Builder) viewAlias(view *view.View) string { var alias string if view.Alias != "" { diff --git a/view/column.go b/view/column.go index 6f83f707e..9a20680a9 100644 --- a/view/column.go +++ b/view/column.go @@ -22,6 +22,7 @@ type ( Tag string `json:",omitempty"` Expression string `json:",omitempty"` + Aggregate bool `json:",omitempty"` Filterable bool `json:",omitempty"` Groupable bool `json:",omitempty"` Nullable bool `json:",omitempty"` @@ -187,6 +188,10 @@ func (c *Column) defaultValue(rType reflect.Type) string { } } +func (c *Column) DefaultValue() string { + return c.defaultValue(c.rType) +} + func (c *Column) FieldName() string { return c._fieldName } diff --git a/view/columns.go b/view/columns.go index 44b5c8f19..a4cf162ad 100644 --- a/view/columns.go +++ b/view/columns.go @@ -197,6 +197,8 @@ func NewColumns(columns sqlparser.Columns, config map[string]*ColumnConfig) Colu } name = item.Identity() column := NewColumn(name, item.Type, item.RawType, item.IsNullable, WithColumnTag(item.Tag)) + column.Expression = item.Expression + column.Aggregate = isAggregateProjection(item.Expression) if item.Name != item.Alias && item.Alias != "" && item.Name != "" { column.Tag += fmt.Sprintf(`source:"%v"`, item.Name) } @@ -210,3 +212,17 @@ func NewColumns(columns sqlparser.Columns, config map[string]*ColumnConfig) Colu } return result } + +func isAggregateProjection(expression string) bool { + expression = strings.ToLower(strings.TrimSpace(expression)) + switch { + case strings.Contains(expression, "count("), + strings.Contains(expression, "sum("), + strings.Contains(expression, "avg("), + strings.Contains(expression, "min("), + strings.Contains(expression, "max("): + return true + default: + return false + } +} diff --git a/view/views.go b/view/views.go index 494b30d2a..9fcc27123 100644 --- a/view/views.go +++ b/view/views.go @@ -72,7 +72,7 @@ func (n *NamespacedView) indexView(aView *View, aPath string) { nsView.Root = true nsView.Namespaces = append(nsView.Namespaces, "") } - if selector.Namespace != "" { + if selector != nil && selector.Namespace != "" { nsView.Namespaces = append(nsView.Namespaces, selector.Namespace) } n.Views = append(n.Views, nsView) From bcd0f0c5bbcd1d56d6391850e6cab20507e796f5 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 13:20:42 -0700 Subject: [PATCH 169/279] added dynamic grouping --- gateway/mcp.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index 4bf053020..a473723a3 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -70,8 +70,7 @@ func (r *Router) mcpToolCallHandler(component *repository.Component, aRoute *Rou // 2) Apply parameters to request URL/query/body for _, p := range allParams { - name := strings.Title(p.Name) - value := params.Arguments[name] + value := toolArgumentValue(p, params.Arguments) pType := p.Schema.Type() if pType.Kind() == reflect.Ptr { pType = pType.Elem() @@ -404,6 +403,10 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty } appendField(name, parameter.Schema.Type(), tag) case state.KindRequestBody: + if parameter.IsAnonymous() { + appendAnonymousBodyFields(&inputFields, uniqueFieldName, parameter.Schema.Type()) + continue + } // If body is a slice, mark optional in schema. var tag reflect.StructTag if parameter.Schema != nil && parameter.Schema.Type().Kind() == reflect.Slice { @@ -445,6 +448,74 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty return reflect.StructOf(inputFields) } +func toolArgumentValue(parameter *state.Parameter, arguments map[string]interface{}) interface{} { + if parameter == nil { + return nil + } + if parameter.In != nil && parameter.In.Kind == state.KindRequestBody && parameter.IsAnonymous() && parameter.Schema != nil { + return anonymousBodyArgumentValue(arguments, parameter.Schema.Type()) + } + return arguments[strings.Title(parameter.Name)] +} + +func appendAnonymousBodyFields(fields *[]reflect.StructField, unique map[string]bool, bodyType reflect.Type) { + bodyType = indirectType(bodyType) + if bodyType == nil || bodyType.Kind() != reflect.Struct { + return + } + for i := 0; i < bodyType.NumField(); i++ { + field := bodyType.Field(i) + if !field.IsExported() { + continue + } + if unique[field.Name] { + continue + } + unique[field.Name] = true + *fields = append(*fields, field) + } +} + +func anonymousBodyArgumentValue(arguments map[string]interface{}, bodyType reflect.Type) interface{} { + bodyType = indirectType(bodyType) + if bodyType == nil || bodyType.Kind() != reflect.Struct { + return nil + } + payload := map[string]interface{}{} + for i := 0; i < bodyType.NumField(); i++ { + field := bodyType.Field(i) + if !field.IsExported() { + continue + } + value, ok := arguments[field.Name] + if !ok { + continue + } + payload[jsonFieldName(field)] = value + } + if len(payload) == 0 { + return nil + } + return payload +} + +func jsonFieldName(field reflect.StructField) string { + if tag := field.Tag.Get("json"); tag != "" { + parts := strings.Split(tag, ",") + if parts[0] != "" && parts[0] != "-" { + return parts[0] + } + } + return strings.ToLower(field.Name[:1]) + field.Name[1:] +} + +func indirectType(rType reflect.Type) reflect.Type { + for rType != nil && rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + return rType +} + func (r *Router) buildTemplateResourceIntegration(item *dpath.Item, aPath *dpath.Path, aRoute *Route, provider *repository.Provider) error { if aPath.Internal { return nil From e78ea4f84309b686207f73e6f0bebbe7dd096eb0 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 13:22:16 -0700 Subject: [PATCH 170/279] added dynamic grouping --- repository/report.go | 111 +++++++++ repository/report_handler.go | 294 ++++++++++++++++++++++ repository/report_handler_test.go | 199 +++++++++++++++ repository/report_runtime.go | 398 ++++++++++++++++++++++++++++++ repository/report_runtime_test.go | 174 +++++++++++++ 5 files changed, 1176 insertions(+) create mode 100644 repository/report.go create mode 100644 repository/report_handler.go create mode 100644 repository/report_handler_test.go create mode 100644 repository/report_runtime.go create mode 100644 repository/report_runtime_test.go diff --git a/repository/report.go b/repository/report.go new file mode 100644 index 000000000..6117ff4f2 --- /dev/null +++ b/repository/report.go @@ -0,0 +1,111 @@ +package repository + +import ( + "fmt" + "reflect" + "strings" + + "github.com/viant/datly/view/state" +) + +type Report struct { + Enabled bool `json:",omitempty" yaml:"Enabled,omitempty"` + Input string `json:",omitempty" yaml:"Input,omitempty"` + Dimensions string `json:",omitempty" yaml:"Dimensions,omitempty"` + Measures string `json:",omitempty" yaml:"Measures,omitempty"` + Filters string `json:",omitempty" yaml:"Filters,omitempty"` + OrderBy string `json:",omitempty" yaml:"OrderBy,omitempty"` + Limit string `json:",omitempty" yaml:"Limit,omitempty"` + Offset string `json:",omitempty" yaml:"Offset,omitempty"` +} + +type ReportMetadata struct { + InputName string + BodyFieldName string + DimensionsKey string + MeasuresKey string + FiltersKey string + Dimensions []*ReportField + Measures []*ReportField + Filters []*ReportFilter + OrderBy string + Limit string + Offset string +} + +type ReportField struct { + Name string + FieldName string + Section string + Description string +} + +type ReportFilter struct { + Name string + FieldName string + Section string + Description string + Parameter *state.Parameter +} + +func (r *Report) Clone() *Report { + if r == nil { + return nil + } + ret := *r + return &ret +} + +func (r *Report) normalize() *Report { + if r == nil { + return nil + } + ret := r.Clone() + ret.Input = strings.TrimSpace(ret.Input) + ret.Dimensions = defaultString(ret.Dimensions, "Dimensions") + ret.Measures = defaultString(ret.Measures, "Measures") + ret.Filters = defaultString(ret.Filters, "Filters") + ret.OrderBy = defaultString(ret.OrderBy, "OrderBy") + ret.Limit = defaultString(ret.Limit, "Limit") + ret.Offset = defaultString(ret.Offset, "Offset") + return ret +} + +func (r *Report) inputTypeName(componentName, inputName, viewName string) string { + if r != nil && strings.TrimSpace(r.Input) != "" { + return strings.TrimSpace(r.Input) + } + switch { + case strings.TrimSpace(inputName) != "": + return state.SanitizeTypeName(strings.TrimSpace(inputName) + "ReportInput") + case strings.TrimSpace(componentName) != "": + return state.SanitizeTypeName(strings.TrimSpace(componentName) + "ReportInput") + default: + return state.SanitizeTypeName(strings.TrimSpace(viewName) + "ReportInput") + } +} + +func (r *ReportMetadata) validateSelection() error { + if r == nil { + return fmt.Errorf("report metadata was empty") + } + if len(r.Dimensions) == 0 && len(r.Measures) == 0 { + return fmt.Errorf("report metadata had no selectable dimensions or measures") + } + return nil +} + +func (r *ReportFilter) schemaType() reflect.Type { + if r == nil || r.Parameter == nil || r.Parameter.Schema == nil { + return nil + } + return r.Parameter.OutputType() +} + +func defaultString(value, fallback string) string { + value = strings.TrimSpace(value) + if value == "" { + return fallback + } + return value +} diff --git a/repository/report_handler.go b/repository/report_handler.go new file mode 100644 index 000000000..a5a157566 --- /dev/null +++ b/repository/report_handler.go @@ -0,0 +1,294 @@ +package repository + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "reflect" + "strconv" + "strings" + + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/view/state" + xhandler "github.com/viant/xdatly/handler" + xdhttp "github.com/viant/xdatly/handler/http" +) + +type reportHandler struct { + Dispatcher contract.Dispatcher + Path *contract.Path + Metadata *ReportMetadata + Original *Component + BodyType reflect.Type +} + +func (r *reportHandler) Exec(ctx context.Context, session xhandler.Session) (interface{}, error) { + if r == nil || r.Dispatcher == nil || r.Path == nil || r.Metadata == nil || r.Original == nil { + return nil, fmt.Errorf("report handler was not initialized") + } + request, err := session.Http().NewRequest(ctx) + if err != nil { + return nil, err + } + input, err := r.reportInput(ctx, request) + if err != nil { + return nil, err + } + query, err := r.buildQuery(input) + if err != nil { + return nil, err + } + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + viewName := "" + namespacedNil := true + if r.Original.View != nil { + viewName = r.Original.View.Name + } + if r.Original.NamespacedView != nil { + namespacedNil = false + } + fmt.Printf("[DATLY_REPORT_HANDLER] original=%s method=%s target=%s view=%s namespaced_nil=%v query=%s\n", r.Original.Path.Key(), r.Path.Method, r.Path.URI, viewName, namespacedNil, query.Encode()) + } + internalReq := request.Clone(ctx) + internalReq.Method = r.Path.Method + internalReq.URL = cloneURL(request.URL) + internalReq.URL.Path = strings.TrimSuffix(request.URL.Path, "/report") + internalReq.URL.RawPath = internalReq.URL.Path + internalReq.URL.RawQuery = query.Encode() + internalReq.RequestURI = internalReq.URL.RequestURI() + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + fmt.Printf("[DATLY_REPORT_HANDLER] dispatch request_uri=%s header_auth=%q\n", internalReq.RequestURI, internalReq.Header.Get("Authorization")) + } + redirect := &xdhttp.Route{URL: r.Path.URI, Method: r.Path.Method} + return nil, session.Http().Redirect(ctx, redirect, internalReq) +} + +func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) (interface{}, error) { + if request != nil && request.Body != nil && r.BodyType != nil { + payload, err := io.ReadAll(request.Body) + if err != nil { + return nil, err + } + if len(payload) > 0 { + target := reflect.New(r.BodyType) + if err := json.Unmarshal(payload, target.Interface()); err != nil { + return nil, err + } + return target.Interface(), nil + } + } + input := ctx.Value(xhandler.InputKey) + if input == nil { + return nil, fmt.Errorf("report input was empty") + } + return input, nil +} + +func (r *reportHandler) buildQuery(input interface{}) (url.Values, error) { + root := indirectValue(reflect.ValueOf(input)) + if !root.IsValid() || root.Kind() != reflect.Struct { + return nil, fmt.Errorf("unsupported report input type %T", input) + } + root = bodyRoot(root, r.Metadata.BodyFieldName) + query := url.Values{} + fields, err := r.collectSelections(root, r.Metadata.Dimensions, r.Metadata.Measures) + if err != nil { + return nil, err + } + if len(fields) == 0 { + return nil, fmt.Errorf("report requires at least one dimension or measure") + } + if fieldsParameter := r.Original.View.Selector.FieldsParameter; fieldsParameter != nil && fieldsParameter.In != nil { + query.Set(fieldsParameter.In.Name, strings.Join(fields, ",")) + } + if err := r.collectFilters(root, query); err != nil { + return nil, err + } + if err := r.collectStrings(root, r.Metadata.OrderBy, query, r.selectorName(r.Original.View.Selector.OrderByParameter, "_orderby")); err != nil { + return nil, err + } + if err := r.collectInts(root, r.Metadata.Limit, query, r.selectorName(r.Original.View.Selector.LimitParameter, "_limit")); err != nil { + return nil, err + } + if err := r.collectInts(root, r.Metadata.Offset, query, r.selectorName(r.Original.View.Selector.OffsetParameter, "_offset")); err != nil { + return nil, err + } + return query, nil +} + +func (r *reportHandler) selectorName(parameter *state.Parameter, fallback string) string { + if parameter != nil && parameter.In != nil && strings.TrimSpace(parameter.In.Name) != "" { + return parameter.In.Name + } + return fallback +} + +func (r *reportHandler) collectSelections(root reflect.Value, groups ...[]*ReportField) ([]string, error) { + var result []string + for _, group := range groups { + for _, field := range group { + section := fieldByName(root, field.Section) + if !section.IsValid() { + continue + } + value := fieldByName(indirectValue(section), field.FieldName) + if !value.IsValid() || value.Kind() != reflect.Bool { + continue + } + if value.Bool() { + result = append(result, field.Name) + } + } + } + return result, nil +} + +func (r *reportHandler) collectFilters(root reflect.Value, query url.Values) error { + filters := fieldByName(root, r.Metadata.FiltersKey) + if !filters.IsValid() { + return nil + } + filters = indirectValue(filters) + for _, filter := range r.Metadata.Filters { + value := fieldByName(filters, filter.FieldName) + if !value.IsValid() || isEmptyValue(value) { + continue + } + if filter.Parameter == nil || filter.Parameter.In == nil { + continue + } + appendQueryValue(query, filter.Parameter.In.Name, value) + } + return nil +} + +func (r *reportHandler) collectStrings(root reflect.Value, fieldName string, query url.Values, key string) error { + if fieldName == "" { + return nil + } + value := fieldByName(root, fieldName) + if !value.IsValid() { + return nil + } + value = indirectValue(value) + if !value.IsValid() || value.Kind() != reflect.Slice { + return nil + } + var parts []string + for i := 0; i < value.Len(); i++ { + item := indirectValue(value.Index(i)) + if item.IsValid() && item.Kind() == reflect.String && item.Len() > 0 { + parts = append(parts, item.String()) + } + } + if len(parts) > 0 { + query.Set(key, strings.Join(parts, ",")) + } + return nil +} + +func (r *reportHandler) collectInts(root reflect.Value, fieldName string, query url.Values, key string) error { + if fieldName == "" { + return nil + } + value := fieldByName(root, fieldName) + if !value.IsValid() { + return nil + } + value = indirectValue(value) + if !value.IsValid() { + return nil + } + switch value.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + query.Set(key, strconv.FormatInt(value.Int(), 10)) + } + return nil +} + +func appendQueryValue(query url.Values, key string, value reflect.Value) { + value = indirectValue(value) + switch value.Kind() { + case reflect.String: + if value.String() != "" { + query.Add(key, value.String()) + } + case reflect.Bool: + query.Add(key, strconv.FormatBool(value.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + query.Add(key, strconv.FormatInt(value.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + query.Add(key, strconv.FormatUint(value.Uint(), 10)) + case reflect.Float32, reflect.Float64: + query.Add(key, strconv.FormatFloat(value.Float(), 'f', -1, 64)) + case reflect.Slice, reflect.Array: + for i := 0; i < value.Len(); i++ { + appendQueryValue(query, key, value.Index(i)) + } + } +} + +func fieldByName(root reflect.Value, name string) reflect.Value { + root = indirectValue(root) + if !root.IsValid() || root.Kind() != reflect.Struct || name == "" { + return reflect.Value{} + } + return root.FieldByName(name) +} + +func indirectValue(value reflect.Value) reflect.Value { + for value.IsValid() && value.Kind() == reflect.Ptr { + if value.IsNil() { + return reflect.Value{} + } + value = value.Elem() + } + return value +} + +func isEmptyValue(value reflect.Value) bool { + value = indirectValue(value) + if !value.IsValid() { + return true + } + switch value.Kind() { + case reflect.String, reflect.Array, reflect.Slice, reflect.Map: + return value.Len() == 0 + case reflect.Bool: + return !value.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return value.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return value.Uint() == 0 + case reflect.Float32, reflect.Float64: + return value.Float() == 0 + } + return false +} + +func cloneURL(source *url.URL) *url.URL { + if source == nil { + return &url.URL{} + } + clone := *source + return &clone +} + +func bodyRoot(root reflect.Value, bodyField string) reflect.Value { + if bodyField == "" { + return root + } + body := fieldByName(root, bodyField) + if !body.IsValid() { + return root + } + body = indirectValue(body) + if !body.IsValid() || body.Kind() != reflect.Struct { + return root + } + return body +} diff --git a/repository/report_handler_test.go b/repository/report_handler_test.go new file mode 100644 index 000000000..e09d14c50 --- /dev/null +++ b/repository/report_handler_test.go @@ -0,0 +1,199 @@ +package repository + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + xhandler "github.com/viant/xdatly/handler" + xdauth "github.com/viant/xdatly/handler/auth" + "github.com/viant/xdatly/handler/differ" + xdhttp "github.com/viant/xdatly/handler/http" + xdlogger "github.com/viant/xdatly/handler/logger" + "github.com/viant/xdatly/handler/mbus" + "github.com/viant/xdatly/handler/sqlx" + xdstate "github.com/viant/xdatly/handler/state" + "github.com/viant/xdatly/handler/validator" +) + +type captureDispatcher struct { + path *contract.Path + options *contract.Options +} + +func (d *captureDispatcher) Dispatch(ctx context.Context, path *contract.Path, options ...contract.Option) (interface{}, error) { + d.path = path + d.options = contract.NewOptions(options...) + return map[string]string{"status": "ok"}, nil +} + +type reportTestHTTP struct { + request *http.Request + redirectRoute *xdhttp.Route + redirectRequest *http.Request +} + +func (h *reportTestHTTP) RequestOf(ctx context.Context, v any) (*http.Request, error) { + return h.request, nil +} +func (h *reportTestHTTP) NewRequest(ctx context.Context, opts ...xdstate.Option) (*http.Request, error) { + return h.request, nil +} +func (h *reportTestHTTP) Redirect(ctx context.Context, route *xdhttp.Route, request *http.Request) error { + h.redirectRoute = route + h.redirectRequest = request + return nil +} +func (h *reportTestHTTP) FailWithCode(statusCode int, err error) error { return err } + +type reportTestLogger struct{} + +func (l *reportTestLogger) IsDebugEnabled() bool { return false } +func (l *reportTestLogger) IsInfoEnabled() bool { return false } +func (l *reportTestLogger) IsWarnEnabled() bool { return false } +func (l *reportTestLogger) IsErrorEnabled() bool { return false } +func (l *reportTestLogger) Info(msg string, args ...any) {} +func (l *reportTestLogger) Debug(msg string, args ...any) {} +func (l *reportTestLogger) Warn(msg string, args ...any) {} +func (l *reportTestLogger) Error(msg string, args ...any) {} +func (l *reportTestLogger) Infoc(ctx context.Context, msg string, args ...any) {} +func (l *reportTestLogger) Debugc(ctx context.Context, msg string, args ...any) {} +func (l *reportTestLogger) DebugJSONc(ctx context.Context, msg string, obj any) {} +func (l *reportTestLogger) Warnc(ctx context.Context, msg string, args ...any) {} +func (l *reportTestLogger) Errorc(ctx context.Context, msg string, args ...any) {} +func (l *reportTestLogger) Infos(ctx context.Context, msg string, attrs ...slog.Attr) {} +func (l *reportTestLogger) Debugs(ctx context.Context, msg string, attrs ...slog.Attr) {} +func (l *reportTestLogger) Warns(ctx context.Context, msg string, attrs ...slog.Attr) {} +func (l *reportTestLogger) Errors(ctx context.Context, msg string, attrs ...slog.Attr) {} + +type reportTestSession struct { + http *reportTestHTTP + logger xdlogger.Logger +} + +type reportHandlerDimensions struct { + AccountID bool +} + +type reportHandlerMeasures struct { + TotalSpend bool +} + +type reportHandlerFilters struct { + AccountID *int +} + +type reportHandlerBody struct { + Dimensions reportHandlerDimensions + Measures reportHandlerMeasures + Filters reportHandlerFilters + OrderBy []string + Limit *int + Offset *int +} + +type reportHandlerInput struct { + Report reportHandlerBody +} + +func (s *reportTestSession) Validator() *validator.Service { return nil } +func (s *reportTestSession) Differ() *differ.Service { return nil } +func (s *reportTestSession) MessageBus() *mbus.Service { return nil } +func (s *reportTestSession) Db(opts ...sqlx.Option) (*sqlx.Service, error) { return nil, nil } +func (s *reportTestSession) Stater() *xdstate.Service { return nil } +func (s *reportTestSession) FlushTemplate(ctx context.Context) error { return nil } +func (s *reportTestSession) Session(ctx context.Context, route *xdhttp.Route, opts ...xdstate.Option) (xhandler.Session, error) { + return s, nil +} +func (s *reportTestSession) Http() xdhttp.Http { return s.http } +func (s *reportTestSession) Auth() xdauth.Auth { return nil } +func (s *reportTestSession) Logger() xdlogger.Logger { return s.logger } + +func testReportHandler() *reportHandler { + return &reportHandler{ + Dispatcher: &captureDispatcher{}, + Path: &contract.Path{Method: http.MethodGet, URI: "/v1/api/vendors"}, + Metadata: &ReportMetadata{ + BodyFieldName: "Report", + DimensionsKey: "Dimensions", + MeasuresKey: "Measures", + FiltersKey: "Filters", + OrderBy: "OrderBy", + Limit: "Limit", + Offset: "Offset", + Dimensions: []*ReportField{{Name: "AccountID", FieldName: "AccountID", Section: "Dimensions"}}, + Measures: []*ReportField{{Name: "TotalSpend", FieldName: "TotalSpend", Section: "Measures"}}, + Filters: []*ReportFilter{{Name: "accountID", FieldName: "AccountID"}}, + }, + Original: &Component{ + View: &view.View{ + Selector: &view.Config{ + FieldsParameter: &state.Parameter{In: state.NewQueryLocation("_fields")}, + OrderByParameter: &state.Parameter{In: state.NewQueryLocation("_orderby")}, + LimitParameter: &state.Parameter{In: state.NewQueryLocation("_limit")}, + OffsetParameter: &state.Parameter{In: state.NewQueryLocation("_offset")}, + }, + }, + }, + } +} + +func testReportInput() reportHandlerInput { + accountID := 101 + limit := 25 + return reportHandlerInput{ + Report: reportHandlerBody{ + Dimensions: reportHandlerDimensions{AccountID: true}, + Measures: reportHandlerMeasures{TotalSpend: true}, + Filters: reportHandlerFilters{AccountID: &accountID}, + OrderBy: []string{"AccountID"}, + Limit: &limit, + }, + } +} + +func TestReportHandler_BuildQuery_FromPostBody(t *testing.T) { + handler := testReportHandler() + handler.Metadata.Filters[0].Parameter = &state.Parameter{In: state.NewQueryLocation("accountID")} + query, err := handler.buildQuery(testReportInput()) + require.NoError(t, err) + assert.Equal(t, "AccountID,TotalSpend", query.Get("_fields")) + assert.Equal(t, "AccountID", query.Get("_orderby")) + assert.Equal(t, "25", query.Get("_limit")) + assert.Equal(t, "101", query.Get("accountID")) +} + +func TestReportHandler_Exec_PreservesAuthorizationHeader(t *testing.T) { + handler := testReportHandler() + handler.Metadata.Filters[0].Parameter = &state.Parameter{In: state.NewQueryLocation("accountID")} + + req := httptest.NewRequest(http.MethodPost, "http://localhost/v1/api/vendors/report", nil) + req.Header.Set("Authorization", "Bearer test-token") + httpSession := &reportTestHTTP{request: req} + session := &reportTestSession{ + http: httpSession, + logger: &reportTestLogger{}, + } + + ctx := context.WithValue(context.Background(), xhandler.InputKey, testReportInput()) + _, err := handler.Exec(ctx, session) + require.NoError(t, err) + require.NotNil(t, httpSession.redirectRoute) + require.NotNil(t, httpSession.redirectRequest) + assert.Equal(t, "Bearer test-token", httpSession.redirectRequest.Header.Get("Authorization")) + assert.Equal(t, "/v1/api/vendors", httpSession.redirectRequest.URL.Path) + assert.Equal(t, http.MethodGet, httpSession.redirectRoute.Method) + assert.Equal(t, "/v1/api/vendors", httpSession.redirectRoute.URL) + query := httpSession.redirectRequest.URL.Query() + assert.Equal(t, "AccountID,TotalSpend", query.Get("_fields")) + assert.Equal(t, "AccountID", query.Get("_orderby")) + assert.Equal(t, "25", query.Get("_limit")) + assert.Equal(t, "101", query.Get("accountID")) +} diff --git a/repository/report_runtime.go b/repository/report_runtime.go new file mode 100644 index 000000000..653d4ebca --- /dev/null +++ b/repository/report_runtime.go @@ -0,0 +1,398 @@ +package repository + +import ( + "context" + "fmt" + "net/http" + "os" + "reflect" + "strconv" + "strings" + + "github.com/viant/datly/repository/contract" + rephandler "github.com/viant/datly/repository/handler" + "github.com/viant/datly/repository/path" + "github.com/viant/datly/service" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/tagly/format/text" +) + +func (s *Service) appendReportProvider(ctx context.Context, item *path.Item, routePath *path.Path, providers []*Provider, provider *Provider) ([]*Provider, error) { + if routePath == nil || routePath.Report == nil || !routePath.Report.Enabled { + return providers, nil + } + component, err := provider.Component(ctx) + if err != nil || component == nil { + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + fmt.Printf("[DATLY_REPORT] skip source=%s path=%s err=%v component_nil=%v\n", item.SourceURL, routePath.Path.Key(), err, component == nil) + } + return providers, err + } + if !isReportEligible(component) { + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + viewName := "" + groupable := false + if component.View != nil { + viewName = component.View.Name + groupable = component.View.Groupable + } + reportEnabled := false + if component.Report != nil { + reportEnabled = component.Report.Enabled + } + fmt.Printf("[DATLY_REPORT] ineligible source=%s uri=%s method=%s report=%v groupable=%v view=%s\n", item.SourceURL, component.URI, component.Method, reportEnabled, groupable, viewName) + } + return providers, nil + } + reportComponent, reportPath, err := buildReportArtifacts(ctx, s.registry.Dispatcher(), component, routePath) + if err != nil { + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + fmt.Printf("[DATLY_REPORT] build_failed source=%s uri=%s err=%v\n", item.SourceURL, component.URI, err) + } + return nil, err + } + reportProvider := &Provider{ + path: contract.Path{Method: reportComponent.Method, URI: reportComponent.URI}, + control: routePath.Version, + newComponent: func(ctx context.Context, opts ...Option) (*Component, error) { + original, err := provider.Component(ctx, opts...) + if err != nil || original == nil { + return nil, err + } + component, _, err := buildReportArtifacts(ctx, s.registry.Dispatcher(), original, routePath) + return component, err + }, + component: reportComponent, + } + item.Paths = append(item.Paths, reportPath) + providers = append(providers, reportProvider) + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + fmt.Printf("[DATLY_REPORT] appended source=%s original=%s report=%s\n", item.SourceURL, component.Path.Key(), reportPath.Path.Key()) + } + return providers, nil +} + +func isReportEligible(component *Component) bool { + if component == nil || component.Report == nil || !component.Report.Enabled { + return false + } + if component.View == nil || !component.View.Groupable { + return false + } + return strings.EqualFold(component.Method, http.MethodGet) +} + +func (s *Service) buildReportComponent(original *Component, routePath *path.Path) (*Component, *path.Path, error) { + return buildReportArtifacts(context.Background(), s.registry.Dispatcher(), original, routePath) +} + +func BuildReportComponent(dispatcher contract.Dispatcher, original *Component) (*Component, error) { + component, _, err := buildReportArtifacts(context.Background(), dispatcher, original, nil) + return component, err +} + +func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, original *Component, routePath *path.Path) (*Component, *path.Path, error) { + config := original.Report.normalize() + metadata, err := buildReportMetadata(original, config) + if err != nil { + return nil, nil, err + } + inputType, err := buildReportInputType(original, metadata, config) + if err != nil { + return nil, nil, err + } + reportURI := strings.TrimSuffix(original.URI, "/") + "/report" + ret := *original + ret.Path = contract.Path{Method: http.MethodPost, URI: reportURI} + ret.Handler = rephandler.NewHandler(&reportHandler{ + Dispatcher: dispatcher, + Path: &original.Path, + Metadata: metadata, + Original: original, + BodyType: inputType.Schema.Type(), + }) + ret.Service = service.TypeExecutor + ret.Report = config + ret.View = buildReportWrapperView(original.View) + ret.Async = nil + ret.Input.Type = *inputType + ret.Input.Type.Parameters = nil + var reportPath *path.Path + if routePath != nil { + pathCopy := *routePath + pathCopy.Path = ret.Path + pathCopy.View = routePath.View + pathCopy.Internal = routePath.Internal + pathCopy.Meta = routePath.Meta + pathCopy.ModelContextProtocol = routePath.ModelContextProtocol + pathCopy.Report = routePath.Report + if pathCopy.Name != "" { + pathCopy.Name += " Report" + } + if pathCopy.Description != "" { + pathCopy.Description += " report" + } + reportPath = &pathCopy + } + return &ret, reportPath, nil +} + +func buildReportWrapperView(original *view.View) *view.View { + if original == nil { + return nil + } + ret := &view.View{ + Name: original.Name + "#report", + Description: original.Description, + Module: original.Module, + Alias: original.Alias, + Mode: view.ModeHandler, + Connector: original.Connector, + CaseFormat: original.CaseFormat, + Groupable: original.Groupable, + Selector: &view.Config{}, + } + if original.Schema != nil { + ret.Schema = original.Schema.Clone() + } + ret.SetResource(original.GetResource()) + return ret +} + +func buildReportMetadata(component *Component, report *Report) (*ReportMetadata, error) { + report = report.normalize() + viewRef := component.View + if viewRef == nil { + return nil, fmt.Errorf("report component view was empty") + } + result := &ReportMetadata{ + InputName: report.inputTypeName(component.Name, component.Input.Type.Name, viewRef.Name), + BodyFieldName: "Report", + DimensionsKey: report.Dimensions, + MeasuresKey: report.Measures, + FiltersKey: report.Filters, + OrderBy: report.OrderBy, + Limit: report.Limit, + Offset: report.Offset, + } + for _, column := range viewRef.Columns { + if column == nil || column.FieldName() == "" { + continue + } + fieldName := exportedReportFieldName(column.FieldName()) + field := &ReportField{Name: column.FieldName(), FieldName: fieldName, Description: column.Name} + switch { + case column.Groupable: + field.Section = report.Dimensions + result.Dimensions = append(result.Dimensions, field) + case column.Aggregate || (viewRef.Groupable && !column.Groupable): + field.Section = report.Measures + result.Measures = append(result.Measures, field) + } + } + for _, parameter := range component.Input.Type.Parameters { + if parameter == nil || len(parameter.Predicates) == 0 || parameter.In == nil { + continue + } + if isSelectorParameter(parameter, viewRef) { + continue + } + result.Filters = append(result.Filters, &ReportFilter{ + Name: parameter.Name, + FieldName: exportedReportFieldName(parameter.Name), + Section: report.Filters, + Description: parameter.Description, + Parameter: parameter, + }) + } + if err := result.validateSelection(); err != nil { + return nil, err + } + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + var filters []string + for _, filter := range result.Filters { + filters = append(filters, filter.Name+":"+filter.FieldName) + } + fmt.Printf("[DATLY_REPORT] metadata input=%s dimensions=%d measures=%d filters=%v\n", result.InputName, len(result.Dimensions), len(result.Measures), filters) + } + return result, nil +} + +func buildReportInputType(component *Component, metadata *ReportMetadata, report *Report) (*state.Type, error) { + if report != nil && report.Input != "" { + schema := state.NewSchema(nil, state.WithSchemaPackage(""), state.WithModulePath("")) + schema.Name = strings.TrimSpace(report.Input) + inputType, err := state.NewType(state.WithSchema(schema), state.WithResource(component.View.Resource())) + if err != nil { + return nil, err + } + if err := inputType.Init(); err != nil { + return nil, err + } + return inputType, validateExplicitReportInput(inputType, metadata) + } + bodyType := synthesizeReportBodyType(metadata) + bodySchema := state.NewSchema(bodyType) + bodySchema.Name = metadata.InputName + bodyParam := state.NewParameter(metadata.BodyFieldName, state.NewBodyLocation(""), state.WithParameterSchema(bodySchema)) + bodyParam.Tag = `anonymous:"true"` + bodyParam.SetTypeNameTag() + inputType, err := state.NewType( + state.WithParameters(state.Parameters{bodyParam}), + state.WithBodyType(true), + state.WithSchema(state.NewSchema(bodyType)), + state.WithResource(component.View.Resource()), + ) + if err != nil { + return nil, err + } + if err := inputType.Init(); err != nil { + return nil, err + } + inputType.Name = metadata.InputName + return inputType, nil +} + +func validateExplicitReportInput(inputType *state.Type, metadata *ReportMetadata) error { + if inputType == nil || inputType.Type() == nil { + return fmt.Errorf("explicit report input type was empty") + } + rType := inputType.Type().Type() + if rType == nil { + return fmt.Errorf("explicit report input state type was empty") + } + rType = reflectTypeOfState(rType) + for _, fieldName := range []string{metadata.DimensionsKey, metadata.MeasuresKey, metadata.FiltersKey, metadata.OrderBy, metadata.Limit, metadata.Offset} { + if fieldName == "" { + continue + } + if _, ok := rType.FieldByName(fieldName); !ok { + return fmt.Errorf("explicit report input %s missing field %s", rType.String(), fieldName) + } + } + return nil +} + +func synthesizeReportBodyType(metadata *ReportMetadata) reflect.Type { + var fields []reflect.StructField + fields = append(fields, reflect.StructField{ + Name: metadata.DimensionsKey, + Type: sectionStructType(metadata.Dimensions), + Tag: buildReportTag(lowerCamel(metadata.DimensionsKey), "Selected grouping dimensions"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.MeasuresKey, + Type: sectionStructType(metadata.Measures), + Tag: buildReportTag(lowerCamel(metadata.MeasuresKey), "Selected aggregate measures"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.FiltersKey, + Type: filterStructType(metadata.Filters), + Tag: buildReportTag(lowerCamel(metadata.FiltersKey), "Report filters derived from original predicate parameters"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.OrderBy, + Type: reflect.TypeOf([]string{}), + Tag: buildReportTag(lowerCamel(metadata.OrderBy), "Ordering expressions applied to the grouped result"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.Limit, + Type: reflect.TypeOf((*int)(nil)), + Tag: buildReportTag(lowerCamel(metadata.Limit), "Maximum number of grouped rows to return"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.Offset, + Type: reflect.TypeOf((*int)(nil)), + Tag: buildReportTag(lowerCamel(metadata.Offset), "Row offset applied to the grouped result"), + }) + return reflect.StructOf(fields) +} + +func sectionStructType(fields []*ReportField) reflect.Type { + if len(fields) == 0 { + return reflect.TypeOf(struct{}{}) + } + structFields := make([]reflect.StructField, 0, len(fields)) + for _, field := range fields { + structFields = append(structFields, reflect.StructField{ + Name: field.FieldName, + Type: reflect.TypeOf(false), + Tag: buildReportTag(lowerCamel(field.Name), field.Description), + }) + } + return reflect.StructOf(structFields) +} + +func filterStructType(filters []*ReportFilter) reflect.Type { + if len(filters) == 0 { + return reflect.TypeOf(struct{}{}) + } + structFields := make([]reflect.StructField, 0, len(filters)) + for _, filter := range filters { + rType := reflect.TypeOf("") + if schemaType := filter.schemaType(); schemaType != nil { + rType = schemaType + } + structFields = append(structFields, reflect.StructField{ + Name: filter.FieldName, + Type: rType, + Tag: buildReportTag(lowerCamel(filter.Name), filter.Description), + }) + } + return reflect.StructOf(structFields) +} + +func buildReportTag(jsonName, description string) reflect.StructTag { + result := fmt.Sprintf(`json:"%s,omitempty"`, jsonName) + if description = strings.TrimSpace(description); description != "" { + result += " desc:" + strconv.Quote(description) + } + return reflect.StructTag(result) +} + +func isSelectorParameter(parameter *state.Parameter, aView *view.View) bool { + if parameter == nil || parameter.In == nil { + return false + } + if aView != nil && aView.Selector != nil { + for _, selector := range []*state.Parameter{ + aView.Selector.FieldsParameter, + aView.Selector.OrderByParameter, + aView.Selector.LimitParameter, + aView.Selector.OffsetParameter, + aView.Selector.PageParameter, + } { + if selector != nil && selector.In != nil && selector.In.Name == parameter.In.Name { + return true + } + } + } + name := strings.ToLower(parameter.In.Name) + return name == "_fields" || name == "_orderby" || name == "_limit" || name == "_offset" || name == "_page" || name == "criteria" +} + +func lowerCamel(value string) string { + if value == "" { + return "" + } + return text.CaseFormatUpperCamel.Format(value, text.CaseFormatLowerCamel) +} + +func exportedReportFieldName(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + return state.SanitizeTypeName(value) +} + +func reflectTypeOfState(rType reflect.Type) reflect.Type { + if rType == nil { + return nil + } + if rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + return rType +} diff --git a/repository/report_runtime_test.go b/repository/report_runtime_test.go new file mode 100644 index 000000000..2ec0709ba --- /dev/null +++ b/repository/report_runtime_test.go @@ -0,0 +1,174 @@ +package repository + +import ( + "context" + "embed" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/repository/path" + "github.com/viant/datly/view" + "github.com/viant/datly/view/extension" + "github.com/viant/datly/view/state" + "github.com/viant/tagly/format/text" + "github.com/viant/xdatly/codec" + "github.com/viant/xreflect" +) + +type reportTestResource struct{} + +func (r *reportTestResource) LookupParameter(name string) (*state.Parameter, error) { return nil, nil } +func (r *reportTestResource) AppendParameter(parameter *state.Parameter) {} +func (r *reportTestResource) ViewSchema(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *reportTestResource) ViewSchemaPointer(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *reportTestResource) LookupType() xreflect.LookupType { return nil } +func (r *reportTestResource) LoadText(ctx context.Context, URL string) (string, error) { + return "", nil +} +func (r *reportTestResource) Codecs() *codec.Registry { return codec.New() } +func (r *reportTestResource) CodecOptions() *codec.Options { return codec.NewOptions(nil) } +func (r *reportTestResource) ExpandSubstitutes(value string) string { return value } +func (r *reportTestResource) ReverseSubstitutes(value string) string { return value } +func (r *reportTestResource) EmbedFS() *embed.FS { return nil } +func (r *reportTestResource) SetFSEmbedder(embedder *state.FSEmbedder) {} + +func TestBuildReportMetadataAndComponent(t *testing.T) { + resource := view.EmptyResource() + columnResource := &reportTestResource{} + rootView := view.NewView("vendor", "VENDOR") + rootView.Groupable = true + rootView.Selector = &view.Config{ + FieldsParameter: &state.Parameter{Name: "fields", In: state.NewQueryLocation("_fields")}, + OrderByParameter: &state.Parameter{Name: "orderBy", In: state.NewQueryLocation("_orderby")}, + LimitParameter: &state.Parameter{Name: "limit", In: state.NewQueryLocation("_limit")}, + OffsetParameter: &state.Parameter{Name: "offset", In: state.NewQueryLocation("_offset")}, + } + rootView.Columns = []*view.Column{ + view.NewColumn("AccountID", "int", reflect.TypeOf(0), false), + view.NewColumn("UserCreated", "int", reflect.TypeOf(0), false), + view.NewColumn("TotalSpend", "float64", reflect.TypeOf(float64(0)), false), + } + rootView.Columns[0].Groupable = true + rootView.Columns[1].Groupable = true + rootView.Columns[2].Aggregate = true + for _, column := range rootView.Columns { + require.NoError(t, column.Init(columnResource, text.CaseFormatUndefined, false)) + } + rootView.SetResource(resource) + resource.AddViews(rootView) + + inputType, err := state.NewType(state.WithParameters(state.Parameters{ + &state.Parameter{Name: "vendorIDs", In: state.NewQueryLocation("vendorIDs"), Schema: state.NewSchema(reflect.TypeOf([]int{})), Description: "Vendor IDs to include"}, + &state.Parameter{Name: "accountID", In: state.NewQueryLocation("accountID"), Schema: state.NewSchema(reflect.TypeOf(0)), Predicates: []*extension.PredicateConfig{{Name: "ByAccount"}}, Description: "Account identifier filter"}, + &state.Parameter{Name: "fields", In: state.NewQueryLocation("_fields"), Schema: state.NewSchema(reflect.TypeOf([]string{}))}, + }), state.WithResource(columnResource)) + require.NoError(t, err) + inputType.Name = "VendorInput" + + component := &Component{ + Path: contract.Path{Method: "GET", URI: "/v1/api/vendors"}, + Meta: contract.Meta{Name: "vendors"}, + View: rootView, + Report: (&Report{Enabled: true}).normalize(), + Contract: contract.Contract{ + Input: contract.Input{Type: *inputType}, + }, + } + + metadata, err := buildReportMetadata(component, component.Report) + require.NoError(t, err) + require.NotNil(t, metadata) + assert.Equal(t, "VendorInputReportInput", metadata.InputName) + require.Len(t, metadata.Dimensions, 2) + require.Len(t, metadata.Measures, 1) + require.Len(t, metadata.Filters, 1) + assert.Equal(t, "AccountID", metadata.Dimensions[0].Name) + assert.Equal(t, "TotalSpend", metadata.Measures[0].Name) + assert.Equal(t, "accountID", metadata.Filters[0].Name) + + service := &Service{registry: NewRegistry("", nil, nil)} + reportComponent, reportPath, err := service.buildReportComponent(component, &path.Path{ + Path: component.Path, + View: &path.ViewRef{Ref: rootView.Name}, + ModelContextProtocol: contract.ModelContextProtocol{ + MCPTool: true, + }, + Meta: contract.Meta{ + Name: "vendors", + Description: "Vendor listing", + }, + Report: &path.Report{Enabled: true}, + }) + require.NoError(t, err) + require.NotNil(t, reportComponent) + require.NotNil(t, reportPath) + assert.Equal(t, "POST", reportComponent.Method) + assert.Equal(t, "/v1/api/vendors/report", reportComponent.URI) + require.NotNil(t, reportComponent.Report) + require.NotNil(t, reportComponent.View) + assert.NotSame(t, component.View, reportComponent.View) + assert.Equal(t, view.ModeHandler, reportComponent.View.Mode) + assert.Nil(t, reportComponent.View.Template) + assert.Equal(t, "/v1/api/vendors/report", reportPath.URI) + assert.Equal(t, "POST", reportPath.Method) + assert.True(t, reportPath.MCPTool) + assert.Equal(t, "vendors Report", reportPath.Name) + assert.Equal(t, "Vendor listing report", reportPath.Description) + reportInputType, err := buildReportInputType(component, metadata, component.Report) + require.NoError(t, err) + require.NotNil(t, reportInputType) + require.NotNil(t, reportInputType.Schema) + require.NotNil(t, reportInputType.Schema.Type()) + bodyType := reportInputType.Schema.Type() + if bodyType.Kind() == reflect.Ptr { + bodyType = bodyType.Elem() + } + _, ok := bodyType.FieldByName("Dimensions") + assert.True(t, ok) + _, ok = bodyType.FieldByName("Measures") + assert.True(t, ok) + _, ok = bodyType.FieldByName("Filters") + assert.True(t, ok) + filtersField, ok := bodyType.FieldByName("Filters") + require.True(t, ok) + filterType := filtersField.Type + require.Greater(t, filterType.NumField(), 0) + filterField := filterType.Field(0) + assert.True(t, strings.Contains(string(filterField.Tag), `desc:"Account identifier filter"`)) +} + +func TestService_InitComponentProviders_RegistersLocalGroupingReportRoute(t *testing.T) { + ctx := context.Background() + baseDir := filepath.Join("..", "e2e", "local", "regression") + if _, err := os.Stat(filepath.Join(baseDir, "paths.yaml")); err != nil { + t.Skipf("missing local regression fixture: %v", err) + } + service, err := New(ctx, + WithComponentURL(baseDir), + WithResourceURL(baseDir), + WithNoPlugin(), + WithRefreshDisabled(true), + ) + require.NoError(t, err) + reportPath := &contract.Path{Method: "POST", URI: "/v1/api/shape/dev/vendors-grouping/report"} + provider, err := service.Registry().LookupProvider(ctx, reportPath) + require.NoError(t, err) + require.NotNil(t, provider) + component, err := provider.Component(ctx) + require.NoError(t, err) + require.NotNil(t, component) + require.NotNil(t, component.Report) + assert.True(t, component.Report.Enabled) + assert.Equal(t, "POST", component.Method) + assert.Equal(t, "/v1/api/shape/dev/vendors-grouping/report", component.URI) +} From 674161fbf452a2549fa02cb8f465cf9a2ac9577f Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 13:24:18 -0700 Subject: [PATCH 171/279] added dynamic grouping --- gateway/mcp_report_test.go | 178 +++++++++++++++++++++++++++ view/grouped_relation_compat_test.go | 56 +++++++++ 2 files changed, 234 insertions(+) create mode 100644 gateway/mcp_report_test.go create mode 100644 view/grouped_relation_compat_test.go diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go new file mode 100644 index 000000000..23daed6ce --- /dev/null +++ b/gateway/mcp_report_test.go @@ -0,0 +1,178 @@ +package gateway + +import ( + "context" + "encoding/json" + "io" + "net/http" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/mcp-protocol/authorization" + "github.com/viant/mcp-protocol/schema" +) + +func TestRouter_buildToolInputType_FlattensAnonymousBody(t *testing.T) { + bodyType := reflect.StructOf([]reflect.StructField{ + { + Name: "Dimensions", + Type: reflect.StructOf([]reflect.StructField{ + {Name: "AccountId", Type: reflect.TypeOf(false), Tag: `json:"accountId,omitempty" desc:"Account identifier"`}, + }), + Tag: `json:"dimensions,omitempty"`, + }, + { + Name: "Measures", + Type: reflect.StructOf([]reflect.StructField{ + {Name: "TotalId", Type: reflect.TypeOf(false), Tag: `json:"totalId,omitempty" desc:"Total identifier"`}, + }), + Tag: `json:"measures,omitempty"`, + }, + { + Name: "Filters", + Type: reflect.StructOf([]reflect.StructField{ + {Name: "VendorIDs", Type: reflect.TypeOf([]int{}), Tag: `json:"vendorIDs,omitempty" desc:"Vendor IDs"`}, + }), + Tag: `json:"filters,omitempty"`, + }, + {Name: "OrderBy", Type: reflect.TypeOf([]string{}), Tag: `json:"orderBy,omitempty"`}, + }) + bodyParam := state.NewParameter("Report", state.NewBodyLocation(""), state.WithParameterSchema(state.NewSchema(bodyType))) + bodyParam.Tag = `anonymous:"true"` + component := &repository.Component{ + Path: contract.Path{Method: "POST", URI: "/v1/api/dev/vendors-grouping/report"}, + View: &view.View{}, + Contract: contract.Contract{ + Input: contract.Input{ + Type: state.Type{Parameters: state.Parameters{bodyParam}}, + }, + }, + } + + rType := (&Router{}).buildToolInputType(component) + require.Equal(t, reflect.Struct, rType.Kind()) + _, ok := rType.FieldByName("Report") + assert.False(t, ok) + for _, name := range []string{"Dimensions", "Measures", "Filters", "OrderBy"} { + _, ok = rType.FieldByName(name) + assert.True(t, ok, name) + } +} + +func TestAnonymousBodyArgumentValue_UsesJSONFieldNames(t *testing.T) { + bodyType := reflect.StructOf([]reflect.StructField{ + {Name: "Dimensions", Type: reflect.StructOf([]reflect.StructField{{Name: "AccountId", Type: reflect.TypeOf(false), Tag: `json:"accountId,omitempty"`}}), Tag: `json:"dimensions,omitempty"`}, + {Name: "Measures", Type: reflect.StructOf([]reflect.StructField{{Name: "TotalId", Type: reflect.TypeOf(false), Tag: `json:"totalId,omitempty"`}}), Tag: `json:"measures,omitempty"`}, + {Name: "Filters", Type: reflect.StructOf([]reflect.StructField{{Name: "VendorIDs", Type: reflect.TypeOf([]int{}), Tag: `json:"vendorIDs,omitempty"`}}), Tag: `json:"filters,omitempty"`}, + {Name: "OrderBy", Type: reflect.TypeOf([]string{}), Tag: `json:"orderBy,omitempty"`}, + {Name: "Limit", Type: reflect.TypeOf((*int)(nil)), Tag: `json:"limit,omitempty"`}, + }) + + value := anonymousBodyArgumentValue(map[string]interface{}{ + "Dimensions": map[string]interface{}{"AccountId": true}, + "Measures": map[string]interface{}{"TotalId": true}, + "Filters": map[string]interface{}{"VendorIDs": []interface{}{1.0, 2.0, 3.0}}, + "OrderBy": []interface{}{"accountId"}, + }, bodyType) + + data, err := json.Marshal(value) + require.NoError(t, err) + assert.JSONEq(t, `{ + "dimensions":{"AccountId":true}, + "measures":{"TotalId":true}, + "filters":{"VendorIDs":[1,2,3]}, + "orderBy":["accountId"] + }`, string(data)) +} + +func TestRouter_addAuthTokenIfPresent_AddsBearerToken(t *testing.T) { + router := &Router{} + req, err := http.NewRequest(http.MethodPost, "http://localhost/v1/api/dev/vendors-grouping/report", nil) + require.NoError(t, err) + + ctx := context.WithValue(context.Background(), authorization.TokenKey, &authorization.Token{Token: "abc123"}) + router.addAuthTokenIfPresent(ctx, req) + + assert.Equal(t, "Bearer abc123", req.Header.Get("Authorization")) +} + +func TestRouter_mcpToolCallHandler_PassesAuthorizationToReportRoute(t *testing.T) { + bodyType := reflect.StructOf([]reflect.StructField{ + { + Name: "Dimensions", + Type: reflect.StructOf([]reflect.StructField{ + {Name: "AccountId", Type: reflect.TypeOf(false), Tag: `json:"accountId,omitempty"`}, + }), + Tag: `json:"dimensions,omitempty"`, + }, + { + Name: "Measures", + Type: reflect.StructOf([]reflect.StructField{ + {Name: "TotalId", Type: reflect.TypeOf(false), Tag: `json:"totalId,omitempty"`}, + }), + Tag: `json:"measures,omitempty"`, + }, + { + Name: "Filters", + Type: reflect.StructOf([]reflect.StructField{ + {Name: "VendorIDs", Type: reflect.TypeOf([]int{}), Tag: `json:"vendorIDs,omitempty"`}, + }), + Tag: `json:"filters,omitempty"`, + }, + {Name: "OrderBy", Type: reflect.TypeOf([]string{}), Tag: `json:"orderBy,omitempty"`}, + }) + bodyParam := state.NewParameter("Report", state.NewBodyLocation(""), state.WithParameterSchema(state.NewSchema(bodyType))) + bodyParam.Tag = `anonymous:"true"` + component := &repository.Component{ + Path: contract.Path{Method: http.MethodPost, URI: "/v1/api/dev/vendors-grouping/report"}, + Contract: contract.Contract{ + Input: contract.Input{ + Type: state.Type{Parameters: state.Parameters{bodyParam}}, + }, + }, + } + + var actualAuth string + var actualBody string + route := &Route{ + Path: &contract.Path{Method: http.MethodPost, URI: "/v1/api/dev/vendors-grouping/report"}, + Handler: func(ctx context.Context, response http.ResponseWriter, req *http.Request) { + actualAuth = req.Header.Get("Authorization") + if req.Body != nil { + payload, _ := io.ReadAll(req.Body) + actualBody = string(payload) + } + response.WriteHeader(http.StatusOK) + _, _ = response.Write([]byte(`{"ok":true}`)) + }, + } + + handler := (&Router{}).mcpToolCallHandler(component, route) + ctx := context.WithValue(context.Background(), authorization.TokenKey, &authorization.Token{Token: "jwt-token"}) + result, rpcErr := handler(ctx, &schema.CallToolRequest{ + Params: schema.CallToolRequestParams{ + Arguments: map[string]interface{}{ + "Dimensions": map[string]interface{}{"AccountId": true}, + "Measures": map[string]interface{}{"TotalId": true}, + "Filters": map[string]interface{}{"VendorIDs": []interface{}{1.0, 2.0}}, + "OrderBy": []interface{}{"accountId"}, + }, + }, + }) + + require.Nil(t, rpcErr) + require.NotNil(t, result) + assert.Equal(t, "Bearer jwt-token", actualAuth) + assert.JSONEq(t, `{ + "dimensions":{"AccountId":true}, + "measures":{"TotalId":true}, + "filters":{"VendorIDs":[1,2]}, + "orderBy":["accountId"] + }`, actualBody) +} diff --git a/view/grouped_relation_compat_test.go b/view/grouped_relation_compat_test.go new file mode 100644 index 000000000..2d1cbfee3 --- /dev/null +++ b/view/grouped_relation_compat_test.go @@ -0,0 +1,56 @@ +package view + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view/state" + "github.com/viant/tagly/format/text" +) + +func TestView_EnsureColumns_UsesTypedSchemaForGroupedRelationAlias(t *testing.T) { + ctx := context.Background() + resource := NewResource(nil) + resource.Types = []*TypeDefinition{ + { + Name: "DisqualifiedView", + Package: "taxonomy", + ModulePath: "github.vianttech.com/viant/platform/pkg/platform/taxonomy", + DataType: `struct{TaxonomyId string ` + "`sqlx:\"TAXONOMY_ID\" source:\"SEGMENT_ID\" velty:\"names=TAXONOMY_ID|TaxonomyId\"`" + `; IsDisqualified int ` + "`sqlx:\"IS_DISQUALIFIED\" internal:\"true\" json:\"-\" velty:\"names=IS_DISQUALIFIED|IsDisqualified\"`" + `; }`, + }, + } + require.NoError(t, resource.Init(ctx)) + + aView := &View{ + Name: "disqualified", + Table: "CI_TAXONOMY_DISQUALIFIED", + Alias: "t", + Mode: ModeQuery, + Schema: &state.Schema{Name: "DisqualifiedView", Package: "taxonomy", Cardinality: state.Many}, + Template: &Template{Source: "SELECT dq.SEGMENT_ID AS TAXONOMY_ID, 1 AS IS_DISQUALIFIED FROM CI_TAXONOMY_DISQUALIFIED dq GROUP BY dq.SEGMENT_ID"}, + ColumnsConfig: map[string]*ColumnConfig{ + "IS_DISQUALIFIED": { + Name: "IS_DISQUALIFIED", + Tag: ptrString(`json:"-" internal:"true"`), + }, + }, + } + + require.NoError(t, aView.ensureColumns(ctx, resource)) + require.Len(t, aView.Columns, 2) + aView.CaseFormat = text.CaseFormatLowerUnderscore + aView._columns = Columns(aView.Columns).Index(aView.CaseFormat) + + column, ok := aView.ColumnByName("TaxonomyId") + require.True(t, ok) + require.Equal(t, "TAXONOMY_ID", column.Name) + + column, ok = aView.ColumnByName("SEGMENT_ID") + require.True(t, ok) + require.Equal(t, "TAXONOMY_ID", column.Name) +} + +func ptrString(value string) *string { + return &value +} From de43b73779af910c3182764ce8bc8c52c95000d2 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 13:28:31 -0700 Subject: [PATCH 172/279] added dynamic grouping --- repository/path/container.go | 1 + repository/report.go | 8 +++ repository/report_runtime.go | 3 + repository/report_runtime_test.go | 112 ++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+) diff --git a/repository/path/container.go b/repository/path/container.go index dd607d7c1..e8e316bbc 100644 --- a/repository/path/container.go +++ b/repository/path/container.go @@ -39,6 +39,7 @@ type ( Report struct { Enabled bool `json:",omitempty" yaml:"Enabled,omitempty"` + MCPTool *bool `json:",omitempty" yaml:"MCPTool,omitempty"` Input string `json:",omitempty" yaml:"Input,omitempty"` Dimensions string `json:",omitempty" yaml:"Dimensions,omitempty"` Measures string `json:",omitempty" yaml:"Measures,omitempty"` diff --git a/repository/report.go b/repository/report.go index 6117ff4f2..4b8c1a5f1 100644 --- a/repository/report.go +++ b/repository/report.go @@ -10,6 +10,7 @@ import ( type Report struct { Enabled bool `json:",omitempty" yaml:"Enabled,omitempty"` + MCPTool *bool `json:",omitempty" yaml:"MCPTool,omitempty"` Input string `json:",omitempty" yaml:"Input,omitempty"` Dimensions string `json:",omitempty" yaml:"Dimensions,omitempty"` Measures string `json:",omitempty" yaml:"Measures,omitempty"` @@ -71,6 +72,13 @@ func (r *Report) normalize() *Report { return ret } +func (r *Report) mcpToolEnabled() bool { + if r == nil || r.MCPTool == nil { + return true + } + return *r.MCPTool +} + func (r *Report) inputTypeName(componentName, inputName, viewName string) string { if r != nil && strings.TrimSpace(r.Input) != "" { return strings.TrimSpace(r.Input) diff --git a/repository/report_runtime.go b/repository/report_runtime.go index 653d4ebca..6537ee9c1 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -126,6 +126,9 @@ func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, o pathCopy.Internal = routePath.Internal pathCopy.Meta = routePath.Meta pathCopy.ModelContextProtocol = routePath.ModelContextProtocol + pathCopy.MCPTool = config.mcpToolEnabled() + pathCopy.MCPResource = false + pathCopy.MCPTemplateResource = false pathCopy.Report = routePath.Report if pathCopy.Name != "" { pathCopy.Name += " Report" diff --git a/repository/report_runtime_test.go b/repository/report_runtime_test.go index 2ec0709ba..f9bf5265f 100644 --- a/repository/report_runtime_test.go +++ b/repository/report_runtime_test.go @@ -147,6 +147,118 @@ func TestBuildReportMetadataAndComponent(t *testing.T) { assert.True(t, strings.Contains(string(filterField.Tag), `desc:"Account identifier filter"`)) } +func TestBuildReportComponent_EnablesMCPToolOnSiblingRoute(t *testing.T) { + resource := view.EmptyResource() + rootView := view.NewView("vendor", "VENDOR") + rootView.Groupable = true + rootView.Columns = []*view.Column{ + view.NewColumn("AccountID", "int", reflect.TypeOf(0), false), + view.NewColumn("TotalSpend", "float64", reflect.TypeOf(float64(0)), false), + } + rootView.Columns[0].Groupable = true + rootView.Columns[1].Aggregate = true + for _, column := range rootView.Columns { + require.NoError(t, column.Init(&reportTestResource{}, text.CaseFormatUndefined, false)) + } + rootView.SetResource(resource) + resource.AddViews(rootView) + + inputType, err := state.NewType(state.WithParameters(state.Parameters{ + &state.Parameter{Name: "accountID", In: state.NewQueryLocation("accountID"), Schema: state.NewSchema(reflect.TypeOf(0)), Predicates: []*extension.PredicateConfig{{Name: "ByAccount"}}, Description: "Account identifier filter"}, + }), state.WithResource(&reportTestResource{})) + require.NoError(t, err) + inputType.Name = "VendorInput" + + component := &Component{ + Path: contract.Path{Method: "GET", URI: "/v1/api/vendors"}, + Meta: contract.Meta{Name: "vendors"}, + View: rootView, + Report: (&Report{Enabled: true}).normalize(), + Contract: contract.Contract{ + Input: contract.Input{Type: *inputType}, + }, + } + + service := &Service{registry: NewRegistry("", nil, nil)} + _, reportPath, err := service.buildReportComponent(component, &path.Path{ + Path: component.Path, + View: &path.ViewRef{Ref: rootView.Name}, + ModelContextProtocol: contract.ModelContextProtocol{ + MCPTool: false, + MCPResource: true, + MCPTemplateResource: true, + }, + Meta: contract.Meta{ + Name: "vendors", + Description: "Vendor listing", + }, + Report: &path.Report{Enabled: true}, + }) + require.NoError(t, err) + require.NotNil(t, reportPath) + assert.True(t, reportPath.MCPTool) + assert.False(t, reportPath.MCPResource) + assert.False(t, reportPath.MCPTemplateResource) +} + +func TestBuildReportComponent_DisablesMCPToolWhenReportFlagIsFalse(t *testing.T) { + resource := view.EmptyResource() + rootView := view.NewView("vendor", "VENDOR") + rootView.Groupable = true + rootView.Columns = []*view.Column{ + view.NewColumn("AccountID", "int", reflect.TypeOf(0), false), + view.NewColumn("TotalSpend", "float64", reflect.TypeOf(float64(0)), false), + } + rootView.Columns[0].Groupable = true + rootView.Columns[1].Aggregate = true + for _, column := range rootView.Columns { + require.NoError(t, column.Init(&reportTestResource{}, text.CaseFormatUndefined, false)) + } + rootView.SetResource(resource) + resource.AddViews(rootView) + + inputType, err := state.NewType(state.WithParameters(state.Parameters{ + &state.Parameter{Name: "accountID", In: state.NewQueryLocation("accountID"), Schema: state.NewSchema(reflect.TypeOf(0)), Predicates: []*extension.PredicateConfig{{Name: "ByAccount"}}, Description: "Account identifier filter"}, + }), state.WithResource(&reportTestResource{})) + require.NoError(t, err) + inputType.Name = "VendorInput" + + disabled := false + component := &Component{ + Path: contract.Path{Method: "GET", URI: "/v1/api/vendors"}, + Meta: contract.Meta{Name: "vendors"}, + View: rootView, + Report: (&Report{ + Enabled: true, + MCPTool: &disabled, + }).normalize(), + Contract: contract.Contract{ + Input: contract.Input{Type: *inputType}, + }, + } + + service := &Service{registry: NewRegistry("", nil, nil)} + _, reportPath, err := service.buildReportComponent(component, &path.Path{ + Path: component.Path, + View: &path.ViewRef{Ref: rootView.Name}, + ModelContextProtocol: contract.ModelContextProtocol{ + MCPTool: true, + MCPResource: true, + MCPTemplateResource: true, + }, + Meta: contract.Meta{ + Name: "vendors", + Description: "Vendor listing", + }, + Report: &path.Report{Enabled: true, MCPTool: &disabled}, + }) + require.NoError(t, err) + require.NotNil(t, reportPath) + assert.False(t, reportPath.MCPTool) + assert.False(t, reportPath.MCPResource) + assert.False(t, reportPath.MCPTemplateResource) +} + func TestService_InitComponentProviders_RegistersLocalGroupingReportRoute(t *testing.T) { ctx := context.Background() baseDir := filepath.Join("..", "e2e", "local", "regression") From 400996bad66ed786b8c985302d18f0b012c20241 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 13:29:15 -0700 Subject: [PATCH 173/279] added dynamic grouping --- gateway/mcp_report_test.go | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go index 23daed6ce..88b6e8581 100644 --- a/gateway/mcp_report_test.go +++ b/gateway/mcp_report_test.go @@ -12,10 +12,13 @@ import ( "github.com/stretchr/testify/require" "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" + dpath "github.com/viant/datly/repository/path" + "github.com/viant/datly/repository/version" "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/mcp-protocol/authorization" "github.com/viant/mcp-protocol/schema" + serverproto "github.com/viant/mcp-protocol/server" ) func TestRouter_buildToolInputType_FlattensAnonymousBody(t *testing.T) { @@ -176,3 +179,74 @@ func TestRouter_mcpToolCallHandler_PassesAuthorizationToReportRoute(t *testing.T "orderBy":["accountId"] }`, actualBody) } + +func TestRouter_buildToolsIntegration_RegistersReportTool(t *testing.T) { + bodyType := reflect.StructOf([]reflect.StructField{ + { + Name: "Dimensions", + Type: reflect.StructOf([]reflect.StructField{ + {Name: "AccountId", Type: reflect.TypeOf(false), Tag: `json:"accountId,omitempty" desc:"Account identifier"`}, + }), + Tag: `json:"dimensions,omitempty" desc:"Selected grouping dimensions"`, + }, + { + Name: "Measures", + Type: reflect.StructOf([]reflect.StructField{ + {Name: "TotalId", Type: reflect.TypeOf(false), Tag: `json:"totalId,omitempty" desc:"Total identifier"`}, + }), + Tag: `json:"measures,omitempty" desc:"Selected aggregate measures"`, + }, + { + Name: "Filters", + Type: reflect.StructOf([]reflect.StructField{ + {Name: "VendorIDs", Type: reflect.TypeOf([]int{}), Tag: `json:"vendorIDs,omitempty" desc:"Vendor IDs to include"`}, + }), + Tag: `json:"filters,omitempty" desc:"Report filters derived from original predicate parameters"`, + }, + {Name: "OrderBy", Type: reflect.TypeOf([]string{}), Tag: `json:"orderBy,omitempty"`}, + }) + bodyParam := state.NewParameter("Report", state.NewBodyLocation(""), state.WithParameterSchema(state.NewSchema(bodyType))) + bodyParam.Tag = `anonymous:"true"` + component := &repository.Component{ + Path: contract.Path{Method: http.MethodPost, URI: "/v1/api/dev/vendors-grouping/report"}, + View: &view.View{Name: "vendor"}, + Contract: contract.Contract{ + Input: contract.Input{ + Type: state.Type{Parameters: state.Parameters{bodyParam}}, + }, + }, + } + provider := repository.NewProvider( + contract.Path{Method: http.MethodPost, URI: "/v1/api/dev/vendors-grouping/report"}, + &version.Control{}, + func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { + return component, nil + }, + ) + route := &Route{ + Path: &contract.Path{Method: http.MethodPost, URI: "/v1/api/dev/vendors-grouping/report"}, + Handler: func(ctx context.Context, response http.ResponseWriter, req *http.Request) { + response.WriteHeader(http.StatusOK) + }, + } + registry := serverproto.NewRegistry() + router := &Router{mcpRegistry: registry} + + err := router.buildToolsIntegration(&dpath.Item{}, &dpath.Path{ + Path: contract.Path{Method: http.MethodPost, URI: "/v1/api/dev/vendors-grouping/report"}, + Meta: contract.Meta{Name: "vendors grouping report", Description: "Vendor grouping report"}, + ModelContextProtocol: contract.ModelContextProtocol{ + MCPTool: true, + }, + View: &dpath.ViewRef{Ref: "vendor"}, + }, route, provider) + require.NoError(t, err) + + tools := registry.ListRegisteredTools() + require.Len(t, tools, 1) + tool := tools[0] + assert.Equal(t, "vendorsgroupingreport", tool.Name) + require.Contains(t, tool.InputSchema.Properties, "dimensions") + require.Contains(t, tool.InputSchema.Properties, "measures") + require.Contains(t, tool.InputSchema.Properties, "filters") +} From 6b9107f4b06ee6fb5cdc43068a236a83f7d69614 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 13:39:33 -0700 Subject: [PATCH 174/279] added dynamic grouping --- e2e/local/regression/cases/010_grouping/vendors_grouping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql index 310c45218..9734a860a 100644 --- a/e2e/local/regression/cases/010_grouping/vendors_grouping.sql +++ b/e2e/local/regression/cases/010_grouping/vendors_grouping.sql @@ -1,4 +1,4 @@ -/* {"URI":"vendors-grouping/"} */ +/* {"URI":"vendors-grouping/","Name":"vendors grouping","MCPTool":true} */ #set( $_ = $report()) #set( $_ = $Data(output/view).Embed()) From 92d6daa2ba6004d4eacac0841f65823eb9d9d5ad Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 13:56:40 -0700 Subject: [PATCH 175/279] added dynamic grouping --- gateway/mcp_report_test.go | 75 +++++++++++++++++++++++++++++++ repository/codegen.go | 22 +++++++++ repository/report_runtime.go | 1 - repository/report_runtime_test.go | 66 +++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 1 deletion(-) diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go index 88b6e8581..f4ed42376 100644 --- a/gateway/mcp_report_test.go +++ b/gateway/mcp_report_test.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "embed" "encoding/json" "io" "net/http" @@ -15,12 +16,39 @@ import ( dpath "github.com/viant/datly/repository/path" "github.com/viant/datly/repository/version" "github.com/viant/datly/view" + "github.com/viant/datly/view/extension" "github.com/viant/datly/view/state" "github.com/viant/mcp-protocol/authorization" "github.com/viant/mcp-protocol/schema" serverproto "github.com/viant/mcp-protocol/server" + "github.com/viant/tagly/format/text" + "github.com/viant/xdatly/codec" + "github.com/viant/xreflect" ) +type repositoryReportTestResource struct{} + +func (r *repositoryReportTestResource) LookupParameter(name string) (*state.Parameter, error) { + return nil, nil +} +func (r *repositoryReportTestResource) AppendParameter(parameter *state.Parameter) {} +func (r *repositoryReportTestResource) ViewSchema(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *repositoryReportTestResource) ViewSchemaPointer(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *repositoryReportTestResource) LookupType() xreflect.LookupType { return nil } +func (r *repositoryReportTestResource) LoadText(ctx context.Context, URL string) (string, error) { + return "", nil +} +func (r *repositoryReportTestResource) Codecs() *codec.Registry { return codec.New() } +func (r *repositoryReportTestResource) CodecOptions() *codec.Options { return codec.NewOptions(nil) } +func (r *repositoryReportTestResource) ExpandSubstitutes(value string) string { return value } +func (r *repositoryReportTestResource) ReverseSubstitutes(value string) string { return value } +func (r *repositoryReportTestResource) EmbedFS() *embed.FS { return nil } +func (r *repositoryReportTestResource) SetFSEmbedder(embedder *state.FSEmbedder) {} + func TestRouter_buildToolInputType_FlattensAnonymousBody(t *testing.T) { bodyType := reflect.StructOf([]reflect.StructField{ { @@ -250,3 +278,50 @@ func TestRouter_buildToolsIntegration_RegistersReportTool(t *testing.T) { require.Contains(t, tool.InputSchema.Properties, "measures") require.Contains(t, tool.InputSchema.Properties, "filters") } + +func TestRouter_buildToolInputType_UsesBuiltReportComponentParameters(t *testing.T) { + resource := view.EmptyResource() + rootView := view.NewView("vendor", "VENDOR") + rootView.Groupable = true + rootView.Columns = []*view.Column{ + view.NewColumn("AccountID", "int", reflect.TypeOf(0), false), + view.NewColumn("TotalSpend", "float64", reflect.TypeOf(float64(0)), false), + } + rootView.Columns[0].Groupable = true + rootView.Columns[1].Aggregate = true + for _, column := range rootView.Columns { + require.NoError(t, column.Init(&repositoryReportTestResource{}, text.CaseFormatUndefined, false)) + } + rootView.SetResource(resource) + resource.AddViews(rootView) + + inputType, err := state.NewType(state.WithParameters(state.Parameters{ + &state.Parameter{Name: "vendorIDs", In: state.NewQueryLocation("vendorIDs"), Schema: state.NewSchema(reflect.TypeOf([]int{})), Predicates: []*extension.PredicateConfig{{Name: "ByVendor"}}, Description: "Vendor IDs to include"}, + }), state.WithResource(&repositoryReportTestResource{})) + require.NoError(t, err) + inputType.Name = "VendorInput" + + component := &repository.Component{ + Path: contract.Path{Method: http.MethodGet, URI: "/v1/api/vendors"}, + Meta: contract.Meta{Name: "vendors"}, + View: rootView, + Report: &repository.Report{Enabled: true}, + Contract: contract.Contract{ + Input: contract.Input{Type: *inputType}, + }, + } + + reportComponent, err := repository.BuildReportComponent(nil, component) + require.NoError(t, err) + require.NotNil(t, reportComponent) + require.Len(t, reportComponent.Input.Type.Parameters, 1) + + rType := (&Router{}).buildToolInputType(reportComponent) + require.Equal(t, reflect.Struct, rType.Kind()) + _, ok := rType.FieldByName("Report") + assert.False(t, ok) + for _, name := range []string{"Dimensions", "Measures", "Filters", "OrderBy", "Limit", "Offset"} { + _, ok = rType.FieldByName(name) + assert.True(t, ok, name) + } +} diff --git a/repository/codegen.go b/repository/codegen.go index 137ac8c34..8813f4f84 100644 --- a/repository/codegen.go +++ b/repository/codegen.go @@ -13,6 +13,7 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/toolbox/data" "github.com/viant/xreflect" + "os" "path" "reflect" "strconv" @@ -29,6 +30,27 @@ func (c *Component) GenerateOutputCode(ctx context.Context, withDefineComponent, builder := strings.Builder{} input := c.Input.Type.Type() registry := c.TypeRegistry() + if os.Getenv("DATLY_DEBUG_CODEGEN") != "" { + outputViewSchema := "" + outputViewType := "" + if viewParameter := c.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); viewParameter != nil && viewParameter.Schema != nil { + outputViewSchema = viewParameter.Schema.Name + if viewParameter.Schema.Type() != nil { + outputViewType = viewParameter.Schema.Type().String() + } + } + defs := 0 + if c.View != nil { + defs = len(c.View.TypeDefinitions()) + } + fmt.Printf("[DATLY_CODEGEN] uri=%s method=%s view=%s defs=%d outputViewSchema=%s outputViewType=%s package=%s\n", + c.URI, c.Method, func() string { + if c.View == nil { + return "" + } + return c.View.Name + }(), defs, outputViewSchema, outputViewType, c.Output.Type.Package) + } if viewParameter := c.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); viewParameter != nil { aTag := &tags.Tag{} diff --git a/repository/report_runtime.go b/repository/report_runtime.go index 6537ee9c1..3dfe6765b 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -117,7 +117,6 @@ func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, o ret.View = buildReportWrapperView(original.View) ret.Async = nil ret.Input.Type = *inputType - ret.Input.Type.Parameters = nil var reportPath *path.Path if routePath != nil { pathCopy := *routePath diff --git a/repository/report_runtime_test.go b/repository/report_runtime_test.go index f9bf5265f..aa6272b91 100644 --- a/repository/report_runtime_test.go +++ b/repository/report_runtime_test.go @@ -119,6 +119,8 @@ func TestBuildReportMetadataAndComponent(t *testing.T) { assert.NotSame(t, component.View, reportComponent.View) assert.Equal(t, view.ModeHandler, reportComponent.View.Mode) assert.Nil(t, reportComponent.View.Template) + require.Len(t, reportComponent.Input.Type.Parameters, 1) + assert.True(t, reportComponent.Input.Type.Parameters[0].IsAnonymous()) assert.Equal(t, "/v1/api/vendors/report", reportPath.URI) assert.Equal(t, "POST", reportPath.Method) assert.True(t, reportPath.MCPTool) @@ -284,3 +286,67 @@ func TestService_InitComponentProviders_RegistersLocalGroupingReportRoute(t *tes assert.Equal(t, "POST", component.Method) assert.Equal(t, "/v1/api/shape/dev/vendors-grouping/report", component.URI) } + +func TestBuildReportComponent_DoesNotStripOriginalViewTypeDefinitionsFromCodegen(t *testing.T) { + resource := view.EmptyResource() + rootView := view.NewView("metrics_view", "metrics_view") + rootView.Groupable = true + rootView.Template = &view.Template{Source: "SELECT agency_id, SUM(total_spend) AS total_spend FROM metrics_view GROUP BY 1"} + rootView.Schema = state.NewSchema(reflect.TypeOf([]*struct { + AgencyId *int `sqlx:"agency_id"` + TotalSpend *float64 `sqlx:"total_spend"` + }{})) + rootView.Columns = []*view.Column{ + view.NewColumn("AgencyId", "int", reflect.TypeOf(0), false), + view.NewColumn("TotalSpend", "float64", reflect.TypeOf(float64(0)), false), + } + rootView.Columns[0].Groupable = true + rootView.Columns[1].Aggregate = true + for _, column := range rootView.Columns { + require.NoError(t, column.Init(&reportTestResource{}, text.CaseFormatUndefined, false)) + } + resource.Types = []*view.TypeDefinition{ + {Name: "MetricsViewView", Package: "metrics", DataType: `struct{AgencyId *int ` + "`sqlx:\"agency_id\"`" + `; TotalSpend *float64 ` + "`sqlx:\"total_spend\"`" + `;}`}, + } + rootView.SetResource(resource) + resource.AddViews(rootView) + + inputType, err := state.NewType(state.WithParameters(state.Parameters{ + &state.Parameter{Name: "agencyID", In: state.NewQueryLocation("agency_id"), Schema: state.NewSchema(reflect.TypeOf(0)), Predicates: []*extension.PredicateConfig{{Name: "ByAgency"}}, Description: "Agency filter"}, + }), state.WithResource(&reportTestResource{})) + require.NoError(t, err) + inputType.Name = "MetricsViewInput" + + outputType, err := state.NewType(state.WithParameters(state.Parameters{ + &state.Parameter{Name: "Data", In: state.NewOutputLocation("view"), Schema: &state.Schema{Name: "MetricsViewView", Package: "metrics", Cardinality: state.Many}}, + })) + require.NoError(t, err) + outputType.Name = "MetricsViewOutput" + + component := &Component{ + Path: contract.Path{Method: "GET", URI: "/v1/api/core/metrics/performance_summary"}, + Meta: contract.Meta{Name: "MetricsPerformance"}, + View: rootView, + Report: (&Report{Enabled: true}).normalize(), + Contract: contract.Contract{ + Input: contract.Input{Type: *inputType}, + Output: contract.Output{Type: *outputType}, + }, + } + + before := component.GenerateOutputCode(context.Background(), true, false, nil) + require.Contains(t, before, "type MetricsViewView struct") + + service := &Service{registry: NewRegistry("", nil, nil)} + _, _, err = service.buildReportComponent(component, &path.Path{ + Path: component.Path, + View: &path.ViewRef{Ref: rootView.Name}, + Report: &path.Report{ + Enabled: true, + }, + }) + require.NoError(t, err) + + after := component.GenerateOutputCode(context.Background(), true, false, nil) + require.Contains(t, after, "type MetricsViewView struct") +} From a53544ec61bd5b9caef5b6ab2d302e13ea77d995 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 14:25:40 -0700 Subject: [PATCH 176/279] added dynamic grouping --- repository/codegen.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repository/codegen.go b/repository/codegen.go index 8813f4f84..ca3042108 100644 --- a/repository/codegen.go +++ b/repository/codegen.go @@ -77,6 +77,9 @@ func (c *Component) GenerateOutputCode(ctx context.Context, withDefineComponent, aTag.SummarySQL = tags.ViewSQLSummary(tags.NewViewSQL(tmpl.Summary.Source, "")) } viewParameter.Tag = string(aTag.UpdateTag(reflect.StructTag(viewParameter.Tag))) + if c.View != nil && c.View.Schema != nil && c.View.Schema.Name != "" && !strings.Contains(viewParameter.Tag, `typeName:"`) { + viewParameter.Tag = strings.TrimSpace(viewParameter.Tag + ` typeName:"` + c.View.Schema.Name + `"`) + } } output, _ := c.Output.Type.Parameters.ReflectType("", registry.Lookup, state.WithRelation(), state.WithSQL(), state.WithVelty(false)) From 2e260feadf18af513fb1bb0ff7b0813653e05094 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 14:45:22 -0700 Subject: [PATCH 177/279] added dynamic grouping --- repository/codegen.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/repository/codegen.go b/repository/codegen.go index ca3042108..137ac8c34 100644 --- a/repository/codegen.go +++ b/repository/codegen.go @@ -13,7 +13,6 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/toolbox/data" "github.com/viant/xreflect" - "os" "path" "reflect" "strconv" @@ -30,27 +29,6 @@ func (c *Component) GenerateOutputCode(ctx context.Context, withDefineComponent, builder := strings.Builder{} input := c.Input.Type.Type() registry := c.TypeRegistry() - if os.Getenv("DATLY_DEBUG_CODEGEN") != "" { - outputViewSchema := "" - outputViewType := "" - if viewParameter := c.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); viewParameter != nil && viewParameter.Schema != nil { - outputViewSchema = viewParameter.Schema.Name - if viewParameter.Schema.Type() != nil { - outputViewType = viewParameter.Schema.Type().String() - } - } - defs := 0 - if c.View != nil { - defs = len(c.View.TypeDefinitions()) - } - fmt.Printf("[DATLY_CODEGEN] uri=%s method=%s view=%s defs=%d outputViewSchema=%s outputViewType=%s package=%s\n", - c.URI, c.Method, func() string { - if c.View == nil { - return "" - } - return c.View.Name - }(), defs, outputViewSchema, outputViewType, c.Output.Type.Package) - } if viewParameter := c.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); viewParameter != nil { aTag := &tags.Tag{} @@ -77,9 +55,6 @@ func (c *Component) GenerateOutputCode(ctx context.Context, withDefineComponent, aTag.SummarySQL = tags.ViewSQLSummary(tags.NewViewSQL(tmpl.Summary.Source, "")) } viewParameter.Tag = string(aTag.UpdateTag(reflect.StructTag(viewParameter.Tag))) - if c.View != nil && c.View.Schema != nil && c.View.Schema.Name != "" && !strings.Contains(viewParameter.Tag, `typeName:"`) { - viewParameter.Tag = strings.TrimSpace(viewParameter.Tag + ` typeName:"` + c.View.Schema.Name + `"`) - } } output, _ := c.Output.Type.Parameters.ReflectType("", registry.Lookup, state.WithRelation(), state.WithSQL(), state.WithVelty(false)) From 5c3592a3d084862e7ff9784149fb6419b7a693ae Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 14:59:07 -0700 Subject: [PATCH 178/279] added dynamic grouping --- repository/codegen.go | 31 +++++++++++ repository/report_runtime.go | 100 ++++++++++++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/repository/codegen.go b/repository/codegen.go index 137ac8c34..60397b032 100644 --- a/repository/codegen.go +++ b/repository/codegen.go @@ -13,6 +13,7 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/toolbox/data" "github.com/viant/xreflect" + "os" "path" "reflect" "strconv" @@ -29,6 +30,36 @@ func (c *Component) GenerateOutputCode(ctx context.Context, withDefineComponent, builder := strings.Builder{} input := c.Input.Type.Type() registry := c.TypeRegistry() + if os.Getenv("DATLY_DEBUG_REPORT_GEN") != "" { + outputViewSchema := "" + outputViewType := "" + outputViewTag := "" + if viewParameter := c.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); viewParameter != nil { + outputViewTag = viewParameter.Tag + if viewParameter.Schema != nil { + outputViewSchema = viewParameter.Schema.Name + if viewParameter.Schema.Type() != nil { + outputViewType = viewParameter.Schema.Type().String() + } + } + } + defs := 0 + viewName := "" + viewSchemaName := "" + viewSchemaType := "" + if c.View != nil { + viewName = c.View.Name + defs = len(c.View.TypeDefinitions()) + if c.View.Schema != nil { + viewSchemaName = c.View.Schema.Name + if c.View.Schema.Type() != nil { + viewSchemaType = c.View.Schema.Type().String() + } + } + } + fmt.Printf("[DATLY_CODEGEN] uri=%s method=%s view=%s viewSchema=%s viewSchemaType=%s defs=%d outputViewSchema=%s outputViewType=%s outputViewTag=%s package=%s\n", + c.URI, c.Method, viewName, viewSchemaName, viewSchemaType, defs, outputViewSchema, outputViewType, outputViewTag, c.Output.Type.Package) + } if viewParameter := c.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); viewParameter != nil { aTag := &tags.Tag{} diff --git a/repository/report_runtime.go b/repository/report_runtime.go index 3dfe6765b..2cc9a8e4a 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -2,6 +2,7 @@ package repository import ( "context" + "embed" "fmt" "net/http" "os" @@ -16,6 +17,8 @@ import ( "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/tagly/format/text" + "github.com/viant/xdatly/codec" + "github.com/viant/xreflect" ) func (s *Service) appendReportProvider(ctx context.Context, item *path.Item, routePath *path.Path, providers []*Provider, provider *Provider) ([]*Provider, error) { @@ -93,6 +96,9 @@ func BuildReportComponent(dispatcher contract.Dispatcher, original *Component) ( } func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, original *Component, routePath *path.Path) (*Component, *path.Path, error) { + if os.Getenv("DATLY_DEBUG_REPORT_GEN") != "" { + debugReportComponent("before_build_report", original) + } config := original.Report.normalize() metadata, err := buildReportMetadata(original, config) if err != nil { @@ -137,9 +143,50 @@ func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, o } reportPath = &pathCopy } + if os.Getenv("DATLY_DEBUG_REPORT_GEN") != "" { + debugReportComponent("after_build_report_original", original) + debugReportComponent("after_build_report_report", &ret) + } return &ret, reportPath, nil } +func debugReportComponent(label string, component *Component) { + if component == nil { + fmt.Printf("[DATLY_REPORT_GEN] %s component=nil\n", label) + return + } + viewName := "" + viewSchemaName := "" + viewSchemaType := "" + viewDefs := 0 + if component.View != nil { + viewName = component.View.Name + viewDefs = len(component.View.TypeDefinitions()) + if component.View.Schema != nil { + viewSchemaName = component.View.Schema.Name + if component.View.Schema.Type() != nil { + viewSchemaType = component.View.Schema.Type().String() + } + } + } + outputSchemaName := "" + outputSchemaType := "" + outputTag := "" + if component.Output.Type.Parameters != nil { + if param := component.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); param != nil { + outputTag = param.Tag + if param.Schema != nil { + outputSchemaName = param.Schema.Name + if param.Schema.Type() != nil { + outputSchemaType = param.Schema.Type().String() + } + } + } + } + fmt.Printf("[DATLY_REPORT_GEN] %s uri=%s method=%s view=%s viewSchema=%s viewSchemaType=%s viewDefs=%d outputSchema=%s outputSchemaType=%s outputTag=%s\n", + label, component.URI, component.Method, viewName, viewSchemaName, viewSchemaType, viewDefs, outputSchemaName, outputSchemaType, outputTag) +} + func buildReportWrapperView(original *view.View) *view.View { if original == nil { return nil @@ -240,11 +287,15 @@ func buildReportInputType(component *Component, metadata *ReportMetadata, report bodyParam := state.NewParameter(metadata.BodyFieldName, state.NewBodyLocation(""), state.WithParameterSchema(bodySchema)) bodyParam.Tag = `anonymous:"true"` bodyParam.SetTypeNameTag() + // Synthetic report input must not initialize against the original component resource. + // Using the shared resource resolves linked named types and mutates the original + // component generation state, which breaks repeated code generation. + inputResource := newReportInputResource(component.View.Resource()) inputType, err := state.NewType( state.WithParameters(state.Parameters{bodyParam}), state.WithBodyType(true), state.WithSchema(state.NewSchema(bodyType)), - state.WithResource(component.View.Resource()), + state.WithResource(inputResource), ) if err != nil { return nil, err @@ -398,3 +449,50 @@ func reflectTypeOfState(rType reflect.Type) reflect.Type { } return rType } + +type reportInputResource struct { + base state.Resource +} + +func newReportInputResource(base state.Resource) state.Resource { + return &reportInputResource{base: base} +} + +func (r *reportInputResource) LookupParameter(name string) (*state.Parameter, error) { return nil, nil } +func (r *reportInputResource) AppendParameter(parameter *state.Parameter) {} +func (r *reportInputResource) ViewSchema(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *reportInputResource) ViewSchemaPointer(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *reportInputResource) LookupType() xreflect.LookupType { return nil } +func (r *reportInputResource) LoadText(ctx context.Context, URL string) (string, error) { + return "", nil +} +func (r *reportInputResource) Codecs() *codec.Registry { + if r.base != nil && r.base.Codecs() != nil { + return r.base.Codecs() + } + return codec.New() +} +func (r *reportInputResource) CodecOptions() *codec.Options { + if r.base != nil && r.base.CodecOptions() != nil { + return r.base.CodecOptions() + } + return codec.NewOptions(nil) +} +func (r *reportInputResource) ExpandSubstitutes(value string) string { + if r.base != nil { + return r.base.ExpandSubstitutes(value) + } + return value +} +func (r *reportInputResource) ReverseSubstitutes(value string) string { + if r.base != nil { + return r.base.ReverseSubstitutes(value) + } + return value +} +func (r *reportInputResource) EmbedFS() *embed.FS { return nil } +func (r *reportInputResource) SetFSEmbedder(embedder *state.FSEmbedder) {} From 40242c5b6666dc6b5876c373222ee3fe3f20bd69 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:01:13 -0700 Subject: [PATCH 179/279] added dynamic grouping --- repository/report_runtime.go | 64 +++++++++++++++++------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/repository/report_runtime.go b/repository/report_runtime.go index 2cc9a8e4a..d0da708ed 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -25,53 +25,26 @@ func (s *Service) appendReportProvider(ctx context.Context, item *path.Item, rou if routePath == nil || routePath.Report == nil || !routePath.Report.Enabled { return providers, nil } - component, err := provider.Component(ctx) - if err != nil || component == nil { - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - fmt.Printf("[DATLY_REPORT] skip source=%s path=%s err=%v component_nil=%v\n", item.SourceURL, routePath.Path.Key(), err, component == nil) - } - return providers, err - } - if !isReportEligible(component) { - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - viewName := "" - groupable := false - if component.View != nil { - viewName = component.View.Name - groupable = component.View.Groupable - } - reportEnabled := false - if component.Report != nil { - reportEnabled = component.Report.Enabled - } - fmt.Printf("[DATLY_REPORT] ineligible source=%s uri=%s method=%s report=%v groupable=%v view=%s\n", item.SourceURL, component.URI, component.Method, reportEnabled, groupable, viewName) - } - return providers, nil - } - reportComponent, reportPath, err := buildReportArtifacts(ctx, s.registry.Dispatcher(), component, routePath) - if err != nil { - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - fmt.Printf("[DATLY_REPORT] build_failed source=%s uri=%s err=%v\n", item.SourceURL, component.URI, err) - } - return nil, err - } + reportPath := buildReportPath(routePath) reportProvider := &Provider{ - path: contract.Path{Method: reportComponent.Method, URI: reportComponent.URI}, + path: reportPath.Path, control: routePath.Version, newComponent: func(ctx context.Context, opts ...Option) (*Component, error) { original, err := provider.Component(ctx, opts...) if err != nil || original == nil { return nil, err } + if !isReportEligible(original) { + return nil, nil + } component, _, err := buildReportArtifacts(ctx, s.registry.Dispatcher(), original, routePath) return component, err }, - component: reportComponent, } item.Paths = append(item.Paths, reportPath) providers = append(providers, reportProvider) if os.Getenv("DATLY_DEBUG_REPORT") != "" { - fmt.Printf("[DATLY_REPORT] appended source=%s original=%s report=%s\n", item.SourceURL, component.Path.Key(), reportPath.Path.Key()) + fmt.Printf("[DATLY_REPORT] appended_lazy source=%s original=%s report=%s\n", item.SourceURL, routePath.Path.Key(), reportPath.Path.Key()) } return providers, nil } @@ -209,6 +182,31 @@ func buildReportWrapperView(original *view.View) *view.View { return ret } +func buildReportPath(routePath *path.Path) *path.Path { + pathCopy := *routePath + pathCopy.Path = contract.Path{ + Method: http.MethodPost, + URI: strings.TrimSuffix(routePath.URI, "/") + "/report", + } + pathCopy.MCPTool = reportPathMCPToolEnabled(routePath.Report) + pathCopy.MCPResource = false + pathCopy.MCPTemplateResource = false + if pathCopy.Name != "" { + pathCopy.Name += " Report" + } + if pathCopy.Description != "" { + pathCopy.Description += " report" + } + return &pathCopy +} + +func reportPathMCPToolEnabled(report *path.Report) bool { + if report == nil || report.MCPTool == nil { + return true + } + return *report.MCPTool +} + func buildReportMetadata(component *Component, report *Report) (*ReportMetadata, error) { report = report.normalize() viewRef := component.View From d97fdfe4addfe0bfef6212a8b047eadd733e5771 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:16:42 -0700 Subject: [PATCH 180/279] added dynamic grouping --- gateway/mcp.go | 3 +++ gateway/mcp_report_test.go | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/gateway/mcp.go b/gateway/mcp.go index a473723a3..6465ff976 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -488,6 +488,9 @@ func anonymousBodyArgumentValue(arguments map[string]interface{}, bodyType refle continue } value, ok := arguments[field.Name] + if !ok { + value, ok = arguments[jsonFieldName(field)] + } if !ok { continue } diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go index f4ed42376..150890ea1 100644 --- a/gateway/mcp_report_test.go +++ b/gateway/mcp_report_test.go @@ -122,6 +122,25 @@ func TestAnonymousBodyArgumentValue_UsesJSONFieldNames(t *testing.T) { }`, string(data)) } +func TestAnonymousBodyArgumentValue_AcceptsJSONStyleTopLevelArgumentNames(t *testing.T) { + bodyType := reflect.StructOf([]reflect.StructField{ + {Name: "Dimensions", Type: reflect.StructOf([]reflect.StructField{{Name: "AdOrderId", Type: reflect.TypeOf(false), Tag: `json:"adOrderId,omitempty"`}}), Tag: `json:"dimensions,omitempty"`}, + {Name: "Measures", Type: reflect.StructOf([]reflect.StructField{{Name: "Bids", Type: reflect.TypeOf(false), Tag: `json:"bids,omitempty"`}}), Tag: `json:"measures,omitempty"`}, + }) + + value := anonymousBodyArgumentValue(map[string]interface{}{ + "dimensions": map[string]interface{}{"adOrderId": true}, + "measures": map[string]interface{}{"bids": true}, + }, bodyType) + + data, err := json.Marshal(value) + require.NoError(t, err) + assert.JSONEq(t, `{ + "dimensions":{"adOrderId":true}, + "measures":{"bids":true} + }`, string(data)) +} + func TestRouter_addAuthTokenIfPresent_AddsBearerToken(t *testing.T) { router := &Router{} req, err := http.NewRequest(http.MethodPost, "http://localhost/v1/api/dev/vendors-grouping/report", nil) From 13eb6710b86373d2030ebc99318a0197cb574662 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:20:44 -0700 Subject: [PATCH 181/279] added dynamic grouping --- gateway/mcp.go | 3 +++ gateway/mcp_report_test.go | 8 ++++++++ repository/report_handler.go | 3 +++ 3 files changed, 14 insertions(+) diff --git a/gateway/mcp.go b/gateway/mcp.go index 6465ff976..0ec47596a 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -276,6 +276,9 @@ func (r *Router) newToolHTTPRequest(method, URL string, body io.Reader) (*http.R if err != nil { return nil, jsonrpc.NewInvalidRequest(err.Error(), nil) } + if body != nil { + httpRequest.Header.Set("Content-Type", "application/json") + } return httpRequest, nil } diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go index 150890ea1..b2e98190b 100644 --- a/gateway/mcp_report_test.go +++ b/gateway/mcp_report_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "reflect" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -227,6 +228,13 @@ func TestRouter_mcpToolCallHandler_PassesAuthorizationToReportRoute(t *testing.T }`, actualBody) } +func TestRouter_newToolHTTPRequest_SetsJSONContentTypeForBody(t *testing.T) { + req, rpcErr := (&Router{}).newToolHTTPRequest(http.MethodPost, "http://localhost/v1/api/dev/vendors-grouping/report", strings.NewReader(`{"dimensions":{"accountId":true}}`)) + require.Nil(t, rpcErr) + require.NotNil(t, req) + assert.Equal(t, "application/json", req.Header.Get("Content-Type")) +} + func TestRouter_buildToolsIntegration_RegistersReportTool(t *testing.T) { bodyType := reflect.StructOf([]reflect.StructField{ { diff --git a/repository/report_handler.go b/repository/report_handler.go index a5a157566..c4ea25159 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -73,6 +73,9 @@ func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) if err != nil { return nil, err } + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + fmt.Printf("[DATLY_REPORT_HANDLER] raw_body=%q\n", string(payload)) + } if len(payload) > 0 { target := reflect.New(r.BodyType) if err := json.Unmarshal(payload, target.Interface()); err != nil { From 944dd93cf645e4bf01f4eb696756d61f92eec30c Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:28:46 -0700 Subject: [PATCH 182/279] added dynamic grouping --- repository/report_handler.go | 6 ++++- repository/report_handler_test.go | 40 ++++++++++++++++++++----------- repository/report_runtime.go | 4 ++-- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/repository/report_handler.go b/repository/report_handler.go index c4ea25159..ebc7dad52 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -77,7 +77,11 @@ func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) fmt.Printf("[DATLY_REPORT_HANDLER] raw_body=%q\n", string(payload)) } if len(payload) > 0 { - target := reflect.New(r.BodyType) + targetType := r.BodyType + for targetType.Kind() == reflect.Ptr { + targetType = targetType.Elem() + } + target := reflect.New(targetType) if err := json.Unmarshal(payload, target.Interface()); err != nil { return nil, err } diff --git a/repository/report_handler_test.go b/repository/report_handler_test.go index e09d14c50..43b86214d 100644 --- a/repository/report_handler_test.go +++ b/repository/report_handler_test.go @@ -1,10 +1,14 @@ package repository import ( + "bytes" "context" + "encoding/json" + "io" "log/slog" "net/http" "net/http/httptest" + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -99,10 +103,6 @@ type reportHandlerBody struct { Offset *int } -type reportHandlerInput struct { - Report reportHandlerBody -} - func (s *reportTestSession) Validator() *validator.Service { return nil } func (s *reportTestSession) Differ() *differ.Service { return nil } func (s *reportTestSession) MessageBus() *mbus.Service { return nil } @@ -121,7 +121,7 @@ func testReportHandler() *reportHandler { Dispatcher: &captureDispatcher{}, Path: &contract.Path{Method: http.MethodGet, URI: "/v1/api/vendors"}, Metadata: &ReportMetadata{ - BodyFieldName: "Report", + BodyFieldName: "", DimensionsKey: "Dimensions", MeasuresKey: "Measures", FiltersKey: "Filters", @@ -145,17 +145,15 @@ func testReportHandler() *reportHandler { } } -func testReportInput() reportHandlerInput { +func testReportInput() reportHandlerBody { accountID := 101 limit := 25 - return reportHandlerInput{ - Report: reportHandlerBody{ - Dimensions: reportHandlerDimensions{AccountID: true}, - Measures: reportHandlerMeasures{TotalSpend: true}, - Filters: reportHandlerFilters{AccountID: &accountID}, - OrderBy: []string{"AccountID"}, - Limit: &limit, - }, + return reportHandlerBody{ + Dimensions: reportHandlerDimensions{AccountID: true}, + Measures: reportHandlerMeasures{TotalSpend: true}, + Filters: reportHandlerFilters{AccountID: &accountID}, + OrderBy: []string{"AccountID"}, + Limit: &limit, } } @@ -197,3 +195,17 @@ func TestReportHandler_Exec_PreservesAuthorizationHeader(t *testing.T) { assert.Equal(t, "25", query.Get("_limit")) assert.Equal(t, "101", query.Get("accountID")) } + +func TestReportHandler_ReportInput_AcceptsUnwrappedBody(t *testing.T) { + handler := testReportHandler() + handler.BodyType = reflect.TypeOf(&reportHandlerBody{}) + payload, err := json.Marshal(testReportInput()) + require.NoError(t, err) + req := httptest.NewRequest(http.MethodPost, "http://localhost/v1/api/vendors/report", io.NopCloser(bytes.NewReader(payload))) + input, err := handler.reportInput(context.Background(), req) + require.NoError(t, err) + body, ok := input.(*reportHandlerBody) + require.True(t, ok) + require.True(t, body.Dimensions.AccountID) + require.True(t, body.Measures.TotalSpend) +} diff --git a/repository/report_runtime.go b/repository/report_runtime.go index d0da708ed..ee907bedd 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -215,7 +215,7 @@ func buildReportMetadata(component *Component, report *Report) (*ReportMetadata, } result := &ReportMetadata{ InputName: report.inputTypeName(component.Name, component.Input.Type.Name, viewRef.Name), - BodyFieldName: "Report", + BodyFieldName: "", DimensionsKey: report.Dimensions, MeasuresKey: report.Measures, FiltersKey: report.Filters, @@ -279,7 +279,7 @@ func buildReportInputType(component *Component, metadata *ReportMetadata, report } return inputType, validateExplicitReportInput(inputType, metadata) } - bodyType := synthesizeReportBodyType(metadata) + bodyType := reflect.PtrTo(synthesizeReportBodyType(metadata)) bodySchema := state.NewSchema(bodyType) bodySchema.Name = metadata.InputName bodyParam := state.NewParameter(metadata.BodyFieldName, state.NewBodyLocation(""), state.WithParameterSchema(bodySchema)) From 5995d6202c8bda8120cd91716e87fa86aa825592 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:30:56 -0700 Subject: [PATCH 183/279] added dynamic grouping --- repository/report_runtime.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/repository/report_runtime.go b/repository/report_runtime.go index ee907bedd..2e96f009c 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -81,6 +81,24 @@ func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, o if err != nil { return nil, nil, err } + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + inputSchemaType := "" + if inputType != nil && inputType.Schema != nil && inputType.Schema.Type() != nil { + inputSchemaType = inputType.Schema.Type().String() + } + paramName, paramIn, paramSchema := "", "", "" + if inputType != nil && len(inputType.Parameters) > 0 && inputType.Parameters[0] != nil { + paramName = inputType.Parameters[0].Name + if inputType.Parameters[0].In != nil { + paramIn = string(inputType.Parameters[0].In.Kind) + ":" + inputType.Parameters[0].In.Name + } + if inputType.Parameters[0].Schema != nil && inputType.Parameters[0].Schema.Type() != nil { + paramSchema = inputType.Parameters[0].Schema.Type().String() + } + } + fmt.Printf("[DATLY_REPORT] built_input uri=%s input_schema=%s body_field=%q param_name=%s in=%s param_schema=%s\n", + original.URI, inputSchemaType, metadata.BodyFieldName, paramName, paramIn, paramSchema) + } reportURI := strings.TrimSuffix(original.URI, "/") + "/report" ret := *original ret.Path = contract.Path{Method: http.MethodPost, URI: reportURI} @@ -282,7 +300,7 @@ func buildReportInputType(component *Component, metadata *ReportMetadata, report bodyType := reflect.PtrTo(synthesizeReportBodyType(metadata)) bodySchema := state.NewSchema(bodyType) bodySchema.Name = metadata.InputName - bodyParam := state.NewParameter(metadata.BodyFieldName, state.NewBodyLocation(""), state.WithParameterSchema(bodySchema)) + bodyParam := state.NewParameter("Report", state.NewBodyLocation(""), state.WithParameterSchema(bodySchema)) bodyParam.Tag = `anonymous:"true"` bodyParam.SetTypeNameTag() // Synthetic report input must not initialize against the original component resource. From 471bc9ab5e2b464ef6e1286b4439fce8ce6a2391 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:31:46 -0700 Subject: [PATCH 184/279] added dynamic grouping --- repository/component.go | 8 ++++++++ repository/report_handler.go | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/repository/component.go b/repository/component.go index 4238105db..1c2c17bb1 100644 --- a/repository/component.go +++ b/repository/component.go @@ -3,6 +3,7 @@ package repository import ( "context" "embed" + stdjson "encoding/json" "fmt" "net/http" "reflect" @@ -441,6 +442,13 @@ func (c *Component) UnmarshalFor(opts ...UnmarshalOption) shared.Unmarshal { } req := options.request // capture for closure + if c != nil && c.Report != nil && c.Report.Enabled && c.Handler != nil { + if parameter := c.Input.Type.AnonymousParameters(); parameter != nil && parameter.In != nil && parameter.In.Kind == state.KindRequestBody { + return func(data []byte, dest interface{}) error { + return stdjson.Unmarshal(data, dest) + } + } + } return func(data []byte, dest interface{}) error { if len(interceptors) > 0 || req != nil { return c.Content.Marshaller.JSON.JsonMarshaller.Unmarshal(data, dest, interceptors, req) diff --git a/repository/report_handler.go b/repository/report_handler.go index ebc7dad52..2483cf3e6 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -68,6 +68,10 @@ func (r *reportHandler) Exec(ctx context.Context, session xhandler.Session) (int } func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) (interface{}, error) { + input := ctx.Value(xhandler.InputKey) + if input != nil { + return input, nil + } if request != nil && request.Body != nil && r.BodyType != nil { payload, err := io.ReadAll(request.Body) if err != nil { @@ -88,7 +92,6 @@ func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) return target.Interface(), nil } } - input := ctx.Value(xhandler.InputKey) if input == nil { return nil, fmt.Errorf("report input was empty") } From 66521bc48c07c9babd484f3640bba7b74559ba92 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:33:24 -0700 Subject: [PATCH 185/279] added dynamic grouping --- repository/report_handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repository/report_handler.go b/repository/report_handler.go index 2483cf3e6..e1f0aaaba 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -70,6 +70,9 @@ func (r *reportHandler) Exec(ctx context.Context, session xhandler.Session) (int func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) (interface{}, error) { input := ctx.Value(xhandler.InputKey) if input != nil { + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + fmt.Printf("[DATLY_REPORT_HANDLER] bound_input_type=%T\n", input) + } return input, nil } if request != nil && request.Body != nil && r.BodyType != nil { From 3944782a3e96cfa24251290a53593ac7a7febcc7 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:34:39 -0700 Subject: [PATCH 186/279] added dynamic grouping --- repository/report_runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/report_runtime.go b/repository/report_runtime.go index 2e96f009c..2dd55182e 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -233,7 +233,7 @@ func buildReportMetadata(component *Component, report *Report) (*ReportMetadata, } result := &ReportMetadata{ InputName: report.inputTypeName(component.Name, component.Input.Type.Name, viewRef.Name), - BodyFieldName: "", + BodyFieldName: "Report", DimensionsKey: report.Dimensions, MeasuresKey: report.Measures, FiltersKey: report.Filters, From 1ee24931a019b14470d4e884ddf36971fb91b7d2 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:38:03 -0700 Subject: [PATCH 187/279] added dynamic grouping --- repository/report_runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/report_runtime.go b/repository/report_runtime.go index 2dd55182e..2e96f009c 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -233,7 +233,7 @@ func buildReportMetadata(component *Component, report *Report) (*ReportMetadata, } result := &ReportMetadata{ InputName: report.inputTypeName(component.Name, component.Input.Type.Name, viewRef.Name), - BodyFieldName: "Report", + BodyFieldName: "", DimensionsKey: report.Dimensions, MeasuresKey: report.Measures, FiltersKey: report.Filters, From 8bdde2b208ee551579e22ea94d59edc656a2696d Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:44:40 -0700 Subject: [PATCH 188/279] added dynamic grouping --- repository/report_handler.go | 15 +++++++++------ service/reader/sql.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/repository/report_handler.go b/repository/report_handler.go index e1f0aaaba..d8b5d5c18 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -69,12 +69,6 @@ func (r *reportHandler) Exec(ctx context.Context, session xhandler.Session) (int func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) (interface{}, error) { input := ctx.Value(xhandler.InputKey) - if input != nil { - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - fmt.Printf("[DATLY_REPORT_HANDLER] bound_input_type=%T\n", input) - } - return input, nil - } if request != nil && request.Body != nil && r.BodyType != nil { payload, err := io.ReadAll(request.Body) if err != nil { @@ -95,6 +89,15 @@ func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) return target.Interface(), nil } } + if input != nil { + if os.Getenv("DATLY_DEBUG_REPORT") != "" { + fmt.Printf("[DATLY_REPORT_HANDLER] bound_input_type=%T\n", input) + if data, err := json.Marshal(input); err == nil { + fmt.Printf("[DATLY_REPORT_HANDLER] bound_input_json=%s\n", string(data)) + } + } + return input, nil + } if input == nil { return nil, fmt.Errorf("report input was empty") } diff --git a/service/reader/sql.go b/service/reader/sql.go index 02568c72b..53af23f7e 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -299,6 +299,7 @@ func (b *Builder) rewriteGroupBy(SQL string, allColumns []*view.Column, projecte groupBy = append(groupBy, query.NewItem(expr.NewIntLiteral(strconv.Itoa(position)))) } parsed.GroupBy = groupBy + parsed.OrderBy = filterGroupedOrderBy(parsed.OrderBy, parsed.List) rewritten := sqlparser.Stringify(parsed) if wrapped { @@ -328,6 +329,38 @@ func projectedColumnPositions(allColumns []*view.Column, projectedColumns []*vie return result } +func filterGroupedOrderBy(orderBy query.List, items query.List) query.List { + if len(orderBy) == 0 || len(items) == 0 { + return orderBy + } + allowed := map[string]bool{} + for _, item := range items { + if item == nil { + continue + } + if item.Expr != nil { + allowed[normalizeExpression(sqlparser.Stringify(item.Expr))] = true + } + if item.Alias != "" { + allowed[normalizeExpression(item.Alias)] = true + } + } + result := make(query.List, 0, len(orderBy)) + for _, item := range orderBy { + if item == nil || item.Expr == nil { + continue + } + if allowed[normalizeExpression(sqlparser.Stringify(item.Expr))] { + result = append(result, item) + } + } + return result +} + +func normalizeExpression(value string) string { + return strings.ToUpper(strings.Join(strings.Fields(strings.TrimSpace(value)), " ")) +} + func projectedGroupByPositions(items query.List, projectedColumns []*view.Column) []int { maxLen := len(items) if len(projectedColumns) < maxLen { From 324a597476d9bb4b3914a94b52659b65a97c4f53 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 9 Mar 2026 15:52:49 -0700 Subject: [PATCH 189/279] added dynamic grouping --- repository/codegen.go | 31 --------------- repository/report_handler.go | 24 ------------ repository/report_runtime.go | 73 ------------------------------------ 3 files changed, 128 deletions(-) diff --git a/repository/codegen.go b/repository/codegen.go index 60397b032..137ac8c34 100644 --- a/repository/codegen.go +++ b/repository/codegen.go @@ -13,7 +13,6 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/toolbox/data" "github.com/viant/xreflect" - "os" "path" "reflect" "strconv" @@ -30,36 +29,6 @@ func (c *Component) GenerateOutputCode(ctx context.Context, withDefineComponent, builder := strings.Builder{} input := c.Input.Type.Type() registry := c.TypeRegistry() - if os.Getenv("DATLY_DEBUG_REPORT_GEN") != "" { - outputViewSchema := "" - outputViewType := "" - outputViewTag := "" - if viewParameter := c.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); viewParameter != nil { - outputViewTag = viewParameter.Tag - if viewParameter.Schema != nil { - outputViewSchema = viewParameter.Schema.Name - if viewParameter.Schema.Type() != nil { - outputViewType = viewParameter.Schema.Type().String() - } - } - } - defs := 0 - viewName := "" - viewSchemaName := "" - viewSchemaType := "" - if c.View != nil { - viewName = c.View.Name - defs = len(c.View.TypeDefinitions()) - if c.View.Schema != nil { - viewSchemaName = c.View.Schema.Name - if c.View.Schema.Type() != nil { - viewSchemaType = c.View.Schema.Type().String() - } - } - } - fmt.Printf("[DATLY_CODEGEN] uri=%s method=%s view=%s viewSchema=%s viewSchemaType=%s defs=%d outputViewSchema=%s outputViewType=%s outputViewTag=%s package=%s\n", - c.URI, c.Method, viewName, viewSchemaName, viewSchemaType, defs, outputViewSchema, outputViewType, outputViewTag, c.Output.Type.Package) - } if viewParameter := c.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); viewParameter != nil { aTag := &tags.Tag{} diff --git a/repository/report_handler.go b/repository/report_handler.go index d8b5d5c18..28ac7ae2f 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -7,7 +7,6 @@ import ( "io" "net/http" "net/url" - "os" "reflect" "strconv" "strings" @@ -42,17 +41,6 @@ func (r *reportHandler) Exec(ctx context.Context, session xhandler.Session) (int if err != nil { return nil, err } - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - viewName := "" - namespacedNil := true - if r.Original.View != nil { - viewName = r.Original.View.Name - } - if r.Original.NamespacedView != nil { - namespacedNil = false - } - fmt.Printf("[DATLY_REPORT_HANDLER] original=%s method=%s target=%s view=%s namespaced_nil=%v query=%s\n", r.Original.Path.Key(), r.Path.Method, r.Path.URI, viewName, namespacedNil, query.Encode()) - } internalReq := request.Clone(ctx) internalReq.Method = r.Path.Method internalReq.URL = cloneURL(request.URL) @@ -60,9 +48,6 @@ func (r *reportHandler) Exec(ctx context.Context, session xhandler.Session) (int internalReq.URL.RawPath = internalReq.URL.Path internalReq.URL.RawQuery = query.Encode() internalReq.RequestURI = internalReq.URL.RequestURI() - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - fmt.Printf("[DATLY_REPORT_HANDLER] dispatch request_uri=%s header_auth=%q\n", internalReq.RequestURI, internalReq.Header.Get("Authorization")) - } redirect := &xdhttp.Route{URL: r.Path.URI, Method: r.Path.Method} return nil, session.Http().Redirect(ctx, redirect, internalReq) } @@ -74,9 +59,6 @@ func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) if err != nil { return nil, err } - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - fmt.Printf("[DATLY_REPORT_HANDLER] raw_body=%q\n", string(payload)) - } if len(payload) > 0 { targetType := r.BodyType for targetType.Kind() == reflect.Ptr { @@ -90,12 +72,6 @@ func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) } } if input != nil { - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - fmt.Printf("[DATLY_REPORT_HANDLER] bound_input_type=%T\n", input) - if data, err := json.Marshal(input); err == nil { - fmt.Printf("[DATLY_REPORT_HANDLER] bound_input_json=%s\n", string(data)) - } - } return input, nil } if input == nil { diff --git a/repository/report_runtime.go b/repository/report_runtime.go index 2e96f009c..47f145b5e 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -5,7 +5,6 @@ import ( "embed" "fmt" "net/http" - "os" "reflect" "strconv" "strings" @@ -43,9 +42,6 @@ func (s *Service) appendReportProvider(ctx context.Context, item *path.Item, rou } item.Paths = append(item.Paths, reportPath) providers = append(providers, reportProvider) - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - fmt.Printf("[DATLY_REPORT] appended_lazy source=%s original=%s report=%s\n", item.SourceURL, routePath.Path.Key(), reportPath.Path.Key()) - } return providers, nil } @@ -69,9 +65,6 @@ func BuildReportComponent(dispatcher contract.Dispatcher, original *Component) ( } func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, original *Component, routePath *path.Path) (*Component, *path.Path, error) { - if os.Getenv("DATLY_DEBUG_REPORT_GEN") != "" { - debugReportComponent("before_build_report", original) - } config := original.Report.normalize() metadata, err := buildReportMetadata(original, config) if err != nil { @@ -81,24 +74,6 @@ func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, o if err != nil { return nil, nil, err } - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - inputSchemaType := "" - if inputType != nil && inputType.Schema != nil && inputType.Schema.Type() != nil { - inputSchemaType = inputType.Schema.Type().String() - } - paramName, paramIn, paramSchema := "", "", "" - if inputType != nil && len(inputType.Parameters) > 0 && inputType.Parameters[0] != nil { - paramName = inputType.Parameters[0].Name - if inputType.Parameters[0].In != nil { - paramIn = string(inputType.Parameters[0].In.Kind) + ":" + inputType.Parameters[0].In.Name - } - if inputType.Parameters[0].Schema != nil && inputType.Parameters[0].Schema.Type() != nil { - paramSchema = inputType.Parameters[0].Schema.Type().String() - } - } - fmt.Printf("[DATLY_REPORT] built_input uri=%s input_schema=%s body_field=%q param_name=%s in=%s param_schema=%s\n", - original.URI, inputSchemaType, metadata.BodyFieldName, paramName, paramIn, paramSchema) - } reportURI := strings.TrimSuffix(original.URI, "/") + "/report" ret := *original ret.Path = contract.Path{Method: http.MethodPost, URI: reportURI} @@ -134,50 +109,9 @@ func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, o } reportPath = &pathCopy } - if os.Getenv("DATLY_DEBUG_REPORT_GEN") != "" { - debugReportComponent("after_build_report_original", original) - debugReportComponent("after_build_report_report", &ret) - } return &ret, reportPath, nil } -func debugReportComponent(label string, component *Component) { - if component == nil { - fmt.Printf("[DATLY_REPORT_GEN] %s component=nil\n", label) - return - } - viewName := "" - viewSchemaName := "" - viewSchemaType := "" - viewDefs := 0 - if component.View != nil { - viewName = component.View.Name - viewDefs = len(component.View.TypeDefinitions()) - if component.View.Schema != nil { - viewSchemaName = component.View.Schema.Name - if component.View.Schema.Type() != nil { - viewSchemaType = component.View.Schema.Type().String() - } - } - } - outputSchemaName := "" - outputSchemaType := "" - outputTag := "" - if component.Output.Type.Parameters != nil { - if param := component.Output.Type.Parameters.LookupByLocation(state.KindOutput, "view"); param != nil { - outputTag = param.Tag - if param.Schema != nil { - outputSchemaName = param.Schema.Name - if param.Schema.Type() != nil { - outputSchemaType = param.Schema.Type().String() - } - } - } - } - fmt.Printf("[DATLY_REPORT_GEN] %s uri=%s method=%s view=%s viewSchema=%s viewSchemaType=%s viewDefs=%d outputSchema=%s outputSchemaType=%s outputTag=%s\n", - label, component.URI, component.Method, viewName, viewSchemaName, viewSchemaType, viewDefs, outputSchemaName, outputSchemaType, outputTag) -} - func buildReportWrapperView(original *view.View) *view.View { if original == nil { return nil @@ -274,13 +208,6 @@ func buildReportMetadata(component *Component, report *Report) (*ReportMetadata, if err := result.validateSelection(); err != nil { return nil, err } - if os.Getenv("DATLY_DEBUG_REPORT") != "" { - var filters []string - for _, filter := range result.Filters { - filters = append(filters, filter.Name+":"+filter.FieldName) - } - fmt.Printf("[DATLY_REPORT] metadata input=%s dimensions=%d measures=%d filters=%v\n", result.InputName, len(result.Dimensions), len(result.Measures), filters) - } return result, nil } From a9e233d882786679b3b35d9b5bb66940e38ad3fb Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 10 Mar 2026 07:13:11 -0700 Subject: [PATCH 190/279] extended grouping e2e --- .../patch_basic_many.sql | 33 +++++ .../patch_basic_one.sql | 33 +++++ .../patch_basic_many_many.sql | 55 +++++++ .../post_basic_many.sql | 28 ++++ .../post_basic_one.sql | 28 ++++ .../post_comprehensive_many.sql | 29 ++++ .../dql/generate_post_except/post_except.sql | 28 ++++ e2e/local/regression/app.yaml | 4 +- .../regression/cases/010_grouping/test.yaml | 73 ++++++++++ .../regression/dev/.meta/vendor_meta.yaml | 137 ------------------ e2e/local/regression/regression.yaml | 2 +- e2e/v1/regression/app.yaml | 2 +- 12 files changed, 311 insertions(+), 141 deletions(-) create mode 100644 e2e/local/dql/generate_patch_basic_many/patch_basic_many.sql create mode 100644 e2e/local/dql/generate_patch_basic_one/patch_basic_one.sql create mode 100644 e2e/local/dql/generate_patch_many_many/patch_basic_many_many.sql create mode 100644 e2e/local/dql/generate_post_basic_many/post_basic_many.sql create mode 100644 e2e/local/dql/generate_post_basic_one/post_basic_one.sql create mode 100644 e2e/local/dql/generate_post_comprehensive_many/post_comprehensive_many.sql create mode 100644 e2e/local/dql/generate_post_except/post_except.sql delete mode 100644 e2e/local/regression/dev/.meta/vendor_meta.yaml diff --git a/e2e/local/dql/generate_patch_basic_many/patch_basic_many.sql b/e2e/local/dql/generate_patch_basic_many/patch_basic_many.sql new file mode 100644 index 000000000..d99b50691 --- /dev/null +++ b/e2e/local/dql/generate_patch_basic_many/patch_basic_many.sql @@ -0,0 +1,33 @@ +/* {"URI":"/v1/api/dev/basic/foos-many","Method":"PATCH","Connector":"dev"} */ + + +import ( + "generate_patch_basic_many.Foos" + ) + + +#set($_ = $Foos<[]Foos>(body/).WithTag('anonymous:"true"').Required()) + #set($_ = $CurFoosId(param/Foos) /* +? SELECT ARRAY_AGG(Id) AS Values FROM `/` LIMIT 1 +*/ +) + #set($_ = $CurFoos<[]*Foos>(view/CurFoos) /* +? SELECT * FROM FOOS +WHERE $criteria.In("ID", $CurFoosId.Values) +*/ +) +#set($_ = $Foos<[]>(body/).WithTag('anonymous:"true" typeName:"Foos"').Required().Output()) + + + +$sequencer.Allocate("FOOS", $Foos, "Id") + +#set($CurFoosById = $CurFoos.IndexBy("Id")) + +#foreach($RecFoos in $Foos) + #if($CurFoosById.HasKey($RecFoos.Id) == true) +$sql.Update($RecFoos, "FOOS"); + #else +$sql.Insert($RecFoos, "FOOS"); + #end +#end \ No newline at end of file diff --git a/e2e/local/dql/generate_patch_basic_one/patch_basic_one.sql b/e2e/local/dql/generate_patch_basic_one/patch_basic_one.sql new file mode 100644 index 000000000..0a1bc8093 --- /dev/null +++ b/e2e/local/dql/generate_patch_basic_one/patch_basic_one.sql @@ -0,0 +1,33 @@ +/* {"URI":"/v1/api/dev/basic/foos","Method":"PATCH","Connector":"dev"} */ + + +import ( + "generate_patch_basic_one.Foos" + ) + + +#set($_ = $Foos(body/).WithTag('anonymous:"true"').Required()) + #set($_ = $CurFoosId(param/Foos) /* +? SELECT ARRAY_AGG(Id) AS Values FROM `/` LIMIT 1 +*/ +) + #set($_ = $CurFoos<[]*Foos>(view/CurFoos) /* +? SELECT * FROM FOOS +WHERE $criteria.In("ID", $CurFoosId.Values) +*/ +) +#set($_ = $Foos<>(body/).WithTag('anonymous:"true" typeName:"Foos"').Required().Output()) + + + +$sequencer.Allocate("FOOS", $Foos, "Id") + +#set($CurFoosById = $CurFoos.IndexBy("Id")) + +#if($Foos) + #if($CurFoosById.HasKey($Foos.Id) == true) +$sql.Update($Foos, "FOOS"); + #else +$sql.Insert($Foos, "FOOS"); + #end +#end \ No newline at end of file diff --git a/e2e/local/dql/generate_patch_many_many/patch_basic_many_many.sql b/e2e/local/dql/generate_patch_many_many/patch_basic_many_many.sql new file mode 100644 index 000000000..c7c6b2588 --- /dev/null +++ b/e2e/local/dql/generate_patch_many_many/patch_basic_many_many.sql @@ -0,0 +1,55 @@ +/* {"URI":"/v1/api/dev/basic/foos-many-many","Method":"PATCH","Connector":"dev"} */ + + +import ( + "generate_patch_many_many.Foos" + "generate_patch_many_many.FoosPerformance" + ) + + +#set($_ = $Foos<[]Foos>(body/).WithTag('anonymous:"true"').Required()) + #set($_ = $CurFoosId(param/Foos) /* +? SELECT ARRAY_AGG(Id) AS Values FROM `/` LIMIT 1 +*/ +) + #set($_ = $CurFoosFoosPerformanceId(param/Foos) /* +? SELECT ARRAY_AGG(Id) AS Values FROM `/FoosPerformance` LIMIT 1 +*/ +) + #set($_ = $CurFoosPerformance<[]*FoosPerformance>(view/CurFoosPerformance) /* +? SELECT * FROM FOOS_PERFORMANCE +WHERE $criteria.In("ID", $CurFoosFoosPerformanceId.Values) +*/ +) + #set($_ = $CurFoos<[]*Foos>(view/CurFoos) /* +? SELECT * FROM FOOS +WHERE $criteria.In("ID", $CurFoosId.Values) +*/ +) +#set($_ = $Foos<[]>(body/).WithTag('anonymous:"true" typeName:"Foos"').Required().Output()) + + + +$sequencer.Allocate("FOOS", $Foos, "Id") + +$sequencer.Allocate("FOOS_PERFORMANCE", $Foos, "FoosPerformance/Id") + +#set($CurFoosById = $CurFoos.IndexBy("Id")) +#set($CurFoosPerformanceById = $CurFoosPerformance.IndexBy("Id")) + +#foreach($RecFoos in $Foos) + #if($CurFoosById.HasKey($RecFoos.Id) == true) +$sql.Update($RecFoos, "FOOS"); + #else +$sql.Insert($RecFoos, "FOOS"); + #end + + #foreach($RecFoosPerformance in $RecFoos.FoosPerformance) + #set($RecFoosPerformance.FooId = $RecFoos.Id) + #if($CurFoosPerformanceById.HasKey($RecFoosPerformance.Id) == true) +$sql.Update($RecFoosPerformance, "FOOS_PERFORMANCE"); + #else +$sql.Insert($RecFoosPerformance, "FOOS_PERFORMANCE"); + #end + #end +#end \ No newline at end of file diff --git a/e2e/local/dql/generate_post_basic_many/post_basic_many.sql b/e2e/local/dql/generate_post_basic_many/post_basic_many.sql new file mode 100644 index 000000000..8e9013974 --- /dev/null +++ b/e2e/local/dql/generate_post_basic_many/post_basic_many.sql @@ -0,0 +1,28 @@ +/* {"URI":"/v1/api/dev/basic/events-many","Method":"POST","Connector":"dev"} */ + + +import ( + "generate_post_basic_many.Events" + ) + + +#set($_ = $Events<[]Events>(body/).WithTag('anonymous:"true"').Required()) + #set($_ = $CurEventsId(param/Events) /* +? SELECT ARRAY_AGG(Id) AS Values FROM `/` LIMIT 1 +*/ +) + #set($_ = $CurEvents<[]*Events>(view/CurEvents) /* +? SELECT * FROM EVENTS +WHERE $criteria.In("ID", $CurEventsId.Values) +*/ +) +#set($_ = $Events<[]>(body/).WithTag('anonymous:"true" typeName:"Events"').Required().Output()) + + + +$sequencer.Allocate("EVENTS", $Events, "Id") + + +#foreach($RecEvents in $Events) +$sql.Insert($RecEvents, "EVENTS"); +#end \ No newline at end of file diff --git a/e2e/local/dql/generate_post_basic_one/post_basic_one.sql b/e2e/local/dql/generate_post_basic_one/post_basic_one.sql new file mode 100644 index 000000000..8e7680390 --- /dev/null +++ b/e2e/local/dql/generate_post_basic_one/post_basic_one.sql @@ -0,0 +1,28 @@ +/* {"URI":"/v1/api/dev/basic/events","Method":"POST","Connector":"dev"} */ + + +import ( + "generate_post_basic_one.Events" + ) + + +#set($_ = $Events(body/).WithTag('anonymous:"true"').Required()) + #set($_ = $CurEventsId(param/Events) /* +? SELECT ARRAY_AGG(Id) AS Values FROM `/` LIMIT 1 +*/ +) + #set($_ = $CurEvents<*Events>(view/CurEvents) /* +? SELECT * FROM EVENTS +WHERE $criteria.In("ID", $CurEventsId.Values) +*/ +) +#set($_ = $Events<>(body/).WithTag('anonymous:"true" typeName:"Events"').Required().Output()) + + + +$sequencer.Allocate("EVENTS", $Events, "Id") + + +#if($Events) +$sql.Insert($Events, "EVENTS"); +#end \ No newline at end of file diff --git a/e2e/local/dql/generate_post_comprehensive_many/post_comprehensive_many.sql b/e2e/local/dql/generate_post_comprehensive_many/post_comprehensive_many.sql new file mode 100644 index 000000000..15dc98aac --- /dev/null +++ b/e2e/local/dql/generate_post_comprehensive_many/post_comprehensive_many.sql @@ -0,0 +1,29 @@ +/* {"URI":"/v1/api/dev/comprehensive/events-many","Method":"POST","Connector":"dev"} */ + + +import ( + "generate_post_comprehensive_many.Events" + ) + + +#set($_ = $Events<[]Events>(body/Data).Required()) + #set($_ = $CurEventsId(param/Events) /* +? SELECT ARRAY_AGG(Id) AS Values FROM `/` LIMIT 1 +*/ +) + #set($_ = $CurEvents<[]*Events>(view/CurEvents) /* +? SELECT * FROM EVENTS +WHERE $criteria.In("ID", $CurEventsId.Values) +*/ +) +#set($_ = $Status<*>(output/status).WithTag('anonymous:"true"').Output()) + #set($_ = $Data<[]>(body/Data).WithTag(' typeName:"Events"').Required().Output()) + + + +$sequencer.Allocate("EVENTS", $Events, "Id") + + +#foreach($RecEvents in $Events) +$sql.Insert($RecEvents, "EVENTS"); +#end \ No newline at end of file diff --git a/e2e/local/dql/generate_post_except/post_except.sql b/e2e/local/dql/generate_post_except/post_except.sql new file mode 100644 index 000000000..d143860ef --- /dev/null +++ b/e2e/local/dql/generate_post_except/post_except.sql @@ -0,0 +1,28 @@ +/* {"URI":"/v1/api/dev/basic/events-except","Method":"POST","Connector":"dev"} */ + + +import ( + "generate_post_except.Events" + ) + + +#set($_ = $Events(body/).WithTag('anonymous:"true"').Required()) + #set($_ = $CurEventsId(param/Events) /* +? SELECT ARRAY_AGG(Id) AS Values FROM `/` LIMIT 1 +*/ +) + #set($_ = $CurEvents<*Events>(view/CurEvents) /* +? SELECT * FROM EVENTS +WHERE $criteria.In("ID", $CurEventsId.Values) +*/ +) +#set($_ = $Events<>(body/).WithTag('anonymous:"true" typeName:"Events"').Required().Output()) + + + +$sequencer.Allocate("EVENTS", $Events, "Id") + + +#if($Events) +$sql.Insert($Events, "EVENTS"); +#end \ No newline at end of file diff --git a/e2e/local/regression/app.yaml b/e2e/local/regression/app.yaml index 33036c290..0d29167db 100644 --- a/e2e/local/regression/app.yaml +++ b/e2e/local/regression/app.yaml @@ -14,7 +14,7 @@ pipeline: immuneToHangups: true env: TEST: 1 - command: ulimit -Sn 10000 && ./datly -c=$appPath/e2e/local/autogen/Datly/config.json -z=/tmp/jobs/datly > /tmp/datly.out + command: ulimit -Sn 10000 && ./datly -c=$appPath/e2e/local/autogen/Datly/config.json -z=/tmp/jobs/datly --mcpPort=8281 > /tmp/datly.out validator: stop: @@ -33,4 +33,4 @@ pipeline: TEST: 1 VALIDATOR_PORT: 8871 - command: ./validator_24 \ No newline at end of file + command: ./validator_24 diff --git a/e2e/local/regression/cases/010_grouping/test.yaml b/e2e/local/regression/cases/010_grouping/test.yaml index 2d0e70000..222546276 100644 --- a/e2e/local/regression/cases/010_grouping/test.yaml +++ b/e2e/local/regression/cases/010_grouping/test.yaml @@ -111,3 +111,76 @@ pipeline: Expect: Code: 200 JSONBody: $LoadJSON('${parentPath}/expect_empty.json') + + mcpInitialize: + action: http/runner:send + requests: + - Method: POST + URL: http://127.0.0.1:8281/mcp + Header: + Content-Type: + - application/json + JSONBody: + jsonrpc: "2.0" + id: 1 + method: initialize + params: + protocolVersion: "2025-06-18" + capabilities: {} + clientInfo: + name: endly + version: "1.0" + Expect: + Code: 200 + post: + protocolVersion: ${Responses[0].JSONBody.result.protocolVersion} + + mcpListTools: + action: http/runner:send + requests: + - Method: POST + URL: http://127.0.0.1:8281/mcp + Header: + Content-Type: + - application/json + MCP-Protocol-Version: + - ${protocolVersion} + JSONBody: + jsonrpc: "2.0" + id: 2 + method: tools/list + params: {} + Expect: + Code: 200 + + mcpCallReport: + action: http/runner:send + requests: + - Method: POST + URL: http://127.0.0.1:8281/mcp + Header: + Content-Type: + - application/json + MCP-Protocol-Version: + - ${protocolVersion} + JSONBody: + jsonrpc: "2.0" + id: 3 + method: tools/call + params: + name: vendorsgroupingReport + arguments: + dimensions: + accountId: true + measures: + totalId: true + maxId: true + filters: + vendorIDs: + - 1 + - 2 + - 3 + orderBy: + - accountId + Expect: + Code: 200 diff --git a/e2e/local/regression/dev/.meta/vendor_meta.yaml b/e2e/local/regression/dev/.meta/vendor_meta.yaml deleted file mode 100644 index 626be2a09..000000000 --- a/e2e/local/regression/dev/.meta/vendor_meta.yaml +++ /dev/null @@ -1,137 +0,0 @@ -items: - products: - - name: ID - datatype: int - tag: "" - expression: "" - filterable: false - nullable: false - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - - name: my_name - datatype: VARCHAR - tag: ' sqlx:"my_name"' - expression: "" - filterable: false - nullable: true - default: "" - formattag: null - codec: null - databasecolumn: my_name - indexedby: "" - - name: VENDOR_ID - datatype: int - tag: "" - expression: "" - filterable: false - nullable: true - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - vendor: - - name: ID - datatype: int - tag: "" - expression: "" - filterable: false - nullable: false - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - - name: NAME - datatype: varchar - tag: "" - expression: "" - filterable: false - nullable: true - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - - name: ACCOUNT_ID - datatype: int - tag: "" - expression: "" - filterable: false - nullable: true - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - - name: CREATED - datatype: datetime - tag: "" - expression: "" - filterable: false - nullable: true - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - - name: USER_CREATED - datatype: int - tag: "" - expression: "" - filterable: false - nullable: true - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - - name: UPDATED - datatype: datetime - tag: "" - expression: "" - filterable: false - nullable: true - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - - name: USER_UPDATED - datatype: int - tag: "" - expression: "" - filterable: false - nullable: true - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - vendor/DataSummary/Meta: - - name: PAGE_CNT - datatype: BIGINT - tag: source:"1 + (COUNT(1) / 1)" - expression: "" - filterable: false - nullable: true - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" - - name: CNT - datatype: BIGINT - tag: "" - expression: "" - filterable: false - nullable: false - default: "" - formattag: null - codec: null - databasecolumn: "" - indexedby: "" -sourceurl: "" diff --git a/e2e/local/regression/regression.yaml b/e2e/local/regression/regression.yaml index 28d26c353..e4e47e667 100644 --- a/e2e/local/regression/regression.yaml +++ b/e2e/local/regression/regression.yaml @@ -30,7 +30,7 @@ pipeline: '[]gen': '@gen' subPath: 'cases/${index}_*' - range: 10..010 + range: 1..015 template: checkSkip: action: nop diff --git a/e2e/v1/regression/app.yaml b/e2e/v1/regression/app.yaml index d46066f08..50f8c3824 100644 --- a/e2e/v1/regression/app.yaml +++ b/e2e/v1/regression/app.yaml @@ -14,4 +14,4 @@ pipeline: immuneToHangups: true env: TEST: 1 - command: pkill -f '${v1Path}/autogen/Datly/config.json -z=/tmp/jobs/datly_v1' >/dev/null 2>&1 || true; ulimit -Sn 10000 && ./datly -c=${v1Path}/autogen/Datly/config.json -z=/tmp/jobs/datly_v1 > /tmp/datly_v1.out 2>&1 + command: pkill -f '${v1Path}/autogen/Datly/config.json -z=/tmp/jobs/datly_v1' >/dev/null 2>&1 || true; ulimit -Sn 10000 && ./datly -c=${v1Path}/autogen/Datly/config.json -z=/tmp/jobs/datly_v1 --mcpPort=8281 > /tmp/datly_v1.out 2>&1 From 1308c349df30c80894d169c58a4f835f6131b499 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 10 Mar 2026 07:14:08 -0700 Subject: [PATCH 191/279] enhanced grouping --- cmd/command/run.go | 12 + cmd/option.go | 9 +- cmd/options/run.go | 4 + internal/inference/tag_test.go | 44 +++ internal/translator/report_runtime_test.go | 78 ++++ repository/report.go | 121 +------ repository/report/build.go | 333 +++++++++++++++++ repository/report/build_test.go | 281 +++++++++++++++ repository/report/model.go | 119 +++++++ repository/report_runtime.go | 110 ++---- repository/report_runtime_test.go | 27 +- repository/shape/dql_engine_test.go | 45 +++ repository/shape/xgen/codegen.go | 1 - service/reader/anonymous_mysql_test.go | 171 +++++++++ service/reader/sql_groupable_test.go | 335 ++++++++++++++++++ .../reader/sql_projection_regression_test.go | 55 +++ service/session/selector_test.go | 54 +++ view/column_lookup_test.go | 27 ++ view/columns.go | 1 - 19 files changed, 1610 insertions(+), 217 deletions(-) create mode 100644 internal/inference/tag_test.go create mode 100644 internal/translator/report_runtime_test.go create mode 100644 repository/report/build.go create mode 100644 repository/report/build_test.go create mode 100644 repository/report/model.go create mode 100644 service/reader/anonymous_mysql_test.go create mode 100644 service/reader/sql_groupable_test.go create mode 100644 service/reader/sql_projection_regression_test.go create mode 100644 service/session/selector_test.go create mode 100644 view/column_lookup_test.go diff --git a/cmd/command/run.go b/cmd/command/run.go index a084275d2..db3ba7a1f 100644 --- a/cmd/command/run.go +++ b/cmd/command/run.go @@ -5,6 +5,7 @@ import ( "github.com/viant/afs/file" "github.com/viant/afs/url" "github.com/viant/datly/cmd/options" + "github.com/viant/datly/gateway" "github.com/viant/datly/gateway/runtime/standalone" "github.com/viant/datly/internal/setter" ) @@ -42,5 +43,16 @@ func (s *Service) run(ctx context.Context, run *options.Run) (*standalone.Server _ = s.fs.Copy(ctx, parent, s.config.Config.PluginsURL) } s.config.Version = run.Version + if run.MCPPort != nil || run.MCPAuthURL != "" || run.MCPIssuerURL != "" || run.MCPAuthMode != "" { + if s.config.Config.MCP == nil { + s.config.Config.MCP = &gateway.ModelContextProtocol{} + } + if run.MCPPort != nil { + s.config.Config.MCP.Port = run.MCPPort + } + setter.SetStringIfEmpty(&s.config.Config.MCP.OAuth2ConfigURL, run.MCPAuthURL) + setter.SetStringIfEmpty(&s.config.Config.MCP.IssuerURL, run.MCPIssuerURL) + setter.SetStringIfEmpty(&s.config.Config.MCP.AuthorizerMode, run.MCPAuthMode) + } return standalone.New(ctx, standalone.WithConfig(s.config)) } diff --git a/cmd/option.go b/cmd/option.go index 94511419a..85072d8d8 100644 --- a/cmd/option.go +++ b/cmd/option.go @@ -41,6 +41,10 @@ type ( cache *view.Cache SubstituesURL []string `long:"substituesURL" description:"substitues URL, expands template before processing"` JobURL string `short:"z" long:"joburl" description:"job url"` + MCPPort int `long:"mcpPort" description:"enable MCP HTTP server on the specified port"` + MCPAuthURL string `long:"mcpAuthClient" description:"auth client url for MCP server"` + MCPIssuerURL string `long:"mcpIssuerURL" description:"issuer url for MCP server"` + MCPAuthMode string `long:"mcpAuth" description:"authorizer S - server authorizer, F fallback authorizer"` } Package struct { @@ -166,7 +170,10 @@ func (o *Options) BuildOption() *options.Options { } if o.ConfigURL != "" && repo == nil { - result.Run = &options.Run{ConfigURL: o.ConfigURL, JobURL: o.JobURL} + result.Run = &options.Run{ConfigURL: o.ConfigURL, JobURL: o.JobURL, MCPAuthURL: o.MCPAuthURL, MCPIssuerURL: o.MCPIssuerURL, MCPAuthMode: o.MCPAuthMode} + if o.MCPPort > 0 { + result.Run.MCPPort = &o.MCPPort + } } return result } diff --git a/cmd/options/run.go b/cmd/options/run.go index fb810b3f7..19c1f991f 100644 --- a/cmd/options/run.go +++ b/cmd/options/run.go @@ -11,6 +11,10 @@ type Run struct { MaxJobs int `short:"W" long:"mjobs" description:"max jobs" default:"40" ` FailedJobURL string `short:"F" long:"fjobs" description:"failed jobs" ` LoadPlugin bool `short:"L" long:"lplugin" description:"load plugin"` + MCPPort *int `long:"mcpPort" description:"enable MCP HTTP server on the specified port"` + MCPAuthURL string `long:"mcpAuthClient" description:"auth client url for MCP server"` + MCPIssuerURL string `long:"mcpIssuerURL" description:"issuer url for MCP server"` + MCPAuthMode string `long:"mcpAuth" description:"authorizer S - server authorizer, F fallback authorizer" choice:"F" choice:"S"` PluginInfo string Version string } diff --git a/internal/inference/tag_test.go b/internal/inference/tag_test.go new file mode 100644 index 000000000..234325e4a --- /dev/null +++ b/internal/inference/tag_test.go @@ -0,0 +1,44 @@ +package inference + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view/state" + "github.com/viant/sqlparser" + "github.com/viant/sqlx/metadata/sink" +) + +func TestSpecBuildType_PreservesSourceTagForAliasedProjection(t *testing.T) { + spec := &Spec{ + Table: "CI_TAXONOMY_DISQUALIFIED", + Columns: sqlparser.Columns{ + &sqlparser.Column{ + Name: "TAXONOMY_ID", + Alias: "TAXONOMY_ID", + Expression: "dq.SEGMENT_ID", + Namespace: "dq", + Type: "string", + }, + &sqlparser.Column{ + Name: "IS_DISQUALIFIED", + Type: "int", + }, + }, + pk: map[string]sink.Key{}, + Fk: map[string]sink.Key{}, + } + + err := spec.BuildType("taxonomy", "DisqualifiedView", state.Many, nil, nil) + require.NoError(t, err) + require.NotNil(t, spec.Type) + require.Len(t, spec.Type.columnFields, 2) + + field := spec.Type.columnFields[0] + require.Equal(t, `sqlx:"TAXONOMY_ID" source:"SEGMENT_ID" validate:"required"`, field.Tag) + + structField := field.StructField(WithStructTag()) + require.Equal(t, "SEGMENT_ID", reflect.StructTag(structField.Tag).Get("source")) + require.Equal(t, "TAXONOMY_ID", reflect.StructTag(structField.Tag).Get("sqlx")) +} diff --git a/internal/translator/report_runtime_test.go b/internal/translator/report_runtime_test.go new file mode 100644 index 000000000..878826ab9 --- /dev/null +++ b/internal/translator/report_runtime_test.go @@ -0,0 +1,78 @@ +package translator + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/afs" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/gateway" + "github.com/viant/datly/gateway/runtime/standalone" + "github.com/viant/datly/repository" + "github.com/viant/datly/view" +) + +func TestService_persistRouterRule_PreservesReportMetadataOnRouteComponent(t *testing.T) { + routeRoot := t.TempDir() + + repoOptions := &options.Repository{ + RepositoryURL: routeRoot, + APIPrefix: "/v1/api", + } + cfg := &Config{ + repository: repoOptions, + Config: &standalone.Config{ + Config: &gateway.Config{ + ExposableConfig: gateway.ExposableConfig{ + RouteURL: routeRoot, + }, + }, + }, + } + svc := &Service{ + Repository: &Repository{ + fs: afs.New(), + Config: cfg, + }, + fs: afs.New(), + } + + ruleOptions := &options.Rule{ + Project: routeRoot, + ModulePrefix: "dev", + Source: []string{routeRoot + "/vendors_grouping.sql"}, + } + require.NoError(t, os.WriteFile(ruleOptions.Source[0], []byte("SELECT 1"), 0o600)) + require.NoError(t, ruleOptions.Init()) + + resource := NewResource(ruleOptions, repoOptions, nil) + resource.Rule.Root = "vendor" + resource.Rule.Route.URI = "/vendors-grouping" + resource.Rule.Route.Method = "GET" + resource.Rule.Report = &repository.Report{Enabled: true} + resource.Rule.Viewlets.Append(&Viewlet{ + Name: "vendor", + View: &View{ + View: view.View{ + Name: "vendor", + }, + }, + }) + + require.NoError(t, svc.persistRouterRule(context.Background(), resource, "Reader")) + require.NotEmpty(t, svc.Repository.Files) + + var persisted string + for _, candidate := range svc.Repository.Files { + if strings.HasSuffix(candidate.URL, "vendors_grouping.yaml") { + persisted = candidate.Content + break + } + } + require.NotEmpty(t, persisted) + require.Contains(t, persisted, "Report:") + require.Contains(t, persisted, "Enabled: true") +} diff --git a/repository/report.go b/repository/report.go index 4b8c1a5f1..37baa30cf 100644 --- a/repository/report.go +++ b/repository/report.go @@ -1,119 +1,8 @@ package repository -import ( - "fmt" - "reflect" - "strings" +import reportmodel "github.com/viant/datly/repository/report" - "github.com/viant/datly/view/state" -) - -type Report struct { - Enabled bool `json:",omitempty" yaml:"Enabled,omitempty"` - MCPTool *bool `json:",omitempty" yaml:"MCPTool,omitempty"` - Input string `json:",omitempty" yaml:"Input,omitempty"` - Dimensions string `json:",omitempty" yaml:"Dimensions,omitempty"` - Measures string `json:",omitempty" yaml:"Measures,omitempty"` - Filters string `json:",omitempty" yaml:"Filters,omitempty"` - OrderBy string `json:",omitempty" yaml:"OrderBy,omitempty"` - Limit string `json:",omitempty" yaml:"Limit,omitempty"` - Offset string `json:",omitempty" yaml:"Offset,omitempty"` -} - -type ReportMetadata struct { - InputName string - BodyFieldName string - DimensionsKey string - MeasuresKey string - FiltersKey string - Dimensions []*ReportField - Measures []*ReportField - Filters []*ReportFilter - OrderBy string - Limit string - Offset string -} - -type ReportField struct { - Name string - FieldName string - Section string - Description string -} - -type ReportFilter struct { - Name string - FieldName string - Section string - Description string - Parameter *state.Parameter -} - -func (r *Report) Clone() *Report { - if r == nil { - return nil - } - ret := *r - return &ret -} - -func (r *Report) normalize() *Report { - if r == nil { - return nil - } - ret := r.Clone() - ret.Input = strings.TrimSpace(ret.Input) - ret.Dimensions = defaultString(ret.Dimensions, "Dimensions") - ret.Measures = defaultString(ret.Measures, "Measures") - ret.Filters = defaultString(ret.Filters, "Filters") - ret.OrderBy = defaultString(ret.OrderBy, "OrderBy") - ret.Limit = defaultString(ret.Limit, "Limit") - ret.Offset = defaultString(ret.Offset, "Offset") - return ret -} - -func (r *Report) mcpToolEnabled() bool { - if r == nil || r.MCPTool == nil { - return true - } - return *r.MCPTool -} - -func (r *Report) inputTypeName(componentName, inputName, viewName string) string { - if r != nil && strings.TrimSpace(r.Input) != "" { - return strings.TrimSpace(r.Input) - } - switch { - case strings.TrimSpace(inputName) != "": - return state.SanitizeTypeName(strings.TrimSpace(inputName) + "ReportInput") - case strings.TrimSpace(componentName) != "": - return state.SanitizeTypeName(strings.TrimSpace(componentName) + "ReportInput") - default: - return state.SanitizeTypeName(strings.TrimSpace(viewName) + "ReportInput") - } -} - -func (r *ReportMetadata) validateSelection() error { - if r == nil { - return fmt.Errorf("report metadata was empty") - } - if len(r.Dimensions) == 0 && len(r.Measures) == 0 { - return fmt.Errorf("report metadata had no selectable dimensions or measures") - } - return nil -} - -func (r *ReportFilter) schemaType() reflect.Type { - if r == nil || r.Parameter == nil || r.Parameter.Schema == nil { - return nil - } - return r.Parameter.OutputType() -} - -func defaultString(value, fallback string) string { - value = strings.TrimSpace(value) - if value == "" { - return fallback - } - return value -} +type Report = reportmodel.Config +type ReportMetadata = reportmodel.Metadata +type ReportField = reportmodel.Field +type ReportFilter = reportmodel.Filter diff --git a/repository/report/build.go b/repository/report/build.go new file mode 100644 index 000000000..6d054e6a0 --- /dev/null +++ b/repository/report/build.go @@ -0,0 +1,333 @@ +package report + +import ( + "context" + "embed" + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/tagly/format/text" + "github.com/viant/xdatly/codec" + "github.com/viant/xreflect" +) + +type Component struct { + Name string + InputName string + Parameters state.Parameters + View *view.View + Resource state.Resource + Report *Config +} + +func AssembleMetadata(component *Component, cfg *Config) (*Metadata, error) { + if component == nil { + return nil, fmt.Errorf("report component was empty") + } + cfg = normalizeConfig(component, cfg) + viewRef := component.View + if viewRef == nil { + return nil, fmt.Errorf("report component view was empty") + } + result := &Metadata{ + InputName: cfg.InputTypeName(component.Name, component.InputName, viewRef.Name), + BodyFieldName: "", + DimensionsKey: cfg.Dimensions, + MeasuresKey: cfg.Measures, + FiltersKey: cfg.Filters, + OrderBy: cfg.OrderBy, + Limit: cfg.Limit, + Offset: cfg.Offset, + } + for _, column := range viewRef.Columns { + if column == nil || column.FieldName() == "" { + continue + } + fieldName := exportedFieldName(column.FieldName()) + field := &Field{Name: column.FieldName(), FieldName: fieldName, Description: column.Name} + switch { + case column.Groupable: + field.Section = cfg.Dimensions + result.Dimensions = append(result.Dimensions, field) + case column.Aggregate || (viewRef.Groupable && !column.Groupable): + field.Section = cfg.Measures + result.Measures = append(result.Measures, field) + } + } + for _, parameter := range component.Parameters { + if parameter == nil || len(parameter.Predicates) == 0 || parameter.In == nil { + continue + } + if isSelectorParameter(parameter, viewRef) { + continue + } + result.Filters = append(result.Filters, &Filter{ + Name: parameter.Name, + FieldName: exportedFieldName(parameter.Name), + Section: cfg.Filters, + Description: parameter.Description, + Parameter: parameter, + }) + } + if err := result.ValidateSelection(); err != nil { + return nil, err + } + return result, nil +} + +func BuildBodyType(metadata *Metadata) reflect.Type { + var fields []reflect.StructField + fields = append(fields, reflect.StructField{ + Name: metadata.DimensionsKey, + Type: sectionStructType(metadata.Dimensions), + Tag: buildTag(lowerCamel(metadata.DimensionsKey), "Selected grouping dimensions"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.MeasuresKey, + Type: sectionStructType(metadata.Measures), + Tag: buildTag(lowerCamel(metadata.MeasuresKey), "Selected aggregate measures"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.FiltersKey, + Type: filterStructType(metadata.Filters), + Tag: buildTag(lowerCamel(metadata.FiltersKey), "Report filters derived from original predicate parameters"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.OrderBy, + Type: reflect.TypeOf([]string{}), + Tag: buildTag(lowerCamel(metadata.OrderBy), "Ordering expressions applied to the grouped result"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.Limit, + Type: reflect.TypeOf((*int)(nil)), + Tag: buildTag(lowerCamel(metadata.Limit), "Maximum number of grouped rows to return"), + }) + fields = append(fields, reflect.StructField{ + Name: metadata.Offset, + Type: reflect.TypeOf((*int)(nil)), + Tag: buildTag(lowerCamel(metadata.Offset), "Row offset applied to the grouped result"), + }) + return reflect.StructOf(fields) +} + +func BuildInputType(component *Component, metadata *Metadata, cfg *Config) (*state.Type, error) { + if component == nil { + return nil, fmt.Errorf("report component was empty") + } + if metadata == nil { + return nil, fmt.Errorf("report metadata was empty") + } + cfg = normalizeConfig(component, cfg) + if cfg.Input != "" { + schema := state.NewSchema(nil, state.WithSchemaPackage(""), state.WithModulePath("")) + schema.Name = strings.TrimSpace(cfg.Input) + inputType, err := state.NewType(state.WithSchema(schema), state.WithResource(component.resource())) + if err != nil { + return nil, err + } + if err := inputType.Init(); err != nil { + return nil, err + } + return inputType, validateExplicitInput(inputType, metadata) + } + bodyType := reflect.PtrTo(BuildBodyType(metadata)) + bodySchema := state.NewSchema(bodyType) + bodySchema.Name = metadata.InputName + bodyParam := state.NewParameter("Report", state.NewBodyLocation(""), state.WithParameterSchema(bodySchema)) + bodyParam.Tag = `anonymous:"true"` + bodyParam.SetTypeNameTag() + inputType, err := state.NewType( + state.WithParameters(state.Parameters{bodyParam}), + state.WithBodyType(true), + state.WithSchema(state.NewSchema(bodyType)), + state.WithResource(newInputResource(component.resource())), + ) + if err != nil { + return nil, err + } + if err := inputType.Init(); err != nil { + return nil, err + } + inputType.Name = metadata.InputName + return inputType, nil +} + +func normalizeConfig(component *Component, cfg *Config) *Config { + if cfg != nil { + return cfg.Normalize() + } + if component == nil || component.Report == nil { + return (&Config{}).Normalize() + } + return component.Report.Normalize() +} + +func (c *Component) resource() state.Resource { + if c == nil { + return nil + } + if c.Resource != nil { + return c.Resource + } + if c.View != nil { + return c.View.Resource() + } + return nil +} + +func exportedFieldName(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + return state.SanitizeTypeName(value) +} + +func isSelectorParameter(parameter *state.Parameter, aView *view.View) bool { + if parameter == nil || parameter.In == nil { + return false + } + if aView != nil && aView.Selector != nil { + for _, selector := range []*state.Parameter{ + aView.Selector.FieldsParameter, + aView.Selector.OrderByParameter, + aView.Selector.LimitParameter, + aView.Selector.OffsetParameter, + aView.Selector.PageParameter, + } { + if selector != nil && selector.In != nil && selector.In.Name == parameter.In.Name { + return true + } + } + } + name := strings.ToLower(parameter.In.Name) + return name == "_fields" || name == "_orderby" || name == "_limit" || name == "_offset" || name == "_page" || name == "criteria" +} + +func validateExplicitInput(inputType *state.Type, metadata *Metadata) error { + if inputType == nil { + return fmt.Errorf("explicit report input type was empty") + } + var rType reflect.Type + if inputType.Schema != nil { + rType = inputType.Schema.Type() + } + if rType == nil && inputType.Type() != nil { + rType = inputType.Type().Type() + } + if rType == nil { + return fmt.Errorf("explicit report input state type was empty") + } + if rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + for _, fieldName := range []string{metadata.DimensionsKey, metadata.MeasuresKey, metadata.FiltersKey, metadata.OrderBy, metadata.Limit, metadata.Offset} { + if fieldName == "" { + continue + } + if _, ok := rType.FieldByName(fieldName); !ok { + return fmt.Errorf("explicit report input %s missing field %s", rType.String(), fieldName) + } + } + return nil +} + +func sectionStructType(fields []*Field) reflect.Type { + if len(fields) == 0 { + return reflect.TypeOf(struct{}{}) + } + structFields := make([]reflect.StructField, 0, len(fields)) + for _, field := range fields { + structFields = append(structFields, reflect.StructField{ + Name: field.FieldName, + Type: reflect.TypeOf(false), + Tag: buildTag(lowerCamel(field.Name), field.Description), + }) + } + return reflect.StructOf(structFields) +} + +func filterStructType(filters []*Filter) reflect.Type { + if len(filters) == 0 { + return reflect.TypeOf(struct{}{}) + } + structFields := make([]reflect.StructField, 0, len(filters)) + for _, filter := range filters { + rType := reflect.TypeOf("") + if schemaType := filter.SchemaType(); schemaType != nil { + rType = schemaType + } + structFields = append(structFields, reflect.StructField{ + Name: filter.FieldName, + Type: rType, + Tag: buildTag(lowerCamel(filter.Name), filter.Description), + }) + } + return reflect.StructOf(structFields) +} + +func buildTag(jsonName, description string) reflect.StructTag { + result := fmt.Sprintf(`json:"%s,omitempty"`, jsonName) + if description = strings.TrimSpace(description); description != "" { + result += " desc:" + strconv.Quote(description) + } + return reflect.StructTag(result) +} + +func lowerCamel(value string) string { + if value == "" { + return "" + } + return text.CaseFormatUpperCamel.Format(value, text.CaseFormatLowerCamel) +} + +type inputResource struct { + base state.Resource +} + +func newInputResource(base state.Resource) state.Resource { + return &inputResource{base: base} +} + +func (r *inputResource) LookupParameter(name string) (*state.Parameter, error) { return nil, nil } +func (r *inputResource) AppendParameter(parameter *state.Parameter) {} +func (r *inputResource) ViewSchema(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *inputResource) ViewSchemaPointer(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *inputResource) LookupType() xreflect.LookupType { return nil } +func (r *inputResource) LoadText(ctx context.Context, URL string) (string, error) { + return "", nil +} +func (r *inputResource) Codecs() *codec.Registry { + if r.base != nil && r.base.Codecs() != nil { + return r.base.Codecs() + } + return codec.New() +} +func (r *inputResource) CodecOptions() *codec.Options { + if r.base != nil && r.base.CodecOptions() != nil { + return r.base.CodecOptions() + } + return codec.NewOptions(nil) +} +func (r *inputResource) ExpandSubstitutes(value string) string { + if r.base != nil { + return r.base.ExpandSubstitutes(value) + } + return value +} +func (r *inputResource) ReverseSubstitutes(value string) string { + if r.base != nil { + return r.base.ReverseSubstitutes(value) + } + return value +} +func (r *inputResource) EmbedFS() *embed.FS { return nil } +func (r *inputResource) SetFSEmbedder(embedder *state.FSEmbedder) {} diff --git a/repository/report/build_test.go b/repository/report/build_test.go new file mode 100644 index 000000000..0551a5412 --- /dev/null +++ b/repository/report/build_test.go @@ -0,0 +1,281 @@ +package report + +import ( + "context" + "embed" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + "github.com/viant/datly/view/extension" + "github.com/viant/datly/view/state" + "github.com/viant/tagly/format/text" + "github.com/viant/xdatly/codec" + "github.com/viant/xreflect" +) + +type testResource struct{} + +type explicitReportInput struct { + Dimensions struct { + AccountID bool + UserCreated bool + } + Measures struct { + TotalSpend bool + } + Filters struct { + AccountId int + } + OrderBy []string + Limit *int + Offset *int +} + +func (r *testResource) LookupParameter(name string) (*state.Parameter, error) { return nil, nil } +func (r *testResource) AppendParameter(parameter *state.Parameter) {} +func (r *testResource) ViewSchema(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *testResource) ViewSchemaPointer(ctx context.Context, name string) (*state.Schema, error) { + return nil, nil +} +func (r *testResource) LookupType() xreflect.LookupType { return nil } +func (r *testResource) LoadText(ctx context.Context, URL string) (string, error) { + return "", nil +} +func (r *testResource) Codecs() *codec.Registry { return codec.New() } +func (r *testResource) CodecOptions() *codec.Options { return codec.NewOptions(nil) } +func (r *testResource) ExpandSubstitutes(value string) string { return value } +func (r *testResource) ReverseSubstitutes(value string) string { return value } +func (r *testResource) EmbedFS() *embed.FS { return nil } +func (r *testResource) SetFSEmbedder(embedder *state.FSEmbedder) { +} + +func TestAssembleMetadata(t *testing.T) { + tests := []struct { + name string + component *Component + config *Config + assertion func(t *testing.T, got *Metadata, err error) + }{ + { + name: "uses component report defaults", + component: newComponentFixture(t, &Config{Enabled: true}), + assertion: func(t *testing.T, got *Metadata, err error) { + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "VendorInputReportInput", got.InputName) + assert.Equal(t, "Dimensions", got.DimensionsKey) + assert.Equal(t, "Measures", got.MeasuresKey) + assert.Equal(t, "Filters", got.FiltersKey) + require.Len(t, got.Dimensions, 2) + require.Len(t, got.Measures, 1) + require.Len(t, got.Filters, 1) + assert.Equal(t, "AccountID", got.Dimensions[0].Name) + assert.Equal(t, "UserCreated", got.Dimensions[1].Name) + assert.Equal(t, "TotalSpend", got.Measures[0].Name) + assert.Equal(t, "accountID", got.Filters[0].Name) + assert.Equal(t, "AccountId", got.Filters[0].FieldName) + }, + }, + { + name: "uses explicit config names", + component: newComponentFixture(t, &Config{Enabled: true}), + config: &Config{ + Input: "CustomReportInput", + Dimensions: "Groups", + Measures: "Metrics", + Filters: "Predicates", + OrderBy: "Sort", + Limit: "PageSize", + Offset: "Cursor", + }, + assertion: func(t *testing.T, got *Metadata, err error) { + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "CustomReportInput", got.InputName) + assert.Equal(t, "Groups", got.DimensionsKey) + assert.Equal(t, "Metrics", got.MeasuresKey) + assert.Equal(t, "Predicates", got.FiltersKey) + assert.Equal(t, "Sort", got.OrderBy) + assert.Equal(t, "PageSize", got.Limit) + assert.Equal(t, "Cursor", got.Offset) + assert.Equal(t, "Groups", got.Dimensions[0].Section) + assert.Equal(t, "Metrics", got.Measures[0].Section) + assert.Equal(t, "Predicates", got.Filters[0].Section) + }, + }, + { + name: "errors on missing view", + component: &Component{Report: &Config{Enabled: true}}, + assertion: func(t *testing.T, got *Metadata, err error) { + require.Error(t, err) + assert.Nil(t, got) + assert.Contains(t, err.Error(), "view was empty") + }, + }, + { + name: "errors when no selectable columns", + component: newComponentWithoutSelectableColumns(t), + assertion: func(t *testing.T, got *Metadata, err error) { + require.Error(t, err) + assert.Nil(t, got) + assert.Contains(t, err.Error(), "no selectable dimensions or measures") + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := AssembleMetadata(test.component, test.config) + test.assertion(t, got, err) + }) + } +} + +func TestBuildInputType(t *testing.T) { + tests := []struct { + name string + component *Component + config *Config + assertion func(t *testing.T, got *state.Type, err error) + }{ + { + name: "builds synthetic anonymous body input", + component: newComponentFixture(t, &Config{Enabled: true}), + assertion: func(t *testing.T, got *state.Type, err error) { + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "VendorInputReportInput", got.Name) + require.Len(t, got.Parameters, 1) + assert.True(t, got.Parameters[0].IsAnonymous()) + require.NotNil(t, got.Schema) + rType := got.Schema.Type() + require.NotNil(t, rType) + assert.Equal(t, reflect.Ptr, rType.Kind()) + bodyType := rType.Elem() + dimensions, ok := bodyType.FieldByName("Dimensions") + require.True(t, ok) + assert.Equal(t, `json:"dimensions,omitempty" desc:"Selected grouping dimensions"`, string(dimensions.Tag)) + measures, ok := bodyType.FieldByName("Measures") + require.True(t, ok) + assert.Equal(t, reflect.Struct, measures.Type.Kind()) + filters, ok := bodyType.FieldByName("Filters") + require.True(t, ok) + filterField, ok := filters.Type.FieldByName("AccountId") + require.True(t, ok) + assert.Contains(t, string(filterField.Tag), `json:"accountId,omitempty"`) + assert.Contains(t, string(filterField.Tag), `desc:"Account identifier filter"`) + limit, ok := bodyType.FieldByName("Limit") + require.True(t, ok) + assert.Equal(t, reflect.TypeOf((*int)(nil)), limit.Type) + }, + }, + { + name: "uses explicit configured input type", + component: newComponentWithExplicitInput(t), + config: (&Config{Input: "ExplicitReportInput"}).Normalize(), + assertion: func(t *testing.T, got *state.Type, err error) { + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.Type()) + require.NotNil(t, got.Type().Type()) + assert.Equal(t, reflect.TypeOf(explicitReportInput{}), got.Type().Type()) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + metadata, err := AssembleMetadata(test.component, test.config) + require.NoError(t, err) + got, err := BuildInputType(test.component, metadata, test.config) + test.assertion(t, got, err) + }) + } +} + +func newComponentFixture(t *testing.T, reportCfg *Config) *Component { + t.Helper() + resource := view.EmptyResource() + columnResource := &testResource{} + rootView := view.NewView("vendor", "VENDOR") + rootView.Groupable = true + rootView.Selector = &view.Config{ + FieldsParameter: &state.Parameter{Name: "fields", In: state.NewQueryLocation("_fields")}, + OrderByParameter: &state.Parameter{Name: "orderBy", In: state.NewQueryLocation("_orderby")}, + LimitParameter: &state.Parameter{Name: "limit", In: state.NewQueryLocation("_limit")}, + OffsetParameter: &state.Parameter{Name: "offset", In: state.NewQueryLocation("_offset")}, + } + rootView.Columns = []*view.Column{ + view.NewColumn("AccountID", "int", reflect.TypeOf(0), false), + view.NewColumn("UserCreated", "int", reflect.TypeOf(0), false), + view.NewColumn("TotalSpend", "float64", reflect.TypeOf(float64(0)), false), + } + rootView.Columns[0].Groupable = true + rootView.Columns[1].Groupable = true + rootView.Columns[2].Aggregate = true + for _, column := range rootView.Columns { + require.NoError(t, column.Init(columnResource, text.CaseFormatUndefined, false)) + } + rootView.SetResource(resource) + resource.AddViews(rootView) + + inputType, err := state.NewType(state.WithParameters(state.Parameters{ + &state.Parameter{Name: "vendorIDs", In: state.NewQueryLocation("vendorIDs"), Schema: state.NewSchema(reflect.TypeOf([]int{})), Description: "Vendor IDs to include"}, + &state.Parameter{Name: "accountID", In: state.NewQueryLocation("accountID"), Schema: state.NewSchema(reflect.TypeOf(0)), Predicates: []*extension.PredicateConfig{{Name: "ByAccount"}}, Description: "Account identifier filter"}, + &state.Parameter{Name: "fields", In: state.NewQueryLocation("_fields"), Schema: state.NewSchema(reflect.TypeOf([]string{}))}, + }), state.WithResource(columnResource)) + require.NoError(t, err) + inputType.Name = "VendorInput" + + return &Component{ + Name: "vendors", + InputName: inputType.Name, + Parameters: inputType.Parameters, + View: rootView, + Resource: rootView.Resource(), + Report: reportCfg, + } +} + +func newComponentWithoutSelectableColumns(t *testing.T) *Component { + t.Helper() + resource := view.EmptyResource() + columnResource := &testResource{} + rootView := view.NewView("vendor", "VENDOR") + rootView.Groupable = false + rootView.Columns = []*view.Column{ + view.NewColumn("PlainValue", "int", reflect.TypeOf(0), false), + } + for _, column := range rootView.Columns { + require.NoError(t, column.Init(columnResource, text.CaseFormatUndefined, false)) + } + rootView.SetResource(resource) + resource.AddViews(rootView) + + inputType, err := state.NewType(state.WithParameters(nil), state.WithResource(columnResource)) + require.NoError(t, err) + + return &Component{ + Name: "vendors", + InputName: inputType.Name, + Parameters: inputType.Parameters, + View: rootView, + Resource: rootView.Resource(), + Report: &Config{Enabled: true}, + } +} + +func newComponentWithExplicitInput(t *testing.T) *Component { + t.Helper() + component := newComponentFixture(t, &Config{Enabled: true, Input: "ExplicitReportInput"}) + resource := view.EmptyResource() + require.NoError(t, resource.TypeRegistry().Register("ExplicitReportInput", xreflect.WithReflectType(reflect.TypeOf(explicitReportInput{})))) + component.View.SetResource(resource) + component.Resource = component.View.Resource() + return component +} diff --git a/repository/report/model.go b/repository/report/model.go new file mode 100644 index 000000000..e1ac447cf --- /dev/null +++ b/repository/report/model.go @@ -0,0 +1,119 @@ +package report + +import ( + "fmt" + "reflect" + "strings" + + "github.com/viant/datly/view/state" +) + +type Config struct { + Enabled bool + MCPTool *bool + Input string + Dimensions string + Measures string + Filters string + OrderBy string + Limit string + Offset string +} + +type Metadata struct { + InputName string + BodyFieldName string + DimensionsKey string + MeasuresKey string + FiltersKey string + Dimensions []*Field + Measures []*Field + Filters []*Filter + OrderBy string + Limit string + Offset string +} + +type Field struct { + Name string + FieldName string + Section string + Description string +} + +type Filter struct { + Name string + FieldName string + Section string + Description string + Parameter *state.Parameter +} + +func (c *Config) Clone() *Config { + if c == nil { + return nil + } + ret := *c + return &ret +} + +func (c *Config) Normalize() *Config { + if c == nil { + return nil + } + ret := c.Clone() + ret.Input = strings.TrimSpace(ret.Input) + ret.Dimensions = defaultString(ret.Dimensions, "Dimensions") + ret.Measures = defaultString(ret.Measures, "Measures") + ret.Filters = defaultString(ret.Filters, "Filters") + ret.OrderBy = defaultString(ret.OrderBy, "OrderBy") + ret.Limit = defaultString(ret.Limit, "Limit") + ret.Offset = defaultString(ret.Offset, "Offset") + return ret +} + +func (c *Config) MCPToolEnabled() bool { + if c == nil || c.MCPTool == nil { + return true + } + return *c.MCPTool +} + +func (c *Config) InputTypeName(componentName, inputName, viewName string) string { + if c != nil && strings.TrimSpace(c.Input) != "" { + return strings.TrimSpace(c.Input) + } + switch { + case strings.TrimSpace(inputName) != "": + return state.SanitizeTypeName(strings.TrimSpace(inputName) + "ReportInput") + case strings.TrimSpace(componentName) != "": + return state.SanitizeTypeName(strings.TrimSpace(componentName) + "ReportInput") + default: + return state.SanitizeTypeName(strings.TrimSpace(viewName) + "ReportInput") + } +} + +func (m *Metadata) ValidateSelection() error { + if m == nil { + return fmt.Errorf("report metadata was empty") + } + if len(m.Dimensions) == 0 && len(m.Measures) == 0 { + return fmt.Errorf("report metadata had no selectable dimensions or measures") + } + return nil +} + +func (f *Filter) SchemaType() reflect.Type { + if f == nil || f.Parameter == nil || f.Parameter.Schema == nil { + return nil + } + return f.Parameter.OutputType() +} + +func defaultString(value, fallback string) string { + value = strings.TrimSpace(value) + if value == "" { + return fallback + } + return value +} diff --git a/repository/report_runtime.go b/repository/report_runtime.go index 47f145b5e..b7d685e88 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -12,6 +12,7 @@ import ( "github.com/viant/datly/repository/contract" rephandler "github.com/viant/datly/repository/handler" "github.com/viant/datly/repository/path" + reportmodel "github.com/viant/datly/repository/report" "github.com/viant/datly/service" "github.com/viant/datly/view" "github.com/viant/datly/view/state" @@ -65,7 +66,7 @@ func BuildReportComponent(dispatcher contract.Dispatcher, original *Component) ( } func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, original *Component, routePath *path.Path) (*Component, *path.Path, error) { - config := original.Report.normalize() + config := original.Report.Normalize() metadata, err := buildReportMetadata(original, config) if err != nil { return nil, nil, err @@ -97,7 +98,7 @@ func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, o pathCopy.Internal = routePath.Internal pathCopy.Meta = routePath.Meta pathCopy.ModelContextProtocol = routePath.ModelContextProtocol - pathCopy.MCPTool = config.mcpToolEnabled() + pathCopy.MCPTool = config.MCPToolEnabled() pathCopy.MCPResource = false pathCopy.MCPTemplateResource = false pathCopy.Report = routePath.Report @@ -160,94 +161,27 @@ func reportPathMCPToolEnabled(report *path.Report) bool { } func buildReportMetadata(component *Component, report *Report) (*ReportMetadata, error) { - report = report.normalize() - viewRef := component.View - if viewRef == nil { - return nil, fmt.Errorf("report component view was empty") - } - result := &ReportMetadata{ - InputName: report.inputTypeName(component.Name, component.Input.Type.Name, viewRef.Name), - BodyFieldName: "", - DimensionsKey: report.Dimensions, - MeasuresKey: report.Measures, - FiltersKey: report.Filters, - OrderBy: report.OrderBy, - Limit: report.Limit, - Offset: report.Offset, - } - for _, column := range viewRef.Columns { - if column == nil || column.FieldName() == "" { - continue - } - fieldName := exportedReportFieldName(column.FieldName()) - field := &ReportField{Name: column.FieldName(), FieldName: fieldName, Description: column.Name} - switch { - case column.Groupable: - field.Section = report.Dimensions - result.Dimensions = append(result.Dimensions, field) - case column.Aggregate || (viewRef.Groupable && !column.Groupable): - field.Section = report.Measures - result.Measures = append(result.Measures, field) - } - } - for _, parameter := range component.Input.Type.Parameters { - if parameter == nil || len(parameter.Predicates) == 0 || parameter.In == nil { - continue - } - if isSelectorParameter(parameter, viewRef) { - continue - } - result.Filters = append(result.Filters, &ReportFilter{ - Name: parameter.Name, - FieldName: exportedReportFieldName(parameter.Name), - Section: report.Filters, - Description: parameter.Description, - Parameter: parameter, - }) - } - if err := result.validateSelection(); err != nil { - return nil, err - } - return result, nil + source := &reportmodel.Component{ + Name: component.Name, + InputName: component.Input.Type.Name, + Parameters: component.Input.Type.Parameters, + View: component.View, + Resource: component.View.Resource(), + Report: report, + } + return reportmodel.AssembleMetadata(source, report) } func buildReportInputType(component *Component, metadata *ReportMetadata, report *Report) (*state.Type, error) { - if report != nil && report.Input != "" { - schema := state.NewSchema(nil, state.WithSchemaPackage(""), state.WithModulePath("")) - schema.Name = strings.TrimSpace(report.Input) - inputType, err := state.NewType(state.WithSchema(schema), state.WithResource(component.View.Resource())) - if err != nil { - return nil, err - } - if err := inputType.Init(); err != nil { - return nil, err - } - return inputType, validateExplicitReportInput(inputType, metadata) - } - bodyType := reflect.PtrTo(synthesizeReportBodyType(metadata)) - bodySchema := state.NewSchema(bodyType) - bodySchema.Name = metadata.InputName - bodyParam := state.NewParameter("Report", state.NewBodyLocation(""), state.WithParameterSchema(bodySchema)) - bodyParam.Tag = `anonymous:"true"` - bodyParam.SetTypeNameTag() - // Synthetic report input must not initialize against the original component resource. - // Using the shared resource resolves linked named types and mutates the original - // component generation state, which breaks repeated code generation. - inputResource := newReportInputResource(component.View.Resource()) - inputType, err := state.NewType( - state.WithParameters(state.Parameters{bodyParam}), - state.WithBodyType(true), - state.WithSchema(state.NewSchema(bodyType)), - state.WithResource(inputResource), - ) - if err != nil { - return nil, err - } - if err := inputType.Init(); err != nil { - return nil, err - } - inputType.Name = metadata.InputName - return inputType, nil + source := &reportmodel.Component{ + Name: component.Name, + InputName: component.Input.Type.Name, + Parameters: component.Input.Type.Parameters, + View: component.View, + Resource: component.View.Resource(), + Report: report, + } + return reportmodel.BuildInputType(source, metadata, report) } func validateExplicitReportInput(inputType *state.Type, metadata *ReportMetadata) error { @@ -327,7 +261,7 @@ func filterStructType(filters []*ReportFilter) reflect.Type { structFields := make([]reflect.StructField, 0, len(filters)) for _, filter := range filters { rType := reflect.TypeOf("") - if schemaType := filter.schemaType(); schemaType != nil { + if schemaType := filter.SchemaType(); schemaType != nil { rType = schemaType } structFields = append(structFields, reflect.StructField{ diff --git a/repository/report_runtime_test.go b/repository/report_runtime_test.go index aa6272b91..c57cbbc8f 100644 --- a/repository/report_runtime_test.go +++ b/repository/report_runtime_test.go @@ -79,7 +79,7 @@ func TestBuildReportMetadataAndComponent(t *testing.T) { Path: contract.Path{Method: "GET", URI: "/v1/api/vendors"}, Meta: contract.Meta{Name: "vendors"}, View: rootView, - Report: (&Report{Enabled: true}).normalize(), + Report: (&Report{Enabled: true}).Normalize(), Contract: contract.Contract{ Input: contract.Input{Type: *inputType}, }, @@ -175,7 +175,7 @@ func TestBuildReportComponent_EnablesMCPToolOnSiblingRoute(t *testing.T) { Path: contract.Path{Method: "GET", URI: "/v1/api/vendors"}, Meta: contract.Meta{Name: "vendors"}, View: rootView, - Report: (&Report{Enabled: true}).normalize(), + Report: (&Report{Enabled: true}).Normalize(), Contract: contract.Contract{ Input: contract.Input{Type: *inputType}, }, @@ -233,7 +233,7 @@ func TestBuildReportComponent_DisablesMCPToolWhenReportFlagIsFalse(t *testing.T) Report: (&Report{ Enabled: true, MCPTool: &disabled, - }).normalize(), + }).Normalize(), Contract: contract.Contract{ Input: contract.Input{Type: *inputType}, }, @@ -263,7 +263,8 @@ func TestBuildReportComponent_DisablesMCPToolWhenReportFlagIsFalse(t *testing.T) func TestService_InitComponentProviders_RegistersLocalGroupingReportRoute(t *testing.T) { ctx := context.Background() - baseDir := filepath.Join("..", "e2e", "local", "regression") + baseDir, err := filepath.Abs(filepath.Join("..", "e2e", "local", "regression")) + require.NoError(t, err) if _, err := os.Stat(filepath.Join(baseDir, "paths.yaml")); err != nil { t.Skipf("missing local regression fixture: %v", err) } @@ -274,7 +275,7 @@ func TestService_InitComponentProviders_RegistersLocalGroupingReportRoute(t *tes WithRefreshDisabled(true), ) require.NoError(t, err) - reportPath := &contract.Path{Method: "POST", URI: "/v1/api/shape/dev/vendors-grouping/report"} + reportPath := &contract.Path{Method: "POST", URI: "/v1/api/dev/vendors-grouping/report"} provider, err := service.Registry().LookupProvider(ctx, reportPath) require.NoError(t, err) require.NotNil(t, provider) @@ -284,13 +285,14 @@ func TestService_InitComponentProviders_RegistersLocalGroupingReportRoute(t *tes require.NotNil(t, component.Report) assert.True(t, component.Report.Enabled) assert.Equal(t, "POST", component.Method) - assert.Equal(t, "/v1/api/shape/dev/vendors-grouping/report", component.URI) + assert.Equal(t, "/v1/api/dev/vendors-grouping/report", component.URI) } func TestBuildReportComponent_DoesNotStripOriginalViewTypeDefinitionsFromCodegen(t *testing.T) { resource := view.EmptyResource() rootView := view.NewView("metrics_view", "metrics_view") rootView.Groupable = true + rootView.Connector = &view.Connector{Connection: view.Connection{DBConfig: view.DBConfig{Name: "dev"}}} rootView.Template = &view.Template{Source: "SELECT agency_id, SUM(total_spend) AS total_spend FROM metrics_view GROUP BY 1"} rootView.Schema = state.NewSchema(reflect.TypeOf([]*struct { AgencyId *int `sqlx:"agency_id"` @@ -308,6 +310,10 @@ func TestBuildReportComponent_DoesNotStripOriginalViewTypeDefinitionsFromCodegen resource.Types = []*view.TypeDefinition{ {Name: "MetricsViewView", Package: "metrics", DataType: `struct{AgencyId *int ` + "`sqlx:\"agency_id\"`" + `; TotalSpend *float64 ` + "`sqlx:\"total_spend\"`" + `;}`}, } + require.NoError(t, resource.TypeRegistry().Register("MetricsViewView", xreflect.WithPackage("metrics"), xreflect.WithReflectType(reflect.TypeOf(struct { + AgencyId *int `sqlx:"agency_id"` + TotalSpend *float64 `sqlx:"total_spend"` + }{})))) rootView.SetResource(resource) resource.AddViews(rootView) @@ -319,7 +325,7 @@ func TestBuildReportComponent_DoesNotStripOriginalViewTypeDefinitionsFromCodegen outputType, err := state.NewType(state.WithParameters(state.Parameters{ &state.Parameter{Name: "Data", In: state.NewOutputLocation("view"), Schema: &state.Schema{Name: "MetricsViewView", Package: "metrics", Cardinality: state.Many}}, - })) + }), state.WithResource(rootView.Resource())) require.NoError(t, err) outputType.Name = "MetricsViewOutput" @@ -327,7 +333,7 @@ func TestBuildReportComponent_DoesNotStripOriginalViewTypeDefinitionsFromCodegen Path: contract.Path{Method: "GET", URI: "/v1/api/core/metrics/performance_summary"}, Meta: contract.Meta{Name: "MetricsPerformance"}, View: rootView, - Report: (&Report{Enabled: true}).normalize(), + Report: (&Report{Enabled: true}).Normalize(), Contract: contract.Contract{ Input: contract.Input{Type: *inputType}, Output: contract.Output{Type: *outputType}, @@ -335,7 +341,7 @@ func TestBuildReportComponent_DoesNotStripOriginalViewTypeDefinitionsFromCodegen } before := component.GenerateOutputCode(context.Background(), true, false, nil) - require.Contains(t, before, "type MetricsViewView struct") + require.Contains(t, before, "type Data struct") service := &Service{registry: NewRegistry("", nil, nil)} _, _, err = service.buildReportComponent(component, &path.Path{ @@ -348,5 +354,6 @@ func TestBuildReportComponent_DoesNotStripOriginalViewTypeDefinitionsFromCodegen require.NoError(t, err) after := component.GenerateOutputCode(context.Background(), true, false, nil) - require.Contains(t, after, "type MetricsViewView struct") + require.Contains(t, after, "type Data struct") + assert.Equal(t, before, after) } diff --git a/repository/shape/dql_engine_test.go b/repository/shape/dql_engine_test.go index 80ed16cf0..a19db4b8d 100644 --- a/repository/shape/dql_engine_test.go +++ b/repository/shape/dql_engine_test.go @@ -355,6 +355,51 @@ func TestDQLCompileLoad_MetaFormatPreservesSummariesWithoutLinkedTypes(t *testin require.NoError(t, err) } +func TestDQLCompileLoad_DistrictPaginationInheritsNestedRelationTypeContextPackages(t *testing.T) { + dqlPath := filepath.Join("..", "..", "e2e", "v1", "dql", "dev", "district", "district_pagination.sql") + dqlPath, err := filepath.Abs(dqlPath) + require.NoError(t, err) + dqlBytes, err := os.ReadFile(dqlPath) + require.NoError(t, err) + + source := &shape.Source{ + Name: "district_pagination", + Path: dqlPath, + DQL: string(dqlBytes), + } + planResult, err := shapeCompile.New().Compile( + context.Background(), + source, + shape.WithLinkedTypes(false), + shape.WithTypeContextPackageDir(filepath.Join("e2e", "v1", "shape", "dev", "district", "pagination")), + shape.WithTypeContextPackageName("pagination"), + ) + require.NoError(t, err) + + componentArtifact, err := shapeLoad.New().LoadComponent(context.Background(), planResult, shape.WithLoadTypeContextPackages(true)) + require.NoError(t, err) + + component, ok := shapeLoad.ComponentFrom(componentArtifact) + require.True(t, ok) + require.NotNil(t, component) + + root, err := componentArtifact.Resource.Views.Index().Lookup(component.RootView) + require.NoError(t, err) + require.NotNil(t, root) + require.NotNil(t, root.Schema) + assert.Equal(t, "pagination", root.Schema.Package) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/district/pagination", root.Schema.PackagePath) + + require.Len(t, root.With, 1) + child := &root.With[0].Of.View + require.NotNil(t, child.Schema) + assert.Equal(t, "pagination", child.Schema.Package) + assert.Equal(t, "github.com/viant/datly/e2e/v1/shape/dev/district/pagination", child.Schema.PackagePath) + + _, err = initTypeRegistryForResource(componentArtifact.Resource) + require.NoError(t, err) +} + func initTypeRegistryForResource(resource *view.Resource) (*xreflect.Types, error) { registry := extension.NewRegistry() imports := view.Imports{} diff --git a/repository/shape/xgen/codegen.go b/repository/shape/xgen/codegen.go index 7734557b1..6716ea7ae 100644 --- a/repository/shape/xgen/codegen.go +++ b/repository/shape/xgen/codegen.go @@ -3358,7 +3358,6 @@ func (g *ComponentCodegen) renderDefineComponent(builder *strings.Builder, compo builder.WriteString(fmt.Sprintf(`, view.WithConnectorRef(%q)`, connectorRef)) } builder.WriteString(")") - builder.WriteString(")") if reportOption := g.reportComponentOption(); reportOption != "" { builder.WriteString(",\n") builder.WriteString("\t\t") diff --git a/service/reader/anonymous_mysql_test.go b/service/reader/anonymous_mysql_test.go new file mode 100644 index 000000000..3a7c761eb --- /dev/null +++ b/service/reader/anonymous_mysql_test.go @@ -0,0 +1,171 @@ +package reader + +import ( + "context" + "database/sql" + "os" + "reflect" + "testing" + + _ "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + vstate "github.com/viant/datly/view/state" + sqlxread "github.com/viant/sqlx/io/read" +) + +func TestSQLXReader_AnonymousVsNamedPatchType(t *testing.T) { + if os.Getenv("TEST") != "1" { + t.Skip("set TEST=1 to run integration reader check") + } + + db, err := sql.Open("mysql", "root:dev@tcp(localhost:3306)/dev?parseTime=true") + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + t.Run("anonymous", func(t *testing.T) { + type anonymousHas struct { + Id bool + Name bool + Quantity bool + } + type anonymousRow struct { + Id int + Name *string + Quantity *int + Has *anonymousHas + } + + reader, err := sqlxread.New(context.Background(), db, "SELECT * FROM FOOS WHERE ID = 4", func() interface{} { + return &anonymousRow{} + }) + require.NoError(t, err) + + var rows []*anonymousRow + err = reader.QueryAll(context.Background(), func(row interface{}) error { + rows = append(rows, row.(*anonymousRow)) + return nil + }) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, 4, rows[0].Id) + }) + + t.Run("named-reflect-structof", func(t *testing.T) { + hasType := reflect.StructOf([]reflect.StructField{ + {Name: "Id", Type: reflect.TypeOf(true)}, + {Name: "Name", Type: reflect.TypeOf(true)}, + {Name: "Quantity", Type: reflect.TypeOf(true)}, + }) + rowType := reflect.StructOf([]reflect.StructField{ + {Name: "Id", Type: reflect.TypeOf(int(0)), Tag: `sqlx:"ID"`}, + {Name: "Name", Type: reflect.TypeOf((*string)(nil)), Tag: `sqlx:"NAME"`}, + {Name: "Quantity", Type: reflect.TypeOf((*int)(nil)), Tag: `sqlx:"QUANTITY"`}, + {Name: "Has", Type: reflect.PtrTo(hasType), Tag: `setMarker:"true" format:"-" sqlx:"-" diff:"-" json:"-"`}, + }) + + reader, err := sqlxread.New(context.Background(), db, "SELECT * FROM FOOS WHERE ID = 4", func() interface{} { + return reflect.New(rowType).Interface() + }) + require.NoError(t, err) + + var rows []interface{} + err = reader.QueryAll(context.Background(), func(row interface{}) error { + rows = append(rows, row) + return nil + }) + require.NoError(t, err) + require.Len(t, rows, 1) + }) + + t.Run("collector-backed-anonymous", func(t *testing.T) { + type anonymousHas struct { + Id bool + Name bool + Quantity bool + } + type anonymousRow struct { + Id int `sqlx:"ID"` + Name *string `sqlx:"NAME"` + Quantity *int `sqlx:"QUANTITY"` + Has *anonymousHas `setMarker:"true" format:"-" sqlx:"-" diff:"-" json:"-"` + } + + aView := &view.View{ + Name: "CurFoos", + Schema: vstate.NewSchema(reflect.TypeOf([]*anonymousRow{})), + } + aView.Schema.Cardinality = vstate.Many + collector := view.NewCollector(aView.Schema.Slice(), aView, &[]*anonymousRow{}, nil, false) + reader, err := sqlxread.New(context.Background(), db, "SELECT * FROM FOOS WHERE ID = 4", collector.NewItem()) + require.NoError(t, err) + + err = reader.QueryAll(context.Background(), collector.Visitor(context.Background())) + require.NoError(t, err) + dest := collector.Dest().([]*anonymousRow) + require.Len(t, dest, 1) + require.Equal(t, 4, dest[0].Id) + }) + + t.Run("collector-backed-reflect-structof", func(t *testing.T) { + hasType := reflect.StructOf([]reflect.StructField{ + {Name: "Id", Type: reflect.TypeOf(true)}, + {Name: "Name", Type: reflect.TypeOf(true)}, + {Name: "Quantity", Type: reflect.TypeOf(true)}, + }) + rowType := reflect.StructOf([]reflect.StructField{ + {Name: "Id", Type: reflect.TypeOf(int(0)), Tag: `sqlx:"ID"`}, + {Name: "Name", Type: reflect.TypeOf((*string)(nil)), Tag: `sqlx:"NAME"`}, + {Name: "Quantity", Type: reflect.TypeOf((*int)(nil)), Tag: `sqlx:"QUANTITY"`}, + {Name: "Has", Type: reflect.PtrTo(hasType), Tag: `setMarker:"true" format:"-" sqlx:"-" diff:"-" json:"-"`}, + }) + sliceType := reflect.SliceOf(reflect.PtrTo(rowType)) + aView := &view.View{ + Name: "CurFoos", + Schema: vstate.NewSchema(sliceType), + } + aView.Schema.Cardinality = vstate.Many + + destPtr := reflect.New(sliceType).Interface() + collector := view.NewCollector(aView.Schema.Slice(), aView, destPtr, nil, false) + reader, err := sqlxread.New(context.Background(), db, "SELECT * FROM FOOS WHERE ID = 4", collector.NewItem()) + require.NoError(t, err) + + err = reader.QueryAll(context.Background(), collector.Visitor(context.Background())) + require.NoError(t, err) + destValue := reflect.ValueOf(collector.Dest()) + require.Equal(t, 1, destValue.Len()) + require.Equal(t, int64(4), destValue.Index(0).Elem().FieldByName("Id").Int()) + }) + + t.Run("collector-backed-reflect-structof-v1-order", func(t *testing.T) { + hasType := reflect.StructOf([]reflect.StructField{ + {Name: "Name", Type: reflect.TypeOf(true)}, + {Name: "Quantity", Type: reflect.TypeOf(true)}, + {Name: "Id", Type: reflect.TypeOf(true)}, + }) + rowType := reflect.StructOf([]reflect.StructField{ + {Name: "Name", Type: reflect.TypeOf((*string)(nil)), Tag: `sqlx:"NAME"`}, + {Name: "Quantity", Type: reflect.TypeOf((*int)(nil)), Tag: `sqlx:"QUANTITY"`}, + {Name: "Id", Type: reflect.TypeOf(int(0)), Tag: `sqlx:"ID"`}, + {Name: "Has", Type: reflect.PtrTo(hasType), Tag: `setMarker:"true" format:"-" sqlx:"-" diff:"-" json:"-"`}, + }) + sliceType := reflect.SliceOf(reflect.PtrTo(rowType)) + aView := &view.View{ + Name: "CurFoos", + Schema: vstate.NewSchema(sliceType), + } + aView.Schema.Cardinality = vstate.Many + + destPtr := reflect.New(sliceType).Interface() + collector := view.NewCollector(aView.Schema.Slice(), aView, destPtr, nil, false) + reader, err := sqlxread.New(context.Background(), db, "SELECT * FROM FOOS WHERE ID = 4", collector.NewItem()) + require.NoError(t, err) + + err = reader.QueryAll(context.Background(), collector.Visitor(context.Background())) + require.NoError(t, err) + destValue := reflect.ValueOf(collector.Dest()) + require.Equal(t, 1, destValue.Len()) + require.Equal(t, int64(4), destValue.Index(0).Elem().FieldByName("Id").Int()) + }) +} diff --git a/service/reader/sql_groupable_test.go b/service/reader/sql_groupable_test.go new file mode 100644 index 000000000..7cb4bbecc --- /dev/null +++ b/service/reader/sql_groupable_test.go @@ -0,0 +1,335 @@ +package reader + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" +) + +func TestBuilder_appendColumns(t *testing.T) { + testView := newGroupableTestView(t) + builder := NewBuilder() + + useCases := []struct { + description string + selector *view.Statelet + expectNames []string + expectNil bool + expectedSQL string + }{ + { + description: "default projection keeps view column order", + selector: view.NewStatelet(), + expectNil: true, + expectedSQL: " t.region_id, t.total_sales, t.country_id", + }, + { + description: "selector projection keeps requested order", + selector: func() *view.Statelet { + selector := view.NewStatelet() + selector.Columns = []string{"country_id", "region_id"} + return selector + }(), + expectNames: []string{"country_id", "region_id"}, + expectedSQL: " country_id, region_id", + }, + { + description: "grouped selector projection uses derived aliases for aggregate columns", + selector: func() *view.Statelet { + selector := view.NewStatelet() + selector.Columns = []string{"account_id", "total_id", "max_id"} + return selector + }(), + expectNames: []string{"account_id", "total_id", "max_id"}, + expectedSQL: " account_id, total_id, max_id", + }, + } + + for _, useCase := range useCases { + t.Run(useCase.description, func(t *testing.T) { + sb := &strings.Builder{} + viewUnderTest := testView + if useCase.description == "grouped selector projection uses derived aliases for aggregate columns" { + viewUnderTest = aggregateSelectorTestView(t) + } + projected, err := builder.appendColumns(sb, viewUnderTest, useCase.selector) + require.NoError(t, err) + require.Equal(t, useCase.expectedSQL, sb.String()) + if useCase.expectNil { + require.Nil(t, projected) + return + } + require.Equal(t, useCase.expectNames, columnNames(projected)) + }) + } +} + +func TestBuilder_rewriteGroupBy(t *testing.T) { + testView := newGroupableTestView(t) + aggregateColumns := aggregateGroupableColumns() + groupedMetrics := groupedMetricsColumns() + builder := NewBuilder() + + useCases := []struct { + description string + sql string + allColumns []*view.Column + projected []*view.Column + expected string + }{ + { + description: "replace existing group by with selected original positions", + sql: "(SELECT region_id, SUM(total_sales) AS total_sales, country_id FROM sales GROUP BY 1, 3)", + allColumns: testView.Columns, + projected: []*view.Column{testView.Columns[2], testView.Columns[1]}, + expected: "(SELECT country_id, SUM(total_sales) AS total_sales FROM sales GROUP BY 1)", + }, + { + description: "remove group by when no selected projected column is groupable", + sql: "(SELECT region_id, SUM(total_sales) AS total_sales, country_id FROM sales GROUP BY 1, 3)", + allColumns: testView.Columns, + projected: []*view.Column{testView.Columns[1]}, + expected: "(SELECT SUM(total_sales) AS total_sales FROM sales)", + }, + { + description: "add group by when absent", + sql: "(SELECT region_id, SUM(total_sales) AS total_sales, country_id FROM sales)", + allColumns: testView.Columns, + projected: []*view.Column{testView.Columns[0], testView.Columns[1]}, + expected: "(SELECT region_id, SUM(total_sales) AS total_sales FROM sales GROUP BY 1)", + }, + { + description: "skip rewrite when no specific projection was selected", + sql: "(SELECT region_id, SUM(total_sales) AS total_sales, country_id FROM sales GROUP BY 1, 3)", + allColumns: testView.Columns, + projected: nil, + expected: "(SELECT region_id, SUM(total_sales) AS total_sales, country_id FROM sales GROUP BY 1, 3)", + }, + { + description: "rewrite grouped aggregates to selected groupable positions only", + sql: "(SELECT account_id, user_created, SUM(id) AS total_id, MAX(id) AS max_id FROM vendor GROUP BY 1, 2)", + allColumns: aggregateColumns, + projected: []*view.Column{aggregateColumns[0], aggregateColumns[2], aggregateColumns[3]}, + expected: "(SELECT account_id, SUM(id) AS total_id, MAX(id) AS max_id FROM vendor GROUP BY 1)", + }, + { + description: "rewrite grouped metrics query prunes unselected dimensions from select list", + sql: "(SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, SUM(p.bids) AS bids, SUM(p.impressions) AS impressions, SUM(p.clicks) AS clicks, SUM(p.conversions) AS conversions, SUM(p.total_spend) AS total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?))) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 LIMIT 1000)", + allColumns: groupedMetrics, + projected: []*view.Column{ + groupedMetrics[0], + groupedMetrics[1], + groupedMetrics[2], + groupedMetrics[3], + groupedMetrics[4], + groupedMetrics[5], + groupedMetrics[6], + }, + expected: "(SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id FROM `viant-mediator.forecaster.fact_perf_daily_mv` p WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?))) GROUP BY 1, 2, 3, 4, 5, 6, 7)", + }, + { + description: "rewrite grouped metrics CTE prunes unselected dimensions and preserves order", + sql: "WITH params AS (SELECT CAST(GREATEST(?, 1) AS INT64) AS date_interval), last_n AS (SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, p.bids, p.impressions, p.clicks, p.conversions, p.total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p JOIN params prm ON TRUE WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL prm.date_interval DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?)))) SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, SUM(p.bids) AS bids, SUM(p.impressions) AS impressions, SUM(p.clicks) AS clicks, SUM(p.conversions) AS conversions, SUM(p.total_spend) AS total_spend FROM last_n p GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ORDER BY p.event_date", + allColumns: groupedMetrics, + projected: []*view.Column{ + groupedMetrics[0], + groupedMetrics[1], + groupedMetrics[2], + groupedMetrics[3], + groupedMetrics[4], + groupedMetrics[5], + groupedMetrics[6], + }, + expected: "WITH params AS (SELECT CAST(GREATEST(?, 1) AS INT64) AS date_interval), last_n AS (SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, p.bids, p.impressions, p.clicks, p.conversions, p.total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p JOIN params prm ON TRUE WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL prm.date_interval DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?)))) SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id FROM last_n p GROUP BY 1, 2, 3, 4, 5, 6, 7 ORDER BY p.event_date", + }, + { + description: "rewrite grouped metrics CTE keeps selected non aggregate site_type in group by", + sql: "WITH params AS (SELECT CAST(GREATEST(?, 1) AS INT64) AS date_interval), last_n AS (SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, p.bids, p.impressions, p.clicks, p.conversions, p.total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p JOIN params prm ON TRUE WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL prm.date_interval DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?)) AND ((p.campaign_id IN (?))))) SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.site_type, SUM(p.bids) AS bids, SUM(p.impressions) AS impressions, SUM(p.clicks) AS clicks, SUM(p.conversions) AS conversions, SUM(p.total_spend) AS total_spend FROM last_n p GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ORDER BY p.event_date LIMIT 1000", + allColumns: func() []*view.Column { + cloned := cloneColumns(groupedMetrics) + cloned[10].Groupable = false + return cloned + }(), + projected: func() []*view.Column { + cloned := cloneColumns(groupedMetrics) + cloned[10].Groupable = false + return []*view.Column{ + cloned[0], + cloned[1], + cloned[2], + cloned[3], + cloned[4], + cloned[5], + cloned[10], + cloned[11], + cloned[12], + cloned[13], + cloned[14], + cloned[15], + } + }(), + expected: "WITH params AS (SELECT CAST(GREATEST(?, 1) AS INT64) AS date_interval), last_n AS (SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, p.bids, p.impressions, p.clicks, p.conversions, p.total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p JOIN params prm ON TRUE WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL prm.date_interval DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?)) AND ((p.campaign_id IN (?))))) SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.site_type, SUM(p.bids) AS bids, SUM(p.impressions) AS impressions, SUM(p.clicks) AS clicks, SUM(p.conversions) AS conversions, SUM(p.total_spend) AS total_spend FROM last_n p GROUP BY 1, 2, 3, 4, 5, 6, 7 ORDER BY p.event_date", + }, + { + description: "rewrite grouped metrics with publisher subset renumbers group by after pruning", + sql: "(SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, SUM(p.bids) AS bids, SUM(p.impressions) AS impressions, SUM(p.clicks) AS clicks, SUM(p.conversions) AS conversions, SUM(p.total_spend) AS total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?))) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 LIMIT 1000)", + allColumns: groupedMetrics, + projected: []*view.Column{ + groupedMetrics[0], + groupedMetrics[1], + groupedMetrics[2], + groupedMetrics[3], + groupedMetrics[4], + groupedMetrics[5], + groupedMetrics[7], + }, + expected: "(SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.publisher_id FROM `viant-mediator.forecaster.fact_perf_daily_mv` p WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?))) GROUP BY 1, 2, 3, 4, 5, 6, 7)", + }, + { + description: "rewrite grouped report projection drops order by on pruned dimension", + sql: "WITH last_n AS (SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, p.bids, p.impressions, p.clicks, p.conversions, p.total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p WHERE p.event_date BETWEEN DATE(DATE_SUB(CURRENT_DATE(), INTERVAL 7 DAY)) AND DATE(CURRENT_DATE()-1)) SELECT p.ad_order_id, SUM(p.bids) AS bids FROM last_n p GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ORDER BY p.event_date LIMIT 1000", + allColumns: groupedMetrics, + projected: []*view.Column{ + groupedMetrics[4], + groupedMetrics[11], + }, + expected: "WITH last_n AS (SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, p.bids, p.impressions, p.clicks, p.conversions, p.total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p WHERE p.event_date BETWEEN DATE(DATE_SUB(CURRENT_DATE(), INTERVAL 7 DAY)) AND DATE(CURRENT_DATE()-1)) SELECT p.ad_order_id, SUM(p.bids) AS bids FROM last_n p GROUP BY 1", + }, + } + + for _, useCase := range useCases { + t.Run(useCase.description, func(t *testing.T) { + actual, err := builder.rewriteGroupBy(useCase.sql, useCase.allColumns, useCase.projected) + require.NoError(t, err) + require.Equal(t, normalizeSQL(useCase.expected), normalizeSQL(actual)) + }) + } +} + +func TestBuilder_appendRelationColumn_UsesProjectedRelationAliasForGroupedDerivedView(t *testing.T) { + builder := NewBuilder() + aView := view.NewView("disqualified", "disqualified", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + &view.Column{Name: "TAXONOMY_ID", DataType: "int"}, + &view.Column{Name: "IS_DISQUALIFIED", DataType: "int"}, + }), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + + relation := &view.Relation{ + Of: &view.ReferenceView{ + On: view.Links{ + &view.Link{Field: "TaxonomyId", Column: "dq.SEGMENT_ID"}, + }, + }, + } + + t.Run("default projection does not append raw source column when projected alias exists", func(t *testing.T) { + sb := &strings.Builder{} + require.NoError(t, builder.checkViewAndAppendRelColumn(sb, aView, relation)) + require.Equal(t, "", sb.String()) + }) + + t.Run("selector projection appends projected alias expression instead of raw source column", func(t *testing.T) { + sb := &strings.Builder{} + selector := view.NewStatelet() + selector.Columns = []string{"IS_DISQUALIFIED"} + selector.Init(aView) + require.NoError(t, builder.checkSelectorAndAppendRelColumn(sb, aView, selector, relation)) + require.Equal(t, ", TAXONOMY_ID", sb.String()) + }) +} + +func newGroupableTestView(t *testing.T) *view.View { + t.Helper() + trueValue := true + aView := view.NewView("sales", "sales", + view.WithGroupable(true), + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + &view.Column{Name: "region_id", DataType: "string"}, + &view.Column{Name: "total_sales", DataType: "float64"}, + &view.Column{Name: "country_id", DataType: "string"}, + }), + ) + aView.ColumnsConfig = map[string]*view.ColumnConfig{ + "region_id": {Name: "region_id", Groupable: &trueValue}, + "country_id": {Name: "country_id", Groupable: &trueValue}, + } + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + return aView +} + +func aggregateGroupableColumns() []*view.Column { + return []*view.Column{ + {Name: "account_id", Groupable: true}, + {Name: "user_created", Groupable: true}, + {Name: "total_id"}, + {Name: "max_id"}, + } +} + +func aggregateSelectorTestView(t *testing.T) *view.View { + t.Helper() + aView := view.NewView("vendor", "vendor", + view.WithGroupable(true), + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + &view.Column{Name: "account_id", DataType: "int", Groupable: true}, + &view.Column{Name: "user_created", DataType: "int", Groupable: true}, + &view.Column{Name: "total_id", DataType: "float64", Expression: "SUM(id)", Aggregate: true}, + &view.Column{Name: "max_id", DataType: "int", Expression: "MAX(id)", Aggregate: true}, + }), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + return aView +} + +func groupedMetricsColumns() []*view.Column { + return []*view.Column{ + {Name: "event_date", Groupable: true}, + {Name: "agency_id", Groupable: true}, + {Name: "advertiser_id", Groupable: true}, + {Name: "campaign_id", Groupable: true}, + {Name: "ad_order_id", Groupable: true}, + {Name: "audience_id", Groupable: true}, + {Name: "deal_id", Groupable: true}, + {Name: "publisher_id", Groupable: true}, + {Name: "channel_id", Groupable: true}, + {Name: "country", Groupable: true}, + {Name: "site_type", Groupable: true}, + {Name: "bids"}, + {Name: "impressions"}, + {Name: "clicks"}, + {Name: "conversions"}, + {Name: "total_spend"}, + } +} + +func cloneColumns(columns []*view.Column) []*view.Column { + result := make([]*view.Column, len(columns)) + for i, column := range columns { + if column == nil { + continue + } + cloned := *column + result[i] = &cloned + } + return result +} + +func columnNames(columns []*view.Column) []string { + result := make([]string, len(columns)) + for i, column := range columns { + result[i] = column.Name + } + return result +} + +func normalizeSQL(SQL string) string { + return strings.Join(strings.Fields(SQL), " ") +} diff --git a/service/reader/sql_projection_regression_test.go b/service/reader/sql_projection_regression_test.go new file mode 100644 index 000000000..9592f741f --- /dev/null +++ b/service/reader/sql_projection_regression_test.go @@ -0,0 +1,55 @@ +package reader + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + "github.com/viant/sqlparser" +) + +func TestBuilder_appendColumns_UsesAliasesForDiscoveredExpressions(t *testing.T) { + builder := NewBuilder() + useCases := []struct { + description string + sql string + expectedSQL string + }{ + { + description: "case expression keeps outer alias projection", + sql: "SELECT (CASE WHEN 'user_name' = 'user_name' THEN u.STR_ID ELSE NULL END) AS VALUE FROM CI_EVENT ev LEFT JOIN CI_CONTACTS u ON ev.CREATED_USER = u.ID", + expectedSQL: " t.VALUE", + }, + { + description: "coalesce expression keeps discovered alias projection", + sql: "SELECT COALESCE(sl.APPROVED_SITE_CNT,0) AS NUMBER_OF_SITES FROM CI_SITE_LIST sl", + expectedSQL: " t.NUMBER_OF_SITES", + }, + } + + for _, useCase := range useCases { + t.Run(useCase.description, func(t *testing.T) { + parsed, err := sqlparser.ParseQuery(useCase.sql) + require.NoError(t, err) + columns := view.NewColumns(sqlparser.NewColumns(parsed.List), nil) + for _, column := range columns { + if strings.TrimSpace(column.DataType) == "" { + column.DataType = "string" + } + } + aView := view.NewView("projection", "projection", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(columns), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + + sb := &strings.Builder{} + projected, err := builder.appendColumns(sb, aView, view.NewStatelet()) + require.NoError(t, err) + require.Nil(t, projected) + require.Equal(t, useCase.expectedSQL, sb.String()) + }) + } +} diff --git a/service/session/selector_test.go b/service/session/selector_test.go new file mode 100644 index 000000000..4907e304e --- /dev/null +++ b/service/session/selector_test.go @@ -0,0 +1,54 @@ +package session + +import ( + "context" + "net/http" + "reflect" + "testing" + + "github.com/viant/datly/repository" + "github.com/viant/datly/view" + vstate "github.com/viant/datly/view/state" + "github.com/viant/datly/view/state/kind/locator" +) + +func TestSessionBind_QuerySelectorErrorDoesNotPanicWithoutCustomParameters(t *testing.T) { + ctx := context.Background() + resource := view.NewResource(nil) + aView := &view.View{ + Name: "v", + Mode: view.ModeQuery, + Selector: &view.Config{ + Constraints: &view.Constraints{}, + }, + } + aView.SetResource(resource) + aView.Template = &view.Template{Schema: vstate.NewSchema(reflect.TypeOf(struct{ Dummy int }{}))} + if err := aView.Template.Init(ctx, resource, aView); err != nil { + t.Fatalf("failed to init template: %v", err) + } + if err := aView.Selector.Init(ctx, resource, aView); err != nil { + t.Fatalf("failed to init selector: %v", err) + } + + component := &repository.Component{View: aView} + outputType, err := vstate.NewType( + vstate.WithSchema(vstate.NewSchema(reflect.TypeOf(struct{ X int }{}))), + vstate.WithResource(aView.Resource()), + ) + if err != nil { + t.Fatalf("failed to build component output type: %v", err) + } + component.Output.Type = *outputType + + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/?_orderby=id", nil) + if err != nil { + t.Fatalf("failed to build request: %v", err) + } + + sess := New(aView, WithComponent(component), WithLocatorOptions(locator.WithRequest(req))) + err = sess.SetViewState(ctx, aView) + if err == nil { + t.Fatal("expected query selector error") + } +} diff --git a/view/column_lookup_test.go b/view/column_lookup_test.go new file mode 100644 index 000000000..db598661d --- /dev/null +++ b/view/column_lookup_test.go @@ -0,0 +1,27 @@ +package view + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestView_ColumnByName_UsesIndexedLookup(t *testing.T) { + aView := NewView("disqualified", "disqualified", + WithConnector(NewConnector("test", "sqlite3", ":memory:")), + WithColumns(Columns{ + &Column{Name: "TAXONOMY_ID", DataType: "int", Tag: `source:"SEGMENT_ID"`}, + &Column{Name: "IS_DISQUALIFIED", DataType: "int"}, + }), + ) + require.NoError(t, aView.Init(context.Background(), EmptyResource())) + + column, ok := aView.ColumnByName("SEGMENT_ID") + require.True(t, ok) + require.Equal(t, "TAXONOMY_ID", column.Name) + + column, ok = aView.ColumnByName("taxonomy_id") + require.True(t, ok) + require.Equal(t, "TAXONOMY_ID", column.Name) +} diff --git a/view/columns.go b/view/columns.go index a4cf162ad..bb9d85dcf 100644 --- a/view/columns.go +++ b/view/columns.go @@ -197,7 +197,6 @@ func NewColumns(columns sqlparser.Columns, config map[string]*ColumnConfig) Colu } name = item.Identity() column := NewColumn(name, item.Type, item.RawType, item.IsNullable, WithColumnTag(item.Tag)) - column.Expression = item.Expression column.Aggregate = isAggregateProjection(item.Expression) if item.Name != item.Alias && item.Alias != "" && item.Name != "" { column.Tag += fmt.Sprintf(`source:"%v"`, item.Name) From a75e070f2ef1641bb23b4b1112c6dece65d4337e Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 10 Mar 2026 07:30:50 -0700 Subject: [PATCH 192/279] enhanced grouping --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 1d47c9ced..1030b2ffe 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,6 @@ module github.com/viant/datly go 1.25.0 -replace github.com/viant/xdatly => ../xdatly require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible From fdefc1afc5fb5d5258094290537d89438fb2d15a Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 10 Mar 2026 07:31:27 -0700 Subject: [PATCH 193/279] enhanced grouping --- go.mod | 3 +-- go.sum | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1030b2ffe..c7c2ac2ab 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,6 @@ module github.com/viant/datly go 1.25.0 - require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible github.com/aws/aws-lambda-go v1.31.0 @@ -55,7 +54,7 @@ require ( github.com/viant/structology v0.8.0 github.com/viant/tagly v0.3.0 github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef - github.com/viant/xdatly v0.5.4-0.20251113181159-0ac8b8b0ff3a + github.com/viant/xdatly v0.5.4-0.20260306062123-17850ac34977 github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 github.com/viant/xdatly/handler v0.0.0-20251208172928-dd34b7f09fd5 github.com/viant/xdatly/types/core v0.0.0-20250307183722-8c84fc717b52 diff --git a/go.sum b/go.sum index fb3ce7263..a71dbbbbf 100644 --- a/go.sum +++ b/go.sum @@ -1212,6 +1212,8 @@ github.com/viant/velty v0.4.0 h1:eesQES/vCpcoPbM+gQLUBuLEL2sEO+A6s6lPpl8eKc4= github.com/viant/velty v0.4.0/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef h1:KqWKMNloyzEg6nIn1pBK4CDEIcaRRhMrMUJr+k+xcPw= github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef/go.mod h1:1TvsnpZFqI9dYVzIkaSYJyJ/UkfxW7fnk0YFafWXrPg= +github.com/viant/xdatly v0.5.4-0.20260306062123-17850ac34977 h1:icW8DECqGoT4UzzOpxBraT/EEC1R0tBw9ev9cF/mrd4= +github.com/viant/xdatly v0.5.4-0.20260306062123-17850ac34977/go.mod h1:lZKZHhVdCZ3U9TU6GUFxKoGN3dPtqt2HkDYzJPq5CEs= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259 h1:9Yry3PUBDzc4rWacOYvAq/TKrTV0agvMF0gwm2gaoHI= github.com/viant/xdatly/extension v0.0.0-20231013204918-ecf3c2edf259/go.mod h1:fb8YgbVadk8X5ZLz49LWGzWmQlZd7Y/I5wE0ru44bIo= github.com/viant/xdatly/handler v0.0.0-20251208172928-dd34b7f09fd5 h1:CrT0HTlQul8FoGN0peylVczAOUEXKVqRAiB35ypRNHY= From 14bdea91fe4ee6d86a51568f408802bea2ae5485 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 14 Mar 2026 12:31:38 -0700 Subject: [PATCH 194/279] - reporting enhancement --- .../cases/001_one_to_many/expect_2.txt | 19 +++--- .../cases/001_one_to_many/vendor_list.dql | 3 +- e2e/local/regression/regression.yaml | 2 +- .../001_relation_one_to_many/expect_2.txt | 17 ++--- go.mod | 2 +- go.sum | 4 +- internal/translator/config.go | 67 +++++++++++++++++-- internal/translator/function.go | 19 ++++++ service.go | 6 +- utils/types/types.go | 7 ++ view/codec.go | 13 +++- 11 files changed, 130 insertions(+), 29 deletions(-) diff --git a/e2e/local/regression/cases/001_one_to_many/expect_2.txt b/e2e/local/regression/cases/001_one_to_many/expect_2.txt index 7796339fb..210665cb9 100644 --- a/e2e/local/regression/cases/001_one_to_many/expect_2.txt +++ b/e2e/local/regression/cases/001_one_to_many/expect_2.txt @@ -5,20 +5,21 @@ import ( ) type GeneratedStruct struct { - Id int `sqlx:"ID" velty:"names=ID|Id"` - Name *string `sqlx:"NAME" velty:"names=NAME|Name"` - AccountId *int `sqlx:"ACCOUNT_ID" velty:"names=ACCOUNT_ID|AccountId"` - Created *time.Time `sqlx:"CREATED" velty:"names=CREATED|Created"` - UserCreated *int `sqlx:"USER_CREATED" velty:"names=USER_CREATED|UserCreated"` - Updated *time.Time `sqlx:"UPDATED" velty:"names=UPDATED|Updated"` - UserUpdated *int `sqlx:"USER_UPDATED" velty:"names=USER_UPDATED|UserUpdated"` - Products []*Products `view:",table=PRODUCT"` + Xmap map[string]interface{} `sqlx:"xmap,type=map[string]interface{}" codec:"JSON" velty:"names=xmap|Xmap"` + Id int `sqlx:"ID" velty:"names=ID|Id"` + Name *string `sqlx:"NAME" velty:"names=NAME|Name"` + AccountId *int `sqlx:"ACCOUNT_ID" velty:"names=ACCOUNT_ID|AccountId"` + Created *time.Time `sqlx:"CREATED" velty:"names=CREATED|Created"` + UserCreated *int `sqlx:"USER_CREATED" velty:"names=USER_CREATED|UserCreated"` + Updated *time.Time `sqlx:"UPDATED" velty:"names=UPDATED|Updated"` + UserUpdated *int `sqlx:"USER_UPDATED" velty:"names=USER_UPDATED|UserUpdated"` + Products []*Products `view:",table=PRODUCT"` } type Products struct { Id int `sqlx:"ID" velty:"names=ID|Id"` Name *string `sqlx:"NAME" velty:"names=NAME|Name"` - VendorId *int `sqlx:"VENDOR_ID" internal:"true" velty:"names=VENDOR_ID|VendorId"` + VendorId *int `sqlx:"VENDOR_ID" internal:"true" velty:"names=VENDOR_ID|VendorId"` Status *int `sqlx:"STATUS" velty:"names=STATUS|Status"` Created *time.Time `sqlx:"CREATED" velty:"names=CREATED|Created"` UserCreated *int `sqlx:"USER_CREATED" velty:"names=USER_CREATED|UserCreated"` diff --git a/e2e/local/regression/cases/001_one_to_many/vendor_list.dql b/e2e/local/regression/cases/001_one_to_many/vendor_list.dql index 5d9a980b9..0ca82ef50 100644 --- a/e2e/local/regression/cases/001_one_to_many/vendor_list.dql +++ b/e2e/local/regression/cases/001_one_to_many/vendor_list.dql @@ -9,8 +9,9 @@ SELECT vendor.*, + cast(vendor.xmap AS map[string]interface{}), products.* EXCEPT VENDOR_ID -FROM (SELECT * FROM VENDOR t ) vendor +FROM (SELECT t.*, '' AS xmap FROM VENDOR t ) vendor JOIN ( SELECT * FROM PRODUCT t WHERE 1 = 1 ${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("AND")} diff --git a/e2e/local/regression/regression.yaml b/e2e/local/regression/regression.yaml index e4e47e667..6df7ab4cd 100644 --- a/e2e/local/regression/regression.yaml +++ b/e2e/local/regression/regression.yaml @@ -30,7 +30,7 @@ pipeline: '[]gen': '@gen' subPath: 'cases/${index}_*' - range: 1..015 + range: 1..001 template: checkSkip: action: nop diff --git a/e2e/v1/cases/001_relation_one_to_many/expect_2.txt b/e2e/v1/cases/001_relation_one_to_many/expect_2.txt index 49c4027be..d1b046e96 100644 --- a/e2e/v1/cases/001_relation_one_to_many/expect_2.txt +++ b/e2e/v1/cases/001_relation_one_to_many/expect_2.txt @@ -5,14 +5,15 @@ import ( ) type GeneratedStruct struct { - Id int `sqlx:"ID"` - Name *string `sqlx:"NAME"` - AccountId *int `sqlx:"ACCOUNT_ID"` - Created *time.Time `sqlx:"CREATED"` - UserCreated *int `sqlx:"USER_CREATED"` - Updated *time.Time `sqlx:"UPDATED"` - UserUpdated *int `sqlx:"USER_UPDATED"` - Products []*Products `view:",table=PRODUCT,connector=dev,selectorNamespace=pr"` + Xmap map[string]interface{} `sqlx:"xmap,type=map[string]interface{}" codec:"JSON"` + Id int `sqlx:"ID"` + Name *string `sqlx:"NAME"` + AccountId *int `sqlx:"ACCOUNT_ID"` + Created *time.Time `sqlx:"CREATED"` + UserCreated *int `sqlx:"USER_CREATED"` + Updated *time.Time `sqlx:"UPDATED"` + UserUpdated *int `sqlx:"USER_UPDATED"` + Products []*Products `view:",table=PRODUCT,connector=dev,selectorNamespace=pr"` } type Products struct { diff --git a/go.mod b/go.mod index c7c2ac2ab..d869cac6f 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.0 - github.com/viant/xreflect v0.7.3 + github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e github.com/viant/xunsafe v0.10.3 golang.org/x/mod v0.28.0 golang.org/x/oauth2 v0.32.0 diff --git a/go.sum b/go.sum index a71dbbbbf..81389715d 100644 --- a/go.sum +++ b/go.sum @@ -1226,8 +1226,8 @@ github.com/viant/xlsy v0.3.1 h1:KwA7PxOTVg+ns4CCPOdfNy5aEA9OUlIByUbuNC9ju0s= github.com/viant/xlsy v0.3.1/go.mod h1:RajfF9HkL/PfIxRCvZSubpNlpdMUNDKYZp8C1o3vF4Q= github.com/viant/xmlify v0.1.1 h1:Kmn7wnsq5APD8uJVP+kM6lIEGhSyjWSNOy4BvyfZQno= github.com/viant/xmlify v0.1.1/go.mod h1:w25+umH6nthlQ8ACT3K2/YJOLlbTXKLQXkdqFs6ky9s= -github.com/viant/xreflect v0.7.3 h1:Oi2ZzSYWvs3lFBNMHEwHm6pu+3sQRlklzllycZyOGHk= -github.com/viant/xreflect v0.7.3/go.mod h1:BwI+lqFjhKv2Vn4E0Jt6nvbwcFOWrM6H+sOMOX3JiU4= +github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e h1:z4uCWPkSCnGwqbIc3ENoYJnYwtR2j/9eI79vO4vK9rQ= +github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e/go.mod h1:BwI+lqFjhKv2Vn4E0Jt6nvbwcFOWrM6H+sOMOX3JiU4= github.com/viant/xunsafe v0.10.3 h1:Fi4N+b5PH7e2iwT1UquAe7wUlTn4Fnb2kBnFLBixX+M= github.com/viant/xunsafe v0.10.3/go.mod h1:V3RCwtqpbNPznhmHysyAOpsyuSVkIYWo1Ewip7qb9/s= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= diff --git a/internal/translator/config.go b/internal/translator/config.go index 656b6f430..1b523cb48 100644 --- a/internal/translator/config.go +++ b/internal/translator/config.go @@ -13,8 +13,11 @@ import ( "github.com/viant/datly/internal/setter" "github.com/viant/datly/internal/translator/parser" dpath "github.com/viant/datly/repository/path" + "github.com/viant/toolbox" + "gopkg.in/yaml.v3" "os" "path" + "path/filepath" "strings" ) @@ -26,10 +29,11 @@ type Config struct { } func (c *Config) Init(ctx context.Context) error { - if len(c.repository.Configs) == 0 { - c.Config = c.inMemoryConfig() - } else if err := c.loadConfig(ctx); err != nil { - return err + c.Config = c.inMemoryConfig() + if len(c.repository.Configs) > 0 { + if err := c.loadConfig(ctx); err != nil { + return err + } } if err := c.updateURIs(); err != nil { return err @@ -109,8 +113,9 @@ func (c *Config) updateURIs() error { func (c *Config) loadConfig(ctx context.Context) error { var configs []interface{} + configs = append(configs, c.Config) for _, URL := range c.repository.Configs.URLs() { - config, err := standalone.NewConfigFromURL(ctx, URL) + config, err := loadConfigFragment(ctx, URL) if err != nil { return err } @@ -124,6 +129,58 @@ func (c *Config) loadConfig(ctx context.Context) error { return json.Unmarshal(merged, c.Config) } +func loadConfigFragment(ctx context.Context, URL string) (*standalone.Config, error) { + data, err := fs.DownloadWithURL(ctx, URL) + if err != nil { + return nil, err + } + aMap := map[string]interface{}{} + if strings.HasSuffix(URL, "yaml") || strings.HasSuffix(URL, "yml") { + if err = yaml.Unmarshal(data, &aMap); err != nil { + return nil, err + } + } else { + if err = json.Unmarshal(data, &aMap); err != nil { + return nil, err + } + } + cfg := &standalone.Config{} + if err = toolbox.DefaultConverter.AssignConverted(cfg, aMap); err != nil { + return nil, err + } + normalizeConfigURLs(cfg, configBaseDir(URL)) + return cfg, nil +} + +func normalizeConfigURLs(cfg *standalone.Config, baseURL string) { + if url.IsRelative(cfg.RouteURL) { + cfg.RouteURL = url.Join(baseURL, cfg.RouteURL) + } + if url.IsRelative(cfg.ContentURL) { + cfg.ContentURL = url.Join(baseURL, cfg.ContentURL) + } + if url.IsRelative(cfg.PluginsURL) { + cfg.PluginsURL = url.Join(baseURL, cfg.PluginsURL) + } + if url.IsRelative(cfg.DependencyURL) { + cfg.DependencyURL = url.Join(baseURL, cfg.DependencyURL) + } + if cfg.JobURL != "" && url.IsRelative(cfg.JobURL) { + cfg.JobURL = url.Join(baseURL, cfg.JobURL) + } + if cfg.FailedJobURL != "" && url.IsRelative(cfg.FailedJobURL) { + cfg.FailedJobURL = url.Join(baseURL, cfg.FailedJobURL) + } +} + +func configBaseDir(URL string) string { + if strings.Contains(URL, "://") { + parent, _ := url.Split(URL, "file") + return parent + } + return filepath.Dir(URL) +} + func (c *Config) inMemoryConfig() *standalone.Config { setter.SetIntIfNil(&c.repository.Port, 8080) return &standalone.Config{ diff --git a/internal/translator/function.go b/internal/translator/function.go index e7b995bce..db41b552b 100644 --- a/internal/translator/function.go +++ b/internal/translator/function.go @@ -5,6 +5,7 @@ import ( "github.com/viant/datly/internal/translator/function" "github.com/viant/datly/utils/types" "github.com/viant/datly/view/extension" + dcodec "github.com/viant/datly/view/extension/codec" "github.com/viant/datly/view/state" "github.com/viant/sqlparser" "reflect" @@ -145,5 +146,23 @@ func (v *Viewlet) applyExplicitCast(column *sqlparser.Column, funcArgs []string) return true, nil } column.RawType = rType + if shouldAttachJSONCodecForCast(rType) && columnConfig.Codec == nil { + columnConfig.Codec = &state.Codec{Name: dcodec.JSON, OutputType: funcArgs[1]} + } return true, nil } + +func shouldAttachJSONCodecForCast(rType reflect.Type) bool { + if rType == nil { + return false + } + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + switch rType.Kind() { + case reflect.Map: + return true + default: + return false + } +} diff --git a/service.go b/service.go index 2c984ddf3..71ea3834a 100644 --- a/service.go +++ b/service.go @@ -42,6 +42,8 @@ import ( //go:embed Version var Version string +const reportSelectionErr = "report metadata had no selectable dimensions or measures" + type ( Service struct { repository *repository.Service @@ -589,7 +591,9 @@ func (s *Service) AddComponent(ctx context.Context, component *repository.Compon registerComponents := append([]*repository.Component{}, components.Components...) if reportComponent, err := repository.BuildReportComponent(s.repository.Registry().Dispatcher(), components.Components[0]); err != nil { - return err + if !strings.Contains(err.Error(), reportSelectionErr) { + return err + } } else if reportComponent != nil { registerComponents = append(registerComponents, reportComponent) } diff --git a/utils/types/types.go b/utils/types/types.go index 8e7ccd784..0411bec1b 100644 --- a/utils/types/types.go +++ b/utils/types/types.go @@ -12,6 +12,13 @@ func LookupType(lookup xreflect.LookupType, typeName string, opts ...xreflect.Op if ok { return rType, nil } + parseOptions := append([]xreflect.Option{}, opts...) + if lookup != nil { + parseOptions = append(parseOptions, xreflect.WithTypeLookup(lookup)) + } + if rType, err := xreflect.Parse(typeName, parseOptions...); err == nil && rType != nil { + return rType, nil + } if lookup == nil { return nil, fmt.Errorf("type %q was not found and no lookup resolver is configured", typeName) } diff --git a/view/codec.go b/view/codec.go index fecdd2eca..2b9e331cc 100644 --- a/view/codec.go +++ b/view/codec.go @@ -53,9 +53,10 @@ func (c *columnsCodec) init(viewType reflect.Type, columns []*Column) error { c.columns = columns codecStructFields := make([]reflect.StructField, len(columns)) for i, column := range columns { + scanType := columnDatabaseScanType(column) codecStructFields[i] = reflect.StructField{ Name: "Col" + strconv.Itoa(i), - Type: column.ColumnType(), + Type: scanType, Tag: reflect.StructTag(fmt.Sprintf(`sqlx:"%v"`, column.Name)), } } @@ -94,6 +95,16 @@ func (c *columnsCodec) init(viewType reflect.Type, columns []*Column) error { return nil } +func columnDatabaseScanType(column *Column) reflect.Type { + if column == nil { + return reflect.TypeOf([]byte{}) + } + if column.Codec != nil && column.Codec.Name == codec2.JSON { + return reflect.TypeOf([]byte{}) + } + return column.ColumnType() +} + func (c *columnsCodec) updateValue(ctx context.Context, value interface{}, record *codec2.ParentValue) error { asPtr := xunsafe.AsPointer(value) for i, column := range c.columns { From 03e2576c054a82913179b000c453e3493f7dd78a Mon Sep 17 00:00:00 2001 From: vc42 Date: Wed, 18 Mar 2026 12:03:55 -0400 Subject: [PATCH 195/279] fixed ReverseReplace incorrect substitution when one subst is a substr of another one --- view/substitutes.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/view/substitutes.go b/view/substitutes.go index b9c54a7e2..c634f106d 100644 --- a/view/substitutes.go +++ b/view/substitutes.go @@ -41,15 +41,25 @@ func (s Substitutes) ReverseReplace(text string) string { if len(s) == 0 { return text } - keys := s.Keys() - for _, k := range keys { - v := s[k] - - key := "${" + k + "}" - if count := strings.Count(text, v); count > 0 { - text = strings.Replace(text, v, key, count) + // Build pairs of (key, value) + pairs := make([]struct{ k, v string }, 0, len(s)) + for k, v := range s { + pairs = append(pairs, struct{ k, v string }{k, v}) + } + // Sort by value length desc, tie-breaker by key asc for stability + sort.SliceStable(pairs, func(i, j int) bool { + if len(pairs[i].v) == len(pairs[j].v) { + return pairs[i].k < pairs[j].k } - + return len(pairs[i].v) > len(pairs[j].v) + }) + // Replace using value-first order + for _, p := range pairs { + if p.v == "" { + continue + } + key := "${" + p.k + "}" + text = strings.ReplaceAll(text, p.v, key) } return text } From c2a111ca1952be5553ed5eda325c251bf38384dd Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Wed, 18 Mar 2026 15:52:17 -0700 Subject: [PATCH 196/279] integrate structology marshaller --- gateway/router/marshal/json/marshal.go | 5 +- gateway/router/route.go | 7 +- go.mod | 4 +- go.sum | 8 +- internal/translator/rule.go | 17 +- internal/translator/service.go | 8 +- repository/component.go | 12 +- repository/content/content.go | 96 ++++++- repository/content/json_marshaller.go | 298 +++++++++++++++++++++ repository/content/json_marshaller_test.go | 219 +++++++++++++++ service.go | 5 +- view/extension/init.go | 3 + view/state/kind/locator/body.go | 29 +- view/state/parameters.go | 21 +- 14 files changed, 687 insertions(+), 45 deletions(-) create mode 100644 repository/content/json_marshaller.go create mode 100644 repository/content/json_marshaller_test.go diff --git a/gateway/router/marshal/json/marshal.go b/gateway/router/marshal/json/marshal.go index c17ffa807..095884dac 100644 --- a/gateway/router/marshal/json/marshal.go +++ b/gateway/router/marshal/json/marshal.go @@ -2,12 +2,13 @@ package json import ( "bytes" + "reflect" + "unsafe" + "github.com/francoispqt/gojay" "github.com/viant/datly/gateway/router/marshal/config" "github.com/viant/tagly/format/text" "github.com/viant/xunsafe" - "reflect" - "unsafe" ) const null = `null` diff --git a/gateway/router/route.go b/gateway/router/route.go index d38b499a6..96d5874ce 100644 --- a/gateway/router/route.go +++ b/gateway/router/route.go @@ -92,7 +92,10 @@ func (r *Route) UnmarshalFunc(request *http.Request) shared.Unmarshal { } return func(bytes []byte, i interface{}) error { - return r.Marshaller.JSON.JsonMarshaller.Unmarshal(bytes, i, jsonPathInterceptor, request) + if r.Marshaller.JSON.CanUnmarshal() { + return r.Marshaller.JSON.Unmarshal(bytes, i) + } + return r.Marshaller.JSON.RuntimeUnmarshallerEngine().Unmarshal(bytes, i, jsonPathInterceptor, request) } } @@ -113,7 +116,7 @@ func (r *Route) Init(ctx context.Context, resource *Resource) error { return nil } r._unmarshallerInterceptors = r.Transforms.FilterByKind(marshal.TransformKindUnmarshal) - if err := r.Component.Content.InitMarshaller(r.Component.IOConfig(), r.Output.Exclude, r.BodyType(), r.OutputType()); err != nil { + if err := r.Component.Content.InitMarshaller(r.Component.IOConfig(), r.Output.Exclude, r.BodyType(), r.OutputType(), resource.Resource.LookupType()); err != nil { return err } if r.APIKey != nil { diff --git a/go.mod b/go.mod index c7c2ac2ab..a49cdbdec 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.0 github.com/viant/xreflect v0.7.3 - github.com/viant/xunsafe v0.10.3 + github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 golang.org/x/mod v0.28.0 golang.org/x/oauth2 v0.32.0 google.golang.org/api v0.201.0 @@ -51,7 +51,7 @@ require ( github.com/viant/jsonrpc v0.17.0 github.com/viant/mcp v0.11.0 github.com/viant/mcp-protocol v0.11.0 - github.com/viant/structology v0.8.0 + github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76 github.com/viant/tagly v0.3.0 github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef github.com/viant/xdatly v0.5.4-0.20260306062123-17850ac34977 diff --git a/go.sum b/go.sum index a71dbbbbf..28c2446cb 100644 --- a/go.sum +++ b/go.sum @@ -1198,8 +1198,8 @@ github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 h1:bnVgWzZzuz2p github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= -github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= -github.com/viant/structology v0.8.0/go.mod h1:Fnm1DyR4gfyPbnhBMkQB5lR6/isYDnncBFO1nCxxmqs= +github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76 h1:LUhy4A9ps2aFP3cBCTw4sMG1QmKWMNG1//vId03utEc= +github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= github.com/viant/structql v0.5.4/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= github.com/viant/tagly v0.3.0 h1:Y8IckveeSrroR8yisq4MBdxhcNqf4v8II01uCpamh4E= @@ -1228,8 +1228,8 @@ github.com/viant/xmlify v0.1.1 h1:Kmn7wnsq5APD8uJVP+kM6lIEGhSyjWSNOy4BvyfZQno= github.com/viant/xmlify v0.1.1/go.mod h1:w25+umH6nthlQ8ACT3K2/YJOLlbTXKLQXkdqFs6ky9s= github.com/viant/xreflect v0.7.3 h1:Oi2ZzSYWvs3lFBNMHEwHm6pu+3sQRlklzllycZyOGHk= github.com/viant/xreflect v0.7.3/go.mod h1:BwI+lqFjhKv2Vn4E0Jt6nvbwcFOWrM6H+sOMOX3JiU4= -github.com/viant/xunsafe v0.10.3 h1:Fi4N+b5PH7e2iwT1UquAe7wUlTn4Fnb2kBnFLBixX+M= -github.com/viant/xunsafe v0.10.3/go.mod h1:V3RCwtqpbNPznhmHysyAOpsyuSVkIYWo1Ewip7qb9/s= +github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 h1:tQOsy7ov3XcTj+OXNF1apq9EKxSj82f5AjJCuhfCkMo= +github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca h1:uvPMDVyP7PXMMioYdyPH+0O+Ta/UO1WFfNYMO3Wz0eg= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= diff --git a/internal/translator/rule.go b/internal/translator/rule.go index 6cea3bbdb..e4c6f10d1 100644 --- a/internal/translator/rule.go +++ b/internal/translator/rule.go @@ -3,6 +3,11 @@ package translator import ( "context" "fmt" + "os" + "path" + "path/filepath" + "strings" + "github.com/viant/afs" "github.com/viant/afs/url" "github.com/viant/datly/gateway/router" @@ -19,10 +24,6 @@ import ( "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/datly/view/state" - "os" - "path" - "path/filepath" - "strings" ) type ( @@ -325,13 +326,15 @@ func (r *Rule) applyDefaults() { setter.SetCaseFormatIfEmpty(&r.Route.Output.CaseFormat, "lc") setter.SetBoolIfFalse(&r.Input.IgnoreEmptyQueryParameters, r.IgnoreEmptyQueryParameters) setter.SetBoolIfFalse(&r.Input.CustomValidation, r.CustomValidation || r.Type != "") + setter.SetStringIfEmpty(&r.Route.Content.Marshaller.JSON.Engine, content.DefaultJSONEngineTypeName) if r.XMLUnmarshalType != "" { r.Route.Content.Marshaller.XML.TypeName = r.XMLUnmarshalType } if r.JSONMarshalType != "" { - r.Route.Content.Marshaller.JSON.TypeName = r.JSONMarshalType - } else if r.JSONUnmarshalType != "" { - r.Route.Content.Marshaller.JSON.TypeName = r.JSONUnmarshalType + r.Route.Content.Marshaller.JSON.MarshalTypeName = r.JSONMarshalType + } + if r.JSONUnmarshalType != "" { + r.Route.Content.Marshaller.JSON.UnmarshalTypeName = r.JSONUnmarshalType } } diff --git a/internal/translator/service.go b/internal/translator/service.go index 7367f6e3b..37cc4f364 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -377,11 +377,11 @@ func (s *Service) persistRouterRule(ctx context.Context, resource *Resource, ser if resource.Rule.XMLUnmarshalType != "" { route.Content.Marshaller.XML.TypeName = resource.Rule.XMLUnmarshalType } - // JSON marshaller/unmarshaller customization: prefer MarshalType if provided, fallback to UnmarshalType. if resource.Rule.JSONMarshalType != "" { - route.Content.Marshaller.JSON.TypeName = resource.Rule.JSONMarshalType - } else if resource.Rule.JSONUnmarshalType != "" { - route.Content.Marshaller.JSON.TypeName = resource.Rule.JSONUnmarshalType + route.Content.Marshaller.JSON.MarshalTypeName = resource.Rule.JSONMarshalType + } + if resource.Rule.JSONUnmarshalType != "" { + route.Content.Marshaller.JSON.UnmarshalTypeName = resource.Rule.JSONUnmarshalType } route.Component.Output.DataFormat = resource.Rule.DataFormat diff --git a/repository/component.go b/repository/component.go index 1c2c17bb1..b78db0e81 100644 --- a/repository/component.go +++ b/repository/component.go @@ -121,11 +121,8 @@ func (c *Component) Init(ctx context.Context, resource *view.Resource) (err erro if err := c.initTransforms(ctx); err != nil { return nil } - if err := c.Content.InitMarshaller(c.IOConfig(), c.Output.Exclude, c.BodyType(), c.OutputType()); err != nil { - return err - } lookupType := resource.LookupType() - if err := c.Content.Marshaller.Init(lookupType); err != nil { + if err := c.Content.InitMarshaller(c.IOConfig(), c.Output.Exclude, c.BodyType(), c.OutputType(), lookupType); err != nil { return err } if err = c.Async.Init(ctx, resource, c.View); err != nil { @@ -450,10 +447,13 @@ func (c *Component) UnmarshalFor(opts ...UnmarshalOption) shared.Unmarshal { } } return func(data []byte, dest interface{}) error { + if c.Content.Marshaller.JSON.CanUnmarshal() { + return c.Content.Marshaller.JSON.Unmarshal(data, dest) + } if len(interceptors) > 0 || req != nil { - return c.Content.Marshaller.JSON.JsonMarshaller.Unmarshal(data, dest, interceptors, req) + return c.Content.Marshaller.JSON.RuntimeUnmarshallerEngine().Unmarshal(data, dest, interceptors, req) } - return c.Content.Marshaller.JSON.JsonMarshaller.Unmarshal(data, dest) + return c.Content.Marshaller.JSON.RuntimeUnmarshallerEngine().Unmarshal(data, dest) } } diff --git a/repository/content/content.go b/repository/content/content.go index a479fbaa1..9183afcec 100644 --- a/repository/content/content.go +++ b/repository/content/content.go @@ -2,6 +2,9 @@ package content import ( "fmt" + "reflect" + "strings" + "github.com/viant/datly/gateway/router/marshal" "github.com/viant/datly/gateway/router/marshal/config" "github.com/viant/datly/gateway/router/marshal/json" @@ -12,8 +15,6 @@ import ( "github.com/viant/xlsy" "github.com/viant/xmlify" "github.com/viant/xreflect" - "reflect" - "strings" ) const ( @@ -59,8 +60,14 @@ type ( } JSON struct { - Codec - JsonMarshaller *json.Marshaller + Engine string `json:",omitempty" yaml:",omitempty"` + MarshalTypeName string `json:",omitempty" yaml:",omitempty"` + UnmarshalTypeName string `json:",omitempty" yaml:",omitempty"` + marshalCodec Codec `json:"-" yaml:"-"` + unmarshalCodec Codec `json:"-" yaml:"-"` + JsonMarshaller *json.Marshaller `json:"-" yaml:"-"` + RuntimeMarshaller JSONMarshallerEngine `json:"-" yaml:"-"` + RuntimeUnmarshaller JSONUnmarshallerEngine `json:"-" yaml:"-"` } XLS struct { @@ -86,6 +93,14 @@ type ( Marshaller interface { Marshal(src interface{}) ([]byte, error) } + + JSONMarshallerEngine interface { + Marshal(src interface{}, options ...interface{}) ([]byte, error) + } + + JSONUnmarshallerEngine interface { + Unmarshal(bytes []byte, dest interface{}, options ...interface{}) error + } ) func (u *Codec) CanUnmarshal() bool { @@ -117,7 +132,7 @@ func (m *Marshallers) Init(lookupType xreflect.LookupType) error { if err := m.JSON.Init(lookupType); err != nil { return err } - if err := m.XML.Init(lookupType); err != nil { + if err := m.XML.init(lookupType, false, true); err != nil { return err } if err := m.CSV.Init(lookupType); err != nil { @@ -127,6 +142,10 @@ func (m *Marshallers) Init(lookupType xreflect.LookupType) error { } func (u *Codec) Init(lookupType xreflect.LookupType) error { + return u.init(lookupType, true, true) +} + +func (u *Codec) init(lookupType xreflect.LookupType, requireMarshal, requireUnmarshal bool) error { if u.TypeName == "" { return nil } @@ -144,12 +163,57 @@ func (u *Codec) Init(lookupType xreflect.LookupType) error { if ok { u.marshal = marshaller.Marshal } - if u.marshal == nil && u.unmarshal == nil { - return fmt.Errorf("invalid type %s: unmarshaller/marshaller were not initialized", u.TypeName) + if requireMarshal && u.marshal == nil { + return fmt.Errorf("invalid type %s: marshaller was not initialized", u.TypeName) + } + if requireUnmarshal && u.unmarshal == nil { + return fmt.Errorf("invalid type %s: unmarshaller was not initialized", u.TypeName) + } + return nil +} + +func (j *JSON) Init(lookupType xreflect.LookupType) error { + j.marshalCodec = Codec{TypeName: j.MarshalTypeName} + j.unmarshalCodec = Codec{TypeName: j.UnmarshalTypeName} + if err := j.marshalCodec.init(lookupType, true, false); err != nil { + return err + } + if err := j.unmarshalCodec.init(lookupType, false, true); err != nil { + return err } return nil } +func (j *JSON) CanMarshal() bool { + return j.marshalCodec.CanMarshal() +} + +func (j *JSON) CanUnmarshal() bool { + return j.unmarshalCodec.CanUnmarshal() +} + +func (j *JSON) Marshal(src interface{}) ([]byte, error) { + return j.marshalCodec.Marshal(src) +} + +func (j *JSON) Unmarshal(bytes []byte, dest interface{}) error { + return j.unmarshalCodec.Unmarshal(bytes, dest) +} + +func (j *JSON) RuntimeMarshallerEngine() JSONMarshallerEngine { + if j.RuntimeMarshaller != nil { + return j.RuntimeMarshaller + } + return j.JsonMarshaller +} + +func (j *JSON) RuntimeUnmarshallerEngine() JSONUnmarshallerEngine { + if j.RuntimeUnmarshaller != nil { + return j.RuntimeUnmarshaller + } + return j.JsonMarshaller +} + func (c *Content) UnmarshallerInterceptors() marshal.Transforms { return c.unmarshallerInterceptors } @@ -176,9 +240,19 @@ func (x *XLSConfig) Options() []xlsy.Option { return options } -func (c *Content) InitMarshaller(config *config.IOConfig, exclude []string, inputType, outputType reflect.Type) error { +func (c *Content) InitMarshaller(config *config.IOConfig, exclude []string, inputType, outputType reflect.Type, lookupType xreflect.LookupType) error { c.unmarshallerInterceptors = c.Transforms.FilterByKind(marshal.TransformKindUnmarshal) - c.Marshaller.JSON.JsonMarshaller = json.New(config) + if err := c.Marshaller.Init(lookupType); err != nil { + return err + } + legacyMarshaller := json.New(config) + c.Marshaller.JSON.JsonMarshaller = legacyMarshaller + runtimeMarshaller, runtimeUnmarshaller, err := newJSONMarshaller(config, c.Marshaller.JSON.Engine, legacyMarshaller, lookupType) + if err != nil { + return err + } + c.Marshaller.JSON.RuntimeMarshaller = runtimeMarshaller + c.Marshaller.JSON.RuntimeUnmarshaller = runtimeUnmarshaller c.Marshaller.XLS.XlsMarshaller = xlsy.NewMarshaller(c.XLS.Options()...) if err := c.initCSVIfNeeded(inputType, outputType); err != nil { @@ -439,14 +513,14 @@ func (c *Content) Marshal(format string, field string, response interface{}, opt if field != "" { responseData := ensureSliceValue(response) tabJSONInterceptors := c.tabJSONInterceptors(field, responseData) - return c.Marshaller.JSON.JsonMarshaller.Marshal(response, tabJSONInterceptors) + return c.Marshaller.JSON.RuntimeMarshallerEngine().Marshal(response, tabJSONInterceptors) } return c.TabularJSON.OutputMarshaller.Marshal(response, options...) case JSONFormat: if c.Marshaller.JSON.CanMarshal() { return c.Marshaller.JSON.Marshal(response) } - return c.Marshaller.JSON.JsonMarshaller.Marshal(response, options...) + return c.Marshaller.JSON.RuntimeMarshallerEngine().Marshal(response, options...) default: return nil, fmt.Errorf("unsupproted readerData format: %s", format) } diff --git a/repository/content/json_marshaller.go b/repository/content/json_marshaller.go new file mode 100644 index 000000000..a360997e4 --- /dev/null +++ b/repository/content/json_marshaller.go @@ -0,0 +1,298 @@ +package content + +import ( + "fmt" + "net/http" + "reflect" + "strings" + + "github.com/viant/datly/gateway/router/marshal/config" + legacyjson "github.com/viant/datly/gateway/router/marshal/json" + structjson "github.com/viant/structology/encoding/json" + "github.com/viant/tagly/format" + "github.com/viant/tagly/format/text" + "github.com/viant/xreflect" + "github.com/viant/xunsafe" +) + +var DefaultJSONEngineTypeName = reflect.TypeOf(StructologyJSONRuntime{}).PkgPath() + "/" + reflect.TypeOf(StructologyJSONRuntime{}).Name() + +func newJSONMarshaller(ioConfig *config.IOConfig, engine string, legacy *legacyjson.Marshaller, lookupType xreflect.LookupType) (JSONMarshallerEngine, JSONUnmarshallerEngine, error) { + typeName := normalizeJSONEngineTypeName(engine) + if rType := xunsafe.LookupType(typeName); rType != nil { + return newJSONMarshallerByReflectType(rType, typeName, ioConfig, legacy) + } + + if lookupType == nil { + return nil, nil, fmt.Errorf("unsupported json marshaller engine: %s", typeName) + } + return newJSONMarshallerByType(lookupType, typeName, ioConfig, legacy) +} + +func normalizeJSONEngineTypeName(engine string) string { + normalized := strings.TrimSpace(engine) + if normalized == "" { + return DefaultJSONEngineTypeName + } + return normalized +} + +func newJSONMarshallerByType(lookupType xreflect.LookupType, typeName string, ioConfig *config.IOConfig, legacy *legacyjson.Marshaller) (JSONMarshallerEngine, JSONUnmarshallerEngine, error) { + rType, err := lookupType(typeName) + if err != nil { + return nil, nil, err + } + return newJSONMarshallerByReflectType(rType, typeName, ioConfig, legacy) +} + +func newJSONMarshallerByReflectType(rType reflect.Type, typeName string, ioConfig *config.IOConfig, legacy *legacyjson.Marshaller) (JSONMarshallerEngine, JSONUnmarshallerEngine, error) { + value := reflect.New(rType).Interface() + if initializer, ok := value.(JSONRuntimeInitializer); ok { + if err := initializer.InitJSONRuntime(ioConfig, legacy); err != nil { + return nil, nil, err + } + } + marshaller, ok := value.(JSONMarshallerEngine) + if !ok { + if codec, ok := value.(Marshaller); ok { + marshaller = marshalCodecAdapter{Marshaller: codec} + } + } + unmarshaller, ok := value.(JSONUnmarshallerEngine) + if !ok { + if codec, ok := value.(Unmarshaller); ok { + unmarshaller = unmarshalCodecAdapter{Unmarshaller: codec} + } + } + if marshaller == nil { + return nil, nil, fmt.Errorf("invalid type %s: json marshaller engine was not initialized", typeName) + } + if unmarshaller == nil { + return nil, nil, fmt.Errorf("invalid type %s: json unmarshaller engine was not initialized", typeName) + } + return marshaller, unmarshaller, nil +} + +type marshalCodecAdapter struct { + Marshaller +} + +func (a marshalCodecAdapter) Marshal(src interface{}, _ ...interface{}) ([]byte, error) { + return a.Marshaller.Marshal(src) +} + +type unmarshalCodecAdapter struct { + Unmarshaller +} + +func (a unmarshalCodecAdapter) Unmarshal(bytes []byte, dest interface{}, _ ...interface{}) error { + return a.Unmarshaller.Unmarshal(bytes, dest) +} + +type JSONRuntimeInitializer interface { + InitJSONRuntime(ioConfig *config.IOConfig, legacy *legacyjson.Marshaller) error +} + +type StructologyJSONRuntime struct { + config *config.IOConfig +} + +func (m *StructologyJSONRuntime) InitJSONRuntime(ioConfig *config.IOConfig, _ *legacyjson.Marshaller) error { + m.config = ioConfig + return nil +} + +func (m *StructologyJSONRuntime) Marshal(src interface{}, options ...interface{}) ([]byte, error) { + structologyOptions, err := m.marshalOptions(options) + if err != nil { + return nil, err + } + return structjson.Marshal(src, structologyOptions...) +} + +func (m *StructologyJSONRuntime) Unmarshal(bytes []byte, dest interface{}, options ...interface{}) error { + structologyOptions, err := m.unmarshalOptions(options) + if err != nil { + return err + } + return structjson.Unmarshal(bytes, dest, structologyOptions...) +} + +func (m *StructologyJSONRuntime) marshalOptions(options []interface{}) ([]structjson.Option, error) { + result := []structjson.Option{ + structjson.WithOmitEmpty(m.config != nil && m.config.OmitEmpty), + structjson.WithNilSlicePolicy(structjson.NilSliceAsEmptyArray), + } + if m.config != nil { + if caseFormat := m.config.CaseFormat; caseFormat.IsDefined() { + result = append(result, structjson.WithPathNameTransformer(datlyPathNameTransformer{caseFormat: caseFormat})) + } + if timeLayout := m.config.GetTimeLayout(); timeLayout != "" { + result = append(result, structjson.WithFormatTag(&format.Tag{TimeLayout: timeLayout})) + } + } + + var filters []*legacyjson.FilterEntry + for _, option := range options { + if option == nil { + continue + } + switch actual := option.(type) { + case []*legacyjson.FilterEntry: + filters = append(filters, actual...) + case legacyjson.MarshalerInterceptors: + if len(actual) > 0 { + return nil, fmt.Errorf("structology engine does not support legacy marshal interceptors") + } + case *legacyjson.MarshallSession: + return nil, fmt.Errorf("structology engine does not support legacy marshal sessions") + default: + return nil, fmt.Errorf("structology engine does not support marshal option %T", option) + } + } + + if excluder := newDatlyPathFieldExcluder(m.config, filters); excluder != nil { + result = append(result, structjson.WithPathFieldExcluder(excluder)) + } + return result, nil +} + +func (m *StructologyJSONRuntime) unmarshalOptions(options []interface{}) ([]structjson.Option, error) { + var result []structjson.Option + if m.config != nil { + if caseFormat := m.config.CaseFormat; caseFormat.IsDefined() { + result = append(result, structjson.WithCaseFormat(caseFormat)) + } + if timeLayout := m.config.GetTimeLayout(); timeLayout != "" { + result = append(result, structjson.WithFormatTag(&format.Tag{TimeLayout: timeLayout})) + } + } + + for _, option := range options { + if option == nil { + continue + } + switch actual := option.(type) { + case legacyjson.UnmarshalerInterceptors: + if len(actual) > 0 { + return nil, fmt.Errorf("structology engine does not support legacy unmarshal interceptors") + } + case *legacyjson.UnmarshalSession: + return nil, fmt.Errorf("structology engine does not support legacy unmarshal sessions") + case *http.Request: + continue + default: + return nil, fmt.Errorf("structology engine does not support unmarshal option %T", option) + } + } + + return result, nil +} + +type LegacyJSONRuntime struct { + marshaller *legacyjson.Marshaller +} + +func (m *LegacyJSONRuntime) InitJSONRuntime(_ *config.IOConfig, legacy *legacyjson.Marshaller) error { + m.marshaller = legacy + return nil +} + +func (m *LegacyJSONRuntime) Marshal(src interface{}, options ...interface{}) ([]byte, error) { + if m.marshaller == nil { + return nil, fmt.Errorf("legacy json runtime was not initialized") + } + return m.marshaller.Marshal(src, options...) +} + +func (m *LegacyJSONRuntime) Unmarshal(bytes []byte, dest interface{}, options ...interface{}) error { + if m.marshaller == nil { + return fmt.Errorf("legacy json runtime was not initialized") + } + return m.marshaller.Unmarshal(bytes, dest, options...) +} + +type datlyPathFieldExcluder struct { + exclude map[string]bool + filters map[string]map[string]bool +} + +type datlyPathNameTransformer struct { + caseFormat text.CaseFormat +} + +func newDatlyPathFieldExcluder(ioConfig *config.IOConfig, entries []*legacyjson.FilterEntry) structjson.PathFieldExcluder { + ret := &datlyPathFieldExcluder{} + if ioConfig != nil && len(ioConfig.Exclude) > 0 { + ret.exclude = ioConfig.Exclude + } + if len(entries) > 0 { + ret.filters = make(map[string]map[string]bool, len(entries)) + for _, entry := range entries { + if entry == nil { + continue + } + fields := make(map[string]bool, len(entry.Fields)) + for _, field := range entry.Fields { + fields[field] = true + } + ret.filters[entry.Path] = fields + normalizedPath := normalizeFilterPath(entry.Path) + if normalizedPath != entry.Path { + ret.filters[normalizedPath] = fields + } + } + } + if len(ret.exclude) == 0 && len(ret.filters) == 0 { + return nil + } + return ret +} + +func (d *datlyPathFieldExcluder) ExcludePath(path []string, fieldName string) bool { + fullPath := fieldName + parentPath := "" + if len(path) > 0 { + parentPath = strings.Join(path, ".") + fullPath = parentPath + "." + fieldName + } + if len(d.exclude) > 0 { + if d.exclude[fullPath] || d.exclude[config.NormalizeExclusionKey(fullPath)] { + return true + } + } + if len(d.filters) == 0 { + return false + } + fields, ok := d.filters[parentPath] + if !ok { + fields, ok = d.filters[normalizeFilterPath(parentPath)] + if !ok { + return false + } + } + return !fields[fieldName] +} + +func normalizeFilterPath(path string) string { + if path == "" { + return "" + } + return strings.ToLower(strings.ReplaceAll(path, "_", "")) +} + +func (d datlyPathNameTransformer) TransformPath(_ []string, fieldName string) string { + if fieldName == "ID" { + switch d.caseFormat { + case text.CaseFormatLower, text.CaseFormatLowerCamel, text.CaseFormatLowerUnderscore: + return "id" + case text.CaseFormatUpperCamel, text.CaseFormatUpper, text.CaseFormatUpperUnderscore: + return "ID" + } + } + fromCaseFormat := text.CaseFormatUpperCamel + if detected := text.DetectCaseFormat(fieldName); detected.IsDefined() { + fromCaseFormat = detected + } + return fromCaseFormat.Format(fieldName, d.caseFormat) +} diff --git a/repository/content/json_marshaller_test.go b/repository/content/json_marshaller_test.go new file mode 100644 index 000000000..81f0b4acb --- /dev/null +++ b/repository/content/json_marshaller_test.go @@ -0,0 +1,219 @@ +package content + +import ( + "fmt" + "reflect" + "testing" + "time" + + "github.com/francoispqt/gojay" + "github.com/stretchr/testify/require" + "github.com/viant/datly/gateway/router/marshal/config" + legacyjson "github.com/viant/datly/gateway/router/marshal/json" + "github.com/viant/tagly/format/text" + "github.com/viant/xreflect" +) + +type runtimeEngineCodec struct{} + +var legacyJSONEngineTypeName = reflect.TypeOf(LegacyJSONRuntime{}).PkgPath() + "/" + reflect.TypeOf(LegacyJSONRuntime{}).Name() + +func (r *runtimeEngineCodec) Marshal(src interface{}, _ ...interface{}) ([]byte, error) { + return []byte(`{"engine":"custom"}`), nil +} + +func (r *runtimeEngineCodec) Unmarshal(bytes []byte, dest interface{}, _ ...interface{}) error { + target := dest.(*map[string]interface{}) + *target = map[string]interface{}{"engine": "custom"} + return nil +} + +func TestNewJSONMarshaller_DefaultsToStructology(t *testing.T) { + cfg := &config.IOConfig{} + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case legacyJSONEngineTypeName: + return reflect.TypeOf(LegacyJSONRuntime{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + marshaller, unmarshaller, err := newJSONMarshaller(cfg, "", legacy, lookup) + + require.NoError(t, err) + _, ok := marshaller.(*StructologyJSONRuntime) + require.True(t, ok) + _, ok = unmarshaller.(*StructologyJSONRuntime) + require.True(t, ok) +} + +func TestNewJSONMarshaller_UsesExplicitLegacyEngine(t *testing.T) { + cfg := &config.IOConfig{} + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case legacyJSONEngineTypeName: + return reflect.TypeOf(LegacyJSONRuntime{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + marshaller, unmarshaller, err := newJSONMarshaller(cfg, legacyJSONEngineTypeName, legacy, lookup) + + require.NoError(t, err) + _, ok := marshaller.(*LegacyJSONRuntime) + require.True(t, ok) + _, ok = unmarshaller.(*LegacyJSONRuntime) + require.True(t, ok) +} + +func TestStructologyMarshaller_MarshalParity(t *testing.T) { + type eventType struct { + ID int + Type string + Extra string `json:"-"` + } + type payload struct { + UserID int + CreatedAt time.Time + Items []int + Name string + EventType eventType + Internal string `internal:"true"` + } + + cfg := &config.IOConfig{ + CaseFormat: text.CaseFormatLowerUnderscore, + TimeLayout: "2006-01-02", + } + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + marshaller, _, err := newJSONMarshaller(cfg, DefaultJSONEngineTypeName, legacy, lookup) + require.NoError(t, err) + + value := payload{ + UserID: 7, + CreatedAt: time.Date(2026, time.March, 5, 0, 0, 0, 0, time.UTC), + EventType: eventType{ID: 11, Type: "alpha", Extra: "ignored"}, + } + filters := []*legacyjson.FilterEntry{ + {Fields: []string{"UserID", "CreatedAt", "Items", "EventType"}}, + {Path: "EventType", Fields: []string{"Type"}}, + } + + actual, err := marshaller.Marshal(value, filters) + require.NoError(t, err) + + expected, err := legacy.Marshal(value, filters) + require.NoError(t, err) + + require.JSONEq(t, string(expected), string(actual)) + require.JSONEq(t, `{"user_id":7,"created_at":"2026-03-05","items":[],"event_type":{"type":"alpha"}}`, string(actual)) +} + +func TestStructologyMarshaller_UnmarshalUsesStructologyForBasicCase(t *testing.T) { + cfg := &config.IOConfig{CaseFormat: text.CaseFormatLowerCamel} + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + _, unmarshaller, err := newJSONMarshaller(cfg, DefaultJSONEngineTypeName, legacy, lookup) + require.NoError(t, err) + + type payload struct { + BuildTimeMs int `json:",omitempty"` + Changed bool + } + + var actual payload + err = unmarshaller.Unmarshal([]byte(`{"buildTimeMs":12,"changed":true}`), &actual) + + require.NoError(t, err) + require.Equal(t, 12, actual.BuildTimeMs) + require.True(t, actual.Changed) +} + +func TestStructologyMarshaller_MarshalRejectsLegacyInterceptors(t *testing.T) { + legacy := legacyjson.New(&config.IOConfig{}) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + marshaller, _, err := newJSONMarshaller(&config.IOConfig{}, DefaultJSONEngineTypeName, legacy, lookup) + require.NoError(t, err) + + type payload struct { + Total int + } + + _, err = marshaller.Marshal(payload{Total: 3}, legacyjson.MarshalerInterceptors{ + "Total": func() ([]byte, error) { + return []byte(`3`), nil + }, + }) + + require.ErrorContains(t, err, "does not support legacy marshal interceptors") +} + +func TestStructologyMarshaller_UnmarshalRejectsLegacyInterceptors(t *testing.T) { + legacy := legacyjson.New(&config.IOConfig{}) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + _, unmarshaller, err := newJSONMarshaller(&config.IOConfig{}, DefaultJSONEngineTypeName, legacy, lookup) + require.NoError(t, err) + + type payload struct { + Total int + } + + var actual payload + err = unmarshaller.Unmarshal([]byte(`{"Total":3}`), &actual, legacyjson.UnmarshalerInterceptors{ + "Total": func(dst interface{}, decoder *gojay.Decoder, options ...interface{}) error { + return decoder.Int(dst.(*int)) + }, + }) + + require.ErrorContains(t, err, "does not support legacy unmarshal interceptors") +} + +func TestNewJSONMarshaller_UsesRegisteredEngineType(t *testing.T) { + cfg := &config.IOConfig{} + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case "pkg.CustomJSONEngine": + return reflect.TypeOf(runtimeEngineCodec{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + marshaller, unmarshaller, err := newJSONMarshaller(cfg, "pkg.CustomJSONEngine", legacy, lookup) + + require.NoError(t, err) + actual, err := marshaller.Marshal(struct{}{}) + require.NoError(t, err) + require.JSONEq(t, `{"engine":"custom"}`, string(actual)) + + var out map[string]interface{} + err = unmarshaller.Unmarshal([]byte(`{}`), &out) + require.NoError(t, err) + require.Equal(t, map[string]interface{}{"engine": "custom"}, out) +} diff --git a/service.go b/service.go index 2c984ddf3..b48504b6f 100644 --- a/service.go +++ b/service.go @@ -463,10 +463,7 @@ func (s *Service) ensureComponentInitialized(comp *repository.Component) error { return nil } // Initialize content marshallers as in Component.Init - if err := comp.Content.InitMarshaller(comp.IOConfig(), comp.Output.Exclude, comp.BodyType(), comp.OutputType()); err != nil { - return err - } - if err := comp.Content.Marshaller.Init(res.LookupType()); err != nil { + if err := comp.Content.InitMarshaller(comp.IOConfig(), comp.Output.Exclude, comp.BodyType(), comp.OutputType(), res.LookupType()); err != nil { return err } return nil diff --git a/view/extension/init.go b/view/extension/init.go index 7e0e679c7..347898e6e 100644 --- a/view/extension/init.go +++ b/view/extension/init.go @@ -6,6 +6,7 @@ import ( "mime/multipart" "net/http" + rcontent "github.com/viant/datly/repository/content" dcodec "github.com/viant/datly/view/extension/codec" "github.com/viant/datly/view/extension/handler" "github.com/viant/datly/view/extension/marshaller" @@ -81,6 +82,8 @@ func InitRegistry() { xreflect.NewType("auth.Token", xreflect.WithReflectType(reflect.TypeOf(&auth.Token{}))), xreflect.NewType("Token", xreflect.WithReflectType(reflect.TypeOf(&auth.Token{}))), xreflect.NewType("time.Location", xreflect.WithReflectType(reflect.TypeOf(&time.Location{}))), + xreflect.NewType("content.StructologyJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.StructologyJSONRuntime{}))), + xreflect.NewType("content.LegacyJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.LegacyJSONRuntime{}))), xreflect.NewType("marshaller.JSON", xreflect.WithReflectType(reflect.TypeOf(marshaller.JSON{}))), xreflect.NewType("marshaller.Gojay", xreflect.WithReflectType(reflect.TypeOf(marshaller.Gojay{}))), )), diff --git a/view/state/kind/locator/body.go b/view/state/kind/locator/body.go index e17af401b..6374362aa 100644 --- a/view/state/kind/locator/body.go +++ b/view/state/kind/locator/body.go @@ -7,6 +7,7 @@ import ( "mime/multipart" "net/http" "reflect" + "strings" "sync" "github.com/viant/datly/shared" @@ -69,7 +70,7 @@ func (r *Body) Value(ctx context.Context, rType reflect.Type, name string) (inte if name == "" { return requestState.State(), true, nil } - sel, err := requestState.Selector(name) + sel, err := r.selectorByName(requestState, name) if err != nil { return nil, false, err } @@ -201,6 +202,32 @@ func (r *Body) ensureRequest(rType reflect.Type) (*structology.State, error) { return requestState, err } +func (r *Body) selectorByName(requestState *structology.State, name string) (*structology.Selector, error) { + sel, err := requestState.Selector(name) + if err == nil { + return sel, nil + } + stateType := requestState.Type() + for _, candidate := range stateType.RootSelectors() { + if jsonFieldName(candidate.Tag()) == name { + return candidate, nil + } + } + return nil, err +} + +func jsonFieldName(tag reflect.StructTag) string { + jsonTag := tag.Get("json") + if jsonTag == "" { + return "" + } + name := strings.SplitN(jsonTag, ",", 2)[0] + if name == "-" { + return "" + } + return name +} + func (r *Body) updateQueryString(ctx context.Context, body interface{}) { var queryParams map[string]string switch actual := body.(type) { diff --git a/view/state/parameters.go b/view/state/parameters.go index 24235aab7..6964ea534 100644 --- a/view/state/parameters.go +++ b/view/state/parameters.go @@ -409,6 +409,10 @@ func (p *Parameter) buildField(pkgPath string, lookupType xreflect.LookupType) ( schema.rType = rType } fieldName := p.Name + tagFieldName := fieldName + if p.In != nil && p.In.Kind == KindRequestBody && p.In.Name != "" { + tagFieldName = p.In.Name + } p.Schema.Cardinality = schema.Cardinality if p.Schema.Cardinality == Many && (rType.Kind() != reflect.Slice && rType.Kind() != reflect.Map) { rType = reflect.SliceOf(rType) @@ -417,11 +421,17 @@ func (p *Parameter) buildField(pkgPath string, lookupType xreflect.LookupType) ( if index := strings.LastIndex(fieldName, "."); index != -1 { fieldName = fieldName[index+1:] } + if p.In != nil && p.In.Kind == KindRequestBody && p.In.Name != "" && fieldName == p.In.Name { + fieldName = SanitizeTypeName(fieldName) + } structField = reflect.StructField{Name: fieldName, Type: rType, PkgPath: xreflect.PkgPath(fieldName, pkgPath), - Tag: p.buildTag(fieldName), + Tag: p.buildTag(tagFieldName), + } + if p.In != nil && p.In.Kind == KindRequestBody && p.In.Name != "" && structField.Tag.Get("json") == "" { + structField.Tag = appendStructTag(structField.Tag, `json:"`+p.In.Name+`,omitempty"`) } if fieldName == rType.Name() && strings.Contains(p.Tag, "anonymous") { @@ -445,13 +455,20 @@ func buildMarkerFieldTag(structField reflect.StructField) stags.Tags { return updated } +func appendStructTag(tag reflect.StructTag, value string) reflect.StructTag { + if tag == "" { + return reflect.StructTag(value) + } + return reflect.StructTag(string(tag) + " " + value) +} + func (p Parameters) BuildBodyType(pkgPath string, lookupType xreflect.LookupType) (reflect.Type, error) { candidates := p.FilterByKind(KindRequestBody) bodyLeafParameters := make(Parameters, 0, len(candidates)) for i, candidate := range candidates { if candidate.In.Name != "" { bodyParameter := *candidates[i] - bodyParameter.Name = candidate.In.Name + bodyParameter.Name = SanitizeTypeName(candidate.In.Name) bodyLeafParameters = append(bodyLeafParameters, &bodyParameter) continue } From b1654616a6e94917356fb7db6a4c8c0e30652051 Mon Sep 17 00:00:00 2001 From: vc42 Date: Sun, 22 Mar 2026 19:43:57 -0400 Subject: [PATCH 197/279] fixed view/codec.go issues --- view/codec.go | 247 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 215 insertions(+), 32 deletions(-) diff --git a/view/codec.go b/view/codec.go index 2b9e331cc..f210b085e 100644 --- a/view/codec.go +++ b/view/codec.go @@ -3,54 +3,79 @@ package view import ( "context" "fmt" + "reflect" + "strconv" + "strings" + "unicode" + codec2 "github.com/viant/datly/view/extension/codec" "github.com/viant/sqlx/io" "github.com/viant/structology" "github.com/viant/xdatly/codec" "github.com/viant/xunsafe" - "reflect" - "strconv" - "strings" ) const ( rawFieldName = "Raw" + shadowFieldName = "Shadow" actualFieldName = "Actual" ) type ( + // columnsCodec builds a wrapper type: + // - Raw: embedded, holds promoted Col{i} with sqlx tags for DB scan of codec sources + // - Shadow: embedded, mirrors Actual's exported fields+tags so sqlx can scan non-codec cols + // - Actual: named (non-embedded) model type to avoid reflect panic when it has methods columnsCodec struct { - fields []*xunsafe.Field - selectors []*structology.Selector - unwrapper *xunsafe.Field + // Promoted Raw.Col{i} fields on OUTER wrapper + fields []*xunsafe.Field + + // structology selectors resolved on OUTER with path "Actual.<...>" (codec targets) + selectors []*structology.Selector + + // OUTER.Shadow (idx=1) and OUTER.Actual (idx=2) + shadowField *xunsafe.Field + unwrapperActual *xunsafe.Field + + // Back-compat alias so view.go references to v._codec.unwrapper still compile + unwrapper *xunsafe.Field + + // OUTER wrapper reflect type actualType reflect.Type - columns []*Column + + // Columns that have codecs + columns []*Column + + // Actual pointer handling + actualIsPtr bool + actualElemTyp reflect.Type + + // Shadow -> Actual copy (safe reflect-based) + shadowFieldNames []string // exported field names mirrored in Shadow and Actual } ) func newColumnsCodec(viewType reflect.Type, columns []*Column) (*columnsCodec, error) { var withCodec []*Column for i, column := range columns { - if column.Codec != nil { + if column != nil && column.Codec != nil { withCodec = append(withCodec, columns[i]) } } - if len(withCodec) == 0 { return nil, nil } - - codec := &columnsCodec{} - - if err := codec.init(viewType, withCodec); err != nil { + cc := &columnsCodec{} + if err := cc.init(viewType, withCodec); err != nil { return nil, err } - - return codec, nil + return cc, nil } func (c *columnsCodec) init(viewType reflect.Type, columns []*Column) error { c.columns = columns + + // Build Raw holder: Col0..ColN with sqlx tags that match result set column names. codecStructFields := make([]reflect.StructField, len(columns)) for i, column := range columns { scanType := columnDatabaseScanType(column) @@ -60,38 +85,100 @@ func (c *columnsCodec) init(viewType reflect.Type, columns []*Column) error { Tag: reflect.StructTag(fmt.Sprintf(`sqlx:"%v"`, column.Name)), } } - rawType := reflect.StructOf(codecStructFields) + + // Determine Actual element type and build Shadow (exported fields only, preserving tags). + c.actualIsPtr = viewType.Kind() == reflect.Ptr + c.actualElemTyp = viewType + if c.actualIsPtr { + c.actualElemTyp = viewType.Elem() + } + shadowFields := make([]reflect.StructField, 0, c.actualElemTyp.NumField()) + c.shadowFieldNames = make([]string, 0, c.actualElemTyp.NumField()) + for i := 0; i < c.actualElemTyp.NumField(); i++ { + f := c.actualElemTyp.Field(i) + // Skip unexported fields + if f.PkgPath != "" { + continue + } + shadowFields = append(shadowFields, reflect.StructField{ + Name: f.Name, + Type: f.Type, + Tag: f.Tag, // preserve sqlx tags for scanning + }) + c.shadowFieldNames = append(c.shadowFieldNames, f.Name) + } + shadowType := reflect.StructOf(shadowFields) + + // OUTER wrapper layout: + // - Raw (embedded, first) + // - Shadow (embedded, second) + // - Actual (named, third) c.actualType = reflect.StructOf([]reflect.StructField{ { Name: rawFieldName, Type: rawType, Anonymous: true, }, + { + Name: shadowFieldName, + Type: shadowType, + Anonymous: true, + }, { Name: actualFieldName, Type: viewType, - Anonymous: true, + Anonymous: false, // non-embedded to avoid reflect panic if model has methods }, }) + // Promoted Raw.Col{i} on OUTER c.fields = make([]*xunsafe.Field, len(columns)) for i := 0; i < len(columns); i++ { - c.fields[i] = xunsafe.FieldByIndex(rawType, i) + colName := "Col" + strconv.Itoa(i) + c.fields[i] = xunsafe.FieldByName(c.actualType, colName) } - c.unwrapper = xunsafe.FieldByIndex(c.actualType, 1) - stateType := structology.NewStateType(c.actualType, structology.WithCustomizedNames(func(name string, tag reflect.StructTag) []string { - sqlxTag := io.ParseTag(tag) - if sqlxTag.Column == "" { - return []string{name} - } - return strings.Split(sqlxTag.Column, "|") - })) + // Shadow and Actual fields on OUTER + // Indexes: 0=Raw, 1=Shadow, 2=Actual + c.shadowField = xunsafe.FieldByIndex(c.actualType, 1) + c.unwrapperActual = xunsafe.FieldByIndex(c.actualType, 2) + // Back-compat alias so view.go can still use v._codec.unwrapper + c.unwrapper = c.unwrapperActual + // Build structology state for OUTER wrapper (honor sqlx tags anywhere) + stateType := structology.NewStateType( + c.actualType, + structology.WithCustomizedNames(func(name string, tag reflect.StructTag) []string { + sqlxTag := io.ParseTag(tag) + if sqlxTag.Column == "" { + return []string{name} + } + return strings.Split(sqlxTag.Column, "|") + }), + ) + + // Build selectors on OUTER using "Actual." (codec targets) for _, column := range columns { - c.selectors = append(c.selectors, stateType.Lookup(actualFieldName+"."+column.Name)) + var sel *structology.Selector + candidates := []string{ + column.Name, // exact alias + strings.ToLower(column.Name), // lowercase alias + toUpperCamel(column.Name), // Go-style name + } + for _, cand := range candidates { + if cand == "" { + continue + } + path := actualFieldName + "." + cand + sel = stateType.Lookup(path) + if sel != nil { + break + } + } + c.selectors = append(c.selectors, sel) } + return nil } @@ -106,17 +193,113 @@ func columnDatabaseScanType(column *Column) reflect.Type { } func (c *columnsCodec) updateValue(ctx context.Context, value interface{}, record *codec2.ParentValue) error { - asPtr := xunsafe.AsPointer(value) + // OUTER wrapper pointer (used by xunsafe/selectors) + outerPtr := xunsafe.AsPointer(value) + + // 1) Ensure OUTER.Actual is non-nil if Actual is a pointer type + if c.actualIsPtr { + curr := c.unwrapperActual.Value(outerPtr) // interface{} of *Elem or nil + needsAlloc := false + if curr == nil { + needsAlloc = true + } else { + rv := reflect.ValueOf(curr) + if rv.Kind() == reflect.Ptr && rv.IsNil() { + needsAlloc = true + } + } + if needsAlloc { + if c.actualElemTyp == nil { + return fmt.Errorf("invalid Actual element type") + } + newVal := reflect.New(c.actualElemTyp).Interface() // *Elem + c.unwrapperActual.SetValue(outerPtr, newVal) + } + } + + // 2) SAFE Shadow -> Actual copy via reflect (avoid unsafe header corruption) + // Build a live reflect.Value view over OUTER + outerRV := reflect.NewAt(c.actualType, outerPtr).Elem() + + // shadowRV is the embedded Shadow struct value + shadowRV := outerRV.FieldByName(shadowFieldName) + + // actualRV is the Actual field (struct or pointer-to-struct) + actualRV := outerRV.FieldByName(actualFieldName) + var actualElemRV reflect.Value + if c.actualIsPtr { + // ensure non-nil (already ensured above) + if actualRV.IsNil() { + actualRV.Set(reflect.New(c.actualElemTyp)) + } + actualElemRV = actualRV.Elem() + } else { + actualElemRV = actualRV + } + + // Copy exported fields by name + for _, name := range c.shadowFieldNames { + dst := actualElemRV.FieldByName(name) + if !dst.IsValid() || !dst.CanSet() { + continue + } + src := shadowRV.FieldByName(name) + if !src.IsValid() { + continue + } + if src.Type().AssignableTo(dst.Type()) { + dst.Set(src) + continue + } + if src.Type().ConvertibleTo(dst.Type()) { + dst.Set(src.Convert(dst.Type())) + continue + } + // Otherwise, skip incompatible types (codec may overwrite later) + } + + // 3) Apply codecs: read raw DB value from promoted Col{i} on OUTER, decode, and set via selectors for i, column := range c.columns { - fieldValue := c.fields[i].Value(asPtr) - decoded, err := column.Codec.Transform(ctx, fieldValue, codec.WithOptions(record)) + if c.fields[i] == nil { + return fmt.Errorf("codec raw field not found for column %q", column.Name) + } + if c.selectors[i] == nil { + return fmt.Errorf("codec selector not found for column %q (tried Actual.)", column.Name) + } + raw := c.fields[i].Value(outerPtr) + decoded, err := column.Codec.Transform(ctx, raw, codec.WithOptions(record)) if err != nil { return err } - if err = c.selectors[i].SetValue(asPtr, decoded); err != nil { + // Selector root is OUTER (path "Actual.<...>") + if err = c.selectors[i].SetValue(outerPtr, decoded); err != nil { return err } } - return nil } + +// toUpperCamel converts snake/space/hyphen/dot separated names to UpperCamel. +// "AD_ORDERS_DATA_INDEX" -> "AdOrdersDataIndex" +func toUpperCamel(s string) string { + if s == "" { + return s + } + var b strings.Builder + b.Grow(len(s)) + capNext := true + for _, r := range s { + switch r { + case '_', '-', ' ', '.': + capNext = true + continue + } + if capNext { + b.WriteRune(unicode.ToUpper(r)) + capNext = false + } else { + b.WriteRune(unicode.ToLower(r)) + } + } + return b.String() +} From b32be4ede550edaf0316fe2e6a45eb076506223d Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Tue, 24 Mar 2026 11:41:49 -0700 Subject: [PATCH 198/279] init --- go.mod | 5 +- go.sum | 11 +- repository/content/content.go | 51 +---- repository/content/tabjson_marshaller.go | 186 ++++++++++++++++++ repository/content/tabjson_marshaller_test.go | 102 ++++++++++ view/extension/init.go | 2 + 6 files changed, 306 insertions(+), 51 deletions(-) create mode 100644 repository/content/tabjson_marshaller.go create mode 100644 repository/content/tabjson_marshaller_test.go diff --git a/go.mod b/go.mod index d48aa85b8..b7fe22188 100644 --- a/go.mod +++ b/go.mod @@ -32,9 +32,8 @@ require ( github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.0 - github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e - + github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 golang.org/x/mod v0.28.0 golang.org/x/oauth2 v0.32.0 google.golang.org/api v0.201.0 @@ -52,7 +51,7 @@ require ( github.com/viant/jsonrpc v0.17.0 github.com/viant/mcp v0.11.0 github.com/viant/mcp-protocol v0.11.0 - github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76 + github.com/viant/structology v0.8.1-0.20260324183544-a0a56cb4c800 github.com/viant/tagly v0.3.0 github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef github.com/viant/xdatly v0.5.4-0.20260306062123-17850ac34977 diff --git a/go.sum b/go.sum index c82510f59..adf6a8628 100644 --- a/go.sum +++ b/go.sum @@ -1198,8 +1198,8 @@ github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 h1:bnVgWzZzuz2p github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= -github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76 h1:LUhy4A9ps2aFP3cBCTw4sMG1QmKWMNG1//vId03utEc= -github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= +github.com/viant/structology v0.8.1-0.20260324183544-a0a56cb4c800 h1:NKLdUFp3tJsRBZrPhajSb1JqPRYWfTuquZBdy8XEOOo= +github.com/viant/structology v0.8.1-0.20260324183544-a0a56cb4c800/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= github.com/viant/structql v0.5.4/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= github.com/viant/tagly v0.3.0 h1:Y8IckveeSrroR8yisq4MBdxhcNqf4v8II01uCpamh4E= @@ -1226,13 +1226,10 @@ github.com/viant/xlsy v0.3.1 h1:KwA7PxOTVg+ns4CCPOdfNy5aEA9OUlIByUbuNC9ju0s= github.com/viant/xlsy v0.3.1/go.mod h1:RajfF9HkL/PfIxRCvZSubpNlpdMUNDKYZp8C1o3vF4Q= github.com/viant/xmlify v0.1.1 h1:Kmn7wnsq5APD8uJVP+kM6lIEGhSyjWSNOy4BvyfZQno= github.com/viant/xmlify v0.1.1/go.mod h1:w25+umH6nthlQ8ACT3K2/YJOLlbTXKLQXkdqFs6ky9s= - -github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 h1:tQOsy7ov3XcTj+OXNF1apq9EKxSj82f5AjJCuhfCkMo= -github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= - github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e h1:z4uCWPkSCnGwqbIc3ENoYJnYwtR2j/9eI79vO4vK9rQ= github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e/go.mod h1:BwI+lqFjhKv2Vn4E0Jt6nvbwcFOWrM6H+sOMOX3JiU4= - +github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 h1:tQOsy7ov3XcTj+OXNF1apq9EKxSj82f5AjJCuhfCkMo= +github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca h1:uvPMDVyP7PXMMioYdyPH+0O+Ta/UO1WFfNYMO3Wz0eg= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= diff --git a/repository/content/content.go b/repository/content/content.go index 9183afcec..c53022988 100644 --- a/repository/content/content.go +++ b/repository/content/content.go @@ -40,9 +40,10 @@ type ( TabularJSONConfig struct { FloatPrecision string + Engine string `json:",omitempty" yaml:",omitempty"` _config *tabjson.Config - InputMarhsaller *tabjson.Marshaller - OutputMarshaller *tabjson.Marshaller + InputMarhsaller TabularJSONUnmarshallerEngine `json:"-" yaml:"-"` + OutputMarshaller TabularJSONMarshallerEngine `json:"-" yaml:"-"` } XMLConfig struct { @@ -258,7 +259,7 @@ func (c *Content) InitMarshaller(config *config.IOConfig, exclude []string, inpu if err := c.initCSVIfNeeded(inputType, outputType); err != nil { return err } - if err := c.initTabJSONIfNeeded(exclude, inputType, outputType); err != nil { + if err := c.initTabJSONIfNeeded(exclude, inputType, outputType, lookupType); err != nil { return err } if err := c.initXMLIfNeeded(exclude, inputType, outputType); err != nil { @@ -307,50 +308,18 @@ func (c *Content) ensureCSV() { c.CSV = &CSVConfig{Separator: ","} } -func (c *Content) initTabJSONIfNeeded(excludedPaths []string, inputType reflect.Type, outputType reflect.Type) error { - - if c.TabularJSON == nil { - c.TabularJSON = &TabularJSONConfig{} - } - - if c.TabularJSON._config == nil { - c.TabularJSON._config = &tabjson.Config{} - } - - if c.TabularJSON._config.FieldSeparator == "" { - c.TabularJSON._config.FieldSeparator = "," - } - +func (c *Content) initTabJSONIfNeeded(excludedPaths []string, inputType reflect.Type, outputType reflect.Type, lookupType xreflect.LookupType) error { + c.TabularJSON = ensureTabularJSONConfig(c.TabularJSON, excludedPaths) if len(c.TabularJSON._config.FieldSeparator) != 1 { return fmt.Errorf("separator has to be a single char, but was %v", c.TabularJSON._config.FieldSeparator) } - - if c.TabularJSON._config.NullValue == "" { - c.TabularJSON._config.NullValue = "null" - } - - if c.TabularJSON.FloatPrecision != "" { - c.TabularJSON._config.StringifierConfig.StringifierFloat32Config.Precision = c.TabularJSON.FloatPrecision - c.TabularJSON._config.StringifierConfig.StringifierFloat64Config.Precision = c.TabularJSON.FloatPrecision - } - - c.TabularJSON._config.ExcludedPaths = excludedPaths - - if outputType.Kind() == reflect.Ptr { - outputType = outputType.Elem() - } - - var err error - c.TabularJSON.OutputMarshaller, err = tabjson.NewMarshaller(outputType, c.TabularJSON._config) + outputMarshaller, inputMarshaller, err := newTabularJSONMarshaller(c.TabularJSON, inputType, outputType, excludedPaths, lookupType) if err != nil { return err } - - if outputType == nil { - return nil - } - c.TabularJSON.InputMarhsaller, err = tabjson.NewMarshaller(inputType, nil) - return err + c.TabularJSON.OutputMarshaller = outputMarshaller + c.TabularJSON.InputMarhsaller = inputMarshaller + return nil } // func (c *Content) initXMLIfNeeded(excludedPaths []string, outputType reflect.Type, inputType reflect.Type) error { diff --git a/repository/content/tabjson_marshaller.go b/repository/content/tabjson_marshaller.go new file mode 100644 index 000000000..a6a02956c --- /dev/null +++ b/repository/content/tabjson_marshaller.go @@ -0,0 +1,186 @@ +package content + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/viant/datly/gateway/router/marshal/tabjson" + structjsontab "github.com/viant/structology/encoding/jsontab" + "github.com/viant/xreflect" + "github.com/viant/xunsafe" +) + +var DefaultTabularJSONEngineTypeName = reflect.TypeOf(StructologyTabularJSONRuntime{}).PkgPath() + "/" + reflect.TypeOf(StructologyTabularJSONRuntime{}).Name() + +type TabularJSONMarshallerEngine interface { + Marshal(src interface{}, options ...interface{}) ([]byte, error) +} + +type TabularJSONUnmarshallerEngine interface { + Unmarshal(bytes []byte, dest interface{}) error +} + +type TabularJSONRuntimeInitializer interface { + InitTabularJSONRuntime(cfg *TabularJSONConfig, excludedPaths []string, inputType, outputType reflect.Type) error +} + +func newTabularJSONMarshaller(cfg *TabularJSONConfig, inputType, outputType reflect.Type, excludedPaths []string, lookupType xreflect.LookupType) (TabularJSONMarshallerEngine, TabularJSONUnmarshallerEngine, error) { + typeName := normalizeTabularJSONEngineTypeName(cfg.Engine) + if rType := xunsafe.LookupType(typeName); rType != nil { + return newTabularJSONMarshallerByReflectType(rType, typeName, cfg, excludedPaths, inputType, outputType) + } + if lookupType == nil { + return nil, nil, fmt.Errorf("unsupported tabular json marshaller engine: %s", typeName) + } + rType, err := lookupType(typeName) + if err != nil { + return nil, nil, err + } + return newTabularJSONMarshallerByReflectType(rType, typeName, cfg, excludedPaths, inputType, outputType) +} + +func normalizeTabularJSONEngineTypeName(engine string) string { + normalized := strings.TrimSpace(engine) + if normalized == "" { + return DefaultTabularJSONEngineTypeName + } + return normalized +} + +func newTabularJSONMarshallerByReflectType(rType reflect.Type, typeName string, cfg *TabularJSONConfig, excludedPaths []string, inputType, outputType reflect.Type) (TabularJSONMarshallerEngine, TabularJSONUnmarshallerEngine, error) { + value := reflect.New(rType).Interface() + if initializer, ok := value.(TabularJSONRuntimeInitializer); ok { + if err := initializer.InitTabularJSONRuntime(cfg, excludedPaths, inputType, outputType); err != nil { + return nil, nil, err + } + } + marshaller, ok := value.(TabularJSONMarshallerEngine) + if !ok { + return nil, nil, fmt.Errorf("invalid type %s: tabular json marshaller engine was not initialized", typeName) + } + unmarshaller, ok := value.(TabularJSONUnmarshallerEngine) + if !ok { + return nil, nil, fmt.Errorf("invalid type %s: tabular json unmarshaller engine was not initialized", typeName) + } + return marshaller, unmarshaller, nil +} + +type StructologyTabularJSONRuntime struct { + config *TabularJSONConfig +} + +func (m *StructologyTabularJSONRuntime) InitTabularJSONRuntime(cfg *TabularJSONConfig, _ []string, _ reflect.Type, _ reflect.Type) error { + m.config = cfg + return nil +} + +func (m *StructologyTabularJSONRuntime) Marshal(src interface{}, options ...interface{}) ([]byte, error) { + jsontabOptions, err := m.marshalOptions(options) + if err != nil { + return nil, err + } + return structjsontab.Marshal(src, jsontabOptions...) +} + +func (m *StructologyTabularJSONRuntime) Unmarshal(bytes []byte, dest interface{}) error { + return structjsontab.Unmarshal(bytes, dest, m.unmarshalOptions()...) +} + +func (m *StructologyTabularJSONRuntime) marshalOptions(options []interface{}) ([]structjsontab.Option, error) { + result := []structjsontab.Option{ + structjsontab.WithTagName(tabjson.TagName), + } + if m.config != nil && m.config.FloatPrecision != "" { + precision, err := strconv.Atoi(strings.TrimSpace(m.config.FloatPrecision)) + if err != nil { + return nil, fmt.Errorf("invalid tabular json float precision %q: %w", m.config.FloatPrecision, err) + } + result = append(result, structjsontab.WithFloatPrecision(precision)) + } + for _, option := range options { + if option == nil { + continue + } + switch actual := option.(type) { + case []*tabjson.Config: + if len(actual) > 0 { + return nil, fmt.Errorf("structology tabular json engine does not support legacy depth configs") + } + default: + return nil, fmt.Errorf("structology tabular json engine does not support marshal option %T", option) + } + } + return result, nil +} + +func (m *StructologyTabularJSONRuntime) unmarshalOptions() []structjsontab.Option { + return []structjsontab.Option{ + structjsontab.WithTagName(tabjson.TagName), + } +} + +type LegacyTabularJSONRuntime struct { + input *tabjson.Marshaller + output *tabjson.Marshaller +} + +func (m *LegacyTabularJSONRuntime) InitTabularJSONRuntime(cfg *TabularJSONConfig, excludedPaths []string, inputType, outputType reflect.Type) error { + configured := ensureTabularJSONConfig(cfg, excludedPaths) + var err error + m.output, err = newLegacyTabularOutputMarshaller(outputType, configured._config) + if err != nil { + return err + } + if inputType == nil { + return nil + } + m.input, err = tabjson.NewMarshaller(inputType, nil) + return err +} + +func (m *LegacyTabularJSONRuntime) Marshal(src interface{}, options ...interface{}) ([]byte, error) { + if m.output == nil { + return nil, fmt.Errorf("legacy tabular json runtime was not initialized") + } + return m.output.Marshal(src, options...) +} + +func (m *LegacyTabularJSONRuntime) Unmarshal(bytes []byte, dest interface{}) error { + if m.input == nil { + return fmt.Errorf("legacy tabular json runtime was not initialized") + } + return m.input.Unmarshal(bytes, dest) +} + +func ensureTabularJSONConfig(cfg *TabularJSONConfig, excludedPaths []string) *TabularJSONConfig { + if cfg == nil { + cfg = &TabularJSONConfig{} + } + if cfg._config == nil { + cfg._config = &tabjson.Config{} + } + if cfg._config.FieldSeparator == "" { + cfg._config.FieldSeparator = "," + } + if cfg._config.NullValue == "" { + cfg._config.NullValue = "null" + } + if cfg.FloatPrecision != "" { + cfg._config.StringifierConfig.StringifierFloat32Config.Precision = cfg.FloatPrecision + cfg._config.StringifierConfig.StringifierFloat64Config.Precision = cfg.FloatPrecision + } + cfg._config.ExcludedPaths = excludedPaths + return cfg +} + +func newLegacyTabularOutputMarshaller(outputType reflect.Type, cfg *tabjson.Config) (*tabjson.Marshaller, error) { + if outputType == nil { + return nil, nil + } + if outputType.Kind() == reflect.Ptr { + outputType = outputType.Elem() + } + return tabjson.NewMarshaller(outputType, cfg) +} diff --git a/repository/content/tabjson_marshaller_test.go b/repository/content/tabjson_marshaller_test.go new file mode 100644 index 000000000..de349f464 --- /dev/null +++ b/repository/content/tabjson_marshaller_test.go @@ -0,0 +1,102 @@ +package content + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/xreflect" +) + +var legacyTabularJSONEngineTypeName = reflect.TypeOf(LegacyTabularJSONRuntime{}).PkgPath() + "/" + reflect.TypeOf(LegacyTabularJSONRuntime{}).Name() + +func TestNewTabularJSONMarshaller_DefaultsToStructology(t *testing.T) { + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case legacyTabularJSONEngineTypeName: + return reflect.TypeOf(LegacyTabularJSONRuntime{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + marshaller, unmarshaller, err := newTabularJSONMarshaller(&TabularJSONConfig{}, reflect.TypeOf(struct{}{}), reflect.TypeOf(struct{}{}), nil, lookup) + + require.NoError(t, err) + _, ok := marshaller.(*StructologyTabularJSONRuntime) + require.True(t, ok) + _, ok = unmarshaller.(*StructologyTabularJSONRuntime) + require.True(t, ok) +} + +func TestNewTabularJSONMarshaller_UsesExplicitLegacyEngine(t *testing.T) { + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case legacyTabularJSONEngineTypeName: + return reflect.TypeOf(LegacyTabularJSONRuntime{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + cfg := &TabularJSONConfig{Engine: legacyTabularJSONEngineTypeName} + marshaller, unmarshaller, err := newTabularJSONMarshaller(cfg, reflect.TypeOf(struct{}{}), reflect.TypeOf(struct{}{}), nil, lookup) + + require.NoError(t, err) + _, ok := marshaller.(*LegacyTabularJSONRuntime) + require.True(t, ok) + _, ok = unmarshaller.(*LegacyTabularJSONRuntime) + require.True(t, ok) +} + +func TestStructologyTabularJSONRuntime_MarshalPrecisionAndNested(t *testing.T) { + type child struct { + ID int `csvName:"id"` + } + type row struct { + ID int `csvName:"id"` + Price float64 `csvName:"price"` + Children []child `csvName:"children"` + } + cfg := &TabularJSONConfig{FloatPrecision: "4"} + runtime := &StructologyTabularJSONRuntime{} + require.NoError(t, runtime.InitTabularJSONRuntime(cfg, nil, nil, nil)) + + actual, err := runtime.Marshal([]row{{ID: 1, Price: 1.123456, Children: []child{{ID: 10}, {ID: 11}}}}) + require.NoError(t, err) + require.JSONEq(t, `[["id","price","children"],[1,1.1235,[["id"],[10],[11]]]]`, string(actual)) +} + +func TestStructologyTabularJSONRuntime_Unmarshal(t *testing.T) { + type row struct { + ID int `csvName:"id"` + Name string `csvName:"name"` + } + runtime := &StructologyTabularJSONRuntime{} + require.NoError(t, runtime.InitTabularJSONRuntime(&TabularJSONConfig{}, nil, nil, nil)) + + var actual []row + err := runtime.Unmarshal([]byte(`[["id","name"],[1,"alpha"],[2,"beta"]]`), &actual) + require.NoError(t, err) + require.Equal(t, []row{{ID: 1, Name: "alpha"}, {ID: 2, Name: "beta"}}, actual) +} + +func TestContentInitTabJSONIfNeeded_DefaultStructology(t *testing.T) { + type row struct { + ID int `csvName:"id"` + } + content := &Content{} + + err := content.initTabJSONIfNeeded(nil, reflect.TypeOf([]row{}), reflect.TypeOf([]row{}), nil) + require.NoError(t, err) + + payload, err := content.TabularJSON.OutputMarshaller.Marshal([]row{{ID: 7}}) + require.NoError(t, err) + + var actual [][]interface{} + require.NoError(t, json.Unmarshal(payload, &actual)) + require.Equal(t, "id", actual[0][0]) + require.EqualValues(t, 7, actual[1][0]) +} diff --git a/view/extension/init.go b/view/extension/init.go index 347898e6e..6b3b8abbc 100644 --- a/view/extension/init.go +++ b/view/extension/init.go @@ -84,6 +84,8 @@ func InitRegistry() { xreflect.NewType("time.Location", xreflect.WithReflectType(reflect.TypeOf(&time.Location{}))), xreflect.NewType("content.StructologyJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.StructologyJSONRuntime{}))), xreflect.NewType("content.LegacyJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.LegacyJSONRuntime{}))), + xreflect.NewType("content.StructologyTabularJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.StructologyTabularJSONRuntime{}))), + xreflect.NewType("content.LegacyTabularJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.LegacyTabularJSONRuntime{}))), xreflect.NewType("marshaller.JSON", xreflect.WithReflectType(reflect.TypeOf(marshaller.JSON{}))), xreflect.NewType("marshaller.Gojay", xreflect.WithReflectType(reflect.TypeOf(marshaller.Gojay{}))), )), From b643bae50ada6279e9766736046c0cf9d8c7fce3 Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Tue, 24 Mar 2026 12:08:09 -0700 Subject: [PATCH 199/279] update structology dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b7fe22188..14aa0fab7 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/viant/jsonrpc v0.17.0 github.com/viant/mcp v0.11.0 github.com/viant/mcp-protocol v0.11.0 - github.com/viant/structology v0.8.1-0.20260324183544-a0a56cb4c800 + github.com/viant/structology v0.9.0 github.com/viant/tagly v0.3.0 github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef github.com/viant/xdatly v0.5.4-0.20260306062123-17850ac34977 diff --git a/go.sum b/go.sum index adf6a8628..403c8d56f 100644 --- a/go.sum +++ b/go.sum @@ -1198,8 +1198,8 @@ github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 h1:bnVgWzZzuz2p github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= -github.com/viant/structology v0.8.1-0.20260324183544-a0a56cb4c800 h1:NKLdUFp3tJsRBZrPhajSb1JqPRYWfTuquZBdy8XEOOo= -github.com/viant/structology v0.8.1-0.20260324183544-a0a56cb4c800/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= +github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= +github.com/viant/structology v0.9.0/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= github.com/viant/structql v0.5.4/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= github.com/viant/tagly v0.3.0 h1:Y8IckveeSrroR8yisq4MBdxhcNqf4v8II01uCpamh4E= From 0efe6759d0ade4f7b14a49a935395f40d00a27cf Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Mar 2026 13:23:58 -0700 Subject: [PATCH 200/279] - reporting enhancement --- service/reader/sql.go | 54 +++++++++++++++++++++++++--- service/reader/sql_groupable_test.go | 20 +++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/service/reader/sql.go b/service/reader/sql.go index 53af23f7e..555b26238 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -15,6 +15,7 @@ import ( "github.com/viant/datly/view/keywords" "github.com/viant/sqlparser" "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/node" "github.com/viant/sqlparser/query" "github.com/viant/sqlx/io/read/cache" ) @@ -384,15 +385,60 @@ func isAggregateSelectItem(item *query.Item) bool { if item == nil || item.Expr == nil { return false } - call, ok := item.Expr.(*expr.Call) - if !ok || call.X == nil { + return containsAggregateNode(item.Expr) +} + +func containsAggregateNode(n node.Node) bool { + switch actual := n.(type) { + case nil: + return false + case *expr.Call: + if actual.X != nil { + switch ident := actual.X.(type) { + case *expr.Ident: + if isAggregateFunction(ident.Name) { + return true + } + case *expr.Selector: + if isAggregateFunction(ident.Name) { + return true + } + } + if containsAggregateNode(actual.X) { + return true + } + } + for _, arg := range actual.Args { + if containsAggregateNode(arg) { + return true + } + } + return false + case *expr.Parenthesis: + return containsAggregateNode(actual.X) + case *expr.Unary: + return containsAggregateNode(actual.X) + case *expr.Binary: + return containsAggregateNode(actual.X) || containsAggregateNode(actual.Y) + case *expr.Switch: + if containsAggregateNode(&actual.Ident) { + return true + } + for _, item := range actual.Cases { + if item == nil { + continue + } + if containsAggregateNode(item.X) || containsAggregateNode(item.Y) { + return true + } + } return false - } - switch actual := call.X.(type) { case *expr.Ident: return isAggregateFunction(actual.Name) case *expr.Selector: return isAggregateFunction(actual.Name) + case *expr.Qualify: + return containsAggregateNode(actual.X) } return false } diff --git a/service/reader/sql_groupable_test.go b/service/reader/sql_groupable_test.go index 7cb4bbecc..fc45d12cd 100644 --- a/service/reader/sql_groupable_test.go +++ b/service/reader/sql_groupable_test.go @@ -115,6 +115,26 @@ func TestBuilder_rewriteGroupBy(t *testing.T) { projected: []*view.Column{aggregateColumns[0], aggregateColumns[2], aggregateColumns[3]}, expected: "(SELECT account_id, SUM(id) AS total_id, MAX(id) AS max_id FROM vendor GROUP BY 1)", }, + { + description: "rewrite grouped aggregates does not group by nested aggregate expressions", + sql: "(SELECT p.channel_id, p.agency_id, ROUND(SUM(p.total_spend), 4) AS total_spend FROM last_n p GROUP BY 1, 2, 3 ORDER BY total_spend DESC LIMIT 200)", + allColumns: func() []*view.Column { + return []*view.Column{ + {Name: "channel_id", Groupable: true}, + {Name: "agency_id", Groupable: true}, + {Name: "total_spend"}, + } + }(), + projected: func() []*view.Column { + columns := []*view.Column{ + {Name: "channel_id", Groupable: true}, + {Name: "agency_id", Groupable: true}, + {Name: "total_spend"}, + } + return columns + }(), + expected: "(SELECT p.channel_id, p.agency_id, ROUND(SUM(p.total_spend), 4) AS total_spend FROM last_n p GROUP BY 1, 2 ORDER BY total_spend DESC)", + }, { description: "rewrite grouped metrics query prunes unselected dimensions from select list", sql: "(SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, SUM(p.bids) AS bids, SUM(p.impressions) AS impressions, SUM(p.clicks) AS clicks, SUM(p.conversions) AS conversions, SUM(p.total_spend) AS total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?))) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 LIMIT 1000)", From 0506aa4f969a92f0022e1fd4dd80fbe3a1f8413e Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 25 Mar 2026 13:41:58 -0700 Subject: [PATCH 201/279] - reporting enhancement --- gateway/dql_bootstrap_test.go | 30 +++++++++++++++++++ gateway/mcp_report_test.go | 6 ++-- internal/translator/resource.go | 2 +- internal/translator/resource_settings_test.go | 19 ++++++++++++ repository/report_runtime.go | 10 +++---- repository/report_runtime_test.go | 5 ++-- .../shape/dql/preprocess/preprocess_test.go | 17 +++++++++++ .../dql/preprocess/settings_directives.go | 6 ++-- 8 files changed, 81 insertions(+), 14 deletions(-) diff --git a/gateway/dql_bootstrap_test.go b/gateway/dql_bootstrap_test.go index 87a56eb4f..daf72b1d1 100644 --- a/gateway/dql_bootstrap_test.go +++ b/gateway/dql_bootstrap_test.go @@ -238,6 +238,36 @@ FROM ( assert.Equal(t, state.Many, component.Output.Cardinality) } +func TestShapeLoadComponent_CubeDirectiveAlias(t *testing.T) { + ctx := context.Background() + dql := ` +#setting($_ = $connector('dev')) +#setting($_ = $route('/v1/api/shape/dev/vendors-cube', 'GET')) +#setting($_ = $cube()) +SELECT vendor.*, + groupable(vendor) +FROM ( + SELECT ACCOUNT_ID, + SUM(ID) AS TOTAL_ID + FROM VENDOR + GROUP BY 1 +) vendor` + + planResult, err := shapeCompile.New().Compile(ctx, &shape.Source{ + Name: "vendors_cube", + Path: "vendors_cube.dql", + DQL: dql, + }) + require.NoError(t, err) + + artifact, err := shapeLoad.New().LoadComponent(ctx, planResult) + require.NoError(t, err) + loaded, ok := artifact.Component.(*shapeLoad.Component) + require.True(t, ok) + require.NotNil(t, loaded.Report) + assert.True(t, loaded.Report.Enabled) +} + func TestCompileBootstrapComponent_MetaFormatOutputTypeMatchesRootView(t *testing.T) { ctx := context.Background() repo, err := repository.New(ctx, repository.WithComponentURL(""), repository.WithNoPlugin()) diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go index b2e98190b..e41ac1c18 100644 --- a/gateway/mcp_report_test.go +++ b/gateway/mcp_report_test.go @@ -235,7 +235,7 @@ func TestRouter_newToolHTTPRequest_SetsJSONContentTypeForBody(t *testing.T) { assert.Equal(t, "application/json", req.Header.Get("Content-Type")) } -func TestRouter_buildToolsIntegration_RegistersReportTool(t *testing.T) { +func TestRouter_buildToolsIntegration_RegistersCubeTool(t *testing.T) { bodyType := reflect.StructOf([]reflect.StructField{ { Name: "Dimensions", @@ -289,7 +289,7 @@ func TestRouter_buildToolsIntegration_RegistersReportTool(t *testing.T) { err := router.buildToolsIntegration(&dpath.Item{}, &dpath.Path{ Path: contract.Path{Method: http.MethodPost, URI: "/v1/api/dev/vendors-grouping/report"}, - Meta: contract.Meta{Name: "vendors grouping report", Description: "Vendor grouping report"}, + Meta: contract.Meta{Name: "vendors grouping cube", Description: "Vendor grouping cube"}, ModelContextProtocol: contract.ModelContextProtocol{ MCPTool: true, }, @@ -300,7 +300,7 @@ func TestRouter_buildToolsIntegration_RegistersReportTool(t *testing.T) { tools := registry.ListRegisteredTools() require.Len(t, tools, 1) tool := tools[0] - assert.Equal(t, "vendorsgroupingreport", tool.Name) + assert.Equal(t, "vendorsgroupingcube", tool.Name) require.Contains(t, tool.InputSchema.Properties, "dimensions") require.Contains(t, tool.InputSchema.Properties, "measures") require.Contains(t, tool.InputSchema.Properties, "filters") diff --git a/internal/translator/resource.go b/internal/translator/resource.go index 974a4e70e..dd806983f 100644 --- a/internal/translator/resource.go +++ b/internal/translator/resource.go @@ -40,7 +40,7 @@ var ( handlerSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$handler\s*\(([^)]*)\)\s*\)\s*$`) inputSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$input\s*\(([^)]*)\)\s*\)\s*$`) outputSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$output\s*\(([^)]*)\)\s*\)\s*$`) - reportSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$report\s*\(([^)]*)\)\s*\)\s*$`) + reportSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$(?:report|cube)\s*\(([^)]*)\)\s*\)\s*$`) marshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$marshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) unmarshalSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$unmarshal\s*\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) formatSettingsLineExpr = regexp.MustCompile(`(?im)^\s*#(?:settings|define|set)\s*\(\s*\$_\s*=\s*\$format\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)\s*$`) diff --git a/internal/translator/resource_settings_test.go b/internal/translator/resource_settings_test.go index 32de4d39b..9830b4d6c 100644 --- a/internal/translator/resource_settings_test.go +++ b/internal/translator/resource_settings_test.go @@ -79,3 +79,22 @@ func TestResource_extractRuleSetting_PackageQualifiesTypes(t *testing.T) { assert.NotContains(t, dSQL, "$input(") assert.NotContains(t, dSQL, "$output(") } + +func TestResource_extractRuleSetting_CubeDirectiveAlias(t *testing.T) { + resource := &Resource{Rule: NewRule(), rule: &options.Rule{}} + dSQL := "#settings($_ = $cube('OrderReportInput','Dims','Metrics','Predicates','Sort','Take','Skip'))\n" + + "SELECT 1" + + err := resource.extractRuleSetting(&dSQL) + require.NoError(t, err) + require.NotNil(t, resource.Rule.Report) + assert.True(t, resource.Rule.Report.Enabled) + assert.Equal(t, "OrderReportInput", resource.Rule.Report.Input) + assert.Equal(t, "Dims", resource.Rule.Report.Dimensions) + assert.Equal(t, "Metrics", resource.Rule.Report.Measures) + assert.Equal(t, "Predicates", resource.Rule.Report.Filters) + assert.Equal(t, "Sort", resource.Rule.Report.OrderBy) + assert.Equal(t, "Take", resource.Rule.Report.Limit) + assert.Equal(t, "Skip", resource.Rule.Report.Offset) + assert.NotContains(t, dSQL, "$cube(") +} diff --git a/repository/report_runtime.go b/repository/report_runtime.go index b7d685e88..720f46e14 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -103,10 +103,10 @@ func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, o pathCopy.MCPTemplateResource = false pathCopy.Report = routePath.Report if pathCopy.Name != "" { - pathCopy.Name += " Report" + pathCopy.Name += " Cube" } if pathCopy.Description != "" { - pathCopy.Description += " report" + pathCopy.Description += " cube" } reportPath = &pathCopy } @@ -118,7 +118,7 @@ func buildReportWrapperView(original *view.View) *view.View { return nil } ret := &view.View{ - Name: original.Name + "#report", + Name: original.Name + "#cube", Description: original.Description, Module: original.Module, Alias: original.Alias, @@ -145,10 +145,10 @@ func buildReportPath(routePath *path.Path) *path.Path { pathCopy.MCPResource = false pathCopy.MCPTemplateResource = false if pathCopy.Name != "" { - pathCopy.Name += " Report" + pathCopy.Name += " Cube" } if pathCopy.Description != "" { - pathCopy.Description += " report" + pathCopy.Description += " cube" } return &pathCopy } diff --git a/repository/report_runtime_test.go b/repository/report_runtime_test.go index c57cbbc8f..8e61a3fb3 100644 --- a/repository/report_runtime_test.go +++ b/repository/report_runtime_test.go @@ -117,6 +117,7 @@ func TestBuildReportMetadataAndComponent(t *testing.T) { require.NotNil(t, reportComponent.Report) require.NotNil(t, reportComponent.View) assert.NotSame(t, component.View, reportComponent.View) + assert.Equal(t, "vendor#cube", reportComponent.View.Name) assert.Equal(t, view.ModeHandler, reportComponent.View.Mode) assert.Nil(t, reportComponent.View.Template) require.Len(t, reportComponent.Input.Type.Parameters, 1) @@ -124,8 +125,8 @@ func TestBuildReportMetadataAndComponent(t *testing.T) { assert.Equal(t, "/v1/api/vendors/report", reportPath.URI) assert.Equal(t, "POST", reportPath.Method) assert.True(t, reportPath.MCPTool) - assert.Equal(t, "vendors Report", reportPath.Name) - assert.Equal(t, "Vendor listing report", reportPath.Description) + assert.Equal(t, "vendors Cube", reportPath.Name) + assert.Equal(t, "Vendor listing cube", reportPath.Description) reportInputType, err := buildReportInputType(component, metadata, component.Report) require.NoError(t, err) require.NotNil(t, reportInputType) diff --git a/repository/shape/dql/preprocess/preprocess_test.go b/repository/shape/dql/preprocess/preprocess_test.go index 29e06bcb4..ba3d61f10 100644 --- a/repository/shape/dql/preprocess/preprocess_test.go +++ b/repository/shape/dql/preprocess/preprocess_test.go @@ -184,6 +184,23 @@ func TestPrepare_SpecialDirectives(t *testing.T) { assert.Equal(t, "patch", pre.Directives.TemplateType) } +func TestPrepare_CubeDirectiveAlias(t *testing.T) { + dql := "#setting($_ = $cube('OrderReportInput','Dims','Metrics','Predicates','Sort','Take','Skip'))\n" + + "SELECT id FROM ORDERS o" + pre := Prepare(dql) + require.NotNil(t, pre) + require.NotNil(t, pre.Directives) + require.NotNil(t, pre.Directives.Report) + assert.True(t, pre.Directives.Report.Enabled) + assert.Equal(t, "OrderReportInput", pre.Directives.Report.Input) + assert.Equal(t, "Dims", pre.Directives.Report.Dimensions) + assert.Equal(t, "Metrics", pre.Directives.Report.Measures) + assert.Equal(t, "Predicates", pre.Directives.Report.Filters) + assert.Equal(t, "Sort", pre.Directives.Report.OrderBy) + assert.Equal(t, "Take", pre.Directives.Report.Limit) + assert.Equal(t, "Skip", pre.Directives.Report.Offset) +} + func TestPrepare_InvalidDestDirectiveDiagnostic(t *testing.T) { dql := "SELECT 1\n#settings($_ = $dest())" pre := Prepare(dql) diff --git a/repository/shape/dql/preprocess/settings_directives.go b/repository/shape/dql/preprocess/settings_directives.go index 4327a7780..47954324c 100644 --- a/repository/shape/dql/preprocess/settings_directives.go +++ b/repository/shape/dql/preprocess/settings_directives.go @@ -18,7 +18,7 @@ var ( cacheDirectiveName = map[string]bool{"cache": true} mcpDirectiveName = map[string]bool{"mcp": true} routeDirectiveName = map[string]bool{"route": true} - reportDirectiveName = map[string]bool{"report": true} + reportDirectiveName = map[string]bool{"report": true, "cube": true} constDirectiveName = map[string]bool{"const": true} marshalDirectiveName = map[string]bool{"marshal": true} unmarshalDirectiveName = map[string]bool{"unmarshal": true} @@ -112,13 +112,13 @@ func parseSettingsDirectives(input, fullDQL string, diagnosticOffset int, direct directives.Route = values[len(values)-1] } } - if strings.Contains(lower, "$report") { + if strings.Contains(lower, "$report") || strings.Contains(lower, "$cube") { calls, parseErrors := scanDollarCallsStrict(input, reportDirectiveName) diagnostics = appendDirectiveParseErrors(diagnostics, parseErrors, dqldiag.CodeDirRoute, fullDQL, diagnosticOffset) values := parseReportDirectiveCalls(calls) if len(values) == 0 { if len(calls) > 0 { - diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirRoute, "invalid $report directive", "expected: #settings($_ = $report()) or #settings($_ = $report('InputType','Dimensions','Measures','Filters','OrderBy','Limit','Offset'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) + diagnostics = append(diagnostics, directiveDiagnostic(dqldiag.CodeDirRoute, "invalid $report/$cube directive", "expected: #settings($_ = $report()) or #settings($_ = $cube()) or #settings($_ = $report('InputType','Dimensions','Measures','Filters','OrderBy','Limit','Offset'))", fullDQL, lastDirectiveCallOffset(calls, diagnosticOffset))) } } else { directives.Report = values[len(values)-1] From 0b6e9f002c83a95eeeb33d56c304455e71495d66 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 26 Mar 2026 04:58:04 -0700 Subject: [PATCH 202/279] - column discovery enhancement --- go.mod | 1 + repository/shape/discoverysql/prepare.go | 699 +++++++++++++++++++++++ view/column/discover.go | 44 +- view/column/discover_test.go | 9 + 4 files changed, 749 insertions(+), 4 deletions(-) create mode 100644 repository/shape/discoverysql/prepare.go diff --git a/go.mod b/go.mod index 14aa0fab7..d39fb497d 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/viant/datly go 1.25.0 + require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible github.com/aws/aws-lambda-go v1.31.0 diff --git a/repository/shape/discoverysql/prepare.go b/repository/shape/discoverysql/prepare.go new file mode 100644 index 000000000..b89d6f97f --- /dev/null +++ b/repository/shape/discoverysql/prepare.go @@ -0,0 +1,699 @@ +package discoverysql + +import ( + "strings" + + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/node" + "github.com/viant/sqlparser/query" +) + +// PrepareDiscoverySQL rewrites SQL for zero-row schema discovery. +// It strips template constructs and injects 1 = 0 into every SELECT it can parse. +func PrepareDiscoverySQL(sql string) (string, bool) { + cleaned := strings.TrimSpace(sql) + if cleaned == "" { + return cleaned, false + } + if hasTemplateVariables(cleaned) { + cleaned = strings.TrimSpace(stripTemplateVariables(cleaned)) + } + if cleaned == "" || !strings.Contains(strings.ToLower(cleaned), "select") { + return cleaned, false + } + if rewritten, ok := falsifyQuery(cleaned); ok { + return rewritten, true + } + return falsifyQueryText(cleaned) +} + +func falsifyQuery(sql string) (string, bool) { + sql = strings.TrimSpace(sql) + if sql == "" { + return sql, false + } + parsed, err := sqlparser.ParseQuery(sql) + if err != nil { + return sql, false + } + originalShape := collectSelectShapes(parsed) + falsifySelect(parsed) + parsed.Limit = nil + parsed.Offset = nil + result, ok := safeStringify(parsed) + if !ok || strings.TrimSpace(result) == "" { + return sql, false + } + rewrittenParsed, err := sqlparser.ParseQuery(result) + if err != nil { + return sql, false + } + if !sameSelectShapes(originalShape, collectSelectShapes(rewrittenParsed)) { + return sql, false + } + return result, true +} + +func falsifySelect(sel *query.Select) { + if sel == nil { + return + } + injectFalsePredicate(sel) + for _, ws := range sel.WithSelects { + if ws != nil && ws.X != nil { + falsifySelect(ws.X) + ws.Raw = "" + } + } + if sel.Union != nil && sel.Union.X != nil { + falsifySelect(sel.Union.X) + } + falsifyFromSubquery(sel) + for _, join := range sel.Joins { + if join != nil { + falsifyJoinSubquery(join) + } + } +} + +func injectFalsePredicate(sel *query.Select) { + if sel == nil { + return + } + if containsFalsePredicate(sel.Qualify) { + return + } + fp := &expr.Binary{ + X: &expr.Literal{Value: "1"}, + Op: "=", + Y: &expr.Literal{Value: "0"}, + } + if sel.Qualify == nil || sel.Qualify.X == nil { + sel.Qualify = &expr.Qualify{X: fp} + return + } + sel.Qualify = &expr.Qualify{ + X: &expr.Binary{ + X: fp, + Op: "AND", + Y: sel.Qualify.X, + }, + } +} + +func containsFalsePredicate(n node.Node) bool { + switch actual := n.(type) { + case nil: + return false + case *expr.Qualify: + if actual == nil { + return false + } + return containsFalsePredicate(actual.X) + case *expr.Parenthesis: + if actual == nil { + return false + } + return containsFalsePredicate(actual.X) + case *expr.Binary: + if actual == nil { + return false + } + if isFalseBinary(actual) { + return true + } + return containsFalsePredicate(actual.X) || containsFalsePredicate(actual.Y) + } + return false +} + +func isFalseBinary(b *expr.Binary) bool { + if b == nil || strings.TrimSpace(b.Op) != "=" { + return false + } + left := literalValue(b.X) + right := literalValue(b.Y) + return (left == "1" && right == "0") || (left == "0" && right == "1") +} + +func literalValue(n node.Node) string { + lit, ok := n.(*expr.Literal) + if !ok || lit == nil { + return "" + } + return strings.TrimSpace(lit.Value) +} + +func falsifyFromSubquery(sel *query.Select) { + if sel == nil || sel.From.X == nil { + return + } + switch sub := sel.From.X.(type) { + case *expr.Parenthesis: + falsifySubqueryExpr(sub) + case *expr.Raw: + falsifyRawSubquery(sub) + } +} + +func falsifyRawSubquery(raw *expr.Raw) { + if raw == nil { + return + } + text := strings.TrimSpace(raw.Raw) + if text == "" && raw.Unparsed != "" { + text = strings.TrimSpace(raw.Unparsed) + } + if len(text) >= 2 && text[0] == '(' && text[len(text)-1] == ')' { + text = text[1 : len(text)-1] + } + if !strings.Contains(strings.ToLower(text), "select") { + return + } + subQuery, err := sqlparser.ParseQuery(text) + if err != nil { + return + } + falsifySelect(subQuery) + if rewritten, ok := safeStringify(subQuery); ok { + raw.Raw = "(" + rewritten + ")" + } +} + +func falsifyJoinSubquery(join *query.Join) { + if join == nil || join.With == nil { + return + } + if sub, ok := join.With.(*expr.Parenthesis); ok { + falsifySubqueryExpr(sub) + } +} + +func falsifySubqueryExpr(paren *expr.Parenthesis) { + if paren == nil || paren.X == nil { + return + } + raw, ok := safeStringify(paren.X) + if !ok { + return + } + if !strings.Contains(strings.ToLower(strings.TrimSpace(raw)), "select") { + return + } + subQuery, err := sqlparser.ParseQuery(raw) + if err != nil { + return + } + falsifySelect(subQuery) + if rewritten, ok := safeStringify(subQuery); ok { + paren.X = expr.NewRaw(rewritten) + } +} + +func safeStringify(n node.Node) (_ string, ok bool) { + defer func() { + if recover() != nil { + ok = false + } + }() + return sqlparser.Stringify(n), true +} + +type selectShape struct { + ListLen int + HasFrom bool + JoinLen int + WithLen int + HasUnion bool +} + +func collectSelectShapes(sel *query.Select) []selectShape { + if sel == nil { + return nil + } + result := []selectShape{{ + ListLen: len(sel.List), + HasFrom: sel.From.X != nil, + JoinLen: len(sel.Joins), + WithLen: len(sel.WithSelects), + HasUnion: sel.Union != nil && sel.Union.X != nil, + }} + for _, ws := range sel.WithSelects { + if ws != nil && ws.X != nil { + result = append(result, collectSelectShapes(ws.X)...) + } + } + if sel.Union != nil && sel.Union.X != nil { + result = append(result, collectSelectShapes(sel.Union.X)...) + } + result = append(result, collectNestedShapes(sel.From.X)...) + for _, join := range sel.Joins { + if join != nil { + result = append(result, collectNestedShapes(join.With)...) + } + } + return result +} + +func collectNestedShapes(n node.Node) []selectShape { + switch actual := n.(type) { + case nil: + return nil + case *expr.Parenthesis: + if sub, ok := parseNestedSelect(actual.X); ok { + return collectSelectShapes(sub) + } + case *expr.Raw: + if sub, ok := parseNestedRawSelect(actual); ok { + return collectSelectShapes(sub) + } + } + return nil +} + +func parseNestedSelect(n node.Node) (*query.Select, bool) { + raw, ok := safeStringify(n) + if !ok { + return nil, false + } + raw = strings.TrimSpace(raw) + if raw == "" || !strings.Contains(strings.ToLower(raw), "select") { + return nil, false + } + sub, err := sqlparser.ParseQuery(raw) + if err != nil || sub == nil { + return nil, false + } + return sub, true +} + +func parseNestedRawSelect(raw *expr.Raw) (*query.Select, bool) { + if raw == nil { + return nil, false + } + text := strings.TrimSpace(raw.Raw) + if text == "" && raw.Unparsed != "" { + text = strings.TrimSpace(raw.Unparsed) + } + if len(text) >= 2 && text[0] == '(' && text[len(text)-1] == ')' { + text = strings.TrimSpace(text[1 : len(text)-1]) + } + if text == "" || !strings.Contains(strings.ToLower(text), "select") { + return nil, false + } + sub, err := sqlparser.ParseQuery(text) + if err != nil || sub == nil { + return nil, false + } + return sub, true +} + +func sameSelectShapes(a, b []selectShape) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func falsifyQueryText(sql string) (string, bool) { + sql = strings.TrimSpace(sql) + if sql == "" { + return sql, false + } + rewrittenNested, ok := rewriteNestedQueries(sql) + if !ok { + return sql, false + } + rewritten, ok := injectTopLevelFalsePredicate(rewrittenNested) + if !ok { + return sql, false + } + return rewritten, true +} + +func rewriteNestedQueries(sql string) (string, bool) { + var b strings.Builder + changed := false + for i := 0; i < len(sql); i++ { + if sql[i] != '(' { + b.WriteByte(sql[i]) + continue + } + end := matchParen(sql, i) + if end == -1 { + return sql, false + } + inner := sql[i+1 : end] + trimmed := strings.TrimSpace(inner) + if startsWithSelectQuery(trimmed) { + if rewritten, ok := falsifyQueryText(trimmed); ok { + b.WriteByte('(') + b.WriteString(rewritten) + b.WriteByte(')') + changed = true + i = end + continue + } + } + b.WriteString(sql[i : end+1]) + i = end + } + if !changed { + return sql, true + } + return b.String(), true +} + +func injectTopLevelFalsePredicate(sql string) (string, bool) { + fromPos := findTopLevelKeyword(sql, "from") + wherePos := findTopLevelKeyword(sql, "where") + groupPos := findTopLevelKeyword(sql, "group by") + havingPos := findTopLevelKeyword(sql, "having") + qualifyPos := findTopLevelKeyword(sql, "qualify") + orderPos := findTopLevelKeyword(sql, "order by") + limitPos := findTopLevelKeyword(sql, "limit") + unionPos := findTopLevelKeyword(sql, "union") + + if fromPos == -1 && wherePos == -1 { + return sql, true + } + + if wherePos != -1 { + endWhere := firstPositive(groupPos, havingPos, qualifyPos, orderPos, limitPos, unionPos, len(sql)) + whereClause := sql[wherePos:endWhere] + if containsFalsePredicateText(whereClause) { + return sql, true + } + return injectWithSpacing(sql, endWhere, " AND 1 = 0"), true + } + + insertPos := firstPositive(groupPos, havingPos, qualifyPos, orderPos, limitPos, unionPos, len(sql)) + if insertPos < 0 { + insertPos = len(sql) + } + return injectWithSpacing(sql, insertPos, " WHERE 1 = 0"), true +} + +func injectWithSpacing(sql string, pos int, injection string) string { + left := sql[:pos] + right := sql[pos:] + if len(left) > 0 { + last := left[len(left)-1] + if last != ' ' && last != '\n' && last != '\t' && last != '\r' { + injection = " " + strings.TrimLeft(injection, " ") + } + } + if len(right) > 0 { + first := right[0] + if first != ' ' && first != '\n' && first != '\t' && first != '\r' { + injection += " " + } + } + return left + injection + right +} + +func containsFalsePredicateText(sql string) bool { + normalized := strings.ToLower(strings.Join(strings.Fields(sql), " ")) + return strings.Contains(normalized, "1 = 0") || strings.Contains(normalized, "0 = 1") +} + +func startsWithSelectQuery(sql string) bool { + lower := strings.ToLower(strings.TrimSpace(sql)) + return strings.HasPrefix(lower, "select") || strings.HasPrefix(lower, "with") +} + +func findTopLevelKeyword(sql, keyword string) int { + lower := strings.ToLower(sql) + depth := 0 + inSingle := false + inDouble := false + inBacktick := false + for i := 0; i < len(lower); i++ { + ch := lower[i] + switch ch { + case '\'': + if !inDouble && !inBacktick { + inSingle = !inSingle + } + case '"': + if !inSingle && !inBacktick { + inDouble = !inDouble + } + case '`': + if !inSingle && !inDouble { + inBacktick = !inBacktick + } + } + if inSingle || inDouble || inBacktick { + continue + } + switch ch { + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + } + if depth == 0 && hasKeywordAt(lower, i, keyword) { + return i + } + } + return -1 +} + +func firstPositive(values ...int) int { + best := -1 + for _, value := range values { + if value < 0 { + continue + } + if best == -1 || value < best { + best = value + } + } + return best +} + +func hasKeywordAt(text string, pos int, keyword string) bool { + if pos < 0 || pos+len(keyword) > len(text) || text[pos:pos+len(keyword)] != keyword { + return false + } + beforeOK := pos == 0 || !isKeywordIdentChar(text[pos-1]) + afterPos := pos + len(keyword) + afterOK := afterPos == len(text) || !isKeywordIdentChar(text[afterPos]) + return beforeOK && afterOK +} + +func isKeywordIdentChar(ch byte) bool { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '$' +} + +func matchParen(sql string, start int) int { + depth := 0 + inSingle := false + inDouble := false + inBacktick := false + for i := start; i < len(sql); i++ { + ch := sql[i] + switch ch { + case '\'': + if !inDouble && !inBacktick { + inSingle = !inSingle + } + case '"': + if !inSingle && !inBacktick { + inDouble = !inDouble + } + case '`': + if !inSingle && !inDouble { + inBacktick = !inBacktick + } + } + if inSingle || inDouble || inBacktick { + continue + } + if ch == '(' { + depth++ + } else if ch == ')' { + depth-- + if depth == 0 { + return i + } + } + } + return -1 +} + +func hasTemplateVariables(sql string) bool { + for i := 0; i < len(sql)-1; i++ { + if sql[i] == '$' && isIdentStart(sql[i+1]) { + return true + } + if sql[i] == '#' && (sql[i+1] == 'i' || sql[i+1] == 'f' || sql[i+1] == 'e' || sql[i+1] == 's') { + return true + } + if sql[i] == '$' && sql[i+1] == '{' { + return true + } + } + return false +} + +func stripTemplateVariables(sql string) string { + var b strings.Builder + b.Grow(len(sql)) + i := 0 + for i < len(sql) { + if sql[i] == '#' && i+1 < len(sql) { + directive := matchDirective(sql, i) + if directive != "" { + end := skipDirective(sql, i, directive) + b.WriteByte(' ') + i = end + continue + } + } + if sql[i] == '$' && i+1 < len(sql) { + next := sql[i+1] + if next == '{' { + depth := 1 + j := i + 2 + for j < len(sql) && depth > 0 { + if sql[j] == '{' { + depth++ + } else if sql[j] == '}' { + depth-- + } + j++ + } + b.WriteString("''") + i = j + continue + } + if isIdentStart(next) { + j := i + 1 + for j < len(sql) && isIdentPart(sql[j]) { + j++ + } + hasMethodCall := false + methodExpr := "" + for j < len(sql) && sql[j] == '.' { + methodStart := j + j++ + for j < len(sql) && isIdentPart(sql[j]) { + j++ + } + if j < len(sql) && sql[j] == '(' { + hasMethodCall = true + methodExpr = sql[methodStart:j] + depth := 1 + j++ + for j < len(sql) && depth > 0 { + if sql[j] == '(' { + depth++ + } else if sql[j] == ')' { + depth-- + } + j++ + } + } + } + if hasMethodCall { + if strings.EqualFold(methodExpr, ".AppendBinding") { + b.WriteString("''") + } + } else { + b.WriteString("''") + } + i = j + continue + } + } + b.WriteByte(sql[i]) + i++ + } + return b.String() +} + +func matchDirective(sql string, pos int) string { + directives := []string{"#foreach", "#if", "#elseif", "#else", "#end", "#set", "#settings", "#setting", "#define", "#package", "#import"} + remaining := sql[pos:] + for _, d := range directives { + if len(remaining) >= len(d) && strings.EqualFold(remaining[:len(d)], d) { + if len(remaining) == len(d) || !isIdentPart(remaining[len(d)]) { + return d + } + } + } + return "" +} + +func skipDirective(sql string, pos int, directive string) int { + switch { + case directive == "#set" || directive == "#settings" || directive == "#setting" || directive == "#define": + j := pos + len(directive) + for j < len(sql) && (sql[j] == ' ' || sql[j] == '\t') { + j++ + } + if j < len(sql) && sql[j] == '(' { + depth := 1 + j++ + for j < len(sql) && depth > 0 { + if sql[j] == '(' { + depth++ + } else if sql[j] == ')' { + depth-- + } + j++ + } + return j + } + for j < len(sql) && sql[j] != '\n' { + j++ + } + if j < len(sql) { + j++ + } + return j + case directive == "#foreach" || directive == "#if": + j := pos + len(directive) + depth := 1 + for j < len(sql) && depth > 0 { + d := matchDirective(sql, j) + if d == "#if" || d == "#foreach" { + depth++ + j += len(d) + } else if d == "#end" { + depth-- + j += len(d) + } else { + j++ + } + } + return j + default: + j := pos + len(directive) + for j < len(sql) && sql[j] != '\n' { + j++ + } + if j < len(sql) { + j++ + } + return j + } +} + +func isIdentStart(ch byte) bool { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' +} + +func isIdentPart(ch byte) bool { + return isIdentStart(ch) || (ch >= '0' && ch <= '9') +} diff --git a/view/column/discover.go b/view/column/discover.go index 17a8736ca..5c779fd9c 100644 --- a/view/column/discover.go +++ b/view/column/discover.go @@ -4,11 +4,13 @@ import ( "context" "database/sql" "fmt" + "github.com/viant/datly/repository/shape/discoverysql" "github.com/viant/datly/shared" "github.com/viant/datly/utils/types" dconfig "github.com/viant/datly/view/extension" "github.com/viant/sqlparser" "github.com/viant/sqlparser/expr" + "github.com/viant/sqlparser/node" "github.com/viant/sqlparser/query" "github.com/viant/sqlx/io" "github.com/viant/sqlx/io/config" @@ -20,16 +22,28 @@ import ( ) func Discover(ctx context.Context, db *sql.DB, table, SQL string, SQLArgs ...interface{}) (sqlparser.Columns, error) { + originalSQL := SQL SQL = strings.ReplaceAll(SQL, "$AND_CRITERIA", "") SQL = strings.ReplaceAll(SQL, "$WHERE_CRITERIA", "") + sanitizedSQL := SQL var columns sqlparser.Columns var err error if table == SQL && !strings.Contains(strings.ToLower(SQL), "select") { SQL = "SELECT * FROM " + table + " WHERE 1 = 0" + } else { + if prepared, ok := discoverysql.PrepareDiscoverySQL(SQL); strings.TrimSpace(prepared) != "" { + sanitizedSQL = prepared + if ok { + SQL = prepared + } + } + } + if sanitizedSQL == "" { + sanitizedSQL = SQL } if SQL != "" { if columns, err = detectColumns(ctx, db, SQL, table, SQLArgs...); err != nil { - return columns, err + return columns, fmt.Errorf("%w\noriginal SQL: %s\nsanitized SQL: %s", err, originalSQL, sanitizedSQL) } } if len(columns) == 0 && table != "" { //TODO mere column types @@ -244,9 +258,9 @@ func parseQuery(SQL string) (string, string, sqlparser.Columns) { var table string var queryColumn sqlparser.Columns if sqlQuery != nil { - queryColumn = sqlparser.NewColumns(sqlQuery.List) + queryColumn, _ = safeNewColumns(sqlQuery.List) if sqlQuery.From.X != nil { - table = sqlparser.Stringify(sqlQuery.From.X) + table, _ = safeStringify(sqlQuery.From.X) } // For CTE-backed queries (WITH ...), SELECT * FROM cte_alias must still be // resolved via SQL execution; the alias is not a physical table. @@ -272,7 +286,11 @@ func parseQuery(SQL string) (string, string, sqlparser.Columns) { sqlQuery.From.Alias = "t" } } - SQL = sqlparser.Stringify(sqlQuery) + var ok bool + SQL, ok = safeStringify(sqlQuery) + if !ok { + return table, "", queryColumn + } if table != "" { SQL += " LIMIT 1" } @@ -280,6 +298,24 @@ func parseQuery(SQL string) (string, string, sqlparser.Columns) { return table, SQL, queryColumn } +func safeNewColumns(list query.List) (_ sqlparser.Columns, ok bool) { + defer func() { + if recover() != nil { + ok = false + } + }() + return sqlparser.NewColumns(list), true +} + +func safeStringify(n node.Node) (_ string, ok bool) { + defer func() { + if recover() != nil { + ok = false + } + }() + return sqlparser.Stringify(n), true +} + func falsePredicate() *expr.Binary { return &expr.Binary{X: &expr.Literal{Value: "1"}, Op: "=", Y: &expr.Literal{Value: "0"}} } diff --git a/view/column/discover_test.go b/view/column/discover_test.go index ef2cf966e..be6d9a772 100644 --- a/view/column/discover_test.go +++ b/view/column/discover_test.go @@ -17,3 +17,12 @@ func TestParseQuery_WithCTEStar_DoesNotShortCircuitToTableMetadata(t *testing.T) require.Contains(t, strings.ToUpper(discoveredSQL), "WITH CTE AS") require.Contains(t, discoveredSQL, "LIMIT 1") } + +func TestParseQuery_WithCaseExpression_DoesNotPanic(t *testing.T) { + table, discoveredSQL, cols := parseQuery( + `SELECT CASE WHEN status_flag = 1 THEN 'active' ELSE 'inactive' END AS status_bucket FROM account_window`, + ) + require.Equal(t, "account_window", strings.TrimSpace(table)) + require.Len(t, cols, 0) + require.Equal(t, "", discoveredSQL) +} From a4dd32dacb1878676316d00bea5263a9b57b3f3d Mon Sep 17 00:00:00 2001 From: arao Date: Thu, 26 Mar 2026 11:32:21 -0700 Subject: [PATCH 203/279] ENG-00000 adding new finalize-with-error --- service/operator/service.go | 10 ++++++++++ view/state/hook.go | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/service/operator/service.go b/service/operator/service.go index a5a61ecb1..ee9300503 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -143,6 +143,16 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes err = injectorFinalizer.Finalize(ctx, lookup) return ret, err } + if finalizer, ok := ret.(state.FinalizerWithError); ok { + finalizeErr := finalizer.Finalize(ctx, err) + if err != nil { + if finalizeErr != nil { + return ret, errors.Join(err, finalizeErr) + } + return ret, err + } + return ret, finalizeErr + } if err != nil { return ret, err } diff --git a/view/state/hook.go b/view/state/hook.go index e871f4d6d..144dd93ae 100644 --- a/view/state/hook.go +++ b/view/state/hook.go @@ -17,6 +17,11 @@ type Finalizer interface { Finalize(ctx context.Context) error } +// FinaliserWithError is an error-aware finalizer that receives an error from previous steps. +type FinalizerWithError interface { + Finalize(ctx context.Context, err error) error +} + type InjectorFinalizer interface { Finalize(ctx context.Context, getInjector func(ctx context.Context, path http.Route) (state.Injector, error)) error } From 28512d5d07a2caad829868ffe06199401ccdf188 Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Thu, 26 Mar 2026 12:34:09 -0700 Subject: [PATCH 204/279] fixed sequencer issue --- go.mod | 2 +- go.sum | 4 +- service/executor/sequencer/service.go | 5 +- service/executor/sequencer/walker.go | 41 +++++++++++++- service/executor/sequencer/walker_test.go | 65 +++++++++++++++++++++++ 5 files changed, 112 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 14aa0fab7..68e589e69 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 - github.com/viant/sqlx v0.21.0 + github.com/viant/sqlx v0.22.1-0.20260326175456-cec446e28372 github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.0 diff --git a/go.sum b/go.sum index 403c8d56f..12881d103 100644 --- a/go.sum +++ b/go.sum @@ -1196,8 +1196,8 @@ github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 h1:bnVgWzZzuz2pTa54e7YozHjYNFSapfU3MSklyMkO+Ag= github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= -github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= +github.com/viant/sqlx v0.22.1-0.20260326175456-cec446e28372 h1:5qW+4AbQ8YA0MsyoUx3uaNgTHl52F0JBoC2vsdwKXIM= +github.com/viant/sqlx v0.22.1-0.20260326175456-cec446e28372/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= github.com/viant/structology v0.9.0/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= diff --git a/service/executor/sequencer/service.go b/service/executor/sequencer/service.go index 371af6388..92ffc66b1 100644 --- a/service/executor/sequencer/service.go +++ b/service/executor/sequencer/service.go @@ -32,10 +32,13 @@ func (s *Service) next(table string, any interface{}, selector string) error { if err != nil || emptyRecordCount == 0 { return err } - record, err := aWalker.Leaf(any) + record, err := aWalker.EmptyLeaf(any) if err != nil { return err } + if record == nil { + return nil + } inserter, err := insert.New(s.ctx, s.db, table) if err != nil { return err diff --git a/service/executor/sequencer/walker.go b/service/executor/sequencer/walker.go index 7d761177e..cd7b468d8 100644 --- a/service/executor/sequencer/walker.go +++ b/service/executor/sequencer/walker.go @@ -19,6 +19,11 @@ func (w *Walker) Leaf(value interface{}) (interface{}, error) { return w.leaf(w.root, value) } +// EmptyLeaf returns the first record whose leaf selector currently has a zero value. +func (w *Walker) EmptyLeaf(value interface{}) (interface{}, error) { + return w.emptyLeaf(w.root, value) +} + // Allocate allocate sequence func (w *Walker) Allocate(value interface{}, seq *Sequence) error { return w.allocate(w.root, value, seq) @@ -106,7 +111,7 @@ func (w *Walker) leaf(aNode *node, value interface{}) (interface{}, error) { return value, nil case nodeKindArray: sliceLen := aNode.xSlice.Len(ptr) - for i := 0; i < sliceLen; { + for i := 0; i < sliceLen; i++ { item := aNode.xSlice.ValuePointerAt(ptr, i) first, err := w.leaf(aNode.children, item) if err != nil { @@ -121,6 +126,40 @@ func (w *Walker) leaf(aNode *node, value interface{}) (interface{}, error) { return item, nil } +func (w *Walker) emptyLeaf(aNode *node, value interface{}) (interface{}, error) { + ptr := xunsafe.AsPointer(value) + var item interface{} + switch aNode.kind { + case nodeKindObject: + item = aNode.xField.Interface(ptr) + return w.emptyLeaf(aNode.children, item) + case nodeKindLeaf: + item = aNode.xField.Addr(ptr) + intPtr, err := int64Ptr(item) + if err != nil { + return nil, err + } + if *intPtr == 0 { + return value, nil + } + return nil, nil + case nodeKindArray: + sliceLen := aNode.xSlice.Len(ptr) + for i := 0; i < sliceLen; i++ { + item := aNode.xSlice.ValuePointerAt(ptr, i) + first, err := w.emptyLeaf(aNode.children, item) + if err != nil { + return nil, err + } + if first != nil { + return first, nil + } + } + return nil, nil + } + return item, nil +} + func int64Ptr(value interface{}) (*int64, error) { switch actual := value.(type) { case *int, *uint, uint64: diff --git a/service/executor/sequencer/walker_test.go b/service/executor/sequencer/walker_test.go index b51ee6a08..a8414136f 100644 --- a/service/executor/sequencer/walker_test.go +++ b/service/executor/sequencer/walker_test.go @@ -152,3 +152,68 @@ func TestWalker_Leaf(t *testing.T) { } } + +func TestWalker_EmptyLeaf(t *testing.T) { + type Foo struct { + ID int + Name string + } + + type Bar struct { + ID int + Foos []Foo + } + + testCases := []struct { + description string + value interface{} + selectors []string + expect interface{} + }{ + { + description: "nested selector returns first empty leaf owner", + value: []*Bar{ + { + ID: 1, + Foos: []Foo{ + {ID: 10, Name: "keep"}, + {ID: 0, Name: "allocate-me"}, + }, + }, + }, + selectors: []string{"Foos", "ID"}, + expect: &Foo{ID: 0, Name: "allocate-me"}, + }, + { + description: "slice selector skips non-empty ids", + value: []*Foo{ + {ID: 101, Name: "already-set"}, + {ID: 0, Name: "needs-id"}, + {ID: 0, Name: "also-needs-id"}, + }, + selectors: []string{"ID"}, + expect: &Foo{ID: 0, Name: "needs-id"}, + }, + { + description: "returns nil when there are no empty ids", + value: []*Foo{ + {ID: 101, Name: "already-set"}, + {ID: 102, Name: "also-set"}, + }, + selectors: []string{"ID"}, + expect: nil, + }, + } + + for _, testCase := range testCases { + aWalker, err := NewWalker(testCase.value, testCase.selectors) + if !assert.Nil(t, err, testCase.description) { + continue + } + actual, err := aWalker.EmptyLeaf(testCase.value) + if !assert.Nil(t, err, testCase.description) { + continue + } + assert.EqualValues(t, testCase.expect, actual, testCase.description) + } +} From f41687b15d2ae6e6148875e0109e6159ff90aeca Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Thu, 26 Mar 2026 15:21:31 -0700 Subject: [PATCH 205/279] remove unecessary init --- internal/translator/rule.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/translator/rule.go b/internal/translator/rule.go index e4c6f10d1..06ba92497 100644 --- a/internal/translator/rule.go +++ b/internal/translator/rule.go @@ -326,7 +326,6 @@ func (r *Rule) applyDefaults() { setter.SetCaseFormatIfEmpty(&r.Route.Output.CaseFormat, "lc") setter.SetBoolIfFalse(&r.Input.IgnoreEmptyQueryParameters, r.IgnoreEmptyQueryParameters) setter.SetBoolIfFalse(&r.Input.CustomValidation, r.CustomValidation || r.Type != "") - setter.SetStringIfEmpty(&r.Route.Content.Marshaller.JSON.Engine, content.DefaultJSONEngineTypeName) if r.XMLUnmarshalType != "" { r.Route.Content.Marshaller.XML.TypeName = r.XMLUnmarshalType } From df5d890b622194b4f3ea26ecdf0946c5ecb3534a Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 31 Mar 2026 08:04:32 -0700 Subject: [PATCH 206/279] - added composite key support --- internal/inference/parameter.go | 77 ++++-- internal/inference/spec.go | 47 ++-- internal/inference/state.go | 7 + internal/inference/tag.go | 34 ++- service/executor/expand/parent.go | 145 ++++++++-- service/reader/service.go | 20 +- service/reader/slice.go | 7 + service/reader/sql.go | 70 ++++- view/batch.go | 24 +- view/collector.go | 446 +++++++++++++++++++++++++----- view/relation.go | 5 + view/template.go | 8 +- 12 files changed, 749 insertions(+), 141 deletions(-) diff --git a/internal/inference/parameter.go b/internal/inference/parameter.go index cc2916e69..203cc0e0f 100644 --- a/internal/inference/parameter.go +++ b/internal/inference/parameter.go @@ -366,24 +366,69 @@ func ParentAlias(join *query.Join) string { } func ExtractRelationColumns(join *query.Join) (string, string) { - relColumn := "" - refColumn := "" - sqlparser.Traverse(join.On, func(n node.Node) bool { - switch actual := n.(type) { - case *qexpr.Selector: - column := sqlparser.Stringify(actual.X) - if actual.Name == join.Alias { - if refColumn == "" { - refColumn = column - } - } else if relColumn == "" { - relColumn = column + pairs := ExtractRelationColumnPairs(join) + if len(pairs) == 0 { + return "", "" + } + return pairs[0][0], pairs[0][1] +} + +func ExtractRelationColumnPairs(join *query.Join) [][2]string { + if join == nil || join.On == nil || join.On.X == nil { + return nil + } + return collectRelationColumnPairs(join.On.X, join.Alias) +} + +func collectRelationColumnPairs(n node.Node, refAlias string) [][2]string { + switch actual := n.(type) { + case *qexpr.Binary: + actual = actual.Normalize() + op := strings.ToUpper(strings.TrimSpace(actual.Op)) + if op == "AND" { + left := collectRelationColumnPairs(actual.X, refAlias) + right := collectRelationColumnPairs(actual.Y, refAlias) + return append(left, right...) + } + if op != "=" { + return nil + } + leftAlias, leftColumn, leftOK := selectorParts(actual.X) + rightAlias, rightColumn, rightOK := selectorParts(actual.Y) + if !leftOK || !rightOK { + return nil + } + switch { + case leftAlias == refAlias: + return [][2]string{{rightColumn, leftColumn}} + case rightAlias == refAlias: + return [][2]string{{leftColumn, rightColumn}} + } + case *qexpr.Parenthesis: + return collectRelationColumnPairs(actual.X, refAlias) + } + return nil +} + +func selectorParts(n node.Node) (string, string, bool) { + switch actual := n.(type) { + case *qexpr.Selector: + return actual.Name, sqlparser.Stringify(actual.X), true + case *qexpr.Parenthesis: + return selectorParts(actual.X) + case *qexpr.Collate: + return selectorParts(actual.X) + case *qexpr.Call: + if alias, column, ok := selectorParts(actual.X); ok { + return alias, column, true + } + for _, arg := range actual.Args { + if alias, column, ok := selectorParts(arg); ok { + return alias, column, true } - return true } - return true - }) - return relColumn, refColumn + } + return "", "", false } func (p *Parameter) EnsureCodec() { diff --git a/internal/inference/spec.go b/internal/inference/spec.go index e6a562199..c5227ffef 100644 --- a/internal/inference/spec.go +++ b/internal/inference/spec.go @@ -20,12 +20,18 @@ import ( ) type ( + RelationPair struct { + ParentField *Field + KeyField *Field + } + //Relation defines relation Relation struct { Name string Join *query.Join ParentField *Field KeyField *Field + Pairs []*RelationPair Cardinality state.Cardinality *Spec } @@ -163,28 +169,35 @@ func (s *Spec) AddRelation(name string, join *query.Join, spec *Spec, cardinalit if IsToOne(join) { cardinality = state.One } - relColumn, refColumn := ExtractRelationColumns(join) - parentField := s.Type.ByColumn(relColumn) - if parentField == nil { - var available []string - for _, item := range s.Type.columnFields { - available = append(available, item.Column.Name) + pairColumns := ExtractRelationColumnPairs(join) + if len(pairColumns) == 0 { + return fmt.Errorf("failed to extract relation columns for %v", join.Alias) + } + pairs := make([]*RelationPair, 0, len(pairColumns)) + for _, pair := range pairColumns { + parentField := s.Type.ByColumn(pair[0]) + if parentField == nil { + var available []string + for _, item := range s.Type.columnFields { + available = append(available, item.Column.Name) + } + return fmt.Errorf("failed to match rel field for %v, available: %v %v", pair[0], s.Type.Name, available) } - return fmt.Errorf("failed to match rel field for %v, available: %v %v", relColumn, s.Type.Name, available) - } - - keyField := spec.Type.ByColumn(refColumn) - if keyField == nil { - var available []string - for _, item := range spec.Type.columnFields { - available = append(available, item.Column.Name) + keyField := spec.Type.ByColumn(pair[1]) + if keyField == nil { + var available []string + for _, item := range spec.Type.columnFields { + available = append(available, item.Column.Name) + } + return fmt.Errorf("failed to ref field for %v, available: %v on %v", pair[1], available, join.Alias) } - return fmt.Errorf("failed to ref field for %v, available: %v on %v", refColumn, available, join.Alias) + pairs = append(pairs, &RelationPair{ParentField: parentField, KeyField: keyField}) } rel := &Relation{Spec: spec, - KeyField: keyField, - ParentField: parentField, + KeyField: pairs[0].KeyField, + ParentField: pairs[0].ParentField, + Pairs: pairs, Name: name, Join: join, Cardinality: cardinality} diff --git a/internal/inference/state.go b/internal/inference/state.go index fa28c984f..880f0be76 100644 --- a/internal/inference/state.go +++ b/internal/inference/state.go @@ -327,6 +327,13 @@ func removeBuilinExpr(query string) string { } query = strings.ReplaceAll(query, fragment, "") } + if index := strings.Index(query, "$View.ParentCompositeJoinOn"); index != -1 { + fragment := query[index:] + if endIndex := strings.Index(fragment, ")"); endIndex != -1 { + fragment = fragment[:endIndex+1] + } + query = strings.ReplaceAll(query, fragment, "") + } if !strings.Contains(query, "${predicate.") { return query diff --git a/internal/inference/tag.go b/internal/inference/tag.go index 2075a485e..74855ca38 100644 --- a/internal/inference/tag.go +++ b/internal/inference/tag.go @@ -180,19 +180,29 @@ func (t *Tags) buildRelation(spec *Spec, relation *Relation) { Table: spec.Table, } joinTag := tags.LinkOn{} - - parentColumn := relation.ParentField.Column.Name - if ns := relation.ParentField.Column.Namespace; ns != "" { - parentColumn = ns + "." + parentColumn - } - keyColumn := relation.KeyField.Column.Name - if ns := relation.KeyField.Column.Namespace; ns != "" { - keyColumn = ns + "." + keyColumn + if len(relation.Pairs) == 0 { + relation.Pairs = []*RelationPair{{ + ParentField: relation.ParentField, + KeyField: relation.KeyField, + }} + } + for _, pair := range relation.Pairs { + if pair == nil || pair.ParentField == nil || pair.KeyField == nil { + continue + } + parentColumn := pair.ParentField.Column.Name + if ns := pair.ParentField.Column.Namespace; ns != "" { + parentColumn = ns + "." + parentColumn + } + keyColumn := pair.KeyField.Column.Name + if ns := pair.KeyField.Column.Namespace; ns != "" { + keyColumn = ns + "." + keyColumn + } + joinTag = joinTag.Append( + tags.WithRelLink(pair.ParentField.Name, parentColumn, nil), + tags.WithRefLink(pair.KeyField.Name, keyColumn), + ) } - joinTag = joinTag.Append( - tags.WithRelLink(relation.ParentField.Name, parentColumn, nil), - tags.WithRefLink(relation.KeyField.Name, keyColumn), - ) sqlTag := TagValue{} if rawSQL := strings.Trim(sqlparser.Stringify(join.With), " )("); rawSQL != "" { rawSQL = strings.Replace(rawSQL, "("+spec.Table+")", spec.Table, 1) diff --git a/service/executor/expand/parent.go b/service/executor/expand/parent.go index 97787429d..0a51b93e9 100644 --- a/service/executor/expand/parent.go +++ b/service/executor/expand/parent.go @@ -1,9 +1,12 @@ package expand import ( + "context" "database/sql" "fmt" "github.com/viant/datly/utils/types" + sqlxconfig "github.com/viant/sqlx/io/config" + "github.com/viant/sqlx/metadata/info" "github.com/viant/xunsafe" "os" "reflect" @@ -21,6 +24,7 @@ type ( ColIn(prefix, column string) (string, error) In(prefix string) (string, error) ParentJoinOn(column string, prepend ...string) (string, error) + ParentCompositeJoinOn(prefix string, columns ...string) (string, error) AndParentJoinOn(column string) (string, error) } @@ -45,18 +49,22 @@ type ( ParentBatch interface { ColIn() []interface{} ColInBatch() []interface{} + CompositeIn() [][]interface{} + CompositeInBatch() [][]interface{} + HasComposite() bool } ViewContext struct { - Name string - Alias string - Table string - Limit int - Offset int - Page int - Args []interface{} - NonWindowSQL string - ParentValues []interface{} + Name string + Alias string + Table string + Limit int + Offset int + Page int + Args []interface{} + NonWindowSQL string + ParentValues []interface{} + ParentCompositeValues [][]interface{} expander Expander `velty:"-"` DataUnit *DataUnit `velty:"-"` @@ -102,6 +110,10 @@ func (e *MockExpander) AndParentJoinOn(column string) (string, error) { return e.ColIn("", column) } +func (e *MockExpander) ParentCompositeJoinOn(prefix string, columns ...string) (string, error) { + return "", nil +} + func (e *MockExpander) ColIn(prefix, column string) (string, error) { return "", nil } @@ -111,16 +123,48 @@ func (e *MockExpander) In(prefix string) (string, error) { } func (m *ViewContext) ParentJoinOn(column string, prepend ...string) (string, error) { + prefix := "AND" + columns := []string{column} if len(prepend) > 0 { - return m.ColIn(column, prepend[0]) + prefix = column + columns = prepend } - return m.ColIn("AND", column) + if len(columns) > 1 { + return m.parentCompositeJoinOn(prefix, columns...) + } + return m.ColIn(prefix, columns[0]) } func (m *ViewContext) AndParentJoinOn(column string) (string, error) { return m.ColIn("AND", column) } +func (m *ViewContext) ParentCompositeJoinOn(prefix string, columns ...string) (string, error) { + return m.parentCompositeJoinOn(prefix, columns...) +} + +func (m *ViewContext) parentCompositeJoinOn(prefix string, columns ...string) (string, error) { + if len(columns) == 0 { + return prefix + " 1 = 0 ", nil + } + if m.expander != nil { + return m.expander.ParentCompositeJoinOn(prefix, columns...) + } + rowCount := len(m.ParentCompositeValues) + if rowCount == 0 { + return prefix + " 1 = 0 ", nil + } + dialect, err := m.dialect() + if err != nil { + return "", err + } + if prefix != "" && !strings.HasSuffix(prefix, " ") { + prefix += " " + } + m.addCompositeBindings(m.ParentCompositeValues) + return prefix + renderCompositePredicate(dialect, columns, rowCount), nil +} + func (m *ViewContext) ColIn(prefix, column string) (string, error) { if m.expander != nil { return m.expander.ColIn(prefix, column) @@ -144,6 +188,26 @@ func (m *ViewContext) addBindings(args []interface{}) string { return bindings } +func (m *ViewContext) addCompositeBindings(rows [][]interface{}) { + for _, row := range rows { + m.DataUnit.addAll(row...) + } +} + +func (m *ViewContext) dialect() (*info.Dialect, error) { + if m == nil || m.DataUnit == nil || m.DataUnit.MetaSource == nil { + return nil, nil + } + db, err := m.DataUnit.MetaSource.Db() + if err != nil { + return nil, err + } + if db == nil { + return nil, nil + } + return sqlxconfig.Dialect(context.Background(), db) +} + func (m *ViewContext) In(prefix string) (string, error) { return m.ColIn(prefix, "") } @@ -177,6 +241,40 @@ func AsBindings(key string, values []interface{}) (column string, bindings strin } } +func defaultCompositePredicate(columns []string, rowCount int) string { + if len(columns) == 0 || rowCount <= 0 { + return "1 = 0" + } + builder := &strings.Builder{} + builder.WriteByte('(') + builder.WriteString(strings.Join(columns, ", ")) + builder.WriteString(") IN (") + for row := 0; row < rowCount; row++ { + if row > 0 { + builder.WriteString(", ") + } + builder.WriteByte('(') + for col := range columns { + if col > 0 { + builder.WriteString(", ") + } + builder.WriteByte('?') + } + builder.WriteByte(')') + } + builder.WriteByte(')') + return builder.String() +} + +func renderCompositePredicate(dialect *info.Dialect, columns []string, rowCount int) string { + if renderer, ok := any(dialect).(interface { + CompositeIn([]string, int) string + }); ok { + return renderer.CompositeIn(columns, rowCount) + } + return defaultCompositePredicate(columns, rowCount) +} + func NewViewContext(metaSource ParentSource, aSelector ParentExtras, batchData ParentBatch, options ...interface{}) *ViewContext { if metaSource == nil { return nil @@ -185,6 +283,7 @@ func NewViewContext(metaSource ParentSource, aSelector ParentExtras, batchData P var sanitizer *DataUnit var expander Expander var colInArgs []interface{} + var compositeArgs [][]interface{} for _, option := range options { switch actual := option.(type) { @@ -197,6 +296,7 @@ func NewViewContext(metaSource ParentSource, aSelector ParentExtras, batchData P if batchData != nil { colInArgs = batchData.ColInBatch() + compositeArgs = batchData.CompositeInBatch() } limit := metaSource.ResultLimit() offset := 0 @@ -215,17 +315,18 @@ func NewViewContext(metaSource ParentSource, aSelector ParentExtras, batchData P SQLExec = sanitizer.TemplateSQL } result := &ViewContext{ - expander: expander, - Name: metaSource.ViewName(), - Alias: metaSource.TableAlias(), - Table: metaSource.TableName(), - Limit: limit, - Page: page, - Offset: offset, - Args: args, - NonWindowSQL: SQLExec, - DataUnit: NewDataUnit(metaSource), - ParentValues: colInArgs, + expander: expander, + Name: metaSource.ViewName(), + Alias: metaSource.TableAlias(), + Table: metaSource.TableName(), + Limit: limit, + Page: page, + Offset: offset, + Args: args, + NonWindowSQL: SQLExec, + DataUnit: NewDataUnit(metaSource), + ParentValues: colInArgs, + ParentCompositeValues: compositeArgs, } return result diff --git a/service/reader/service.go b/service/reader/service.go index d362ec899..560749775 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -235,8 +235,12 @@ func (s *Service) afterReadAll(collectorFetchEmitted bool, collector *view.Colle func (s *Service) batchData(collector *view.Collector) *view.BatchData { batchData := &view.BatchData{} - batchData.Values, batchData.ColumnNames = collector.ParentPlaceholders() - batchData.ParentReadSize = len(batchData.Values) + batchData.Values, batchData.CompositeValues, batchData.ColumnNames = collector.ParentPlaceholders() + if batchData.HasComposite() { + batchData.ParentReadSize = len(batchData.CompositeValues) + } else { + batchData.ParentReadSize = len(batchData.Values) + } return batchData } @@ -257,7 +261,11 @@ func (s *Service) exhaustRead(ctx context.Context, view *view.View, selector *vi } func (s *Service) readObjects(ctx context.Context, session *Session, batchData *view.BatchData, view *view.View, collector *view.Collector, selector *view.Statelet, info *response.SQLExecutions) error { - batchData.ValuesBatch, batchData.Size = sliceWithLimit(batchData.Values, batchData.Size, batchData.Size+view.Batch.Size) + if batchData.HasComposite() { + batchData.CompositeValuesBatch, batchData.Size = sliceCompositeWithLimit(batchData.CompositeValues, batchData.Size, batchData.Size+view.Batch.Size) + } else { + batchData.ValuesBatch, batchData.Size = sliceWithLimit(batchData.Values, batchData.Size, batchData.Size+view.Batch.Size) + } visitor := collector.Visitor(ctx) for { err := s.queryInBatches(ctx, session, view, collector, visitor, info, batchData, selector) @@ -268,7 +276,11 @@ func (s *Service) readObjects(ctx context.Context, session *Session, batchData * break } var nextParents int - batchData.ValuesBatch, nextParents = sliceWithLimit(batchData.Values, batchData.Size, batchData.Size+view.Batch.Size) + if batchData.HasComposite() { + batchData.CompositeValuesBatch, nextParents = sliceCompositeWithLimit(batchData.CompositeValues, batchData.Size, batchData.Size+view.Batch.Size) + } else { + batchData.ValuesBatch, nextParents = sliceWithLimit(batchData.Values, batchData.Size, batchData.Size+view.Batch.Size) + } batchData.Size += nextParents } return nil diff --git a/service/reader/slice.go b/service/reader/slice.go index 07699951d..129e3dc13 100644 --- a/service/reader/slice.go +++ b/service/reader/slice.go @@ -7,3 +7,10 @@ func sliceWithLimit(aSlice []interface{}, from, to int) ([]interface{}, int) { return aSlice[from:], len(aSlice) - from } + +func sliceCompositeWithLimit(aSlice [][]interface{}, from, to int) ([][]interface{}, int) { + if len(aSlice) > to { + return aSlice[from:to], to - from + } + return aSlice[from:], len(aSlice) - from +} diff --git a/service/reader/sql.go b/service/reader/sql.go index 555b26238..d2ed875c8 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -18,6 +18,7 @@ import ( "github.com/viant/sqlparser/node" "github.com/viant/sqlparser/query" "github.com/viant/sqlx/io/read/cache" + "github.com/viant/sqlx/metadata/info" ) const ( @@ -46,6 +47,47 @@ func NewBuilder() *Builder { return &Builder{} } +func compositeDialect(ctx context.Context, aView *view.View) (*info.Dialect, error) { + if aView == nil || aView.Connector == nil { + return nil, nil + } + return aView.Connector.Dialect(ctx) +} + +func defaultCompositeIn(columns []string, rowCount int) string { + if len(columns) == 0 || rowCount <= 0 { + return "1 = 0" + } + builder := &strings.Builder{} + builder.WriteByte('(') + builder.WriteString(strings.Join(columns, ", ")) + builder.WriteString(") IN (") + for row := 0; row < rowCount; row++ { + if row > 0 { + builder.WriteString(", ") + } + builder.WriteByte('(') + for col := range columns { + if col > 0 { + builder.WriteString(", ") + } + builder.WriteByte('?') + } + builder.WriteByte(')') + } + builder.WriteByte(')') + return builder.String() +} + +func renderCompositeIn(dialect *info.Dialect, columns []string, rowCount int) string { + if renderer, ok := any(dialect).(interface { + CompositeIn([]string, int) string + }); ok { + return renderer.CompositeIn(columns, rowCount) + } + return defaultCompositeIn(columns, rowCount) +} + // Build builds SQL Select statement func (b *Builder) Build(ctx context.Context, opts ...BuilderOption) (*cache.ParmetrizedQuery, error) { options := newBuilderOptions(opts...) @@ -112,7 +154,9 @@ func (b *Builder) Build(ctx context.Context, opts ...BuilderOption) (*cache.Parm criteriaMeta := hasKeyword(state.Expanded, keywords.Criteria) hasCriteria := criteriaMeta.has() - b.updateColumnsIn(&commonParams, &batchData, exclude) + if err = b.updateColumnsIn(ctx, aView, &commonParams, &batchData, exclude); err != nil { + return nil, err + } if err = b.updatePagination(&commonParams, aView, statelet, exclude); err != nil { return nil, err @@ -520,18 +564,33 @@ func (b *Builder) appendCriteria(sb *strings.Builder, criteria string, addAnd bo } } -func (b *Builder) updateColumnsIn(params *view.CriteriaParam, batchData *view.BatchData, exclude *Exclude) { +func (b *Builder) updateColumnsIn(ctx context.Context, aView *view.View, params *view.CriteriaParam, batchData *view.BatchData, exclude *Exclude) error { if exclude.ColumnsIn { - return + return nil } if batchData == nil || len(batchData.ColumnNames) == 0 { - return + return nil } sb := strings.Builder{} sb.WriteString(" ") columns := len(batchData.ColumnNames) + if batchData.HasComposite() { + rowCount := len(batchData.CompositeValuesBatch) + if rowCount == 0 { + params.ColumnsIn = " 1 = 0" + return nil + } + dialect, err := compositeDialect(ctx, aView) + if err != nil { + return err + } + sb.WriteString(renderCompositeIn(dialect, batchData.ColumnNames, rowCount)) + params.ColumnsIn = sb.String() + return nil + } + switch columns { case 1: sb.WriteString(batchData.ColumnNames[0]) @@ -539,7 +598,7 @@ func (b *Builder) updateColumnsIn(params *view.CriteriaParam, batchData *view.Ba sb.WriteString("(") for i, column := range batchData.ColumnNames { if i > 0 { - sb.WriteString(",") + sb.WriteString(", ") } sb.WriteString(column) } @@ -566,6 +625,7 @@ func (b *Builder) updateColumnsIn(params *view.CriteriaParam, batchData *view.Ba } sb.WriteString(encloseFragment) params.ColumnsIn = sb.String() + return nil } func (b *Builder) appendOrderBy(sb *strings.Builder, aView *view.View, selector *view.Statelet) error { diff --git a/view/batch.go b/view/batch.go index 68f4c3858..ab036de15 100644 --- a/view/batch.go +++ b/view/batch.go @@ -5,8 +5,10 @@ type BatchData struct { Size int ParentReadSize int - Values []interface{} //all values from parent - ValuesBatch []interface{} //batched values defined view.Batch.Size + Values []interface{} // all scalar values from parent + ValuesBatch []interface{} // batched scalar values + CompositeValues [][]interface{} // all composite parent tuples + CompositeValuesBatch [][]interface{} // batched composite tuples } func (b *BatchData) ColIn() []interface{} { @@ -16,3 +18,21 @@ func (b *BatchData) ColIn() []interface{} { func (b *BatchData) ColInBatch() []interface{} { return b.ValuesBatch } + +func (b *BatchData) HasComposite() bool { + return b != nil && len(b.CompositeValues) > 0 +} + +func (b *BatchData) CompositeIn() [][]interface{} { + if b == nil { + return nil + } + return b.CompositeValues +} + +func (b *BatchData) CompositeInBatch() [][]interface{} { + if b == nil { + return nil + } + return b.CompositeValuesBatch +} diff --git a/view/collector.go b/view/collector.go index 8e43229f8..02dc32dcd 100644 --- a/view/collector.go +++ b/view/collector.go @@ -10,6 +10,7 @@ import ( "github.com/viant/xdatly/handler" "github.com/viant/xunsafe" "reflect" + "strings" "sync" "unsafe" ) @@ -17,20 +18,23 @@ import ( // VisitorFn represents visitor function type VisitorFn func(value interface{}) error +type compositeKey string + // Collector collects and build result from View fetched from Database // If View or any of the View.With MatchStrategy support Parallel fetching, it is important to call MergeData // when all needed View was fetched type Collector struct { - Id string - mutex sync.Mutex - parent *Collector - destValue reflect.Value - appender *xunsafe.Appender - valuePosition map[string]map[string]map[interface{}][]int //stores positions in main slice, based on _field name, indexed by _field value. - types map[string]*xunsafe.Type - relation *Relation - dataSync *handler.DataSync - values map[string]*[]interface{} //acts like a buffer. Output resolved with Resolve method can't be put to the value position map + Id string + mutex sync.Mutex + parent *Collector + destValue reflect.Value + appender *xunsafe.Appender + valuePosition map[string]map[string]map[interface{}][]int //stores positions in main slice, based on _field name, indexed by _field value. + compositeValuePosition map[string]map[compositeKey][]int + types map[string]*xunsafe.Type + relation *Relation + dataSync *handler.DataSync + values map[string]*[]interface{} //acts like a buffer. Output resolved with Resolve method can't be put to the value position map // because value fetched from database was not scanned into yet. Putting value to the map as a key, would create key as a pointer to the zero value. slice *xunsafe.Slice @@ -49,6 +53,80 @@ type Collector struct { viewMetaHandler viewSummaryHandlerFn } +func relationCompositeSignature(links Links) string { + parts := make([]string, 0, len(links)) + for _, link := range links { + if link == nil { + continue + } + parts = append(parts, link.Namespace+"."+link.Column) + } + return strings.Join(parts, "|") +} + +func buildCompositeKey(values []interface{}) compositeKey { + parts := make([]string, len(values)) + for i, value := range values { + parts[i] = fmt.Sprintf("%#v", io.NormalizeKey(value)) + } + return compositeKey(strings.Join(parts, "\x1f")) +} + +func normalizeValues(value interface{}) []interface{} { + switch actual := value.(type) { + case []int: + result := make([]interface{}, 0, len(actual)) + for _, item := range actual { + result = append(result, io.NormalizeKey(item)) + } + return result + case []*int64: + result := make([]interface{}, 0, len(actual)) + for _, item := range actual { + if item == nil { + continue + } + result = append(result, io.NormalizeKey(int(*item))) + } + return result + case []int64: + result := make([]interface{}, 0, len(actual)) + for _, item := range actual { + result = append(result, io.NormalizeKey(int(item))) + } + return result + case []string: + result := make([]interface{}, 0, len(actual)) + for _, item := range actual { + result = append(result, io.NormalizeKey(item)) + } + return result + default: + return []interface{}{io.NormalizeKey(value)} + } +} + +func compositeRows(parts [][]interface{}) [][]interface{} { + if len(parts) == 0 { + return nil + } + result := make([][]interface{}, 1) + for _, values := range parts { + if len(values) == 0 { + return nil + } + next := make([][]interface{}, 0, len(result)*len(values)) + for _, existing := range result { + for _, value := range values { + row := append(append([]interface{}{}, existing...), value) + next = append(next, row) + } + } + result = next + } + return result +} + func (r *Collector) SetDest(dest interface{}) { destValue := reflect.ValueOf(dest) if destValue.Kind() == reflect.Ptr { @@ -63,27 +141,28 @@ func (r *Collector) Clone() *Collector { dest := reflect.MakeSlice(r.view.Schema.SliceType(), 0, 1) slicePtrValue.Elem().Set(dest) return &Collector{ - Id: uuid.New().String(), - parent: r.parent, - destValue: slicePtrValue, - appender: r.slice.Appender(xunsafe.ValuePointer(&slicePtrValue)), - valuePosition: r.valuePosition, - types: r.types, - relation: r.relation, - values: r.values, - slice: r.slice, - view: r.view, - relations: r.relations, - dataSync: r.dataSync, - wg: r.wg, - readAll: r.readAll, - wgDelta: r.wgDelta, - indexCounter: r.indexCounter, - manyCounter: r.manyCounter, - codecSlice: r.codecSlice, - codecSliceDest: r.codecSliceDest, - codecAppender: r.codecAppender, - viewMetaHandler: r.viewMetaHandler, + Id: uuid.New().String(), + parent: r.parent, + destValue: slicePtrValue, + appender: r.slice.Appender(xunsafe.ValuePointer(&slicePtrValue)), + valuePosition: r.valuePosition, + compositeValuePosition: r.compositeValuePosition, + types: r.types, + relation: r.relation, + values: r.values, + slice: r.slice, + view: r.view, + relations: r.relations, + dataSync: r.dataSync, + wg: r.wg, + readAll: r.readAll, + wgDelta: r.wgDelta, + indexCounter: r.indexCounter, + manyCounter: r.manyCounter, + codecSlice: r.codecSlice, + codecSliceDest: r.codecSliceDest, + codecAppender: r.codecAppender, + viewMetaHandler: r.viewMetaHandler, } } @@ -150,25 +229,36 @@ func (r *Collector) parentValuesPositions(ns string, columnName string) map[inte return result } +func (r *Collector) parentCompositePositions(relation *Relation) map[compositeKey][]int { + signature := relationCompositeSignature(relation.On) + result, ok := r.parent.compositeValuePosition[signature] + if !ok || len(result) == 0 { + r.indexParentCompositePositions(relation) + result = r.parent.compositeValuePosition[signature] + } + return result +} + // NewCollector creates a collector func NewCollector(slice *xunsafe.Slice, view *View, dest interface{}, viewMetaHandler viewSummaryHandlerFn, readAll bool) *Collector { ensuredDest := ensureDest(dest, view) wg := sync.WaitGroup{} wg.Add(1) return &Collector{ - Id: uuid.New().String(), - destValue: reflect.ValueOf(ensuredDest), - valuePosition: make(map[string]map[string]map[interface{}][]int), - appender: slice.Appender(xunsafe.AsPointer(ensuredDest)), - slice: slice, - view: view, - types: make(map[string]*xunsafe.Type), - values: make(map[string]*[]interface{}), - readAll: readAll, - wg: &wg, - dataSync: handler.NewDataSync(), - wgDelta: 1, - viewMetaHandler: viewMetaHandler, + Id: uuid.New().String(), + destValue: reflect.ValueOf(ensuredDest), + valuePosition: make(map[string]map[string]map[interface{}][]int), + compositeValuePosition: make(map[string]map[compositeKey][]int), + appender: slice.Appender(xunsafe.AsPointer(ensuredDest)), + slice: slice, + view: view, + types: make(map[string]*xunsafe.Type), + values: make(map[string]*[]interface{}), + readAll: readAll, + wg: &wg, + dataSync: handler.NewDataSync(), + wgDelta: 1, + viewMetaHandler: viewMetaHandler, } } @@ -186,6 +276,13 @@ func (r *Collector) Visitor(ctx context.Context) VisitorFn { relation := r.relation visitorRelations := RelationsSlice(r.view.With).PopulateWithVisitor() for _, rel := range visitorRelations { + if rel.IsComposite() { + signature := relationCompositeSignature(rel.On) + if _, ok := r.compositeValuePosition[signature]; !ok { + r.compositeValuePosition[signature] = map[compositeKey][]int{} + } + continue + } for _, item := range rel.On { if _, ok := r.valuePosition[item.Namespace]; !ok { r.valuePosition[item.Namespace] = map[string]map[interface{}][]int{} @@ -219,8 +316,18 @@ func (r *Collector) Visitor(ctx context.Context) VisitorFn { func (r *Collector) valueIndexer(ctx context.Context, visitorRelations []*Relation) func(value interface{}) error { distinctRelations := make([]*Relation, 0) presenceMap := map[string]map[string]bool{} + compositePresence := map[string]bool{} for i := range visitorRelations { + if visitorRelations[i].IsComposite() { + signature := relationCompositeSignature(visitorRelations[i].On) + if compositePresence[signature] { + continue + } + distinctRelations = append(distinctRelations, visitorRelations[i]) + compositePresence[signature] = true + continue + } for _, item := range visitorRelations[i].On { if _, ok := presenceMap[item.Namespace]; !ok { presenceMap[item.Namespace] = map[string]bool{} @@ -236,6 +343,10 @@ func (r *Collector) valueIndexer(ctx context.Context, visitorRelations []*Relati return func(value interface{}) error { ptr := xunsafe.AsPointer(value) for _, rel := range distinctRelations { + if rel.IsComposite() { + r.indexCompositeValueByRel(ptr, rel, r.indexCounter) + continue + } for _, link := range rel.On { if field := link.xField; field != nil { fieldValue := field.Value(ptr) @@ -252,6 +363,25 @@ func (r *Collector) valueIndexer(ctx context.Context, visitorRelations []*Relati } } +func (r *Collector) indexCompositeValueByRel(ptr unsafe.Pointer, rel *Relation, counter int) { + signature := relationCompositeSignature(rel.On) + index := r.compositeValuePosition[signature] + if index == nil { + index = map[compositeKey][]int{} + r.compositeValuePosition[signature] = index + } + valueSets := make([][]interface{}, 0, len(rel.On)) + for _, link := range rel.On { + if link == nil || link.xField == nil { + return + } + valueSets = append(valueSets, normalizeValues(link.xField.Value(ptr))) + } + for _, row := range compositeRows(valueSets) { + index[buildCompositeKey(row)] = append(index[buildCompositeKey(row)], counter) + } +} + func (r *Collector) indexValueByRel(fieldValue interface{}, rel *Relation, counter int) { switch actual := fieldValue.(type) { case []int: @@ -307,6 +437,24 @@ func (r *Collector) visitorOne(relation *Relation) func(value interface{}) error var aKey interface{} return func(owner interface{}) error { + if relation.IsComposite() { + keyParts := make([]interface{}, 0, len(links)) + for _, link := range links { + if link.xField == nil { + return fmt.Errorf("link %v field %v is not found", relation.Name, link.Column) + } + keyParts = append(keyParts, io.NormalizeKey(link.xField.Interface(xunsafe.AsPointer(owner)))) + } + positions, ok := r.parentCompositePositions(relation)[buildCompositeKey(keyParts)] + if !ok { + return nil + } + for _, index := range positions { + item := r.parent.slice.ValuePointerAt(destPtr, index) + holderField.SetValue(xunsafe.AsPointer(item), owner) + } + return nil + } for j, link := range links { if link.xField == nil { return fmt.Errorf("link %v field %v is not found", relation.Name, link.Column) @@ -374,6 +522,31 @@ func (r *Collector) ParentRow(relation *Relation) func(value interface{}) (inter } return func(child interface{}) (interface{}, error) { + if relation.IsComposite() { + keyParts := make([]interface{}, 0, len(links)) + for _, link := range links { + keyField := link.xField + if keyField == nil && xType == nil { + xType = r.types[link.Column] + values = r.values[link.Column] + } + var key interface{} + if keyField != nil { + key = keyField.Interface(xunsafe.AsPointer(child)) + } else { + key = xType.Deref((*values)[r.manyCounter]) + } + keyParts = append(keyParts, io.NormalizeKey(key)) + } + positions, ok := r.parentCompositePositions(relation)[buildCompositeKey(keyParts)] + if !ok { + return nil, fmt.Errorf(`composite key "%v" is not found`, keyParts) + } + if len(positions) > 1 { + return nil, fmt.Errorf(`composite key "%v" has more than one value`, keyParts) + } + return r.parent.slice.ValuePointerAt(destPtr, positions[0]), nil + } var key interface{} var parentPosition int for i, link := range links { @@ -413,6 +586,39 @@ func (r *Collector) visitorMany(relation *Relation) func(value interface{}) erro destPtr := xunsafe.AsPointer(dest) return func(owner interface{}) error { + if relation.IsComposite() { + keyParts := make([]interface{}, 0, len(links)) + for _, link := range links { + keyField := link.xField + if keyField == nil && xType == nil { + xType = r.types[link.Column] + values = r.values[link.Column] + } + var key interface{} + if keyField != nil { + key = keyField.Interface(xunsafe.AsPointer(owner)) + } else { + key = xType.Deref((*values)[r.manyCounter]) + r.manyCounter++ + } + keyParts = append(keyParts, io.NormalizeKey(key)) + } + positions, ok := r.parentCompositePositions(relation)[buildCompositeKey(keyParts)] + if !ok { + return nil + } + for _, index := range positions { + parentItem := r.parent.slice.ValuePointerAt(destPtr, index) + r.Lock().Lock() + sliceAddPtr := holderField.Pointer(xunsafe.AsPointer(parentItem)) + slice := relation.Of.Schema.Slice() + appender := slice.Appender(sliceAddPtr) + appender.Append(owner) + r.Lock().Unlock() + r.view.Logger.ObjectReconciling(dest, owner, parentItem, index) + } + return nil + } var key interface{} for i, link := range links { keyField := link.xField @@ -476,6 +682,13 @@ func (r *Collector) indexParentPositions(ns, name string) { r.parent.indexPositions(ns, name) } +func (r *Collector) indexParentCompositePositions(relation *Relation) { + if r.parent == nil || relation == nil { + return + } + r.parent.indexCompositePositions(relation) +} + func (r *Collector) indexPositions(ns, name string) { values := r.values[name] if values == nil { @@ -508,6 +721,46 @@ func (r *Collector) indexPositions(ns, name string) { } } +func (r *Collector) indexCompositePositions(relation *Relation) { + if relation == nil { + return + } + signature := relationCompositeSignature(relation.On) + index := r.compositeValuePosition[signature] + if index == nil { + index = map[compositeKey][]int{} + r.compositeValuePosition[signature] = index + } + destPtr := xunsafe.AsPointer(r.DestPtr()) + for position := 0; position < r.slice.Len(destPtr); position++ { + parent := r.slice.ValuePointerAt(destPtr, position) + valueSets := make([][]interface{}, 0, len(relation.On)) + for _, link := range relation.On { + if link == nil { + continue + } + if link.xField != nil { + valueSets = append(valueSets, normalizeValues(link.xField.Value(xunsafe.AsPointer(parent)))) + continue + } + values := r.values[link.Column] + if values == nil || position >= len(*values) { + valueSets = nil + break + } + xType := r.types[link.Column] + if xType == nil { + valueSets = nil + break + } + valueSets = append(valueSets, normalizeValues(xType.Deref((*values)[position]))) + } + for _, row := range compositeRows(valueSets) { + index[buildCompositeKey(row)] = append(index[buildCompositeKey(row)], position) + } + } +} + // Relations creates and register new Collector for each Relation present in the Template.Columns if View allows use Template.Columns func (r *Collector) Relations(selector *Statelet) ([]*Collector, error) { result := make([]*Collector, len(r.view.With)) @@ -539,21 +792,22 @@ func (r *Collector) Relations(selector *Statelet) ([]*Collector, error) { return nil, err } result[counter] = &Collector{ - Id: uuid.New().String(), - parent: r, - viewMetaHandler: aHandler, - destValue: destPtr, - dataSync: handler.NewDataSync(), - appender: slice.Appender(xunsafe.ValuePointer(&destPtr)), - valuePosition: make(map[string]map[string]map[interface{}][]int), - types: make(map[string]*xunsafe.Type), - values: make(map[string]*[]interface{}), - slice: slice, - view: &r.view.With[i].Of.View, - relation: r.view.With[i], - readAll: r.view.With[i].Of.MatchStrategy.ReadAll(), - wg: &wg, - wgDelta: delta, + Id: uuid.New().String(), + parent: r, + viewMetaHandler: aHandler, + destValue: destPtr, + dataSync: handler.NewDataSync(), + appender: slice.Appender(xunsafe.ValuePointer(&destPtr)), + valuePosition: make(map[string]map[string]map[interface{}][]int), + compositeValuePosition: make(map[string]map[compositeKey][]int), + types: make(map[string]*xunsafe.Type), + values: make(map[string]*[]interface{}), + slice: slice, + view: &r.view.With[i].Of.View, + relation: r.view.With[i], + readAll: r.view.With[i].Of.MatchStrategy.ReadAll(), + wg: &wg, + wgDelta: delta, } counter++ } @@ -661,6 +915,40 @@ func (r *Collector) MergeData() { func (r *Collector) mergeToParent() { links := r.relation.Of.On + if r.relation.IsComposite() { + destPtr := xunsafe.AsPointer(r.DestPtr()) + holderField := r.relation.holderField + parentSlice := r.parent.slice + parentDestPtr := xunsafe.AsPointer(r.parent.DestPtr()) + valuePositions := r.parentCompositePositions(r.relation) + + for i := 0; i < r.slice.Len(destPtr); i++ { + value := r.slice.ValuePointerAt(destPtr, i) + keyParts := make([]interface{}, 0, len(links)) + for _, link := range links { + keyParts = append(keyParts, io.NormalizeKey(link.xField.Value(xunsafe.AsPointer(value)))) + } + positions, ok := valuePositions[buildCompositeKey(keyParts)] + if !ok { + continue + } + for _, position := range positions { + parentValue := parentSlice.ValuePointerAt(parentDestPtr, position) + if r.relation.Cardinality == state.One { + at := r.slice.ValuePointerAt(destPtr, i) + holderField.SetValue(xunsafe.AsPointer(parentValue), at) + } else if r.relation.Cardinality == state.Many { + r.Lock().Lock() + appender := r.slice.Appender(holderField.ValuePointer(xunsafe.AsPointer(parentValue))) + appender.Append(value) + r.Lock().Unlock() + r.view.Logger.ObjectReconciling(r.Dest(), value, parentValue, position) + } + } + } + return + } + for i, link := range links { valuePositions := r.parentValuesPositions(r.relation.On[i].Namespace, r.relation.On[i].Column) destPtr := xunsafe.AsPointer(r.DestPtr()) @@ -698,12 +986,46 @@ func (r *Collector) mergeToParent() { // that the relation was created from, otherwise empty slice and empty string // i.e. if locators Collector collects Employee{AccountId: int}, Column.Name is account_id and Collector collects Account // it will extract and return all the AccountId that were accumulated and account_id -func (r *Collector) ParentPlaceholders() ([]interface{}, []string) { +func (r *Collector) ParentPlaceholders() ([]interface{}, [][]interface{}, []string) { if r.parent == nil || r.ReadAll() { - return []interface{}{}, nil + return []interface{}{}, nil, nil } destPtr := xunsafe.AsPointer(r.parent.DestPtr()) sliceLen := r.parent.slice.Len(destPtr) + if r.relation.IsComposite() { + result := make([][]interface{}, 0) + unique := map[compositeKey]bool{} + for i := 0; i < sliceLen; i++ { + parent := r.parent.slice.ValuePointerAt(destPtr, i) + valueSets := make([][]interface{}, 0, len(r.relation.On)) + for _, link := range r.relation.On { + field := link.xField + if field != nil { + valueSets = append(valueSets, normalizeValues(field.Value(xunsafe.AsPointer(parent)))) + continue + } + positions := r.parentValuesPositions(link.Namespace, link.Column) + if len(positions) == 0 { + valueSets = nil + break + } + values := make([]interface{}, 0, len(positions)) + for key := range positions { + values = append(values, key) + } + valueSets = append(valueSets, values) + } + for _, row := range compositeRows(valueSets) { + key := buildCompositeKey(row) + if unique[key] { + continue + } + unique[key] = true + result = append(result, row) + } + } + return nil, result, r.relation.Of.On.InColumnExpression() + } result := make([]interface{}, 0) var unique = make(map[any]bool) outer: @@ -772,7 +1094,7 @@ outer: continue outer } } - return result, r.relation.Of.On.InColumnExpression() + return result, nil, r.relation.Of.On.InColumnExpression() } func (r *Collector) WaitIfNeeded() { diff --git a/view/relation.go b/view/relation.go index b20631286..eaeb838dc 100644 --- a/view/relation.go +++ b/view/relation.go @@ -22,6 +22,7 @@ type ( Of *ReferenceView `json:",omitempty"` Caser text.CaseFormat `json:",omitempty"` Cardinality state.Cardinality `json:",omitempty"` //IsToOne, or Many + Composite bool `json:",omitempty"` On Links Holder string `json:",omitempty"` //Represents column created due to the merging. In our example it would be Employee#Account IncludeColumn bool `json:",omitempty"` //tells if Column _field should be kept in the struct type. In our example, if set false in produced Employee would be also AccountId _field @@ -267,6 +268,10 @@ func (r *Relation) TagLink() tags.LinkOn { return links } +func (r *Relation) IsComposite() bool { + return r != nil && (r.Composite || len(r.On) > 1) +} + func (l *Link) EncodeLinkTag() string { result := "" if l.Field != "" { diff --git a/view/template.go b/view/template.go index abf549165..91a2e9ade 100644 --- a/view/template.go +++ b/view/template.go @@ -460,7 +460,13 @@ func (t *Template) replacementEntry(key string, params CriteriaParam, selector * return key, criteriaExpanded, nil case keywords.ColumnsIn[1:]: - *placeholders = append(*placeholders, batchData.ValuesBatch...) + if batchData != nil && batchData.HasComposite() { + for _, row := range batchData.CompositeValuesBatch { + *placeholders = append(*placeholders, row...) + } + } else { + *placeholders = append(*placeholders, batchData.ValuesBatch...) + } return key, params.ColumnsIn, nil case keywords.SelectorCriteria[1:]: *placeholders = append(*placeholders, selector.Placeholders...) From be22e55d56ea2015f8495567d272003d14f745e3 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 31 Mar 2026 08:44:37 -0700 Subject: [PATCH 207/279] - added composite key support --- .../executor/expand/parent_composite_test.go | 68 +++++++++++++ service/reader/sql_composite_test.go | 37 +++++++ service/reader/sql_groupable_test.go | 37 +++++++ view/collector_composite_test.go | 99 +++++++++++++++++++ view/column_lookup_test.go | 19 ++++ view/columns.go | 3 + 6 files changed, 263 insertions(+) create mode 100644 service/executor/expand/parent_composite_test.go create mode 100644 service/reader/sql_composite_test.go create mode 100644 view/collector_composite_test.go diff --git a/service/executor/expand/parent_composite_test.go b/service/executor/expand/parent_composite_test.go new file mode 100644 index 000000000..2899f54ef --- /dev/null +++ b/service/executor/expand/parent_composite_test.go @@ -0,0 +1,68 @@ +package expand + +import ( + "database/sql" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type compositeBatch struct { + rows [][]interface{} +} + +type mockParentSource struct{} + +func (m *mockParentSource) Db() (*sql.DB, error) { return nil, nil } +func (m *mockParentSource) ViewName() string { return "test" } +func (m *mockParentSource) TableAlias() string { return "t" } +func (m *mockParentSource) TableName() string { return "TEST" } +func (m *mockParentSource) ResultLimit() int { return 100 } + +func (b *compositeBatch) ColIn() []interface{} { return nil } +func (b *compositeBatch) ColInBatch() []interface{} { return nil } +func (b *compositeBatch) CompositeIn() [][]interface{} { return b.rows } +func (b *compositeBatch) CompositeInBatch() [][]interface{} { return b.rows } +func (b *compositeBatch) HasComposite() bool { return len(b.rows) > 0 } + +func TestViewContext_ParentCompositeJoinOn(t *testing.T) { + viewCtx := NewViewContext(&mockParentSource{}, nil, &compositeBatch{ + rows: [][]interface{}{ + {101, "A"}, + {202, "B"}, + }, + }, &DataUnit{}) + require.NotNil(t, viewCtx) + require.NotNil(t, viewCtx.DataUnit) + + sqlFragment, err := viewCtx.ParentCompositeJoinOn("AND", "t.advertiser_id", "t.val") + require.NoError(t, err) + assert.Equal(t, "AND (t.advertiser_id, t.val) IN ((?, ?), (?, ?))", sqlFragment) + assert.Equal(t, []interface{}{101, "A", 202, "B"}, viewCtx.DataUnit.ParamsGroup) +} + +func TestViewContext_ParentJoinOn_CompositeArgs(t *testing.T) { + viewCtx := NewViewContext(&mockParentSource{}, nil, &compositeBatch{ + rows: [][]interface{}{ + {101, "A"}, + {202, "B"}, + }, + }, &DataUnit{}) + require.NotNil(t, viewCtx) + + sqlFragment, err := viewCtx.ParentJoinOn("AND", "t.advertiser_id", "t.val") + require.NoError(t, err) + assert.Equal(t, "AND (t.advertiser_id, t.val) IN ((?, ?), (?, ?))", sqlFragment) + assert.Equal(t, []interface{}{101, "A", 202, "B"}, viewCtx.DataUnit.ParamsGroup) +} + +func TestViewContext_ParentCompositeJoinOn_EmptyRows(t *testing.T) { + viewCtx := NewViewContext(&mockParentSource{}, nil, &compositeBatch{}, &DataUnit{}) + require.NotNil(t, viewCtx) + + sqlFragment, err := viewCtx.ParentCompositeJoinOn("AND", "t.advertiser_id", "t.val") + require.NoError(t, err) + assert.True(t, strings.Contains(sqlFragment, "1 = 0")) +} diff --git a/service/reader/sql_composite_test.go b/service/reader/sql_composite_test.go new file mode 100644 index 000000000..b111c4e46 --- /dev/null +++ b/service/reader/sql_composite_test.go @@ -0,0 +1,37 @@ +package reader + +import ( + "context" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" +) + +func TestBuilder_Build_CompositeColumnsIn_SQLite(t *testing.T) { + aView := view.NewView("adobe", "adobe", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + &view.Column{Name: "ADVERTISER_ID", DataType: "int"}, + &view.Column{Name: "DMP_ADOBE_VALUE", DataType: "string"}, + &view.Column{Name: "ID", DataType: "int"}, + }), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + + query, err := NewBuilder().Build(context.Background(), + WithBuilderView(aView), + WithBuilderStatelet(view.NewStatelet()), + WithBuilderBatchData(&view.BatchData{ + ColumnNames: []string{"ADVERTISER_ID", "DMP_ADOBE_VALUE"}, + CompositeValues: [][]interface{}{{101, "A"}, {202, "B"}}, + CompositeValuesBatch: [][]interface{}{{101, "A"}, {202, "B"}}, + }), + ) + require.NoError(t, err) + require.NotNil(t, query) + assert.Contains(t, query.SQL, `(ADVERTISER_ID, DMP_ADOBE_VALUE) IN ((?, ?), (?, ?))`) + assert.Equal(t, []interface{}{101, "A", 202, "B"}, query.Args) +} diff --git a/service/reader/sql_groupable_test.go b/service/reader/sql_groupable_test.go index fc45d12cd..30947ecd6 100644 --- a/service/reader/sql_groupable_test.go +++ b/service/reader/sql_groupable_test.go @@ -264,6 +264,43 @@ func TestBuilder_appendRelationColumn_UsesProjectedRelationAliasForGroupedDerive }) } +func TestBuilder_appendRelationColumn_UsesProjectedAliasForQualifiedSourceRelation(t *testing.T) { + builder := NewBuilder() + aView := view.NewView("comscoreContextual", "comscoreContextual", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + &view.Column{Name: "COMSCORE_CONTEXTUAL_VALUE", DataType: "string", Tag: `source:"t2.SEGMENT_ID"`}, + &view.Column{Name: "NAME", DataType: "string"}, + }), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + + relation := &view.Relation{ + Of: &view.ReferenceView{ + On: view.Links{ + &view.Link{Field: "ComscoreContextualValue", Column: "t2.SEGMENT_ID"}, + }, + }, + } + + require.NoError(t, relation.Of.On.Init("comscoreContextual", aView)) + + t.Run("default projection does not append raw unqualified source column", func(t *testing.T) { + sb := &strings.Builder{} + require.NoError(t, builder.checkViewAndAppendRelColumn(sb, aView, relation)) + require.Equal(t, "", sb.String()) + }) + + t.Run("selector projection appends projected alias instead of raw source column", func(t *testing.T) { + sb := &strings.Builder{} + selector := view.NewStatelet() + selector.Columns = []string{"NAME"} + selector.Init(aView) + require.NoError(t, builder.checkSelectorAndAppendRelColumn(sb, aView, selector, relation)) + require.Equal(t, ", COMSCORE_CONTEXTUAL_VALUE", sb.String()) + }) +} + func newGroupableTestView(t *testing.T) *view.View { t.Helper() trueValue := true diff --git a/view/collector_composite_test.go b/view/collector_composite_test.go new file mode 100644 index 000000000..2e49412d3 --- /dev/null +++ b/view/collector_composite_test.go @@ -0,0 +1,99 @@ +package view + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/view/state" + "github.com/viant/xunsafe" +) + +type compositeParentRow struct { + AdvertiserID int + DmpAdobeValues string + Adobe []*compositeChildRow +} + +type compositeChildRow struct { + AdvertiserID int + DmpAdobeValue string +} + +func TestCollector_ParentPlaceholders_Composite(t *testing.T) { + parentView := &View{Schema: state.NewSchema(reflect.TypeOf([]*compositeParentRow{}))} + parentDest := []*compositeParentRow{ + {AdvertiserID: 101, DmpAdobeValues: "A"}, + {AdvertiserID: 202, DmpAdobeValues: "B"}, + } + parentCollector := NewCollector(parentView.Schema.Slice(), parentView, &parentDest, nil, false) + + relation := &Relation{ + Composite: true, + On: Links{ + &Link{Field: "AdvertiserID", Column: "ADVERTISER_ID", xField: xunsafe.FieldByName(reflect.TypeOf(compositeParentRow{}), "AdvertiserID")}, + &Link{Field: "DmpAdobeValues", Column: "DMP_ADOBE_VALUES", xField: xunsafe.FieldByName(reflect.TypeOf(compositeParentRow{}), "DmpAdobeValues")}, + }, + Of: &ReferenceView{ + On: Links{ + &Link{Field: "AdvertiserID", Column: "ADVERTISER_ID"}, + &Link{Field: "DmpAdobeValue", Column: "DMP_ADOBE_VALUE"}, + }, + }, + } + childCollector := &Collector{parent: parentCollector, relation: relation} + + values, composite, columns := childCollector.ParentPlaceholders() + assert.Nil(t, values) + assert.Equal(t, []string{"ADVERTISER_ID", "DMP_ADOBE_VALUE"}, columns) + assert.Equal(t, [][]interface{}{{101, "A"}, {202, "B"}}, composite) +} + +func TestCollector_MergeToParent_Composite(t *testing.T) { + parentView := &View{Schema: state.NewSchema(reflect.TypeOf([]*compositeParentRow{}))} + parentDest := []*compositeParentRow{ + {AdvertiserID: 101, DmpAdobeValues: "A"}, + {AdvertiserID: 202, DmpAdobeValues: "B"}, + } + parentCollector := NewCollector(parentView.Schema.Slice(), parentView, &parentDest, nil, false) + + childView := &View{ + Schema: state.NewSchema(reflect.TypeOf([]*compositeChildRow{})), + } + relation := &Relation{ + Composite: true, + Cardinality: state.Many, + Holder: "Adobe", + holderField: xunsafe.FieldByName(reflect.TypeOf(compositeParentRow{}), "Adobe"), + On: Links{ + &Link{Field: "AdvertiserID", Column: "ADVERTISER_ID", xField: xunsafe.FieldByName(reflect.TypeOf(compositeParentRow{}), "AdvertiserID")}, + &Link{Field: "DmpAdobeValues", Column: "DMP_ADOBE_VALUES", xField: xunsafe.FieldByName(reflect.TypeOf(compositeParentRow{}), "DmpAdobeValues")}, + }, + Of: &ReferenceView{ + View: View{Schema: state.NewSchema(reflect.TypeOf([]*compositeChildRow{}))}, + On: Links{ + &Link{Field: "AdvertiserID", Column: "ADVERTISER_ID", xField: xunsafe.FieldByName(reflect.TypeOf(compositeChildRow{}), "AdvertiserID")}, + &Link{Field: "DmpAdobeValue", Column: "DMP_ADOBE_VALUE", xField: xunsafe.FieldByName(reflect.TypeOf(compositeChildRow{}), "DmpAdobeValue")}, + }, + }, + } + + childDest := []*compositeChildRow{ + {AdvertiserID: 101, DmpAdobeValue: "A"}, + {AdvertiserID: 202, DmpAdobeValue: "B"}, + {AdvertiserID: 101, DmpAdobeValue: "Z"}, + } + childCollector := NewCollector(childView.Schema.Slice(), childView, &childDest, nil, true) + childCollector.parent = parentCollector + childCollector.relation = relation + childCollector.view = childView + childCollector.slice = childView.Schema.Slice() + + childCollector.mergeToParent() + + require.Len(t, parentDest[0].Adobe, 1) + assert.Equal(t, "A", parentDest[0].Adobe[0].DmpAdobeValue) + require.Len(t, parentDest[1].Adobe, 1) + assert.Equal(t, "B", parentDest[1].Adobe[0].DmpAdobeValue) +} diff --git a/view/column_lookup_test.go b/view/column_lookup_test.go index db598661d..3e6377116 100644 --- a/view/column_lookup_test.go +++ b/view/column_lookup_test.go @@ -25,3 +25,22 @@ func TestView_ColumnByName_UsesIndexedLookup(t *testing.T) { require.True(t, ok) require.Equal(t, "TAXONOMY_ID", column.Name) } + +func TestView_ColumnByName_UsesUnqualifiedSourceLookup(t *testing.T) { + aView := NewView("comscore", "comscore", + WithConnector(NewConnector("test", "sqlite3", ":memory:")), + WithColumns(Columns{ + &Column{Name: "COMSCORE_CONTEXTUAL_VALUE", DataType: "string", Tag: `source:"t2.SEGMENT_ID"`}, + &Column{Name: "NAME", DataType: "string"}, + }), + ) + require.NoError(t, aView.Init(context.Background(), EmptyResource())) + + column, ok := aView.ColumnByName("t2.SEGMENT_ID") + require.True(t, ok) + require.Equal(t, "COMSCORE_CONTEXTUAL_VALUE", column.Name) + + column, ok = aView.ColumnByName("SEGMENT_ID") + require.True(t, ok) + require.Equal(t, "COMSCORE_CONTEXTUAL_VALUE", column.Name) +} diff --git a/view/columns.go b/view/columns.go index bb9d85dcf..0d6e762ec 100644 --- a/view/columns.go +++ b/view/columns.go @@ -23,6 +23,9 @@ func (c Columns) Index(formatCase text.CaseFormat) NamedColumns { if aTag := c[i].Tag; aTag != "" { if src := reflect.StructTag(aTag).Get("source"); src != "" { result[strings.ToLower(src)] = c[i] + if index := strings.LastIndex(src, "."); index != -1 && index+1 < len(src) { + result.RegisterWithName(src[index+1:], c[i]) + } } } result.Register(formatCase, c[i]) From e50e922eaaad405c5c0e954886cd98dddf15197e Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 31 Mar 2026 09:55:18 -0700 Subject: [PATCH 208/279] - added composite key support --- internal/inference/tag.go | 14 ++++++++++++-- internal/translator/view.go | 3 +++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/inference/tag.go b/internal/inference/tag.go index 74855ca38..6e0560aca 100644 --- a/internal/inference/tag.go +++ b/internal/inference/tag.go @@ -190,11 +190,11 @@ func (t *Tags) buildRelation(spec *Spec, relation *Relation) { if pair == nil || pair.ParentField == nil || pair.KeyField == nil { continue } - parentColumn := pair.ParentField.Column.Name + parentColumn := relationColumnName(pair.ParentField.Column) if ns := pair.ParentField.Column.Namespace; ns != "" { parentColumn = ns + "." + parentColumn } - keyColumn := pair.KeyField.Column.Name + keyColumn := relationColumnName(pair.KeyField.Column) if ns := pair.KeyField.Column.Namespace; ns != "" { keyColumn = ns + "." + keyColumn } @@ -215,6 +215,16 @@ func (t *Tags) buildRelation(spec *Spec, relation *Relation) { t.Set(tags.SQLTag, sqlTag) } +func relationColumnName(column *sqlparser.Column) string { + if column == nil { + return "" + } + if column.Name != "" { + return column.Name + } + return column.Alias +} + // Stringify return text representation of struct tag func (t *Tags) Stringify() string { if len(t.order) == 0 { diff --git a/internal/translator/view.go b/internal/translator/view.go index 09732a307..20db586ca 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -299,6 +299,9 @@ func (v *View) buildRelations(parentNamespace *Viewlet, rule *Rule) error { relNamespace.Holder = viewRelation.Holder refViewName := relNamespace.View.Name refColumn := relation.KeyField.Column.Name + if refColumn == "" { + refColumn = relation.KeyField.Column.Alias + } if ns := relation.KeyField.Column.Namespace; ns != "" { refColumn = ns + "." + refColumn } From 0648d91816241a0bb880b1d6bd065cf688ee5a36 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 31 Mar 2026 10:05:25 -0700 Subject: [PATCH 209/279] - added composite key support --- internal/inference/tag_relation_test.go | 69 +++++++++++++++++++++++++ view/tags/tag.go | 13 ++++- view/tags/tag_custom_test.go | 21 ++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 internal/inference/tag_relation_test.go create mode 100644 view/tags/tag_custom_test.go diff --git a/internal/inference/tag_relation_test.go b/internal/inference/tag_relation_test.go new file mode 100644 index 000000000..342d2f17b --- /dev/null +++ b/internal/inference/tag_relation_test.go @@ -0,0 +1,69 @@ +package inference + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + vstate "github.com/viant/datly/view/state" + "github.com/viant/datly/view/tags" + "github.com/viant/sqlparser" + "github.com/viant/sqlparser/query" +) + +func TestTags_buildRelation_UsesColumnAliasWhenNameIsEmpty(t *testing.T) { + selectQuery, err := sqlparser.ParseQuery("SELECT 'app' AS SITE_TYPE_VALUE") + require.NoError(t, err) + + parentColumn := &sqlparser.Column{Name: "SITE_TYPE_VALUES"} + childColumn := &sqlparser.Column{Alias: "SITE_TYPE_VALUE"} + + parentField := &Field{ + Field: view.Field{Name: "SiteTypeValues", Schema: &vstate.Schema{}}, + Column: parentColumn, + } + keyField := &Field{ + Field: view.Field{Name: "SiteTypeValue", Schema: &vstate.Schema{}}, + Column: childColumn, + } + + spec := &Spec{Table: "ignored"} + relation := &Relation{ + Name: "siteType", + Join: &query.Join{Alias: "siteType", With: selectQuery}, + ParentField: parentField, + KeyField: keyField, + Pairs: []*RelationPair{{ + ParentField: parentField, + KeyField: keyField, + }}, + } + + field := &Field{Field: view.Field{Name: "SiteType", Schema: &vstate.Schema{}}, Tags: Tags{}} + field.Tags.buildRelation(spec, relation) + tagString := field.Tags.Stringify() + + parsed, err := tags.Parse(reflect.StructTag(tagString), nil, tags.LinkOnTag) + require.NoError(t, err) + require.Len(t, parsed.LinkOn, 1) + + var relField, relColumn, refField, refColumn string + require.NoError(t, parsed.LinkOn.ForEach(func(rf, rc, kf, kc string, include *bool) error { + relField, relColumn, refField, refColumn = rf, rc, kf, kc + return nil + })) + + require.Equal(t, "SiteTypeValues", relField) + require.Equal(t, "SITE_TYPE_VALUES", relColumn) + require.Equal(t, "SiteTypeValue", refField) + require.Equal(t, "SITE_TYPE_VALUE", refColumn) +} + +func TestType_ByColumn_MatchesAlias(t *testing.T) { + typ := &Type{columnFields: []*Field{{ + Field: view.Field{Name: "SiteTypeValue", Schema: &vstate.Schema{}}, + Column: &sqlparser.Column{Alias: "SITE_TYPE_VALUE"}, + }}} + require.NotNil(t, typ.ByColumn("SITE_TYPE_VALUE")) +} diff --git a/view/tags/tag.go b/view/tags/tag.go index 0cb515032..ea3c4fce5 100644 --- a/view/tags/tag.go +++ b/view/tags/tag.go @@ -114,7 +114,7 @@ func (t *Tag) UpdateTag(tag reflect.StructTag) reflect.StructTag { if t.View != nil { t.appendTag(t.View, &ret) if t.View.CustomTag != "" { - rawTag = t.View.CustomTag + rawTag = normalizeCustomTag(t.View.CustomTag) } } t.appendTag(t.LinkOn, &ret) @@ -144,6 +144,17 @@ func (t *Tag) UpdateTag(tag reflect.StructTag) reflect.StructTag { return reflect.StructTag(structTag) } +func normalizeCustomTag(tag string) string { + tag = strings.TrimSpace(tag) + tag = strings.Trim(tag, "`") + if len(tag) >= 2 { + if (tag[0] == '\'' && tag[len(tag)-1] == '\'') || (tag[0] == '"' && tag[len(tag)-1] == '"') { + tag = tag[1 : len(tag)-1] + } + } + return strings.TrimSpace(tag) +} + func getTagPriority(tag *tags.Tag) int { switch tag.Name { case ParameterTag: diff --git a/view/tags/tag_custom_test.go b/view/tags/tag_custom_test.go new file mode 100644 index 000000000..09b74e961 --- /dev/null +++ b/view/tags/tag_custom_test.go @@ -0,0 +1,21 @@ +package tags + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTag_UpdateTag_NormalizesCustomTagQuotes(t *testing.T) { + tag := &Tag{ + View: &View{ + Name: "siteType", + CustomTag: `'json:",omitempty"'`, + }, + } + + actual := string(tag.UpdateTag(reflect.StructTag(""))) + require.Contains(t, actual, `json:",omitempty"`) + require.NotContains(t, actual, `'json:",omitempty"`) +} From 14192a693da3edb1f0bbd8813704b6b0c8d392f2 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 31 Mar 2026 10:08:14 -0700 Subject: [PATCH 210/279] - added composite key support --- view/tags/tag.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/view/tags/tag.go b/view/tags/tag.go index ea3c4fce5..d25415927 100644 --- a/view/tags/tag.go +++ b/view/tags/tag.go @@ -147,10 +147,14 @@ func (t *Tag) UpdateTag(tag reflect.StructTag) reflect.StructTag { func normalizeCustomTag(tag string) string { tag = strings.TrimSpace(tag) tag = strings.Trim(tag, "`") - if len(tag) >= 2 { - if (tag[0] == '\'' && tag[len(tag)-1] == '\'') || (tag[0] == '"' && tag[len(tag)-1] == '"') { - tag = tag[1 : len(tag)-1] - } + if strings.HasPrefix(tag, "'") { + tag = tag[1:] + } + if strings.HasSuffix(tag, "'") { + tag = tag[:len(tag)-1] + } + if len(tag) >= 2 && strings.HasPrefix(tag, "\"") && strings.HasSuffix(tag, "\"") { + tag = tag[1 : len(tag)-1] } return strings.TrimSpace(tag) } From 5e41c4eebcb5707ba2753f5f5ec5e20c978a31a7 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 2 Apr 2026 11:55:44 -0700 Subject: [PATCH 211/279] - fixed multi preficate builders --- internal/translator/service.go | 6 ++-- internal/translator/view.go | 4 +-- internal/translator/viewlets.go | 50 ++++++++++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/internal/translator/service.go b/internal/translator/service.go index 37cc4f364..1f8ed2aa6 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -194,7 +194,8 @@ func (s *Service) buildExecutorView(ctx context.Context, resource *Resource, DSQ } func (s *Service) translateReaderDSQL(ctx context.Context, resource *Resource, dSQL string) error { - aQuery, err := sqlparser.ParseQuery(dSQL, parser.OnVeltyExpression()) + parseSQL := resource.State.Expand(dSQL) + aQuery, err := sqlparser.ParseQuery(parseSQL, parser.OnVeltyExpression()) if err != nil { return err } @@ -206,7 +207,7 @@ func (s *Service) translateReaderDSQL(ctx context.Context, resource *Resource, d if err = s.updateCodecParameters(ctx, resource); err != nil { return err } - if err = resource.Rule.Viewlets.Init(ctx, aQuery, resource, s.initReaderViewlet, s.buildQueryViewletType); err != nil { + if err = resource.Rule.Viewlets.Init(ctx, aQuery, dSQL, resource, s.initReaderViewlet, s.buildQueryViewletType); err != nil { return err } @@ -586,7 +587,6 @@ func (s *Service) buildQueryViewletType(ctx context.Context, viewlet *Viewlet) e } func (s *Service) buildViewletType(ctx context.Context, db *sql.DB, viewlet *Viewlet) (err error) { - shared.EnsureArgs(viewlet.Expanded.Query, &viewlet.Expanded.Args) viewlet.Spec, err = inference.NewSpec(ctx, db, &s.Repository.Messages, viewlet.Table.Name, viewlet.ColumnConfig, viewlet.Expanded.Query, viewlet.Expanded.Args...) if err != nil { diff --git a/internal/translator/view.go b/internal/translator/view.go index 20db586ca..69d62b802 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -251,8 +251,8 @@ func (v *View) buildTemplate(namespace *Viewlet, rule *Rule) { isRoot := rule.Root == v.Name resource := namespace.Resource v.EnsureTemplate() - v.Template.Source = namespace.SanitizedSQL - v.Template.Parameters = v.matchParameters(namespace.SanitizedSQL, resource.State, isRoot) + v.Template.Source = namespace.SQL + v.Template.Parameters = v.matchParameters(namespace.SQL, resource.State, isRoot) } // matchParameters matches parameter used by SQL, and add explicit parameter for root view diff --git a/internal/translator/viewlets.go b/internal/translator/viewlets.go index ddeb349aa..7902ca63e 100644 --- a/internal/translator/viewlets.go +++ b/internal/translator/viewlets.go @@ -42,12 +42,15 @@ func (n *Viewlets) Append(viewlet *Viewlet) { n.registry[viewlet.Name] = viewlet n.keys = append(n.keys, viewlet.Name) } -func (n *Viewlets) Init(ctx context.Context, aQuery *query.Select, resource *Resource, initFn, setType func(ctx context.Context, n *Viewlet) error) error { +func (n *Viewlets) Init(ctx context.Context, aQuery *query.Select, rootSQL string, resource *Resource, initFn, setType func(ctx context.Context, n *Viewlet) error) error { SQL, err := SafeQueryStringify(aQuery) if err != nil { return err } + if extracted := extractRootViewletSQL(rootSQL, aQuery.From.Alias); extracted != "" { + SQL = extracted + } root := NewViewlet(aQuery.From.Alias, SQL, nil, resource) root.ViewJSONHint = aQuery.From.Comments if root.ViewJSONHint == "" && aQuery.From.X != nil { @@ -123,6 +126,51 @@ func SafeQueryStringify(aQuery *query.Select) (SQL string, err error) { return SQL, err } +func extractRootViewletSQL(SQL, alias string) string { + SQL = strings.TrimSpace(SQL) + alias = strings.TrimSpace(alias) + if SQL == "" || alias == "" { + return "" + } + lowerSQL := strings.ToLower(SQL) + lowerAlias := strings.ToLower(alias) + aliasPos := strings.LastIndex(lowerSQL, lowerAlias) + if aliasPos == -1 { + return "" + } + closePos := aliasPos - 1 + for closePos >= 0 { + switch SQL[closePos] { + case ' ', '\n', '\t', '\r': + closePos-- + continue + case ')': + goto scan + default: + return "" + } + } + return "" + +scan: + if closePos < 0 { + return "" + } + depth := 1 + for i := closePos - 1; i >= 0; i-- { + switch SQL[i] { + case ')': + depth++ + case '(': + depth-- + if depth == 0 { + return strings.TrimSpace(SQL[i : closePos+1]) + } + } + } + return "" +} + func (n *Viewlets) applyViewHintSettings() error { return n.Each(func(namespace *Viewlet) error { return namespace.View.applyHintSettings(namespace) From 219b01b9c5e0d1dcaa0fce363f0fbeb2fe091ebb Mon Sep 17 00:00:00 2001 From: adrianwit Date: Tue, 7 Apr 2026 07:59:27 -0700 Subject: [PATCH 212/279] - fixed multi preficate builders --- go.mod | 3 +-- go.sum | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 77d381533..10263ab1d 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,6 @@ module github.com/viant/datly go 1.25.0 - require ( github.com/aerospike/aerospike-client-go v4.5.2+incompatible github.com/aws/aws-lambda-go v1.31.0 @@ -34,7 +33,7 @@ require ( github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.0 github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e - github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 + github.com/viant/xunsafe v0.11.0 golang.org/x/mod v0.28.0 golang.org/x/oauth2 v0.32.0 google.golang.org/api v0.201.0 diff --git a/go.sum b/go.sum index 12881d103..1e8333b26 100644 --- a/go.sum +++ b/go.sum @@ -1230,6 +1230,8 @@ github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e h1:z4uCWPkSCnGwqb github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e/go.mod h1:BwI+lqFjhKv2Vn4E0Jt6nvbwcFOWrM6H+sOMOX3JiU4= github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 h1:tQOsy7ov3XcTj+OXNF1apq9EKxSj82f5AjJCuhfCkMo= github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= +github.com/viant/xunsafe v0.11.0 h1:Yp5n4JR/6dTZ5asx+hmLGT7UXh5NPz7O9NKmYsk2k0o= +github.com/viant/xunsafe v0.11.0/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca h1:uvPMDVyP7PXMMioYdyPH+0O+Ta/UO1WFfNYMO3Wz0eg= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= From 84b76c71ac5629bd3d0a6cd63448fadfc41075a8 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 7 Apr 2026 10:18:05 -0700 Subject: [PATCH 213/279] - updated pointer handling --- cmd/command/translate_shape_test.go | 70 +++++++++++++++++++++++ internal/converter/repeated.go | 19 ++++-- view/collector.go | 4 +- view/extension/codec/xmlfilter/service.go | 2 +- 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/cmd/command/translate_shape_test.go b/cmd/command/translate_shape_test.go index b76fac4eb..e369dba79 100644 --- a/cmd/command/translate_shape_test.go +++ b/cmd/command/translate_shape_test.go @@ -1,7 +1,10 @@ package command import ( + "context" + "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -28,3 +31,70 @@ func TestRoutePathForShape(t *testing.T) { assert.Equal(t, filepath.ToSlash("platform/campaign"), relDir) assert.Equal(t, "post", stem) } + +func TestTranslateShape_PreservesPredicateBuilderBlocksInGeneratedSQL(t *testing.T) { + ctx := context.Background() + projectDir := t.TempDir() + repoDir := filepath.Join(projectDir, "repo", "dev") + dqlDir := filepath.Join(projectDir, "dql", "opaque") + sqlDir := filepath.Join(dqlDir, "sql") + require.NoError(t, os.MkdirAll(sqlDir, 0o755)) + require.NoError(t, os.MkdirAll(repoDir, 0o755)) + + sqlSource := `SELECT + x0.k_a, + x0.k_b, + z9.m_q, + SUM(x0.v_n) AS agg_alpha, + AVG(z9.v_r) AS agg_beta +FROM + data_alpha x0 +LEFT JOIN data_beta z9 + ON x0.k_b = z9.k_b + +${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")} +GROUP BY + x0.k_a, + x0.k_b, + z9.m_q + ${predicate.Builder().CombineOr($predicate.FilterGroup(1, "HAVING")).Build("HAVING")}` + require.NoError(t, os.WriteFile(filepath.Join(sqlDir, "opaque_source.sql"), []byte(sqlSource), 0o600)) + + dqlSource := `/* {"URI":"/opaque/report","Name":"OpaqueReport"} */ +#set($_ = $cube()) +#set($_ = $Cutoff(query/cutoff).Optional().WithPredicate(0, 'greater_or_equal', 'x0', 'k_a')) +#set($_ = $Threshold(query/threshold).Optional().WithPredicate(1, 'expr', '(SUM(v_n) >= ?)')) + +SELECT opaque_root.*, + grouping_enabled(opaque_root), + allow_nulls(opaque_root), + set_limit(opaque_root, 25) +FROM (${embed:sql/opaque_source.sql}) opaque_root` + dqlPath := filepath.Join(dqlDir, "opaque_report.dql") + require.NoError(t, os.WriteFile(dqlPath, []byte(dqlSource), 0o600)) + + opts := &options.Options{ + Translate: &options.Translate{ + Rule: options.Rule{ + Project: projectDir, + Source: []string{dqlPath}, + Engine: options.EngineShape, + }, + Repository: options.Repository{ + RepositoryURL: repoDir, + APIPrefix: "/v1/api", + }, + }, + } + require.NoError(t, opts.Init(ctx)) + + svc := New() + require.NoError(t, svc.translateShape(ctx, opts)) + + generatedSQLPath := filepath.Join(repoDir, "Datly", "routes", "opaque", "opaque_report", "opaque_report.sql") + data, err := os.ReadFile(generatedSQLPath) + require.NoError(t, err) + generated := string(data) + assert.True(t, strings.Contains(generated, `${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")}`)) + assert.True(t, strings.Contains(generated, `${predicate.Builder().CombineOr($predicate.FilterGroup(1, "HAVING")).Build("HAVING")}`)) +} diff --git a/internal/converter/repeated.go b/internal/converter/repeated.go index 1947d00cd..a7fe8b32a 100644 --- a/internal/converter/repeated.go +++ b/internal/converter/repeated.go @@ -4,7 +4,6 @@ import ( "fmt" "strconv" "strings" - "unsafe" ) type Repeated []string @@ -32,7 +31,11 @@ func (r Repeated) AsUInts() ([]uint, error) { if err != nil { return nil, err } - return *(*[]uint)(unsafe.Pointer(&v)), nil + result := make([]uint, len(v)) + for i, item := range v { + result[i] = uint(item) + } + return result, nil } func (r Repeated) AsInt64s() ([]int64, error) { @@ -40,7 +43,11 @@ func (r Repeated) AsInt64s() ([]int64, error) { if err != nil { return nil, err } - return *(*[]int64)(unsafe.Pointer(&v)), nil + result := make([]int64, len(v)) + for i, item := range v { + result[i] = int64(item) + } + return result, nil } func (r Repeated) AsUInt64s() ([]uint64, error) { @@ -48,7 +55,11 @@ func (r Repeated) AsUInt64s() ([]uint64, error) { if err != nil { return nil, err } - return *(*[]uint64)(unsafe.Pointer(&v)), nil + result := make([]uint64, len(v)) + for i, item := range v { + result[i] = uint64(item) + } + return result, nil } func (r Repeated) AsFloats64() ([]float64, error) { diff --git a/view/collector.go b/view/collector.go index 02dc32dcd..36ad9d5f1 100644 --- a/view/collector.go +++ b/view/collector.go @@ -1115,8 +1115,8 @@ func (r *Collector) Fetched() { } func (r *Collector) Len() int { - if r.DestPtr() != nil { - return (*reflect.SliceHeader)(xunsafe.AsPointer(r.DestPtr())).Len + if r.DestPtr() != nil && r.slice != nil { + return r.slice.Len(xunsafe.AsPointer(r.DestPtr())) } return 0 } diff --git a/view/extension/codec/xmlfilter/service.go b/view/extension/codec/xmlfilter/service.go index 9333b58ef..401e7a131 100644 --- a/view/extension/codec/xmlfilter/service.go +++ b/view/extension/codec/xmlfilter/service.go @@ -42,7 +42,7 @@ func (t *Service) Transfer(aStruct interface{}) (*xml.FilterHolder, error) { xFilterPtr = *(*unsafe.Pointer)(ownerAddr) fieldType = fieldType.Elem() } else { - xFilterPtr = unsafe.Pointer(uintptr(ptr) + field.Offset) + xFilterPtr = unsafe.Add(ptr, field.Offset) } if fieldType.Kind() != reflect.Struct { From beecea767a198019d11a69bbac9d3e768b0bbbf0 Mon Sep 17 00:00:00 2001 From: adranwit Date: Tue, 7 Apr 2026 10:26:57 -0700 Subject: [PATCH 214/279] - updated pointer handling --- internal/converter/repeated.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/converter/repeated.go b/internal/converter/repeated.go index a7fe8b32a..4c30442a1 100644 --- a/internal/converter/repeated.go +++ b/internal/converter/repeated.go @@ -4,10 +4,13 @@ import ( "fmt" "strconv" "strings" + "unsafe" ) type Repeated []string +var intIs64Bit = unsafe.Sizeof(int(0)) == unsafe.Sizeof(uint64(0)) + func (r Repeated) AsInts() ([]int, error) { var result = make([]int, 0, len(r)) for _, item := range r { @@ -31,6 +34,9 @@ func (r Repeated) AsUInts() ([]uint, error) { if err != nil { return nil, err } + if intIs64Bit { + return *(*[]uint)(unsafe.Pointer(&v)), nil + } result := make([]uint, len(v)) for i, item := range v { result[i] = uint(item) @@ -43,6 +49,9 @@ func (r Repeated) AsInt64s() ([]int64, error) { if err != nil { return nil, err } + if intIs64Bit { + return *(*[]int64)(unsafe.Pointer(&v)), nil + } result := make([]int64, len(v)) for i, item := range v { result[i] = int64(item) @@ -55,6 +64,9 @@ func (r Repeated) AsUInt64s() ([]uint64, error) { if err != nil { return nil, err } + if intIs64Bit { + return *(*[]uint64)(unsafe.Pointer(&v)), nil + } result := make([]uint64, len(v)) for i, item := range v { result[i] = uint64(item) From 112713279a45f7d3cc4dc766e26476a4c7d708a4 Mon Sep 17 00:00:00 2001 From: Badr Ezzir Date: Wed, 8 Apr 2026 01:09:59 +0100 Subject: [PATCH 215/279] ENG-54618 Fix concurrent map write bug in `seedFormFromMultipart` and optimize form value handling --- view/state/kind/locator/form.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/view/state/kind/locator/form.go b/view/state/kind/locator/form.go index 2b9190533..cc8563ab1 100644 --- a/view/state/kind/locator/form.go +++ b/view/state/kind/locator/form.go @@ -97,19 +97,18 @@ func NewForm(opts ...Option) (kind.Locator, error) { return ret, nil } -// seedFormFromMultipart parses multipart/form-data (if needed) and copies textual values to the shared form +// seedFormFromMultipart parses multipart/form-data and copies values into shared maps. +// Mutex is required because multiple Form locators (one per parameter) can call this +// concurrently on the same request. Uses form.Values directly instead of form.Set to +// avoid deadlock (form.Set locks the same mutex). func (r *Form) seedFormFromMultipart() { if r.request == nil || r.form == nil { return } if r.request.MultipartForm == nil && len(r.form.Values) == 0 { - // Only ParseMultipartForm for form-data; other multipart types aren't - // supported by ParseMultipartForm. If the shared form already has - // values, treat it as authoritative and avoid parsing. ct := r.request.Header.Get("Content-Type") if ct != "" { if mediaType, _, err := mime.ParseMediaType(ct); err == nil && shared.IsFormData(mediaType) { - // Use the same default memory threshold as Body locator const maxMultipartMemory = 32 << 20 // 32 MiB _ = r.request.ParseMultipartForm(maxMultipartMemory) } @@ -118,14 +117,18 @@ func (r *Form) seedFormFromMultipart() { if r.request.MultipartForm == nil { return } - if len(r.request.Form) == 0 { + // BUG FIX (concurrent map writes): + mu := r.form.Mutex() + mu.Lock() + defer mu.Unlock() + if r.request.Form == nil { r.request.Form = url.Values{} } for k, vs := range r.request.MultipartForm.Value { if len(vs) == 0 { continue } - r.form.Set(k, vs...) + r.form.Values[k] = vs r.request.Form[k] = vs } } From 82c50a7641dc3c6d47a8cc7ae642cd8493c052d9 Mon Sep 17 00:00:00 2001 From: Himanshu Shishir Shah Date: Tue, 7 Apr 2026 17:24:58 -0700 Subject: [PATCH 216/279] ENG-54255: enable call to finalize for patch apis even during errors --- service/operator/executor.go | 2 +- service/session/stater.go | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/service/operator/executor.go b/service/operator/executor.go index 94d7fab94..7fbba3ef1 100644 --- a/service/operator/executor.go +++ b/service/operator/executor.go @@ -41,7 +41,7 @@ func (s *Service) execute(ctx context.Context, aComponent *repository.Component, onDone(time.Now(), err) } if err != nil { - return nil, err + return response, err } return response, nil } diff --git a/service/session/stater.go b/service/session/stater.go index 392a6d3a8..02b48002e 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -251,12 +251,20 @@ func (s *Session) handleComponentOutputType(ctx context.Context, dest interface{ return err } s.Options = sessionOpt - reflectDestValue := reflect.ValueOf(destValue) - - if reflectDestValue.Kind() == reflect.Ptr { - destPtr.Elem().Set(reflectDestValue.Elem()) - } else { - destPtr.Elem().Set(reflectDestValue) + if destValue != nil { + reflectDestValue := reflect.ValueOf(destValue) + if reflectDestValue.Kind() == reflect.Ptr { + destPtr.Elem().Set(reflectDestValue.Elem()) + } else { + destPtr.Elem().Set(reflectDestValue) + } + } + if err != nil { + if errorSetter, ok := dest.(response.StatusSetter); ok { + errorSetter.SetError(err) + return nil + } + return err } return nil } From 748d4421cecdc7f6cf3633d4c36bd89999d24fa5 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 8 Apr 2026 15:20:25 -0700 Subject: [PATCH 217/279] - fixed multi preficate builders --- .../router/marshal/json/marshaller_custom.go | 2 +- .../marshal/json/marshaller_gojay_object.go | 5 +++-- service/executor/expand/predicate.go | 15 +++++++++++++-- service/executor/expand/predicate_test.go | 18 +++++++++++++++++- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/gateway/router/marshal/json/marshaller_custom.go b/gateway/router/marshal/json/marshaller_custom.go index 9dcda9c12..6ce699b49 100644 --- a/gateway/router/marshal/json/marshaller_custom.go +++ b/gateway/router/marshal/json/marshaller_custom.go @@ -50,7 +50,7 @@ func (c *customMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder *goj value := c.valueType.Interface(pointer) asUnmarshaler, ok := value.(UnmarshalerInto) if ok { - dst := c.addrType.Value(pointer) + dst := reflect.NewAt(c.valueType.Type(), pointer).Interface() return asUnmarshaler.UnmarshalJSONWithOptions(dst, decoder, session.Options...) } diff --git a/gateway/router/marshal/json/marshaller_gojay_object.go b/gateway/router/marshal/json/marshaller_gojay_object.go index af3cbbec3..26fc7db6a 100644 --- a/gateway/router/marshal/json/marshaller_gojay_object.go +++ b/gateway/router/marshal/json/marshaller_gojay_object.go @@ -3,6 +3,7 @@ package json import ( "github.com/francoispqt/gojay" "github.com/viant/xunsafe" + "reflect" "unsafe" ) @@ -34,7 +35,7 @@ func (g *gojayObjectMarshaller) MarshallObject(ptr unsafe.Pointer, session *Mars if g.useMarshal { // Prefer pointer receiver if (*T) implements MarshalerJSONObject - if m, ok := g.addrType.Value(ptr).(gojay.MarshalerJSONObject); ok { + if m, ok := reflect.NewAt(g.valueType.Type(), ptr).Interface().(gojay.MarshalerJSONObject); ok { enc := gojay.NewEncoder(session.Buffer) return enc.EncodeObject(m) } @@ -59,7 +60,7 @@ func (g *gojayObjectMarshaller) UnmarshallObject(pointer unsafe.Pointer, decoder } // Prefer pointer receiver only; value receiver cannot mutate destination reliably. - if u, ok := g.addrType.Value(pointer).(gojay.UnmarshalerJSONObject); ok { + if u, ok := reflect.NewAt(g.valueType.Type(), pointer).Interface().(gojay.UnmarshalerJSONObject); ok { return d.Object(u) } diff --git a/service/executor/expand/predicate.go b/service/executor/expand/predicate.go index aae51c8bb..e44c7dceb 100644 --- a/service/executor/expand/predicate.go +++ b/service/executor/expand/predicate.go @@ -80,6 +80,12 @@ func (b *PredicateBuilder) CombineAnd(fragments ...string) *PredicateBuilder { } func (b *PredicateBuilder) combine(keyword string, fragments []string) *PredicateBuilder { + if b == nil { + b = &PredicateBuilder{} + } + if b.output == nil { + b.output = &strings.Builder{} + } builder := &strings.Builder{} for _, fragment := range fragments { if strings.TrimSpace(fragment) == "" { @@ -117,10 +123,9 @@ func (b *PredicateBuilder) combine(keyword string, fragments []string) *Predicat } func (b *PredicateBuilder) Build(keyword string) string { - if b.output.Len() == 0 { + if b == nil || b.output == nil || b.output.Len() == 0 { return "" } - return " " + keyword + " " + b.output.String() } @@ -206,11 +211,17 @@ func (p *Predicate) appendFilter(selector *structology.Selector, value []interfa } func (b *PredicateBuilder) And() *PredicateBuilder { + if b == nil { + b = &PredicateBuilder{} + } b.lastKeyword = "AND" return b } func (b *PredicateBuilder) Or() *PredicateBuilder { + if b == nil { + b = &PredicateBuilder{} + } b.lastKeyword = "OR" return b } diff --git a/service/executor/expand/predicate_test.go b/service/executor/expand/predicate_test.go index f722c37e2..bee20e8bb 100644 --- a/service/executor/expand/predicate_test.go +++ b/service/executor/expand/predicate_test.go @@ -1,4 +1,20 @@ -package expand_test +package expand + +import "testing" + +func TestPredicateBuilder_NilReceiver(t *testing.T) { + var builder *PredicateBuilder + + got := builder.CombineOr("x = ?").Build("WHERE") + if got == "" { + t.Fatalf("expected combined predicate, got empty string") + } + + got = builder.And().CombineAnd("y = ?").Build("WHERE") + if got == "" { + t.Fatalf("expected predicate after And on nil receiver, got empty string") + } +} //func TestPredicate(t *testing.T) { // type Foo struct { From cd0fe18cadfd0bd34fe98360344c09f3d640d863 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 8 Apr 2026 15:46:34 -0700 Subject: [PATCH 218/279] - fixed multi predicaet builders/velthy pointer issue --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 10263ab1d..f18468d0a 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/viant/sqlx v0.22.1-0.20260326175456-cec446e28372 github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 - github.com/viant/velty v0.4.0 + github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e github.com/viant/xunsafe v0.11.0 golang.org/x/mod v0.28.0 diff --git a/go.sum b/go.sum index 1e8333b26..01838e90f 100644 --- a/go.sum +++ b/go.sum @@ -1210,6 +1210,8 @@ github.com/viant/toolbox v0.37.0 h1:+zwSdbQh6I6ZEyxokQJr+1gQKbLEw6erc+Av5dwKtLU= github.com/viant/toolbox v0.37.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/viant/velty v0.4.0 h1:eesQES/vCpcoPbM+gQLUBuLEL2sEO+A6s6lPpl8eKc4= github.com/viant/velty v0.4.0/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= +github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 h1:SIYKYU4d4A+yNS3G0rJN7ShTEW4AEAQPvRn+acfXhA0= +github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef h1:KqWKMNloyzEg6nIn1pBK4CDEIcaRRhMrMUJr+k+xcPw= github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef/go.mod h1:1TvsnpZFqI9dYVzIkaSYJyJ/UkfxW7fnk0YFafWXrPg= github.com/viant/xdatly v0.5.4-0.20260306062123-17850ac34977 h1:icW8DECqGoT4UzzOpxBraT/EEC1R0tBw9ev9cF/mrd4= From f1fb176e79823193f443a8e64dfd199cd0a20a59 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 8 Apr 2026 16:39:03 -0700 Subject: [PATCH 219/279] - fix predicate extractor --- internal/inference/state.go | 12 +++++++++++- internal/translator/resource.go | 2 +- internal/translator/service.go | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/inference/state.go b/internal/inference/state.go index 880f0be76..ed8e9aa35 100644 --- a/internal/inference/state.go +++ b/internal/inference/state.go @@ -298,6 +298,14 @@ func (s State) Explicit() State { } func (s State) Expand(text string) string { + return s.expand(text, false) +} + +func (s State) ExpandPreserveBuiltins(text string) string { + return s.expand(text, true) +} + +func (s State) expand(text string, preserveBuiltins bool) string { expander := data.Map{} if parameters := s.FilterByKind(state.KindConst); len(parameters) > 0 { for _, literal := range parameters { @@ -305,7 +313,9 @@ func (s State) Expand(text string) string { } } - text = removeBuilinExpr(text) + if !preserveBuiltins { + text = removeBuilinExpr(text) + } return expander.ExpandAsText(text) } diff --git a/internal/translator/resource.go b/internal/translator/resource.go index dd806983f..133162301 100644 --- a/internal/translator/resource.go +++ b/internal/translator/resource.go @@ -859,7 +859,7 @@ func (r *Resource) expandSQL(viewlet *Viewlet) (*sqlx.SQL, error) { sourceView.Summary = viewlet } - sourceSQL = viewlet.Resource.State.Expand(sourceSQL) + sourceSQL = viewlet.Resource.State.ExpandPreserveBuiltins(sourceSQL) templateParameters := sqlState.Parameters() if strings.Contains(sourceSQL, "$View.ParentJoinOn") { //TODO adjust parameter value type diff --git a/internal/translator/service.go b/internal/translator/service.go index 1f8ed2aa6..14851c55e 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -194,7 +194,7 @@ func (s *Service) buildExecutorView(ctx context.Context, resource *Resource, DSQ } func (s *Service) translateReaderDSQL(ctx context.Context, resource *Resource, dSQL string) error { - parseSQL := resource.State.Expand(dSQL) + parseSQL := resource.State.ExpandPreserveBuiltins(dSQL) aQuery, err := sqlparser.ParseQuery(parseSQL, parser.OnVeltyExpression()) if err != nil { return err From 99a9ba5381fa502d1575db114ab85c117ee8fe85 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 8 Apr 2026 18:38:05 -0700 Subject: [PATCH 220/279] - fix predicate extractor --- go.mod | 2 +- go.sum | 8 +- internal/inference/state_expand_test.go | 72 ++++++++++++++++++ .../translator/parser/supply_sanitize_test.go | 28 +++++++ internal/translator/resource.go | 2 +- internal/translator/service.go | 5 +- internal/translator/viewlets.go | 74 +++++++++++++------ internal/translator/viewlets_extract_test.go | 24 ++++++ view/sql.go | 3 + 9 files changed, 185 insertions(+), 33 deletions(-) create mode 100644 internal/inference/state_expand_test.go create mode 100644 internal/translator/parser/supply_sanitize_test.go create mode 100644 internal/translator/viewlets_extract_test.go diff --git a/go.mod b/go.mod index f18468d0a..83185bbe3 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( require ( github.com/viant/govalidator v0.3.1 - github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 + github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 ) require ( diff --git a/go.sum b/go.sum index 01838e90f..31f7ee131 100644 --- a/go.sum +++ b/go.sum @@ -1194,8 +1194,8 @@ github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= github.com/viant/pgo v0.11.0/go.mod h1:MFzHmkRFZlciugEgUvpl/3grK789PBSH4dUVSLOSo+Q= github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= -github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 h1:bnVgWzZzuz2pTa54e7YozHjYNFSapfU3MSklyMkO+Ag= -github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= +github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQnwf69oc86xW1e4wEEzKUg5OC6tOYegE= +github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.22.1-0.20260326175456-cec446e28372 h1:5qW+4AbQ8YA0MsyoUx3uaNgTHl52F0JBoC2vsdwKXIM= github.com/viant/sqlx v0.22.1-0.20260326175456-cec446e28372/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= @@ -1208,8 +1208,6 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI github.com/viant/toolbox v0.34.5/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/viant/toolbox v0.37.0 h1:+zwSdbQh6I6ZEyxokQJr+1gQKbLEw6erc+Av5dwKtLU= github.com/viant/toolbox v0.37.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/viant/velty v0.4.0 h1:eesQES/vCpcoPbM+gQLUBuLEL2sEO+A6s6lPpl8eKc4= -github.com/viant/velty v0.4.0/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 h1:SIYKYU4d4A+yNS3G0rJN7ShTEW4AEAQPvRn+acfXhA0= github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87/go.mod h1:Q/UXviI2Nli8WROEpYd/BELMCSvnulQeyNrbPmMiS/Y= github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef h1:KqWKMNloyzEg6nIn1pBK4CDEIcaRRhMrMUJr+k+xcPw= @@ -1230,8 +1228,6 @@ github.com/viant/xmlify v0.1.1 h1:Kmn7wnsq5APD8uJVP+kM6lIEGhSyjWSNOy4BvyfZQno= github.com/viant/xmlify v0.1.1/go.mod h1:w25+umH6nthlQ8ACT3K2/YJOLlbTXKLQXkdqFs6ky9s= github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e h1:z4uCWPkSCnGwqbIc3ENoYJnYwtR2j/9eI79vO4vK9rQ= github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e/go.mod h1:BwI+lqFjhKv2Vn4E0Jt6nvbwcFOWrM6H+sOMOX3JiU4= -github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 h1:tQOsy7ov3XcTj+OXNF1apq9EKxSj82f5AjJCuhfCkMo= -github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= github.com/viant/xunsafe v0.11.0 h1:Yp5n4JR/6dTZ5asx+hmLGT7UXh5NPz7O9NKmYsk2k0o= github.com/viant/xunsafe v0.11.0/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= diff --git a/internal/inference/state_expand_test.go b/internal/inference/state_expand_test.go new file mode 100644 index 000000000..2b3faaa55 --- /dev/null +++ b/internal/inference/state_expand_test.go @@ -0,0 +1,72 @@ +package inference + +import ( + "os" + "strings" + "testing" +) + +func TestStateExpandPreserveBuiltins(t *testing.T) { + state := State{} + state.Append(NewConstParameter("dataset", "ci_ads")) + + input := `SELECT * FROM ${dataset}.CI_AD_ORDER ao ${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")}` + + got := state.ExpandPreserveBuiltins(input) + if !strings.Contains(got, "ci_ads.CI_AD_ORDER") { + t.Fatalf("expected const expansion, got: %s", got) + } + if !strings.Contains(got, `${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")}`) { + t.Fatalf("expected predicate builder to be preserved, got: %s", got) + } +} + +func TestStateExpandStripsBuiltins(t *testing.T) { + state := State{} + state.Append(NewConstParameter("dataset", "ci_ads")) + + input := `SELECT * FROM ${dataset}.CI_AD_ORDER ao ${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")}` + + got := state.Expand(input) + if !strings.Contains(got, "ci_ads.CI_AD_ORDER") { + t.Fatalf("expected const expansion, got: %s", got) + } + if strings.Contains(got, `${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")}`) { + t.Fatalf("expected predicate builder to be stripped, got: %s", got) + } +} + +func TestStateExpand_StripsSupplyPerformancePredicateBuilders(t *testing.T) { + data, err := os.ReadFile("/Users/awitas/go/src/github.vianttech.com/viant/steward/dql/inventory/sql/supply_performance.sql") + if err != nil { + t.Fatalf("read sql: %v", err) + } + state := State{} + got := state.Expand(string(data)) + if strings.Contains(got, `${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")}`) { + t.Fatalf("expected WHERE predicate builder to be stripped, got: %s", got) + } + if strings.Contains(got, `${predicate.Builder().CombineOr($predicate.FilterGroup(1, "HAVING")).Build("HAVING")}`) { + t.Fatalf("expected HAVING predicate builder to be stripped, got: %s", got) + } +} + +func TestStateExpand_StripsPredicateBuildersInWrappedSupplyPerformanceDQL(t *testing.T) { + sqlData, err := os.ReadFile("/Users/awitas/go/src/github.vianttech.com/viant/steward/dql/inventory/sql/supply_performance.sql") + if err != nil { + t.Fatalf("read sql: %v", err) + } + dqlData, err := os.ReadFile("/Users/awitas/go/src/github.vianttech.com/viant/steward/dql/inventory/supply_performance.dql") + if err != nil { + t.Fatalf("read dql: %v", err) + } + combined := strings.Replace(string(dqlData), "${embed:sql/supply_performance.sql}", string(sqlData), 1) + state := State{} + got := state.Expand(combined) + if strings.Contains(got, `${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")}`) { + t.Fatalf("expected wrapped WHERE predicate builder to be stripped, got: %s", got) + } + if strings.Contains(got, `${predicate.Builder().CombineOr($predicate.FilterGroup(1, "HAVING")).Build("HAVING")}`) { + t.Fatalf("expected wrapped HAVING predicate builder to be stripped, got: %s", got) + } +} diff --git a/internal/translator/parser/supply_sanitize_test.go b/internal/translator/parser/supply_sanitize_test.go new file mode 100644 index 000000000..eb0155849 --- /dev/null +++ b/internal/translator/parser/supply_sanitize_test.go @@ -0,0 +1,28 @@ +package parser + +import ( + "os" + "strings" + "testing" + + "github.com/viant/datly/internal/inference" +) + +func TestTemplate_Sanitize_SupplyPerformancePredicateBuilderPreserved(t *testing.T) { + data, err := os.ReadFile("/Users/awitas/go/src/github.vianttech.com/viant/steward/dql/inventory/sql/supply_performance.sql") + if err != nil { + t.Fatalf("read sql: %v", err) + } + state := inference.State{} + tmpl, err := NewTemplate(string(data), &state) + if err != nil { + t.Fatalf("new template: %v", err) + } + actual := tmpl.Sanitize() + if !strings.Contains(actual, `${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")}`) { + t.Fatalf("expected WHERE predicate builder to survive sanitize, got: %s", actual) + } + if !strings.Contains(actual, `${predicate.Builder().CombineOr($predicate.FilterGroup(1, "HAVING")).Build("HAVING")}`) { + t.Fatalf("expected HAVING predicate builder to survive sanitize, got: %s", actual) + } +} diff --git a/internal/translator/resource.go b/internal/translator/resource.go index 133162301..dd806983f 100644 --- a/internal/translator/resource.go +++ b/internal/translator/resource.go @@ -859,7 +859,7 @@ func (r *Resource) expandSQL(viewlet *Viewlet) (*sqlx.SQL, error) { sourceView.Summary = viewlet } - sourceSQL = viewlet.Resource.State.ExpandPreserveBuiltins(sourceSQL) + sourceSQL = viewlet.Resource.State.Expand(sourceSQL) templateParameters := sqlState.Parameters() if strings.Contains(sourceSQL, "$View.ParentJoinOn") { //TODO adjust parameter value type diff --git a/internal/translator/service.go b/internal/translator/service.go index 14851c55e..c4376fa6e 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -194,7 +194,7 @@ func (s *Service) buildExecutorView(ctx context.Context, resource *Resource, DSQ } func (s *Service) translateReaderDSQL(ctx context.Context, resource *Resource, dSQL string) error { - parseSQL := resource.State.ExpandPreserveBuiltins(dSQL) + parseSQL := resource.State.Expand(dSQL) aQuery, err := sqlparser.ParseQuery(parseSQL, parser.OnVeltyExpression()) if err != nil { return err @@ -588,7 +588,8 @@ func (s *Service) buildQueryViewletType(ctx context.Context, viewlet *Viewlet) e func (s *Service) buildViewletType(ctx context.Context, db *sql.DB, viewlet *Viewlet) (err error) { shared.EnsureArgs(viewlet.Expanded.Query, &viewlet.Expanded.Args) - viewlet.Spec, err = inference.NewSpec(ctx, db, &s.Repository.Messages, viewlet.Table.Name, viewlet.ColumnConfig, viewlet.Expanded.Query, viewlet.Expanded.Args...) + queryForSpec := viewlet.Resource.State.Expand(viewlet.Expanded.Query) + viewlet.Spec, err = inference.NewSpec(ctx, db, &s.Repository.Messages, viewlet.Table.Name, viewlet.ColumnConfig, queryForSpec, viewlet.Expanded.Args...) if err != nil { return fmt.Errorf("failed to create spec for %v, %w", viewlet.Name, err) } diff --git a/internal/translator/viewlets.go b/internal/translator/viewlets.go index 7902ca63e..ba28e633d 100644 --- a/internal/translator/viewlets.go +++ b/internal/translator/viewlets.go @@ -134,41 +134,69 @@ func extractRootViewletSQL(SQL, alias string) string { } lowerSQL := strings.ToLower(SQL) lowerAlias := strings.ToLower(alias) - aliasPos := strings.LastIndex(lowerSQL, lowerAlias) - if aliasPos == -1 { - return "" - } - closePos := aliasPos - 1 - for closePos >= 0 { - switch SQL[closePos] { - case ' ', '\n', '\t', '\r': - closePos-- + searchFrom := 0 + for { + fromPos := strings.Index(lowerSQL[searchFrom:], "from") + if fromPos == -1 { + return "" + } + fromPos += searchFrom + afterFrom := fromPos + len("from") + for afterFrom < len(SQL) && isSQLWhitespace(SQL[afterFrom]) { + afterFrom++ + } + if afterFrom >= len(SQL) || SQL[afterFrom] != '(' { + searchFrom = afterFrom continue - case ')': - goto scan - default: + } + closePos := matchClosingParen(SQL, afterFrom) + if closePos == -1 { + return "" + } + aliasPos := closePos + 1 + for aliasPos < len(SQL) && isSQLWhitespace(SQL[aliasPos]) { + aliasPos++ + } + if aliasPos >= len(SQL) { return "" } + aliasEnd := aliasPos + for aliasEnd < len(SQL) && isSQLIdentifierChar(SQL[aliasEnd]) { + aliasEnd++ + } + if strings.EqualFold(lowerAlias, strings.ToLower(SQL[aliasPos:aliasEnd])) { + return strings.TrimSpace(SQL[afterFrom : closePos+1]) + } + searchFrom = aliasEnd } - return "" +} -scan: - if closePos < 0 { - return "" - } - depth := 1 - for i := closePos - 1; i >= 0; i-- { +func matchClosingParen(SQL string, openPos int) int { + depth := 0 + for i := openPos; i < len(SQL); i++ { switch SQL[i] { - case ')': - depth++ case '(': + depth++ + case ')': depth-- if depth == 0 { - return strings.TrimSpace(SQL[i : closePos+1]) + return i } } } - return "" + return -1 +} + +func isSQLWhitespace(b byte) bool { + switch b { + case ' ', '\n', '\t', '\r': + return true + } + return false +} + +func isSQLIdentifierChar(b byte) bool { + return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_' || b == '$' } func (n *Viewlets) applyViewHintSettings() error { diff --git a/internal/translator/viewlets_extract_test.go b/internal/translator/viewlets_extract_test.go new file mode 100644 index 000000000..adac8679a --- /dev/null +++ b/internal/translator/viewlets_extract_test.go @@ -0,0 +1,24 @@ +package translator + +import ( + "strings" + "testing" +) + +func TestExtractRootViewletSQL_UsesFromAliasNotLastAliasOccurrence(t *testing.T) { + sql := ` +SELECT + adConfig.*, + set_limit(adConfig, 1) +FROM ( + ${embed:sql/ad_config/config.sql} +) adConfig +JOIN ( + SELECT 1 +) agency ON 1 = 1` + + got := extractRootViewletSQL(sql, "adConfig") + if !strings.Contains(got, "${embed:sql/ad_config/config.sql}") { + t.Fatalf("expected embedded root sql, got: %s", got) + } +} diff --git a/view/sql.go b/view/sql.go index b99b67bdb..9d277b1cc 100644 --- a/view/sql.go +++ b/view/sql.go @@ -135,6 +135,9 @@ func ensureSelectStatement(evaluation *TemplateEvaluation, v *View) string { source := evaluation.SQL if source != v.Name && source != v.Table { + if strings.Contains(source, "${predicate.") { + return source + } if query, _ := sqlparser.ParseQuery(source); query != nil && query.From.X == nil { return wrapWithSelect(v, source) } From 8cacd881c5971c9c3959b430cd137dfb9766f683 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 9 Apr 2026 12:41:21 -0700 Subject: [PATCH 221/279] - fix predicate extractor --- view/state/kind/locator/form.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/view/state/kind/locator/form.go b/view/state/kind/locator/form.go index cc8563ab1..9b37fdb65 100644 --- a/view/state/kind/locator/form.go +++ b/view/state/kind/locator/form.go @@ -71,18 +71,23 @@ func (r *Form) Value(ctx context.Context, rType reflect.Type, name string) (inte } return nil, false, nil } - // Non-multipart: use standard FormValue fallback + // Non-multipart: parse form/query values and preserve repeated values. r.form.Mutex().Lock() defer r.form.Mutex().Unlock() - value := r.request.FormValue(name) - if value == "" { - if r.request.Form == nil { - return nil, false, nil - } - _, ok := r.request.Form[name] - return "", ok, nil + if err := r.request.ParseForm(); err != nil { + return nil, false, err + } + values, ok := r.request.Form[name] + if !ok { + return nil, false, nil + } + if len(values) > 1 { + return values, true, nil + } + if len(values) == 1 { + return values[0], true, nil } - return value, true, nil + return "", true, nil } if len(values) > 1 { return values, true, nil From e40dcc4467ab32ddb4b4981285f0199898f01ecf Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 9 Apr 2026 12:41:31 -0700 Subject: [PATCH 222/279] - fix predicate extractor --- view/state/kind/locator/form_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 view/state/kind/locator/form_test.go diff --git a/view/state/kind/locator/form_test.go b/view/state/kind/locator/form_test.go new file mode 100644 index 000000000..b7b78d264 --- /dev/null +++ b/view/state/kind/locator/form_test.go @@ -0,0 +1,24 @@ +package locator + +import ( + "context" + "net/http/httptest" + "reflect" + "testing" + + "github.com/stretchr/testify/require" + hstate "github.com/viant/xdatly/handler/state" +) + +func TestForm_Value_PreservesRepeatedQueryValues(t *testing.T) { + req := httptest.NewRequest("GET", "http://localhost/test?site_id=1&site_id=2&site_id=3", nil) + locator := &Form{ + form: hstate.NewForm(), + request: req, + } + + value, ok, err := locator.Value(context.Background(), reflect.TypeOf([]int{}), "site_id") + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []string{"1", "2", "3"}, value) +} From 4da7479dbd89cbf5cca78884ca779afb50a5a537 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 10 Apr 2026 12:35:32 -0700 Subject: [PATCH 223/279] - fix predicate extractor --- gateway/mcp.go | 109 +++++++++++++++++++++++++++++-------- gateway/mcp_report_test.go | 75 +++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 24 deletions(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index 0ec47596a..d12d9352e 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -1,6 +1,8 @@ package gateway import ( + "bytes" + "compress/gzip" "context" "encoding/json" "fmt" @@ -8,6 +10,7 @@ import ( "net/http" "net/url" "reflect" + "strconv" "strings" furl "github.com/viant/afs/url" @@ -236,10 +239,11 @@ func (r *Router) applyParamToRequest(baseURL string, values url.Values, p *state } baseURL = strings.ReplaceAll(baseURL, "{"+p.In.Name+"}", fmt.Sprintf("%v", value)) case state.KindQuery, state.KindForm: - if uniqueQuery[p.In.Name] { + queryName := requestParamName(p) + if uniqueQuery[queryName] { return baseURL, body, nil } - uniqueQuery[p.In.Name] = true + uniqueQuery[queryName] = true if value == nil || value == "" { return baseURL, body, nil } @@ -252,9 +256,9 @@ func (r *Router) applyParamToRequest(baseURL string, values url.Values, p *state items = append(items, fmt.Sprintf("%v", item)) } } - values.Add(p.In.Name, strings.Join(items, ",")) + values.Add(queryName, strings.Join(items, ",")) } else { - values.Add(p.In.Name, fmt.Sprintf("%v", value)) + values.Add(queryName, fmt.Sprintf("%v", value)) } case state.KindRequestBody: if text, ok := value.(string); ok { @@ -270,6 +274,34 @@ func (r *Router) applyParamToRequest(baseURL string, values url.Values, p *state return baseURL, body, nil } +func requestParamName(p *state.Parameter) string { + if p == nil || p.In == nil { + return "" + } + if public, ok := selectorPublicParamName(p); ok { + return public + } + return p.In.Name +} + +func selectorPublicParamName(p *state.Parameter) (string, bool) { + switch strings.TrimSpace(p.Name) { + case "Limit": + return "limit", true + case "Offset": + return "offset", true + case "Page": + return "page", true + case "Fields": + return "fields", true + case "OrderBy": + return "orderBy", true + case "Criteria": + return "criteria", true + } + return "", false +} + // newToolHTTPRequest constructs an HTTP request for routed tool invocation. func (r *Router) newToolHTTPRequest(method, URL string, body io.Reader) (*http.Request, *jsonrpc.Error) { httpRequest, err := http.NewRequest(method, URL, body) @@ -282,6 +314,24 @@ func (r *Router) newToolHTTPRequest(method, URL string, body io.Reader) (*http.R return httpRequest, nil } +func decodeToolResponseBody(responseWriter *proxy.Writer) ([]byte, error) { + data := responseWriter.Body.Bytes() + encoding := strings.TrimSpace(responseWriter.HeaderMap.Get("Content-Encoding")) + if !strings.EqualFold(encoding, "gzip") { + return data, nil + } + reader, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("failed to create gzip reader: %w", err) + } + defer reader.Close() + decoded, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to decompress gzip body: %w", err) + } + return decoded, nil +} + // buildToolCallResult composes a CallToolResult with text content and structured error info if status is not OK. func (r *Router) buildToolCallResult(responseWriter *proxy.Writer, URL, method string) *schema.CallToolResult { var result = &schema.CallToolResult{} @@ -289,7 +339,10 @@ func (r *Router) buildToolCallResult(responseWriter *proxy.Writer, URL, method s if mimeType == "" { mimeType = "application/json" } - data := responseWriter.Body.Bytes() + data, err := decodeToolResponseBody(responseWriter) + if err != nil { + data = []byte(err.Error()) + } result.Content = append(result.Content, schema.CallToolResultContentElem( schema.TextContent{ Type: "text", @@ -303,7 +356,7 @@ func (r *Router) buildToolCallResult(responseWriter *proxy.Writer, URL, method s result.StructuredContent = map[string]interface{}{ "status": responseWriter.Code, "error": true, - "message": responseWriter.Body.String(), + "message": string(data), "headers": responseWriter.HeaderMap, "uri": URL, "method": method, @@ -384,11 +437,7 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty continue } uniquePath[parameter.In.Name] = true - // If parameter is a slice, make it optional in schema via `omitempty` and optional:"true". - var tag reflect.StructTag - if parameter.Schema != nil && parameter.Schema.Type().Kind() == reflect.Slice { - tag = `json:",omitempty" optional:"true"` - } + tag := buildMCPFieldTag(parameter, false) appendField(name, parameter.Schema.Type(), tag) case state.KindQuery, state.KindForm: @@ -396,25 +445,14 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty continue } uniqueQuery[parameter.In.Name] = true - // Repeated (slice) params are optional regardless of "required" tag. - // Otherwise, respect explicit required; default to optional. - tag := reflect.StructTag(parameter.Tag) - if parameter.Schema != nil && parameter.Schema.Type().Kind() == reflect.Slice { - tag = `json:",omitempty" optional:"true"` - } else if !strings.Contains(parameter.Tag, "required") { - tag = `json:",omitempty"` - } + tag := buildMCPFieldTag(parameter, true) appendField(name, parameter.Schema.Type(), tag) case state.KindRequestBody: if parameter.IsAnonymous() { appendAnonymousBodyFields(&inputFields, uniqueFieldName, parameter.Schema.Type()) continue } - // If body is a slice, mark optional in schema. - var tag reflect.StructTag - if parameter.Schema != nil && parameter.Schema.Type().Kind() == reflect.Slice { - tag = `json:",omitempty" optional:"true"` - } + tag := buildMCPFieldTag(parameter, false) appendField(name, parameter.Schema.Type(), tag) } } @@ -451,6 +489,29 @@ func (r *Router) buildToolInputType(components *repository.Component) reflect.Ty return reflect.StructOf(inputFields) } +func buildMCPFieldTag(parameter *state.Parameter, defaultOptional bool) reflect.StructTag { + if parameter == nil { + return reflect.StructTag(`json:",omitempty"`) + } + var parts []string + jsonTag := `json:",omitempty"` + if parameter.Schema != nil && parameter.Schema.Type() != nil && parameter.Schema.Type().Kind() == reflect.Slice { + parts = append(parts, jsonTag, `optional:"true"`) + } else { + parts = append(parts, jsonTag) + if strings.Contains(parameter.Tag, "optional") || strings.Contains(parameter.Tag, `required:"false"`) || defaultOptional { + parts = append(parts, `optional:"true"`) + } + } + if description := strings.TrimSpace(parameter.Description); description != "" { + parts = append(parts, `description:`+strconv.Quote(description)) + } + if example := strings.TrimSpace(parameter.Example); example != "" { + parts = append(parts, `example:`+strconv.Quote(example)) + } + return reflect.StructTag(strings.Join(parts, " ")) +} + func toolArgumentValue(parameter *state.Parameter, arguments map[string]interface{}) interface{} { if parameter == nil { return nil diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go index e41ac1c18..3a3be22d9 100644 --- a/gateway/mcp_report_test.go +++ b/gateway/mcp_report_test.go @@ -1,17 +1,21 @@ package gateway import ( + "bytes" + "compress/gzip" "context" "embed" "encoding/json" "io" "net/http" + "net/url" "reflect" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/viant/datly/gateway/router/proxy" "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" dpath "github.com/viant/datly/repository/path" @@ -306,6 +310,77 @@ func TestRouter_buildToolsIntegration_RegistersCubeTool(t *testing.T) { require.Contains(t, tool.InputSchema.Properties, "filters") } +func TestRouter_buildToolCallResult_DecompressesGzipBody(t *testing.T) { + var compressed bytes.Buffer + gzw := gzip.NewWriter(&compressed) + _, err := gzw.Write([]byte(`{"data":{"rows":[1,2,3]},"status":"ok"}`)) + require.NoError(t, err) + require.NoError(t, gzw.Close()) + + writer := proxy.NewWriter() + writer.Code = http.StatusOK + writer.HeaderMap.Set("Content-Type", "application/json") + writer.HeaderMap.Set("Content-Encoding", "gzip") + _, err = writer.Body.Write(compressed.Bytes()) + require.NoError(t, err) + + result := (&Router{}).buildToolCallResult(writer, "http://localhost/test", http.MethodPost) + require.NotNil(t, result) + require.Len(t, result.Content, 1) + + text, ok := result.Content[0].(schema.TextContent) + require.True(t, ok) + assert.JSONEq(t, `{"data":{"rows":[1,2,3]},"status":"ok"}`, text.Text) + assert.Equal(t, map[string]interface{}{ + "data": map[string]interface{}{"rows": []interface{}{float64(1), float64(2), float64(3)}}, + "status": "ok", + }, result.StructuredContent) +} + +func TestRouter_buildToolInputType_UsesParameterMetadataTags(t *testing.T) { + fieldParam := state.NewParameter("Field", state.NewPathLocation("field"), state.WithParameterSchema(state.NewSchema(reflect.TypeOf("")))) + fieldParam.Description = "Targeting field key." + fieldParam.Example = "IRIS_SEGMENTS" + + operationParam := state.NewParameter("Operation", state.NewPathLocation("operation"), state.WithParameterSchema(state.NewSchema(reflect.TypeOf("")))) + operationParam.Description = "Targeting tree operation." + operationParam.Example = "children" + + component := &repository.Component{ + Path: contract.Path{Method: http.MethodPost, URI: "/v1/api/platform/targeting/tree/{field}/{operation}"}, + View: &view.View{}, + Contract: contract.Contract{ + Input: contract.Input{ + Type: state.Type{Parameters: state.Parameters{fieldParam, operationParam}}, + }, + }, + } + + rType := (&Router{}).buildToolInputType(component) + field, ok := rType.FieldByName("Field") + require.True(t, ok) + assert.Equal(t, "Targeting field key.", field.Tag.Get("description")) + assert.Equal(t, "IRIS_SEGMENTS", field.Tag.Get("example")) + + operation, ok := rType.FieldByName("Operation") + require.True(t, ok) + assert.Equal(t, "Targeting tree operation.", operation.Tag.Get("description")) + assert.Equal(t, "children", operation.Tag.Get("example")) +} + +func TestRouter_applyParamToRequest_UsesPublicSelectorQueryNames(t *testing.T) { + router := &Router{} + values := url.Values{} + param := state.NewParameter("Limit", state.NewQueryLocation("lm_limit"), state.WithParameterSchema(state.NewSchema(reflect.TypeOf(0)))) + + baseURL, body, rpcErr := router.applyParamToRequest("http://localhost/test", values, param, 20, map[string]bool{}, map[string]bool{}, nil) + require.Nil(t, rpcErr) + assert.Equal(t, "http://localhost/test", baseURL) + assert.Nil(t, body) + assert.Equal(t, "20", values.Get("limit")) + assert.Empty(t, values.Get("lm_limit")) +} + func TestRouter_buildToolInputType_UsesBuiltReportComponentParameters(t *testing.T) { resource := view.EmptyResource() rootView := view.NewView("vendor", "VENDOR") From 0b34ec995faeaf6ac433d3ed8f9dcb112e0b84af Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 11 Apr 2026 14:24:05 -0700 Subject: [PATCH 224/279] - patched mcp mapping --- gateway/mcp.go | 83 +++++++++++++++++++++++++++- gateway/mcp_report_test.go | 108 +++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 1 deletion(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index d12d9352e..a47a5989a 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -519,7 +519,88 @@ func toolArgumentValue(parameter *state.Parameter, arguments map[string]interfac if parameter.In != nil && parameter.In.Kind == state.KindRequestBody && parameter.IsAnonymous() && parameter.Schema != nil { return anonymousBodyArgumentValue(arguments, parameter.Schema.Type()) } - return arguments[strings.Title(parameter.Name)] + for _, candidate := range toolArgumentCandidates(parameter) { + if value, ok := arguments[candidate]; ok { + return value + } + } + return nil +} + +func toolArgumentCandidates(parameter *state.Parameter) []string { + if parameter == nil { + return nil + } + var result []string + seen := map[string]bool{} + appendCandidate := func(value string) { + value = strings.TrimSpace(value) + if value == "" || seen[value] { + return + } + seen[value] = true + result = append(result, value) + } + + appendCandidate(strings.Title(parameter.Name)) + appendCandidate(parameter.Name) + appendCandidate(toPascalIdentifier(parameter.Name)) + appendCandidate(toLowerCamelIdentifier(parameter.Name)) + if public := requestParamName(parameter); public != "" { + appendCandidate(public) + appendCandidate(strings.Title(public)) + appendCandidate(toPascalIdentifier(public)) + appendCandidate(toLowerCamelIdentifier(public)) + } + return result +} + +func toPascalIdentifier(value string) string { + parts := splitIdentifierParts(value) + for i, part := range parts { + parts[i] = strings.ToUpper(part[:1]) + strings.ToLower(part[1:]) + } + return strings.Join(parts, "") +} + +func toLowerCamelIdentifier(value string) string { + pascal := toPascalIdentifier(value) + if pascal == "" { + return "" + } + return strings.ToLower(pascal[:1]) + pascal[1:] +} + +func splitIdentifierParts(value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + var result []string + var current []rune + flush := func() { + if len(current) == 0 { + return + } + result = append(result, string(current)) + current = current[:0] + } + for i, r := range value { + switch { + case r == '_' || r == '-' || r == ' ': + flush() + case i > 0 && r >= 'A' && r <= 'Z': + prev := rune(value[i-1]) + if (prev >= 'a' && prev <= 'z') || (prev >= '0' && prev <= '9') { + flush() + } + current = append(current, r) + default: + current = append(current, r) + } + } + flush() + return result } func appendAnonymousBodyFields(fields *[]reflect.StructField, unique map[string]bool, bodyType reflect.Type) { diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go index 3a3be22d9..2ca1534c1 100644 --- a/gateway/mcp_report_test.go +++ b/gateway/mcp_report_test.go @@ -146,6 +146,71 @@ func TestAnonymousBodyArgumentValue_AcceptsJSONStyleTopLevelArgumentNames(t *tes }`, string(data)) } +func TestToolArgumentValue_AcceptsSnakeCaseAliases(t *testing.T) { + parameter := state.NewParameter("audience_id", state.NewQueryLocation("audience_id"), state.WithParameterSchema(state.NewSchema(reflect.TypeOf([]int{})))) + + testCases := []struct { + name string + arguments map[string]interface{} + }{ + { + name: "title alias", + arguments: map[string]interface{}{"Audience_id": []interface{}{7180287.0}}, + }, + { + name: "raw query name", + arguments: map[string]interface{}{"audience_id": []interface{}{7180287.0}}, + }, + { + name: "pascal case alias", + arguments: map[string]interface{}{"AudienceId": []interface{}{7180287.0}}, + }, + { + name: "lower camel alias", + arguments: map[string]interface{}{"audienceId": []interface{}{7180287.0}}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + value := toolArgumentValue(parameter, testCase.arguments) + require.Equal(t, []interface{}{7180287.0}, value) + }) + } +} + +func TestToolArgumentValue_AcceptsSelectorAliases(t *testing.T) { + parameter := &state.Parameter{ + Name: "Limit", + In: state.NewQueryLocation("limit"), + Schema: state.NewSchema(reflect.TypeOf(0)), + } + + testCases := []struct { + name string + arguments map[string]interface{} + want interface{} + }{ + { + name: "exported field name", + arguments: map[string]interface{}{"Limit": 25.0}, + want: 25.0, + }, + { + name: "public query name", + arguments: map[string]interface{}{"limit": 25.0}, + want: 25.0, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + value := toolArgumentValue(parameter, testCase.arguments) + require.Equal(t, testCase.want, value) + }) + } +} + func TestRouter_addAuthTokenIfPresent_AddsBearerToken(t *testing.T) { router := &Router{} req, err := http.NewRequest(http.MethodPost, "http://localhost/v1/api/dev/vendors-grouping/report", nil) @@ -232,6 +297,49 @@ func TestRouter_mcpToolCallHandler_PassesAuthorizationToReportRoute(t *testing.T }`, actualBody) } +func TestRouter_mcpToolCallHandler_MapsComponentAndSelectorArgumentsToHTTPQuery(t *testing.T) { + component := &repository.Component{ + Path: contract.Path{Method: http.MethodGet, URI: "/v1/api/steward/metadata/ad_profile"}, + Contract: contract.Contract{ + Input: contract.Input{ + Type: state.Type{Parameters: state.Parameters{ + state.NewParameter("AudienceId", state.NewFormLocation("audience_id"), state.WithParameterSchema(state.NewSchema(reflect.TypeOf([]int{})))), + state.NewParameter("Limit", state.NewQueryLocation("lm_limit"), state.WithParameterSchema(state.NewSchema(reflect.TypeOf(0)))), + }}, + }, + }, + } + + var actualQuery string + route := &Route{ + Path: &contract.Path{Method: http.MethodGet, URI: "/v1/api/steward/metadata/ad_profile"}, + Handler: func(ctx context.Context, response http.ResponseWriter, req *http.Request) { + actualQuery = req.URL.RawQuery + response.WriteHeader(http.StatusOK) + _, _ = response.Write([]byte(`{"ok":true}`)) + }, + } + + handler := (&Router{}).mcpToolCallHandler(component, route) + result, rpcErr := handler(context.Background(), &schema.CallToolRequest{ + Params: schema.CallToolRequestParams{ + Arguments: map[string]interface{}{ + "AudienceId": []interface{}{7180287.0}, + "limit": 25.0, + }, + }, + }) + + require.Nil(t, rpcErr) + require.NotNil(t, result) + values, err := url.ParseQuery(actualQuery) + require.NoError(t, err) + assert.Equal(t, "7180287", values.Get("audience_id")) + assert.Equal(t, "25", values.Get("limit")) + assert.Empty(t, values.Get("AudienceId")) + assert.Empty(t, values.Get("lm_limit")) +} + func TestRouter_newToolHTTPRequest_SetsJSONContentTypeForBody(t *testing.T) { req, rpcErr := (&Router{}).newToolHTTPRequest(http.MethodPost, "http://localhost/v1/api/dev/vendors-grouping/report", strings.NewReader(`{"dimensions":{"accountId":true}}`)) require.Nil(t, rpcErr) From 0b6ae50d0498f6370ff0aecb80f04f24c01a7507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Filipowicz?= Date: Wed, 22 Apr 2026 16:28:38 +0200 Subject: [PATCH 225/279] fix(session): handle float selector values for limit page and offset --- service/session/selector.go | 10 +++- service/session/selector_numeric_test.go | 62 ++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 service/session/selector_numeric_test.go diff --git a/service/session/selector.go b/service/session/selector.go index bfcb0c2dd..78818d033 100644 --- a/service/session/selector.go +++ b/service/session/selector.go @@ -210,7 +210,10 @@ func (s *Session) populateContentFormat(ctx context.Context, ns *view.NamespaceV } func (s *Session) setPageQuerySelector(value interface{}, ns *view.NamespaceView) error { - page := value.(int) + page, err := toInt(value) + if err != nil { + return fmt.Errorf("invalid page value: %v", err) + } selector := s.state.Lookup(ns.View) actualLimit := selector.Limit if actualLimit == 0 { @@ -301,7 +304,10 @@ func (s *Session) setOffsetQuerySelector(value interface{}, ns *view.NamespaceVi return fmt.Errorf("can't use Offset on view %v", ns.View.Name) } selector := s.state.Lookup(ns.View) - offset := value.(int) + offset, err := toInt(value) + if err != nil { + return fmt.Errorf("invalid offset value: %v", err) + } if offset <= ns.View.Selector.Limit || ns.View.Selector.Limit == 0 { selector.Offset = offset } diff --git a/service/session/selector_numeric_test.go b/service/session/selector_numeric_test.go new file mode 100644 index 000000000..3a4d99102 --- /dev/null +++ b/service/session/selector_numeric_test.go @@ -0,0 +1,62 @@ +package session + +import ( + "context" + "reflect" + "testing" + + "github.com/viant/datly/view" + vstate "github.com/viant/datly/view/state" +) + +func TestSessionQuerySelectorNumericConversions(t *testing.T) { + ctx := context.Background() + resource := view.NewResource(nil) + trueValue := true + aView := &view.View{ + Name: "audience", + Mode: view.ModeQuery, + Selector: func() *view.Config { + cfg := view.QueryStateParameters.Clone() + cfg.Limit = 25 + cfg.Constraints = &view.Constraints{ + Limit: true, + Offset: true, + Page: &trueValue, + } + return cfg + }(), + } + aView.SetResource(resource) + aView.Template = &view.Template{Schema: vstate.NewSchema(reflect.TypeOf(struct{ Dummy int }{}))} + if err := aView.Template.Init(ctx, resource, aView); err != nil { + t.Fatalf("failed to init template: %v", err) + } + if err := aView.Selector.Init(ctx, resource, aView); err != nil { + t.Fatalf("failed to init selector: %v", err) + } + + sess := New(aView) + ns := &view.NamespaceView{View: aView} + + if err := sess.setLimitQuerySelector(float64(1), ns); err != nil { + t.Fatalf("setLimitQuerySelector() error: %v", err) + } + if err := sess.setOffsetQuerySelector(float64(1), ns); err != nil { + t.Fatalf("setOffsetQuerySelector() error: %v", err) + } + if err := sess.setPageQuerySelector(float64(2), ns); err != nil { + t.Fatalf("setPageQuerySelector() error: %v", err) + } + + selector := sess.State().Lookup(aView) + if selector.Limit != 1 { + t.Fatalf("expected Limit=1, got %d", selector.Limit) + } + if selector.Offset != 1 { + t.Fatalf("expected Offset=1, got %d", selector.Offset) + } + if selector.Page != 2 { + t.Fatalf("expected Page=2, got %d", selector.Page) + } +} From 6a1892ce5656461c6b0ce064d2583a7c012b0cc3 Mon Sep 17 00:00:00 2001 From: vc42 Date: Wed, 22 Apr 2026 12:28:21 -0400 Subject: [PATCH 226/279] concurrent map fix in seedFormFromMultipart --- view/state/kind/locator/form.go | 53 +++++++++++++++++---------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/view/state/kind/locator/form.go b/view/state/kind/locator/form.go index 9b37fdb65..4c4ee643a 100644 --- a/view/state/kind/locator/form.go +++ b/view/state/kind/locator/form.go @@ -71,23 +71,18 @@ func (r *Form) Value(ctx context.Context, rType reflect.Type, name string) (inte } return nil, false, nil } - // Non-multipart: parse form/query values and preserve repeated values. + // Non-multipart: use standard FormValue fallback r.form.Mutex().Lock() defer r.form.Mutex().Unlock() - if err := r.request.ParseForm(); err != nil { - return nil, false, err - } - values, ok := r.request.Form[name] - if !ok { - return nil, false, nil - } - if len(values) > 1 { - return values, true, nil - } - if len(values) == 1 { - return values[0], true, nil + value := r.request.FormValue(name) + if value == "" { + if r.request.Form == nil { + return nil, false, nil + } + _, ok := r.request.Form[name] + return "", ok, nil } - return "", true, nil + return value, true, nil } if len(values) > 1 { return values, true, nil @@ -102,18 +97,26 @@ func NewForm(opts ...Option) (kind.Locator, error) { return ret, nil } -// seedFormFromMultipart parses multipart/form-data and copies values into shared maps. -// Mutex is required because multiple Form locators (one per parameter) can call this -// concurrently on the same request. Uses form.Values directly instead of form.Set to -// avoid deadlock (form.Set locks the same mutex). +// seedFormFromMultipart parses multipart/form-data (if needed) and copies textual values to the shared form func (r *Form) seedFormFromMultipart() { if r.request == nil || r.form == nil { return } + + // Session parameter binding runs in parallel. Multipart seeding mutates the + // shared form and request maps, so serialize that work under the form mutex. + mu := r.form.Mutex() + mu.Lock() + defer mu.Unlock() + if r.request.MultipartForm == nil && len(r.form.Values) == 0 { + // Only ParseMultipartForm for form-data; other multipart types aren't + // supported by ParseMultipartForm. If the shared form already has + // values, treat it as authoritative and avoid parsing. ct := r.request.Header.Get("Content-Type") if ct != "" { if mediaType, _, err := mime.ParseMediaType(ct); err == nil && shared.IsFormData(mediaType) { + // Use the same default memory threshold as Body locator const maxMultipartMemory = 32 << 20 // 32 MiB _ = r.request.ParseMultipartForm(maxMultipartMemory) } @@ -122,18 +125,18 @@ func (r *Form) seedFormFromMultipart() { if r.request.MultipartForm == nil { return } - // BUG FIX (concurrent map writes): - mu := r.form.Mutex() - mu.Lock() - defer mu.Unlock() - if r.request.Form == nil { + if r.form.Values == nil { + r.form.Values = url.Values{} + } + if len(r.request.Form) == 0 { r.request.Form = url.Values{} } for k, vs := range r.request.MultipartForm.Value { if len(vs) == 0 { continue } - r.form.Values[k] = vs - r.request.Form[k] = vs + seeded := append([]string(nil), vs...) + r.form.Values[k] = seeded + r.request.Form[k] = append([]string(nil), seeded...) } } From 06c97eb8c73618e3f00a389f91bf93864a9f029e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Filipowicz?= Date: Wed, 22 Apr 2026 18:33:13 +0200 Subject: [PATCH 227/279] updated versjion --- Version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Version b/Version index fcc9d59a4..9a5b249c7 100644 --- a/Version +++ b/Version @@ -1 +1 @@ -v0.21.0 \ No newline at end of file +v0.38.0 \ No newline at end of file From 704013dc252be9b164a055404d4ffe61ad1395f7 Mon Sep 17 00:00:00 2001 From: vc42 Date: Wed, 22 Apr 2026 12:48:42 -0400 Subject: [PATCH 228/279] concurrent map fix in seedFormFromMultipart 2 --- view/state/kind/locator/form.go | 34 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/view/state/kind/locator/form.go b/view/state/kind/locator/form.go index 4c4ee643a..35c7695f7 100644 --- a/view/state/kind/locator/form.go +++ b/view/state/kind/locator/form.go @@ -5,7 +5,6 @@ import ( "mime" "mime/multipart" "net/http" - "net/url" "reflect" "sync" @@ -71,18 +70,23 @@ func (r *Form) Value(ctx context.Context, rType reflect.Type, name string) (inte } return nil, false, nil } - // Non-multipart: use standard FormValue fallback + // Non-multipart: parse form/query values and preserve repeated values. r.form.Mutex().Lock() defer r.form.Mutex().Unlock() - value := r.request.FormValue(name) - if value == "" { - if r.request.Form == nil { - return nil, false, nil - } - _, ok := r.request.Form[name] - return "", ok, nil + if err := r.request.ParseForm(); err != nil { + return nil, false, err + } + values, ok = r.request.Form[name] + if !ok { + return nil, false, nil + } + if len(values) > 1 { + return values, true, nil } - return value, true, nil + if len(values) == 1 { + return values[0], true, nil + } + return "", true, nil } if len(values) > 1 { return values, true, nil @@ -125,18 +129,10 @@ func (r *Form) seedFormFromMultipart() { if r.request.MultipartForm == nil { return } - if r.form.Values == nil { - r.form.Values = url.Values{} - } - if len(r.request.Form) == 0 { - r.request.Form = url.Values{} - } for k, vs := range r.request.MultipartForm.Value { if len(vs) == 0 { continue } - seeded := append([]string(nil), vs...) - r.form.Values[k] = seeded - r.request.Form[k] = append([]string(nil), seeded...) + r.form.Set(k, append([]string(nil), vs...)...) } } From c3ef5e303f08ee369edc0654d53079215b20ef1d Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 22 Apr 2026 15:35:30 -0700 Subject: [PATCH 229/279] - patched mcp mapping --- internal/translator/service.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/translator/service.go b/internal/translator/service.go index c4376fa6e..f858a1e53 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -32,7 +32,6 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/xreflect" "golang.org/x/mod/modfile" - "gopkg.in/yaml.v3" ) type Service struct { @@ -194,8 +193,7 @@ func (s *Service) buildExecutorView(ctx context.Context, resource *Resource, DSQ } func (s *Service) translateReaderDSQL(ctx context.Context, resource *Resource, dSQL string) error { - parseSQL := resource.State.Expand(dSQL) - aQuery, err := sqlparser.ParseQuery(parseSQL, parser.OnVeltyExpression()) + aQuery, err := sqlparser.ParseQuery(dSQL, parser.OnVeltyExpression()) if err != nil { return err } From e1883b2edff926077d8eb21cd60ae9cb9f1c5ba2 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 22 Apr 2026 15:40:31 -0700 Subject: [PATCH 230/279] - patched mcp mapping --- internal/translator/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/translator/service.go b/internal/translator/service.go index f858a1e53..f77678fdf 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -32,6 +32,7 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/xreflect" "golang.org/x/mod/modfile" + "gopkg.in/yaml.v3" ) type Service struct { From ab29a244b6e497bf50f272e72971cf5f2a0aa374 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 22 Apr 2026 19:52:46 -0700 Subject: [PATCH 231/279] - patched mcp mapping --- view/sql.go | 40 +++++++++++++++++++++++++++++++++++++++- view/sql_test.go | 9 +++++++++ view/summary.go | 10 +++++++++- view/view.go | 15 ++++++++++++++- 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/view/sql.go b/view/sql.go index 9d277b1cc..ad5276829 100644 --- a/view/sql.go +++ b/view/sql.go @@ -12,6 +12,7 @@ import ( "github.com/viant/sqlx/io/config" rdata "github.com/viant/toolbox/data" "reflect" + "regexp" "strings" ) @@ -108,9 +109,17 @@ func columnsMetadata(ctx context.Context, db *sql.DB, v *View, columns []io.Colu func detectColumnsSQL(evaluation *TemplateEvaluation, v *View) (string, []interface{}, error) { SQL := ensureSelectStatement(evaluation, v) + if evaluation.Expander != nil { + SQL = neutralizePredicateBuilderForDiscovery(SQL) + SQL = stripVeltyConditionalsForDiscovery(SQL) + SQL = stripDanglingSelectorsForDiscovery(SQL) + SQL = ExpandWithFalseCondition(SQL) + SQL = strings.ReplaceAll(SQL, keywords.Pagination, "") + return SQL, evaluation.Args, nil + } + var placeholders []interface{} var err error - if evaluation.Expander != nil { SQL, err = v.Expand(&placeholders, SQL, &Statelet{}, CriteriaParam{}, &BatchData{}, NewMockSanitizer()) if err != nil { @@ -131,6 +140,35 @@ func detectColumnsSQL(evaluation *TemplateEvaluation, v *View) (string, []interf return SQL, placeholders, nil } +var predicateBuilderBlock = regexp.MustCompile(`(?s)\$\{predicate\.Builder\(\).*?Build\("([A-Z]+)"\)\}`) +var veltyIfBlock = regexp.MustCompile(`(?s)#if\([^\n]*\)\s*.*?#end`) +var danglingSelectorLine = regexp.MustCompile(`(?m)^[ \t]*\$[A-Za-z0-9_.]+[ \t]*\n?`) + +func neutralizePredicateBuilderForDiscovery(SQL string) string { + return predicateBuilderBlock.ReplaceAllStringFunc(SQL, func(fragment string) string { + match := predicateBuilderBlock.FindStringSubmatch(fragment) + if len(match) != 2 { + return " AND 1=0 " + } + switch match[1] { + case "WHERE": + return " WHERE 1=0 " + case "HAVING": + return " HAVING 1=0 " + default: + return " AND 1=0 " + } + }) +} + +func stripVeltyConditionalsForDiscovery(SQL string) string { + return veltyIfBlock.ReplaceAllString(SQL, "") +} + +func stripDanglingSelectorsForDiscovery(SQL string) string { + return danglingSelectorLine.ReplaceAllString(SQL, "") +} + func ensureSelectStatement(evaluation *TemplateEvaluation, v *View) string { source := evaluation.SQL diff --git a/view/sql_test.go b/view/sql_test.go index d25e939d8..f343be31f 100644 --- a/view/sql_test.go +++ b/view/sql_test.go @@ -2,6 +2,8 @@ package view import ( "testing" + + "github.com/stretchr/testify/require" ) func TestDetectColumnsSQL(t *testing.T) { @@ -66,3 +68,10 @@ GROUP BY 1`, //assert.Equal(t, testcase.sql, sql, testcase.description) } } + +func TestNeutralizePredicateBuilderForDiscovery(t *testing.T) { + input := `SELECT * FROM FOO t WHERE 1=1 ${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("AND")}` + actual := neutralizePredicateBuilderForDiscovery(input) + require.NotContains(t, actual, `${predicate.Builder()`) + require.Contains(t, actual, `AND 1=0`) +} diff --git a/view/summary.go b/view/summary.go index 10df9a857..dda4601b6 100644 --- a/view/summary.go +++ b/view/summary.go @@ -362,8 +362,16 @@ func (m *TemplateSummary) prepareSQL(ctx context.Context, owner *Template) (stri stateValue := owner.stateType.NewState() viewParam := AsViewParam(owner._view, nil, nil) + if viewParam.Limit == 0 { + viewParam.Limit = 1 + } + + evaluator, err := NewEvaluator(owner.Parameters, owner.stateType, owner.Source, owner._view._resource.LookupType(), nil) + if err != nil { + return "", nil, err + } - state, err := Evaluate(ctx, owner.sqlEvaluator, expand.WithParameterState(stateValue), expand.WithViewParam(viewParam)) + state, err := Evaluate(ctx, evaluator, expand.WithParameterState(stateValue), expand.WithViewParam(viewParam)) if err != nil { return "", nil, err } diff --git a/view/view.go b/view/view.go index f09073e6b..86f13b389 100644 --- a/view/view.go +++ b/view/view.go @@ -973,6 +973,16 @@ func (v *View) ensureColumns(ctx context.Context, resource *Resource) error { } func (v *View) detectColumns(ctx context.Context, resource *Resource) error { + defer func() { + if r := recover(); r != nil { + panic(fmt.Errorf("detectColumns panic for view=%s ref=%s table=%s source=%s templateURL=%s: %v", v.Name, v.Ref, v.Table, v.Source(), func() string { + if v.Template == nil { + return "" + } + return v.Template.SourceURL + }(), r)) + } + }() SQL := v.Source() var aState state.Parameters if v.Template != nil { @@ -990,10 +1000,13 @@ func (v *View) detectColumns(ctx context.Context, resource *Resource) error { options = append(options, expand2.WithViewParam(&expand2.ViewContext{ParentValues: []interface{}{0}, DataUnit: &expand2.DataUnit{}})) } query, err := v.BuildParametrizedSQL(aState, resource.TypeRegistry(), SQL, bindingArguments, options...) - v.Logger.ColumnsDetection(query.Query, v.Source()) if err != nil { return fmt.Errorf("failed to build parameterized query: %v due to %w", SQL, err) } + if query == nil { + return fmt.Errorf("failed to build parameterized query: %v produced nil query", SQL) + } + v.Logger.ColumnsDetection(query.Query, v.Source()) db, err := v.Connector.DB() if err != nil { return err From f9df71fe7e1f1780a74e71ae909ee3173d2870d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Filipowicz?= Date: Thu, 23 Apr 2026 16:59:59 +0200 Subject: [PATCH 232/279] Restore sanitized translator templates for SQL fragment params --- internal/translator/view.go | 14 ++- internal/translator/view_template_test.go | 127 ++++++++++++++++++++++ 2 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 internal/translator/view_template_test.go diff --git a/internal/translator/view.go b/internal/translator/view.go index 69d62b802..9408acaf3 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -251,8 +251,18 @@ func (v *View) buildTemplate(namespace *Viewlet, rule *Rule) { isRoot := rule.Root == v.Name resource := namespace.Resource v.EnsureTemplate() - v.Template.Source = namespace.SQL - v.Template.Parameters = v.matchParameters(namespace.SQL, resource.State, isRoot) + // Emit sanitized SQL as the runtime template so SQL fragments stay inline; + // matching still uses raw SQL to preserve the post-5e41c4ee root/predicate detection. + sourceSQL := namespace.SanitizedSQL + if sourceSQL == "" { + sourceSQL = namespace.SQL + } + matchSQL := namespace.SQL + if matchSQL == "" { + matchSQL = sourceSQL + } + v.Template.Source = sourceSQL + v.Template.Parameters = v.matchParameters(matchSQL, resource.State, isRoot) } // matchParameters matches parameter used by SQL, and add explicit parameter for root view diff --git a/internal/translator/view_template_test.go b/internal/translator/view_template_test.go new file mode 100644 index 000000000..87d9f54de --- /dev/null +++ b/internal/translator/view_template_test.go @@ -0,0 +1,127 @@ +package translator + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/viant/datly/internal/inference" + "github.com/viant/datly/view/extension" + "github.com/viant/datly/view/state" +) + +func TestViewBuildTemplate_UsesSanitizedSQLAndRetainsPredicateParameters(t *testing.T) { + // Regression coverage for the translator change that must keep WHERE/HAVING + // predicate builders discoverable without regressing SQL-fragment placeholders to '?' binds. + rawSQL := `SELECT opaque_root.* +FROM ($table) opaque_root +${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")} +GROUP BY opaque_root.id +${predicate.Builder().CombineOr($predicate.FilterGroup(1, "HAVING")).Build("HAVING")} +WHERE opaque_root.dstamp BETWEEN $From AND $To` + + sanitizedSQL := `SELECT opaque_root.* +FROM ($Unsafe.table) opaque_root +${predicate.Builder().CombineOr($predicate.FilterGroup(0, "AND")).Build("WHERE")} +GROUP BY opaque_root.id +${predicate.Builder().CombineOr($predicate.FilterGroup(1, "HAVING")).Build("HAVING")} +WHERE opaque_root.dstamp BETWEEN $criteria.AppendBinding($Unsafe.From) AND $criteria.AppendBinding($Unsafe.To)` + + resourceState := inference.State{ + inference.NewConstParameter("table", "ci_event.audience_event_v1"), + { + Parameter: state.Parameter{ + Name: "From", + In: state.NewQueryLocation("from"), + Schema: state.NewSchema(reflect.TypeOf("")), + }, + }, + { + Parameter: state.Parameter{ + Name: "To", + In: state.NewQueryLocation("to"), + Schema: state.NewSchema(reflect.TypeOf("")), + }, + }, + { + Parameter: state.Parameter{ + Name: "Cutoff", + In: state.NewQueryLocation("cutoff"), + Schema: state.NewSchema(reflect.TypeOf("")), + Predicates: []*extension.PredicateConfig{{Group: 0, Name: "greater_or_equal"}}, + }, + }, + { + Parameter: state.Parameter{ + Name: "Threshold", + In: state.NewQueryLocation("threshold"), + Schema: state.NewSchema(reflect.TypeOf(0)), + Predicates: []*extension.PredicateConfig{{Group: 1, Name: "expr"}}, + }, + }, + } + + namespace := &Viewlet{ + Name: "opaque_root", + SQL: rawSQL, + SanitizedSQL: sanitizedSQL, + Resource: &Resource{ + State: resourceState, + }, + } + rule := &Rule{Root: "opaque_root"} + subject := &View{ + Namespace: "opaque_root", + View: View{}.View, + } + subject.Name = "opaque_root" + + subject.buildTemplate(namespace, rule) + + assert.Equal(t, sanitizedSQL, subject.Template.Source) + assert.ElementsMatch(t, []string{"table", "From", "To", "Cutoff", "Threshold"}, templateParameterNames(subject.Template.Parameters)) +} + +func TestViewBuildTemplate_FallsBackToRawSQLWhenSanitizedSQLMissing(t *testing.T) { + rawSQL := `SELECT * FROM $table WHERE created_at >= $From` + + namespace := &Viewlet{ + Name: "vendor", + SQL: rawSQL, + Resource: &Resource{ + State: inference.State{ + inference.NewConstParameter("table", "ci_ads.vendor"), + { + Parameter: state.Parameter{ + Name: "From", + In: state.NewQueryLocation("from"), + Schema: state.NewSchema(reflect.TypeOf("")), + }, + }, + }, + }, + } + rule := &Rule{Root: "vendor"} + subject := &View{ + Namespace: "vendor", + View: View{}.View, + } + subject.Name = "vendor" + + subject.buildTemplate(namespace, rule) + + assert.Equal(t, rawSQL, subject.Template.Source) + assert.ElementsMatch(t, []string{"table", "From"}, templateParameterNames(subject.Template.Parameters)) +} + +func templateParameterNames(params []*state.Parameter) []string { + result := make([]string, 0, len(params)) + for _, param := range params { + name := param.Name + if name == "" { + name = param.Ref + } + result = append(result, name) + } + return result +} From 72d783ff5202e8b61e53f3ea5ca351bed01e1c67 Mon Sep 17 00:00:00 2001 From: vc42 Date: Thu, 23 Apr 2026 14:21:34 -0400 Subject: [PATCH 233/279] mutex corruption fix --- service/reader/service.go | 46 +++++---------------------------- view/state.go | 41 +++++++++++++++++++++++++++++ view/state/kind/locator/form.go | 6 ++++- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/service/reader/service.go b/service/reader/service.go index 560749775..7cfaa7903 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -4,9 +4,7 @@ import ( "context" "database/sql" "fmt" - "os" "reflect" - "runtime/debug" "strings" "sync" "sync/atomic" @@ -37,14 +35,6 @@ type Service struct { // ReadInto reads Data into provided destination, * dDest` is required. It has to be a pointer to `interface{}` or pointer to slice of `T` or `*T` func (s *Service) ReadInto(ctx context.Context, dest interface{}, aView *view.View, opts ...Option) error { - if os.Getenv("DATLY_DEBUG_READER") == "1" { - defer func() { - if r := recover(); r != nil { - fmt.Printf("[READER DEBUG] panic view=%s dest=%T err=%v\n%s\n", aView.Name, dest, r, debug.Stack()) - panic(r) - } - }() - } session, err := NewSession(dest, aView, opts...) if err != nil { return err @@ -287,10 +277,9 @@ func (s *Service) readObjects(ctx context.Context, session *Session, batchData * } func (s *Service) querySummary(ctx context.Context, session *Session, aView *view.View, statelet *view.Statelet, batchDataCopy *view.BatchData, collector *view.Collector, parentViewMetaParam *expand.ViewContext) (*response.SQLExecution, error) { - selectorDeref := *statelet - selectorDeref.Fields = []string{} - selectorDeref.Columns = []string{} - selector := &selectorDeref + selector := statelet.CloneForSummary() + selector.Fields = []string{} + selector.Columns = []string{} var indexed *cache.ParmetrizedQuery var cacheStats *cache.Stats @@ -430,9 +419,6 @@ func (s *Service) BuildCriteria(ctx context.Context, value interface{}, options } func (s *Service) queryInBatches(ctx context.Context, session *Session, aView *view.View, collector *view.Collector, visitor view.VisitorFn, info *response.SQLExecutions, batchData *view.BatchData, selector *view.Statelet) error { - if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { - fmt.Printf("[QUERY DEBUG] queryInBatches view=%s selectorTemplateNil=%v batchValues=%d\n", aView.Name, selector == nil || selector.Template == nil, len(batchData.ValuesBatch)) - } wg := &sync.WaitGroup{} db, err := aView.Db() if err != nil { @@ -471,19 +457,10 @@ func (s *Service) queryObjects(ctx context.Context, session *Session, aView *vie return s.queryWithPartitions(ctx, session, aView, selector, batchData, db, collector, visitor, partitioned) } readData := 0 - if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { - fmt.Printf("[QUERY DEBUG] queryObjects view=%s schema=%v slice=%v collectorView=%s\n", aView.Name, aView.Schema.Type(), aView.Schema.SliceType(), collector.View().Name) - } parametrizedSQL, columnInMatcher, err := s.buildParametrizedSQL(ctx, aView, selector, batchData, collector, session, nil) if err != nil { - if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { - fmt.Printf("[QUERY DEBUG] buildParametrizedSQL error view=%s err=%v\n", aView.Name, err) - } return nil, err } - if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { - fmt.Printf("[QUERY DEBUG] builtSQL view=%s sql=%s args=%#v\n", aView.Name, parametrizedSQL.SQL, parametrizedSQL.Args) - } var parentProvider func(value interface{}) (interface{}, error) handler := func(row interface{}) error { @@ -505,7 +482,8 @@ func (s *Service) queryObjects(ctx context.Context, session *Session, aView *vie } return visitor(row) } - return s.queryWithHandler(ctx, session, aView, collector, columnInMatcher, parametrizedSQL, db, handler, &readData) + execs, err := s.queryWithHandler(ctx, session, aView, collector, columnInMatcher, parametrizedSQL, db, handler, &readData) + return execs, err } func (s *Service) getParentContext(ctx context.Context, row interface{}, collector *view.Collector, parentProvider func(value interface{}) (interface{}, error)) (context.Context, error) { @@ -548,9 +526,6 @@ func (s *Service) queryWithHandler(ctx context.Context, session *Session, aView stats, onDone := NewExecutionInfo(parametrizedSQL, cacheStats, collector) defer onDone() - if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { - fmt.Printf("[QUERY HANDLER] view=%s sql=%s args=%#v\n", aView.Name, parametrizedSQL.SQL, parametrizedSQL.Args) - } if session.DryRun { return []*response.SQLExecution{stats}, nil } @@ -580,16 +555,7 @@ BEGIN: } _ = stmt.Close() }() - debugHandler := handler - if os.Getenv("DATLY_DEBUG_QUERY_HANDLER") == "1" { - debugHandler = func(row interface{}) error { - fmt.Printf("[QUERY HANDLER] view=%s before unwrap row=%T readData=%d\n", aView.Name, row, *readData) - err := handler(row) - fmt.Printf("[QUERY HANDLER] view=%s after handler row=%T readData=%d err=%v\n", aView.Name, row, *readData, err) - return err - } - } - err = reader.QueryAll(ctx, debugHandler, parametrizedSQL.Args...) + err = reader.QueryAll(ctx, handler, parametrizedSQL.Args...) isInvalidConnection = err != nil && strings.Contains(err.Error(), "invalid connection") if isInvalidConnection && atomic.AddUint32(&retires, 1) < 3 { diff --git a/view/state.go b/view/state.go index a187cd4b5..370b9d386 100644 --- a/view/state.go +++ b/view/state.go @@ -151,3 +151,44 @@ func (s *State) Init(aView *View) { func (s *Statelet) IgnoreRead() { s.Ignore = true } + +// CloneForSummary creates a lock-safe copy of Statelet state for summary/meta work. +// It intentionally does not copy mutex or lock-owner bookkeeping. +func (s *Statelet) CloneForSummary() *Statelet { + if s == nil { + return NewStatelet() + } + + ret := &Statelet{ + DatabaseFormat: s.DatabaseFormat, + OutputFormat: s.OutputFormat, + Template: s.Template, + QuerySelector: s.QuerySelector, + QuerySettings: s.QuerySettings, + initialized: s.initialized, + result: s.result, + Ignore: s.Ignore, + } + + if s._columnNames != nil { + ret._columnNames = make(map[string]bool, len(s._columnNames)) + for k, v := range s._columnNames { + ret._columnNames[k] = v + } + } else { + ret._columnNames = map[string]bool{} + } + + if len(s.Filters) > 0 { + ret.Filters = append(predicate.Filters(nil), s.Filters...) + } + + if len(s.Fields) > 0 { + ret.Fields = append([]string(nil), s.Fields...) + } + if len(s.Columns) > 0 { + ret.Columns = append([]string(nil), s.Columns...) + } + + return ret +} diff --git a/view/state/kind/locator/form.go b/view/state/kind/locator/form.go index 35c7695f7..b4fee35bc 100644 --- a/view/state/kind/locator/form.go +++ b/view/state/kind/locator/form.go @@ -5,6 +5,7 @@ import ( "mime" "mime/multipart" "net/http" + "net/url" "reflect" "sync" @@ -129,10 +130,13 @@ func (r *Form) seedFormFromMultipart() { if r.request.MultipartForm == nil { return } + if r.form.Values == nil { + r.form.Values = url.Values{} + } for k, vs := range r.request.MultipartForm.Value { if len(vs) == 0 { continue } - r.form.Set(k, append([]string(nil), vs...)...) + r.form.Values[k] = append([]string(nil), vs...) } } From 080a6fb5ca8cb4b8efab7a1aaa7b7de3dc1a45ef Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 29 Apr 2026 07:27:27 -0700 Subject: [PATCH 234/279] - added orig query parameters to cube request --- repository/report_handler.go | 28 +++++++++++++++++----------- repository/report_handler_test.go | 4 ++-- repository/report_runtime.go | 6 +++--- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/repository/report_handler.go b/repository/report_handler.go index 28ac7ae2f..2145ef2df 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -17,7 +17,7 @@ import ( xdhttp "github.com/viant/xdatly/handler/http" ) -type reportHandler struct { +type cubeHandler struct { Dispatcher contract.Dispatcher Path *contract.Path Metadata *ReportMetadata @@ -25,7 +25,7 @@ type reportHandler struct { BodyType reflect.Type } -func (r *reportHandler) Exec(ctx context.Context, session xhandler.Session) (interface{}, error) { +func (r *cubeHandler) Exec(ctx context.Context, session xhandler.Session) (interface{}, error) { if r == nil || r.Dispatcher == nil || r.Path == nil || r.Metadata == nil || r.Original == nil { return nil, fmt.Errorf("report handler was not initialized") } @@ -37,14 +37,14 @@ func (r *reportHandler) Exec(ctx context.Context, session xhandler.Session) (int if err != nil { return nil, err } - query, err := r.buildQuery(input) + query, err := r.buildQuery(input, request) if err != nil { return nil, err } internalReq := request.Clone(ctx) internalReq.Method = r.Path.Method internalReq.URL = cloneURL(request.URL) - internalReq.URL.Path = strings.TrimSuffix(request.URL.Path, "/report") + internalReq.URL.Path = strings.TrimSuffix(request.URL.Path, "/cube") internalReq.URL.RawPath = internalReq.URL.Path internalReq.URL.RawQuery = query.Encode() internalReq.RequestURI = internalReq.URL.RequestURI() @@ -52,7 +52,7 @@ func (r *reportHandler) Exec(ctx context.Context, session xhandler.Session) (int return nil, session.Http().Redirect(ctx, redirect, internalReq) } -func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) (interface{}, error) { +func (r *cubeHandler) reportInput(ctx context.Context, request *http.Request) (interface{}, error) { input := ctx.Value(xhandler.InputKey) if request != nil && request.Body != nil && r.BodyType != nil { payload, err := io.ReadAll(request.Body) @@ -80,13 +80,19 @@ func (r *reportHandler) reportInput(ctx context.Context, request *http.Request) return input, nil } -func (r *reportHandler) buildQuery(input interface{}) (url.Values, error) { +func (r *cubeHandler) buildQuery(input interface{}, request *http.Request) (url.Values, error) { root := indirectValue(reflect.ValueOf(input)) if !root.IsValid() || root.Kind() != reflect.Struct { return nil, fmt.Errorf("unsupported report input type %T", input) } + + _ = request.ParseForm() + root = bodyRoot(root, r.Metadata.BodyFieldName) query := url.Values{} + for k, v := range request.Form { + query[k] = v + } fields, err := r.collectSelections(root, r.Metadata.Dimensions, r.Metadata.Measures) if err != nil { return nil, err @@ -112,14 +118,14 @@ func (r *reportHandler) buildQuery(input interface{}) (url.Values, error) { return query, nil } -func (r *reportHandler) selectorName(parameter *state.Parameter, fallback string) string { +func (r *cubeHandler) selectorName(parameter *state.Parameter, fallback string) string { if parameter != nil && parameter.In != nil && strings.TrimSpace(parameter.In.Name) != "" { return parameter.In.Name } return fallback } -func (r *reportHandler) collectSelections(root reflect.Value, groups ...[]*ReportField) ([]string, error) { +func (r *cubeHandler) collectSelections(root reflect.Value, groups ...[]*ReportField) ([]string, error) { var result []string for _, group := range groups { for _, field := range group { @@ -139,7 +145,7 @@ func (r *reportHandler) collectSelections(root reflect.Value, groups ...[]*Repor return result, nil } -func (r *reportHandler) collectFilters(root reflect.Value, query url.Values) error { +func (r *cubeHandler) collectFilters(root reflect.Value, query url.Values) error { filters := fieldByName(root, r.Metadata.FiltersKey) if !filters.IsValid() { return nil @@ -158,7 +164,7 @@ func (r *reportHandler) collectFilters(root reflect.Value, query url.Values) err return nil } -func (r *reportHandler) collectStrings(root reflect.Value, fieldName string, query url.Values, key string) error { +func (r *cubeHandler) collectStrings(root reflect.Value, fieldName string, query url.Values, key string) error { if fieldName == "" { return nil } @@ -183,7 +189,7 @@ func (r *reportHandler) collectStrings(root reflect.Value, fieldName string, que return nil } -func (r *reportHandler) collectInts(root reflect.Value, fieldName string, query url.Values, key string) error { +func (r *cubeHandler) collectInts(root reflect.Value, fieldName string, query url.Values, key string) error { if fieldName == "" { return nil } diff --git a/repository/report_handler_test.go b/repository/report_handler_test.go index 43b86214d..58ee75703 100644 --- a/repository/report_handler_test.go +++ b/repository/report_handler_test.go @@ -116,8 +116,8 @@ func (s *reportTestSession) Http() xdhttp.Http { return s.http } func (s *reportTestSession) Auth() xdauth.Auth { return nil } func (s *reportTestSession) Logger() xdlogger.Logger { return s.logger } -func testReportHandler() *reportHandler { - return &reportHandler{ +func testReportHandler() *cubeHandler { + return &cubeHandler{ Dispatcher: &captureDispatcher{}, Path: &contract.Path{Method: http.MethodGet, URI: "/v1/api/vendors"}, Metadata: &ReportMetadata{ diff --git a/repository/report_runtime.go b/repository/report_runtime.go index 720f46e14..e02f36ae1 100644 --- a/repository/report_runtime.go +++ b/repository/report_runtime.go @@ -75,10 +75,10 @@ func buildReportArtifacts(ctx context.Context, dispatcher contract.Dispatcher, o if err != nil { return nil, nil, err } - reportURI := strings.TrimSuffix(original.URI, "/") + "/report" + reportURI := strings.TrimSuffix(original.URI, "/") + "/cube" ret := *original ret.Path = contract.Path{Method: http.MethodPost, URI: reportURI} - ret.Handler = rephandler.NewHandler(&reportHandler{ + ret.Handler = rephandler.NewHandler(&cubeHandler{ Dispatcher: dispatcher, Path: &original.Path, Metadata: metadata, @@ -139,7 +139,7 @@ func buildReportPath(routePath *path.Path) *path.Path { pathCopy := *routePath pathCopy.Path = contract.Path{ Method: http.MethodPost, - URI: strings.TrimSuffix(routePath.URI, "/") + "/report", + URI: strings.TrimSuffix(routePath.URI, "/") + "/cube", } pathCopy.MCPTool = reportPathMCPToolEnabled(routePath.Report) pathCopy.MCPResource = false From c5ebca76ef1412ef55dd6ae0bb334acb9f15bf4a Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 29 Apr 2026 11:38:32 -0700 Subject: [PATCH 235/279] - added orig query parameters to cube request --- repository/component.go | 4 + repository/report_handler.go | 211 ++++++++++++++++++++++++++++-- repository/report_handler_test.go | 123 ++++++++++++++++- 3 files changed, 327 insertions(+), 11 deletions(-) diff --git a/repository/component.go b/repository/component.go index b78db0e81..0aba879c7 100644 --- a/repository/component.go +++ b/repository/component.go @@ -442,6 +442,10 @@ func (c *Component) UnmarshalFor(opts ...UnmarshalOption) shared.Unmarshal { if c != nil && c.Report != nil && c.Report.Enabled && c.Handler != nil { if parameter := c.Input.Type.AnonymousParameters(); parameter != nil && parameter.In != nil && parameter.In.Kind == state.KindRequestBody { return func(data []byte, dest interface{}) error { + + if c.Content.Marshaller.JSON.RuntimeUnmarshaller != nil { + return c.Content.Marshaller.JSON.RuntimeUnmarshaller.Unmarshal(data, dest) + } return stdjson.Unmarshal(data, dest) } } diff --git a/repository/report_handler.go b/repository/report_handler.go index 2145ef2df..57ccf291f 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -2,7 +2,7 @@ package repository import ( "context" - "encoding/json" + "encoding" "fmt" "io" "net/http" @@ -10,6 +10,11 @@ import ( "reflect" "strconv" "strings" + "time" + + "github.com/viant/structology/encoding/json" + tagformat "github.com/viant/tagly/format" + ftime "github.com/viant/tagly/format/time" "github.com/viant/datly/repository/contract" "github.com/viant/datly/view/state" @@ -17,6 +22,12 @@ import ( xdhttp "github.com/viant/xdatly/handler/http" ) +var ( + textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() + stringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem() + timeType = reflect.TypeOf(time.Time{}) +) + type cubeHandler struct { Dispatcher contract.Dispatcher Path *contract.Path @@ -44,6 +55,7 @@ func (r *cubeHandler) Exec(ctx context.Context, session xhandler.Session) (inter internalReq := request.Clone(ctx) internalReq.Method = r.Path.Method internalReq.URL = cloneURL(request.URL) + internalReq.Form = query internalReq.URL.Path = strings.TrimSuffix(request.URL.Path, "/cube") internalReq.URL.RawPath = internalReq.URL.Path internalReq.URL.RawQuery = query.Encode() @@ -152,14 +164,20 @@ func (r *cubeHandler) collectFilters(root reflect.Value, query url.Values) error } filters = indirectValue(filters) for _, filter := range r.Metadata.Filters { - value := fieldByName(filters, filter.FieldName) - if !value.IsValid() || isEmptyValue(value) { + value, field, ok := fieldByNameDetails(filters, filter.FieldName) + if !ok || shouldOmitFilterValue(value) { continue } if filter.Parameter == nil || filter.Parameter.In == nil { continue } - appendQueryValue(query, filter.Parameter.In.Name, value) + formatTag, err := resolveFilterFormatTag(filter, field) + if err != nil { + return err + } + if err := appendQueryValue(query, filter.Parameter.In.Name, value, filter, formatTag); err != nil { + return err + } } return nil } @@ -208,26 +226,70 @@ func (r *cubeHandler) collectInts(root reflect.Value, fieldName string, query ur return nil } -func appendQueryValue(query url.Values, key string, value reflect.Value) { - value = indirectValue(value) +func appendQueryValue(query url.Values, key string, value reflect.Value, filter *ReportFilter, formatTag *tagformat.Tag) error { + for value.IsValid() && value.Kind() == reflect.Interface { + if value.IsNil() { + return nil + } + value = value.Elem() + } + if !value.IsValid() { + return nil + } + if value.Kind() == reflect.Ptr { + if value.IsNil() { + return nil + } + return appendQueryValue(query, key, value.Elem(), filter, formatTag) + } + if value.Kind() == reflect.Struct && value.Type() == timeType { + aTime := value.Interface().(time.Time) + if aTime.IsZero() { + return nil + } + query.Add(key, normalizeTimeFormatTag(formatTag).FormatTime(&aTime)) + return nil + } + + if text, ok, err := marshalTextQueryValue(value); ok || err != nil { + if err != nil { + return wrapUnsupportedFilterValue(filter, key, value, err) + } + query.Add(key, text) + return nil + } + if text, ok := stringifyQueryValue(value); ok { + query.Add(key, text) + return nil + } + switch value.Kind() { case reflect.String: if value.String() != "" { query.Add(key, value.String()) } + return nil case reflect.Bool: query.Add(key, strconv.FormatBool(value.Bool())) + return nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: query.Add(key, strconv.FormatInt(value.Int(), 10)) + return nil case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: query.Add(key, strconv.FormatUint(value.Uint(), 10)) + return nil case reflect.Float32, reflect.Float64: query.Add(key, strconv.FormatFloat(value.Float(), 'f', -1, 64)) + return nil case reflect.Slice, reflect.Array: for i := 0; i < value.Len(); i++ { - appendQueryValue(query, key, value.Index(i)) + if err := appendQueryValue(query, key, value.Index(i), filter, formatTag); err != nil { + return err + } } + return nil } + return wrapUnsupportedFilterValue(filter, key, value, nil) } func fieldByName(root reflect.Value, name string) reflect.Value { @@ -238,6 +300,18 @@ func fieldByName(root reflect.Value, name string) reflect.Value { return root.FieldByName(name) } +func fieldByNameDetails(root reflect.Value, name string) (reflect.Value, reflect.StructField, bool) { + root = indirectValue(root) + if !root.IsValid() || root.Kind() != reflect.Struct || name == "" { + return reflect.Value{}, reflect.StructField{}, false + } + field, ok := root.Type().FieldByName(name) + if !ok { + return reflect.Value{}, reflect.StructField{}, false + } + return root.FieldByName(name), field, true +} + func indirectValue(value reflect.Value) reflect.Value { for value.IsValid() && value.Kind() == reflect.Ptr { if value.IsNil() { @@ -248,11 +322,19 @@ func indirectValue(value reflect.Value) reflect.Value { return value } -func isEmptyValue(value reflect.Value) bool { - value = indirectValue(value) +func shouldOmitFilterValue(value reflect.Value) bool { + for value.IsValid() && value.Kind() == reflect.Interface { + if value.IsNil() { + return true + } + value = value.Elem() + } if !value.IsValid() { return true } + if value.Kind() == reflect.Ptr { + return value.IsNil() + } switch value.Kind() { case reflect.String, reflect.Array, reflect.Slice, reflect.Map: return value.Len() == 0 @@ -264,6 +346,11 @@ func isEmptyValue(value reflect.Value) bool { return value.Uint() == 0 case reflect.Float32, reflect.Float64: return value.Float() == 0 + case reflect.Struct: + if value.Type() == timeType { + return value.Interface().(time.Time).IsZero() + } + return value.IsZero() } return false } @@ -290,3 +377,109 @@ func bodyRoot(root reflect.Value, bodyField string) reflect.Value { } return body } + +func resolveFilterFormatTag(filter *ReportFilter, field reflect.StructField) (*tagformat.Tag, error) { + if field.Tag != "" { + parsed, err := tagformat.Parse(field.Tag) + if err != nil { + return nil, fmt.Errorf("invalid format tag on report filter field %s: %w", field.Name, err) + } + if hasTimeFormat(parsed) { + return normalizeTimeFormatTag(parsed), nil + } + } + if filter == nil || filter.Parameter == nil { + return normalizeTimeFormatTag(nil), nil + } + if strings.TrimSpace(filter.Parameter.Tag) != "" { + parsed, err := tagformat.Parse(reflect.StructTag(filter.Parameter.Tag)) + if err != nil { + return nil, fmt.Errorf("invalid format tag on report filter parameter %s: %w", filter.FieldName, err) + } + if hasTimeFormat(parsed) { + return normalizeTimeFormatTag(parsed), nil + } + } + if dateFormat := strings.TrimSpace(filter.Parameter.DateFormat); dateFormat != "" { + return &tagformat.Tag{ + DateFormat: dateFormat, + TimeLayout: ftime.DateFormatToTimeLayout(dateFormat), + }, nil + } + return normalizeTimeFormatTag(nil), nil +} + +func hasTimeFormat(tag *tagformat.Tag) bool { + if tag == nil { + return false + } + return strings.TrimSpace(tag.DateFormat) != "" || strings.TrimSpace(tag.TimeLayout) != "" || strings.TrimSpace(tag.Timezone) != "" +} + +func normalizeTimeFormatTag(tag *tagformat.Tag) *tagformat.Tag { + if tag == nil { + return &tagformat.Tag{} + } + if tag.TimeLayout == "" && tag.DateFormat != "" { + tag.TimeLayout = ftime.DateFormatToTimeLayout(tag.DateFormat) + } + return tag +} + +func marshalTextQueryValue(value reflect.Value) (string, bool, error) { + if !value.IsValid() { + return "", false, nil + } + if value.Type().Implements(textMarshalerType) && value.CanInterface() { + text, err := value.Interface().(encoding.TextMarshaler).MarshalText() + return string(text), true, err + } + if value.CanAddr() && value.Addr().Type().Implements(textMarshalerType) { + text, err := value.Addr().Interface().(encoding.TextMarshaler).MarshalText() + return string(text), true, err + } + if value.Type().Kind() != reflect.Ptr { + ptr := reflect.New(value.Type()) + ptr.Elem().Set(value) + if ptr.Type().Implements(textMarshalerType) { + text, err := ptr.Interface().(encoding.TextMarshaler).MarshalText() + return string(text), true, err + } + } + return "", false, nil +} + +func stringifyQueryValue(value reflect.Value) (string, bool) { + if !value.IsValid() { + return "", false + } + if value.Type().Implements(stringerType) && value.CanInterface() { + return value.Interface().(fmt.Stringer).String(), true + } + if value.CanAddr() && value.Addr().Type().Implements(stringerType) { + return value.Addr().Interface().(fmt.Stringer).String(), true + } + if value.Type().Kind() != reflect.Ptr { + ptr := reflect.New(value.Type()) + ptr.Elem().Set(value) + if ptr.Type().Implements(stringerType) { + return ptr.Interface().(fmt.Stringer).String(), true + } + } + return "", false +} + +func wrapUnsupportedFilterValue(filter *ReportFilter, key string, value reflect.Value, cause error) error { + filterName := "" + if filter != nil { + filterName = filter.FieldName + } + if filterName == "" { + filterName = key + } + err := fmt.Errorf("unable to serialize report filter %q as query param %q: unsupported value type %s (kind=%s)", filterName, key, value.Type().String(), value.Kind()) + if cause != nil { + return fmt.Errorf("%w: %v", err, cause) + } + return err +} diff --git a/repository/report_handler_test.go b/repository/report_handler_test.go index 58ee75703..1ea816fe6 100644 --- a/repository/report_handler_test.go +++ b/repository/report_handler_test.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "log/slog" "net/http" "net/http/httptest" "reflect" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -103,6 +105,30 @@ type reportHandlerBody struct { Offset *int } +type reportHandlerAdvancedFilters struct { + Created *time.Time `format:"dateFormat=YYYY-MM-DD"` + Enabled *bool + Count *int + Code reportHandlerStringerFilter + Metadata *reportHandlerUnsupportedFilter +} + +type reportHandlerAdvancedBody struct { + Dimensions reportHandlerDimensions + Measures reportHandlerMeasures + Filters reportHandlerAdvancedFilters +} + +type reportHandlerStringerFilter string + +func (f reportHandlerStringerFilter) String() string { + return "stringer:" + string(f) +} + +type reportHandlerUnsupportedFilter struct { + Value string +} + func (s *reportTestSession) Validator() *validator.Service { return nil } func (s *reportTestSession) Differ() *differ.Service { return nil } func (s *reportTestSession) MessageBus() *mbus.Service { return nil } @@ -160,7 +186,8 @@ func testReportInput() reportHandlerBody { func TestReportHandler_BuildQuery_FromPostBody(t *testing.T) { handler := testReportHandler() handler.Metadata.Filters[0].Parameter = &state.Parameter{In: state.NewQueryLocation("accountID")} - query, err := handler.buildQuery(testReportInput()) + req := httptest.NewRequest(http.MethodPost, "http://localhost/v1/api/vendors/cube", nil) + query, err := handler.buildQuery(testReportInput(), req) require.NoError(t, err) assert.Equal(t, "AccountID,TotalSpend", query.Get("_fields")) assert.Equal(t, "AccountID", query.Get("_orderby")) @@ -172,7 +199,7 @@ func TestReportHandler_Exec_PreservesAuthorizationHeader(t *testing.T) { handler := testReportHandler() handler.Metadata.Filters[0].Parameter = &state.Parameter{In: state.NewQueryLocation("accountID")} - req := httptest.NewRequest(http.MethodPost, "http://localhost/v1/api/vendors/report", nil) + req := httptest.NewRequest(http.MethodPost, "http://localhost/v1/api/vendors/cube", nil) req.Header.Set("Authorization", "Bearer test-token") httpSession := &reportTestHTTP{request: req} session := &reportTestSession{ @@ -209,3 +236,95 @@ func TestReportHandler_ReportInput_AcceptsUnwrappedBody(t *testing.T) { require.True(t, body.Dimensions.AccountID) require.True(t, body.Measures.TotalSpend) } + +func TestReportHandler_BuildQuery_FilterSerialization_DataDriven(t *testing.T) { + created := time.Date(2026, time.April, 29, 14, 30, 0, 0, time.UTC) + enabled := false + count := 0 + cases := []struct { + name string + filter *ReportFilter + filters reportHandlerAdvancedFilters + wantQuery map[string]string + wantMissing []string + wantErrSubstr string + }{ + { + name: "formats time with field format tag", + filter: &ReportFilter{Name: "created", FieldName: "Created", Parameter: &state.Parameter{In: state.NewQueryLocation("created")}}, + filters: reportHandlerAdvancedFilters{ + Created: &created, + }, + wantQuery: map[string]string{"created": "2026-04-29"}, + }, + { + name: "preserves explicit false pointer", + filter: &ReportFilter{Name: "enabled", FieldName: "Enabled", Parameter: &state.Parameter{In: state.NewQueryLocation("enabled")}}, + filters: reportHandlerAdvancedFilters{ + Enabled: &enabled, + }, + wantQuery: map[string]string{"enabled": "false"}, + }, + { + name: "preserves explicit zero pointer", + filter: &ReportFilter{Name: "count", FieldName: "Count", Parameter: &state.Parameter{In: state.NewQueryLocation("count")}}, + filters: reportHandlerAdvancedFilters{ + Count: &count, + }, + wantQuery: map[string]string{"count": "0"}, + }, + { + name: "uses stringer for named value", + filter: &ReportFilter{Name: "code", FieldName: "Code", Parameter: &state.Parameter{In: state.NewQueryLocation("code")}}, + filters: reportHandlerAdvancedFilters{ + Code: reportHandlerStringerFilter("A1"), + }, + wantQuery: map[string]string{"code": "stringer:A1"}, + }, + { + name: "omits nil pointer filter", + filter: &ReportFilter{Name: "created", FieldName: "Created", Parameter: &state.Parameter{In: state.NewQueryLocation("created")}}, + filters: reportHandlerAdvancedFilters{ + Created: nil, + }, + wantMissing: []string{"created"}, + }, + { + name: "errors on unsupported present struct", + filter: &ReportFilter{Name: "metadata", FieldName: "Metadata", Parameter: &state.Parameter{In: state.NewQueryLocation("metadata")}}, + filters: reportHandlerAdvancedFilters{ + Metadata: &reportHandlerUnsupportedFilter{Value: "x"}, + }, + wantErrSubstr: `report filter "Metadata"`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + handler := testReportHandler() + handler.Metadata.Filters = []*ReportFilter{tc.filter} + + input := reportHandlerAdvancedBody{ + Dimensions: reportHandlerDimensions{AccountID: true}, + Filters: tc.filters, + } + + req := httptest.NewRequest(http.MethodPost, "http://localhost/v1/api/vendors/cube", nil) + query, err := handler.buildQuery(input, req) + if tc.wantErrSubstr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErrSubstr) + assert.Contains(t, err.Error(), fmt.Sprintf(`query param %q`, tc.filter.Parameter.In.Name)) + return + } + + require.NoError(t, err) + for key, want := range tc.wantQuery { + assert.Equal(t, want, query.Get(key)) + } + for _, key := range tc.wantMissing { + assert.Empty(t, query.Get(key)) + } + }) + } +} From ae441ab27ff880d6ff8ec020398a8b6fbe3cd2e7 Mon Sep 17 00:00:00 2001 From: adranwit Date: Thu, 30 Apr 2026 08:51:51 -0700 Subject: [PATCH 236/279] - added orig query parameters to cube request --- service/reader/sql.go | 30 ++++++++++++++++++++- service/reader/sql_groupable_test.go | 40 ++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/service/reader/sql.go b/service/reader/sql.go index d2ed875c8..f3bddf4ed 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -464,6 +464,11 @@ func containsAggregateNode(n node.Node) bool { return containsAggregateNode(actual.X) case *expr.Binary: return containsAggregateNode(actual.X) || containsAggregateNode(actual.Y) + case *expr.Raw: + if containsAggregateNode(actual.X) { + return true + } + return containsAggregateText(actual.Raw) || containsAggregateText(actual.Unparsed) case *expr.Switch: if containsAggregateNode(&actual.Ident) { return true @@ -487,9 +492,32 @@ func containsAggregateNode(n node.Node) bool { return false } +func containsAggregateText(text string) bool { + upper := strings.ToUpper(strings.TrimSpace(text)) + if upper == "" { + return false + } + for _, name := range []string{ + "SUM", + "COUNT", + "AVG", + "MIN", + "MAX", + "ARRAY_AGG", + "STRING_AGG", + "ANY_VALUE", + "APPROX_COUNT_DISTINCT", + } { + if strings.Contains(upper, name+"(") { + return true + } + } + return false +} + func isAggregateFunction(name string) bool { switch strings.ToUpper(strings.TrimSpace(name)) { - case "SUM", "COUNT", "AVG", "MIN", "MAX", "ARRAY_AGG", "STRING_AGG", "ANY_VALUE": + case "SUM", "COUNT", "AVG", "MIN", "MAX", "ARRAY_AGG", "STRING_AGG", "ANY_VALUE", "APPROX_COUNT_DISTINCT": return true default: return false diff --git a/service/reader/sql_groupable_test.go b/service/reader/sql_groupable_test.go index 30947ecd6..ca648cdbf 100644 --- a/service/reader/sql_groupable_test.go +++ b/service/reader/sql_groupable_test.go @@ -135,6 +135,46 @@ func TestBuilder_rewriteGroupBy(t *testing.T) { }(), expected: "(SELECT p.channel_id, p.agency_id, ROUND(SUM(p.total_spend), 4) AS total_spend FROM last_n p GROUP BY 1, 2 ORDER BY total_spend DESC)", }, + { + description: "rewrite grouped aggregates recognizes approx_count_distinct as aggregate", + sql: "(SELECT (9100 * APPROX_COUNT_DISTINCT(IF(avails = 100, COALESCE(alias_ip, IF(hhip_flag = 1, ip, NULL)), NULL))) AS hh_uniqs FROM audience_event_v1 v GROUP BY 1 LIMIT 200)", + allColumns: []*view.Column{ + {Name: "hh_uniqs"}, + }, + projected: []*view.Column{ + {Name: "hh_uniqs"}, + }, + expected: "(SELECT (9100 * APPROX_COUNT_DISTINCT(IF(avails = 100, COALESCE(alias_ip, IF(hhip_flag = 1, ip, NULL)), NULL))) AS hh_uniqs FROM audience_event_v1 v)", + }, + { + description: "rewrite grouped aggregates removes group by for active forecasting measures", + sql: "(SELECT IFNULL(STRING_AGG(DISTINCT IAB[SAFE_OFFSET(0)], ', ' LIMIT 20), '') AS iab_cats, " + + "(99 * SUM(avails)) + MOD(SUM(avails), 99) AS avails, " + + "(9100 * APPROX_COUNT_DISTINCT(IF(avails = 100, COALESCE(alias_ip, IF(hhip_flag = 1, ip, NULL)), NULL))) AS hh_uniqs, " + + "(101 * SUM(IF(avails != 100, avails, 0))) AS bids, " + + "(9100 * APPROX_COUNT_DISTINCT(uid)) AS device_uniqs " + + "FROM audience_event_v1 v GROUP BY 1, 2, 3, 4, 5 LIMIT 200)", + allColumns: []*view.Column{ + {Name: "iab_cats"}, + {Name: "avails"}, + {Name: "hh_uniqs"}, + {Name: "bids"}, + {Name: "device_uniqs"}, + }, + projected: []*view.Column{ + {Name: "iab_cats"}, + {Name: "avails"}, + {Name: "hh_uniqs"}, + {Name: "bids"}, + {Name: "device_uniqs"}, + }, + expected: "(SELECT IFNULL(STRING_AGG(DISTINCT IAB[SAFE_OFFSET(0)], ', ' LIMIT 20), '') AS iab_cats, " + + "(99 * SUM(avails)) + MOD(SUM(avails), 99) AS avails, " + + "(9100 * APPROX_COUNT_DISTINCT(IF(avails = 100, COALESCE(alias_ip, IF(hhip_flag = 1, ip, NULL)), NULL))) AS hh_uniqs, " + + "(101 * SUM(IF(avails != 100, avails, 0))) AS bids, " + + "(9100 * APPROX_COUNT_DISTINCT(uid)) AS device_uniqs " + + "FROM audience_event_v1 v)", + }, { description: "rewrite grouped metrics query prunes unselected dimensions from select list", sql: "(SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, SUM(p.bids) AS bids, SUM(p.impressions) AS impressions, SUM(p.clicks) AS clicks, SUM(p.conversions) AS conversions, SUM(p.total_spend) AS total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?))) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 LIMIT 1000)", From 9601df7b003acee44cb018e58b26c0881d4ce83b Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 1 May 2026 08:52:31 -0700 Subject: [PATCH 237/279] - added orig query parameters to cube request --- service/reader/service.go | 2 + view/collector.go | 76 +++++++++++++++++ view/collector_bootstrap_test.go | 140 +++++++++++++++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 view/collector_bootstrap_test.go diff --git a/service/reader/service.go b/service/reader/service.go index 7cfaa7903..490402095 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -166,6 +166,8 @@ func (s *Service) readAll(ctx context.Context, session *Session, collector *view return } + collector.BootstrapFromParentHolder() + collectorFetchEmitted = true collector.Fetched() relationGroup.Wait() diff --git a/view/collector.go b/view/collector.go index 36ad9d5f1..b6c5fab5f 100644 --- a/view/collector.go +++ b/view/collector.go @@ -890,6 +890,82 @@ func (r *Collector) ReadAll() bool { return r.readAll } +// BootstrapFromParentHolder seeds this collector from already-materialized parent holder data. +// This is useful when parent OnFetch/OnRelation hooks populate a virtual relation in-memory, +// and nested relations still need a real collector source to continue fetching. +func (r *Collector) BootstrapFromParentHolder() bool { + if r == nil || r.parent == nil || r.relation == nil || r.relation.holderField == nil { + return false + } + if r.Len() > 0 { + return false + } + + parentPtr := xunsafe.AsPointer(r.parent.DestPtr()) + if parentPtr == nil { + return false + } + parentLen := r.parent.slice.Len(parentPtr) + if parentLen == 0 { + return false + } + + visitorRelations := RelationsSlice(r.view.With).PopulateWithVisitor() + indexer := r.valueIndexer(context.Background(), visitorRelations) + appended := 0 + + for i := 0; i < parentLen; i++ { + parentItem := r.parent.slice.ValuePointerAt(parentPtr, i) + if parentItem == nil { + continue + } + holderValue := r.relation.holderField.Value(xunsafe.AsPointer(parentItem)) + appended += r.appendBootstrapHolder(holderValue, indexer) + } + + return appended > 0 +} + +func (r *Collector) appendBootstrapHolder(holderValue interface{}, indexer func(value interface{}) error) int { + if holderValue == nil { + return 0 + } + + value := reflect.ValueOf(holderValue) + if !value.IsValid() { + return 0 + } + + for value.Kind() == reflect.Interface || value.Kind() == reflect.Ptr { + if value.IsNil() { + return 0 + } + value = value.Elem() + } + + switch value.Kind() { + case reflect.Slice, reflect.Array: + appended := 0 + for i := 0; i < value.Len(); i++ { + item := value.Index(i) + if !item.IsValid() { + continue + } + if item.Kind() == reflect.Ptr && item.IsNil() { + continue + } + r.appender.Append(item.Interface()) + _ = indexer(item.Interface()) + appended++ + } + return appended + default: + r.appender.Append(holderValue) + _ = indexer(holderValue) + return 1 + } +} + func (r *Collector) Unlock() { if r.parent == nil { return diff --git a/view/collector_bootstrap_test.go b/view/collector_bootstrap_test.go new file mode 100644 index 000000000..bade2534c --- /dev/null +++ b/view/collector_bootstrap_test.go @@ -0,0 +1,140 @@ +package view + +import ( + "reflect" + "testing" + + "github.com/viant/datly/view/state" + "github.com/viant/xdatly/handler" + "github.com/viant/xunsafe" +) + +type bootstrapAudience struct { + ID int + SignalValues []*bootstrapSignalValue +} + +type bootstrapSignalValue struct { + AudienceID int + FeatureType string + FeatureValue string + SignalPerformance *bootstrapSignalPerformance +} + +type bootstrapSignalPerformance struct { + FeatureType string + Value string +} + +func TestCollector_BootstrapFromParentHolder_SeedsNestedCompositeRelation(t *testing.T) { + parentDest := []*bootstrapAudience{ + { + ID: 1, + SignalValues: []*bootstrapSignalValue{ + {AudienceID: 1, FeatureType: "ias.brand.safety", FeatureValue: "4001"}, + {AudienceID: 1, FeatureType: "ias.fraud", FeatureValue: "402"}, + }, + }, + } + + parentView := &View{ + Name: "audience", + Schema: state.NewSchema(reflect.TypeOf(&bootstrapAudience{})), + } + parentCollector := &Collector{ + destValue: reflect.ValueOf(&parentDest), + slice: xunsafe.NewSlice(reflect.TypeOf(parentDest)), + view: parentView, + dataSync: handler.NewDataSync(), + valuePosition: map[string]map[string]map[interface{}][]int{}, + compositeValuePosition: map[string]map[compositeKey][]int{}, + types: map[string]*xunsafe.Type{}, + values: map[string]*[]interface{}{}, + } + + perfView := &View{ + Name: "signalPerformance", + Schema: state.NewSchema(reflect.TypeOf(&bootstrapSignalPerformance{})), + Template: &Template{}, + } + perfRelation := &Relation{ + Name: "SignalPerformance", + Cardinality: state.One, + Holder: "SignalPerformance", + On: Links{ + {Field: "FeatureType", Column: "FEATURE_TYPE", xField: xunsafe.FieldByName(reflect.TypeOf(bootstrapSignalValue{}), "FeatureType")}, + {Field: "FeatureValue", Column: "FEATURE_VALUE", xField: xunsafe.FieldByName(reflect.TypeOf(bootstrapSignalValue{}), "FeatureValue")}, + }, + Of: &ReferenceView{ + View: *perfView, + On: Links{ + {Column: "FeatureType"}, + {Column: "Value"}, + }, + }, + holderField: xunsafe.FieldByName(reflect.TypeOf(bootstrapSignalValue{}), "SignalPerformance"), + } + + signalView := &View{ + Name: "signalValues", + Schema: state.NewSchema(reflect.TypeOf(&bootstrapSignalValue{})), + Template: &Template{}, + With: []*Relation{perfRelation}, + } + signalDest := make([]*bootstrapSignalValue, 0) + signalCollector := &Collector{ + parent: parentCollector, + destValue: reflect.ValueOf(&signalDest), + appender: xunsafe.NewSlice(reflect.TypeOf(signalDest)).Appender(xunsafe.AsPointer(&signalDest)), + slice: xunsafe.NewSlice(reflect.TypeOf(signalDest)), + view: signalView, + relation: &Relation{Holder: "SignalValues", holderField: xunsafe.FieldByName(reflect.TypeOf(bootstrapAudience{}), "SignalValues")}, + dataSync: handler.NewDataSync(), + valuePosition: map[string]map[string]map[interface{}][]int{}, + compositeValuePosition: map[string]map[compositeKey][]int{}, + types: map[string]*xunsafe.Type{}, + values: map[string]*[]interface{}{}, + } + + children, err := signalCollector.Relations(nil) + if err != nil { + t.Fatalf("relations: %v", err) + } + if len(children) != 1 { + t.Fatalf("expected one nested relation collector, got %d", len(children)) + } + perfCollector := children[0] + + if !signalCollector.BootstrapFromParentHolder() { + t.Fatalf("expected bootstrap from parent holder") + } + if signalCollector.Len() != 2 { + t.Fatalf("bootstrap length = %d, want 2", signalCollector.Len()) + } + + _, composite, columns := perfCollector.ParentPlaceholders() + if len(composite) != 2 { + t.Fatalf("composite placeholder rows = %d, want 2", len(composite)) + } + if !reflect.DeepEqual(columns, []string{"FeatureType", "Value"}) { + t.Fatalf("columns = %v, want %v", columns, []string{"FeatureType", "Value"}) + } + + expected := map[compositeKey]bool{ + buildCompositeKey([]interface{}{"ias.brand.safety", "4001"}): true, + buildCompositeKey([]interface{}{"ias.fraud", "402"}): true, + } + actual := map[compositeKey]bool{} + for _, row := range composite { + actual[buildCompositeKey(row)] = true + } + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("composite rows = %v, want %v", actual, expected) + } + + signature := relationCompositeSignature(perfRelation.On) + indexed := signalCollector.compositeValuePosition[signature] + if len(indexed) != 2 { + t.Fatalf("composite index size = %d, want 2", len(indexed)) + } +} From e5b06a387c84b8a19c799895918f8905f45247a0 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 1 May 2026 09:19:32 -0700 Subject: [PATCH 238/279] - added orig query parameters to cube request --- internal/translator/view.go | 61 ++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/internal/translator/view.go b/internal/translator/view.go index 9408acaf3..f6601b9b1 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -292,43 +292,62 @@ func (v *View) buildRelations(parentNamespace *Viewlet, rule *Rule) error { if relation.KeyField == nil { return fmt.Errorf("failed to add relation: %v, unknown reference", relation.Name) } - columnName := relation.ParentField.Column.Name - if columnName == "" { - columnName = relation.ParentField.Column.Alias - } - - viewRelation.On = append(viewRelation.On, &view.Link{ - Column: columnName, - Namespace: relation.ParentField.Column.Namespace, - Field: relation.ParentField.Name, - }) - holderFormat := text.DetectCaseFormat(relNamespace.Name) viewRelation.Holder = holderFormat.Format(relNamespace.Name, text.CaseFormatUpperCamel) viewRelation.IncludeColumn = true relNamespace.Holder = viewRelation.Holder refViewName := relNamespace.View.Name - refColumn := relation.KeyField.Column.Name - if refColumn == "" { - refColumn = relation.KeyField.Column.Alias - } - if ns := relation.KeyField.Column.Namespace; ns != "" { - refColumn = ns + "." + refColumn - } if relNamespace.View.AllowNulls == nil { relNamespace.View.AllowNulls = v.View.AllowNulls } - - refField := relation.KeyField.Name aRefView := view.NewRefView(refViewName) aRefView.Name = refViewName + "#" - viewRelation.Of = view.NewReferenceView(view.JoinOn(view.WithLink(refField, refColumn)), aRefView) + relLinks, refLinks := relationLinks(relation) + viewRelation.On = relLinks + viewRelation.Of = view.NewReferenceView(refLinks, aRefView) viewRelation.Cardinality = relation.Cardinality v.View.With = append(v.View.With, viewRelation) } return nil } +func relationLinks(relation *inference.Relation) (view.Links, view.Links) { + pairs := relation.Pairs + if len(pairs) == 0 && relation.ParentField != nil && relation.KeyField != nil { + pairs = []*inference.RelationPair{{ + ParentField: relation.ParentField, + KeyField: relation.KeyField, + }} + } + + var relLinks view.Links + var refLinks view.Links + for _, pair := range pairs { + if pair == nil || pair.ParentField == nil || pair.KeyField == nil { + continue + } + columnName := pair.ParentField.Column.Name + if columnName == "" { + columnName = pair.ParentField.Column.Alias + } + relLinks = append(relLinks, &view.Link{ + Column: columnName, + Namespace: pair.ParentField.Column.Namespace, + Field: pair.ParentField.Name, + }) + + refColumn := pair.KeyField.Column.Name + if refColumn == "" { + refColumn = pair.KeyField.Column.Alias + } + if ns := pair.KeyField.Column.Namespace; ns != "" { + refColumn = ns + "." + refColumn + } + refLinks = append(refLinks, view.WithLink(pair.KeyField.Name, refColumn)) + } + return relLinks, refLinks +} + func (v *View) GenerateFiles(baseURL string, ruleName string, files *asset.Files, substitutes view.Substitutes) { if v.View.Template.Source != "" { source := substitutes.ReverseReplace(v.View.Template.Source) From 161179a79a26f0769933b324001246e922597696 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 1 May 2026 09:53:58 -0700 Subject: [PATCH 239/279] - added orig query parameters to cube request --- service/reader/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/reader/service.go b/service/reader/service.go index 490402095..f8ca76675 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -150,7 +150,7 @@ func (s *Service) readAll(ctx context.Context, session *Session, collector *view } batchData := s.batchData(collector) - if len(batchData.ColumnNames) != 0 && len(batchData.Values) == 0 { + if len(batchData.ColumnNames) != 0 && len(batchData.Values) == 0 && len(batchData.CompositeValues) == 0 { return } From cc8d9a5c303c545365fa7bb4050f787ed5928e4b Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 1 May 2026 12:42:34 -0700 Subject: [PATCH 240/279] - added orig query parameters to cube request --- internal/translator/view_relation_test.go | 105 ++++++++++++++++++++++ repository/report_handler.go | 62 ++++++++++++- repository/report_handler_test.go | 55 ++++++++++++ 3 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 internal/translator/view_relation_test.go diff --git a/internal/translator/view_relation_test.go b/internal/translator/view_relation_test.go new file mode 100644 index 000000000..3ca870275 --- /dev/null +++ b/internal/translator/view_relation_test.go @@ -0,0 +1,105 @@ +package translator + +import ( + "context" + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/internal/inference" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/sqlparser" +) + +func TestRelationLinks_UsesAllRelationPairs(t *testing.T) { + relation := testCompositeRelation() + + relLinks, refLinks := relationLinks(relation) + require.Len(t, relLinks, 2) + require.Len(t, refLinks, 2) + + require.Equal(t, "FeatureType", relLinks[0].Field) + require.Equal(t, "FEATURE_TYPE", relLinks[0].Column) + require.Equal(t, "FeatureValue", relLinks[1].Field) + require.Equal(t, "FEATURE_VALUE", relLinks[1].Column) + + require.Equal(t, "FeatureType", refLinks[0].Field) + require.Equal(t, "FeatureType", refLinks[0].Column) + require.Equal(t, "Value", refLinks[1].Field) + require.Equal(t, "Value", refLinks[1].Column) +} + +type translatorParentRecord struct { + FeatureType string + FeatureValue string + SignalPerformance *translatorChildRecord +} + +type translatorChildRecord struct { + FeatureType string + Value string +} + +func testCompositeRelation() *inference.Relation { + return &inference.Relation{ + Name: "signalPerformance", + Cardinality: state.One, + Pairs: []*inference.RelationPair{ + { + ParentField: &inference.Field{ + Field: view.Field{Name: "FeatureType", Schema: state.NewSchema(reflect.TypeOf(""))}, + Column: &sqlparser.Column{Name: "FEATURE_TYPE"}, + }, + KeyField: &inference.Field{ + Field: view.Field{Name: "FeatureType", Schema: state.NewSchema(reflect.TypeOf(""))}, + Column: &sqlparser.Column{Name: "FeatureType"}, + }, + }, + { + ParentField: &inference.Field{ + Field: view.Field{Name: "FeatureValue", Schema: state.NewSchema(reflect.TypeOf(""))}, + Column: &sqlparser.Column{Name: "FEATURE_VALUE"}, + }, + KeyField: &inference.Field{ + Field: view.Field{Name: "Value", Schema: state.NewSchema(reflect.TypeOf(""))}, + Column: &sqlparser.Column{Name: "Value"}, + }, + }, + }, + ParentField: &inference.Field{ + Field: view.Field{Name: "FeatureType", Schema: state.NewSchema(reflect.TypeOf(""))}, + Column: &sqlparser.Column{Name: "FEATURE_TYPE"}, + }, + KeyField: &inference.Field{ + Field: view.Field{Name: "FeatureType", Schema: state.NewSchema(reflect.TypeOf(""))}, + Column: &sqlparser.Column{Name: "FeatureType"}, + }, + Spec: &inference.Spec{ + Namespace: "signalPerformance", + Type: &inference.Type{Name: "SignalPerformanceView"}, + }, + } +} + +func TestRelationLinks_PreservesCompositeJoinPairs_Init(t *testing.T) { + // Ensure the translator-emitted links initialize into a real datly relation object + // with composite join semantics intact. + parent := view.NewView("audience", "", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithViewType(reflect.TypeOf(&translatorParentRecord{})), + view.WithTemplate(&view.Template{}), + ) + child := view.NewView("signalPerformance", "viant-mediator.forecaster.signal_performance", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithViewType(reflect.TypeOf(&translatorChildRecord{})), + view.WithTemplate(&view.Template{}), + ) + relLinks, refLinks := relationLinks(testCompositeRelation()) + require.NoError(t, view.WithOneToOne("SignalPerformance", relLinks, view.NewReferenceView(refLinks, child))(parent)) + require.NoError(t, parent.Init(context.Background(), view.EmptyResource())) + require.Len(t, parent.With, 1) + require.Len(t, parent.With[0].On, 2) + require.Len(t, parent.With[0].Of.On, 2) + require.True(t, parent.With[0].IsComposite()) +} diff --git a/repository/report_handler.go b/repository/report_handler.go index 57ccf291f..8bc7cc77c 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -17,6 +17,7 @@ import ( ftime "github.com/viant/tagly/format/time" "github.com/viant/datly/repository/contract" + "github.com/viant/datly/view" "github.com/viant/datly/view/state" xhandler "github.com/viant/xdatly/handler" xdhttp "github.com/viant/xdatly/handler/http" @@ -105,10 +106,16 @@ func (r *cubeHandler) buildQuery(input interface{}, request *http.Request) (url. for k, v := range request.Form { query[k] = v } - fields, err := r.collectSelections(root, r.Metadata.Dimensions, r.Metadata.Measures) + dimensions, err := r.collectSelections(root, r.Metadata.Dimensions) if err != nil { return nil, err } + measures, err := r.collectSelections(root, r.Metadata.Measures) + if err != nil { + return nil, err + } + fields := append(append([]string{}, dimensions...), measures...) + fields = appendAutoIncludedRelationHolders(r.Original.View, fields, dimensions) if len(fields) == 0 { return nil, fmt.Errorf("report requires at least one dimension or measure") } @@ -130,6 +137,59 @@ func (r *cubeHandler) buildQuery(input interface{}, request *http.Request) (url. return query, nil } +func appendAutoIncludedRelationHolders(rootView *view.View, fields, dimensions []string) []string { + if rootView == nil || len(rootView.With) == 0 || len(dimensions) == 0 { + return fields + } + selected := make(map[string]bool, len(dimensions)) + for _, dimension := range dimensions { + selected[normalizeRelationSelectionKey(dimension)] = true + } + + seen := make(map[string]bool, len(fields)) + for _, field := range fields { + seen[field] = true + } + + for _, relation := range rootView.With { + if relation == nil || strings.TrimSpace(relation.Holder) == "" { + continue + } + if seen[relation.Holder] { + continue + } + if relationMatchesSelectedDimensions(relation, selected) { + fields = append(fields, relation.Holder) + seen[relation.Holder] = true + } + } + return fields +} + +func relationMatchesSelectedDimensions(relation *view.Relation, selected map[string]bool) bool { + if relation == nil { + return false + } + for _, link := range relation.On { + if link == nil { + continue + } + if link.Field != "" && selected[normalizeRelationSelectionKey(link.Field)] { + return true + } + if link.Column != "" && selected[normalizeRelationSelectionKey(link.Column)] { + return true + } + } + return false +} + +func normalizeRelationSelectionKey(value string) string { + value = strings.TrimSpace(strings.ToLower(value)) + replacer := strings.NewReplacer("_", "", ".", "", "-", "", " ", "") + return replacer.Replace(value) +} + func (r *cubeHandler) selectorName(parameter *state.Parameter, fallback string) string { if parameter != nil && parameter.In != nil && strings.TrimSpace(parameter.In.Name) != "" { return parameter.In.Name diff --git a/repository/report_handler_test.go b/repository/report_handler_test.go index 1ea816fe6..953ac4d76 100644 --- a/repository/report_handler_test.go +++ b/repository/report_handler_test.go @@ -92,6 +92,14 @@ type reportHandlerMeasures struct { TotalSpend bool } +type reportHandlerForecastingDimensions struct { + AgegroupId bool +} + +type reportHandlerForecastingMeasures struct { + Avails bool +} + type reportHandlerFilters struct { AccountID *int } @@ -195,6 +203,53 @@ func TestReportHandler_BuildQuery_FromPostBody(t *testing.T) { assert.Equal(t, "101", query.Get("accountID")) } +func TestReportHandler_BuildQuery_AutoIncludesRelationHolderForSelectedDimension(t *testing.T) { + handler := &cubeHandler{ + Metadata: &ReportMetadata{ + DimensionsKey: "Dimensions", + MeasuresKey: "Measures", + FiltersKey: "Filters", + OrderBy: "OrderBy", + Limit: "Limit", + Offset: "Offset", + Dimensions: []*ReportField{ + {Name: "AgegroupId", FieldName: "AgegroupId", Section: "Dimensions"}, + }, + Measures: []*ReportField{ + {Name: "Avails", FieldName: "Avails", Section: "Measures"}, + }, + }, + Original: &Component{ + View: &view.View{ + With: []*view.Relation{ + { + Holder: "AgeGroup", + On: view.Links{ + &view.Link{Field: "AgegroupId", Column: "agegroup_id"}, + }, + }, + }, + Selector: &view.Config{ + FieldsParameter: &state.Parameter{In: state.NewQueryLocation("_fields")}, + }, + }, + }, + } + + input := struct { + Dimensions reportHandlerForecastingDimensions + Measures reportHandlerForecastingMeasures + }{ + Dimensions: reportHandlerForecastingDimensions{AgegroupId: true}, + Measures: reportHandlerForecastingMeasures{Avails: true}, + } + + req := httptest.NewRequest(http.MethodPost, "http://localhost/v1/api/steward/inventory/forecasting/cube", nil) + query, err := handler.buildQuery(input, req) + require.NoError(t, err) + assert.Equal(t, "AgegroupId,Avails,AgeGroup", query.Get("_fields")) +} + func TestReportHandler_Exec_PreservesAuthorizationHeader(t *testing.T) { handler := testReportHandler() handler.Metadata.Filters[0].Parameter = &state.Parameter{In: state.NewQueryLocation("accountID")} From 61b9acb1fbe0730506f5b9ad629ca94d63ce2811 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sat, 2 May 2026 21:45:23 -0700 Subject: [PATCH 241/279] - added orig query parameters to cube request --- view/extension/predicates.go | 13 +++++++++---- view/predicate.go | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/view/extension/predicates.go b/view/extension/predicates.go index b7303d637..626eda97c 100644 --- a/view/extension/predicates.go +++ b/view/extension/predicates.go @@ -193,18 +193,23 @@ func NewDurationPredicate() *Predicate { }, { Name: "WeekDayExpression", Position: 5, + }, { + Name: "MonthDayExpression", + Position: 6, }, } clause := ` -#if($FilterValue == "hour") +#if($FilterValue == "hour" || $FilterValue == "HOUR") ${DayExpression} = ${CurrentDayExpression} AND ${HourExpression} = ${CurrentHourExpression} -#elseif($FilterValue == "day") +#elseif($FilterValue == "day" || $FilterValue == "DAY" || $FilterValue == "today" || $FilterValue == "TODAY") ${DayExpression} = ${CurrentDayExpression} -#elseif($FilterValue == "yesterday") +#elseif($FilterValue == "yesterday" || $FilterValue == "YESTERDAY") ${DayExpression} = ${YesterdayDayExpression} - #elseif($FilterValue == "week") + #elseif($FilterValue == "week" || $FilterValue == "WEEK" || $FilterValue == "seven_days" || $FilterValue == "SEVEN_DAYS") ${DayExpression} BETWEEN ${WeekDayExpression} AND ${CurrentDayExpression} + #elseif($FilterValue == "month" || $FilterValue == "MONTH" || $FilterValue == "thirty_days" || $FilterValue == "THIRTY_DAYS") + ${DayExpression} BETWEEN ${MonthDayExpression} AND ${CurrentDayExpression} #end ` return &Predicate{ diff --git a/view/predicate.go b/view/predicate.go index 415550c28..b46c5c19c 100644 --- a/view/predicate.go +++ b/view/predicate.go @@ -2,6 +2,7 @@ package view import ( "context" + "errors" "fmt" "reflect" "strings" @@ -44,6 +45,8 @@ type ( valueState *expand.NamedVariable hasValueState *expand.NamedVariable stateType *structology.StateType + name string + args []string } ) @@ -52,6 +55,9 @@ func (e *PredicateEvaluator) Compute(ctx context.Context, value interface{}) (*c if !ok { panic("not found custom ctx") } + if err := validatePredicateArgs(e.name, value, e.args); err != nil { + return nil, err + } val := ctx.Value(expand.PredicateState) var aState *structology.State @@ -122,6 +128,23 @@ func (c *predicateCache) get(resource *Resource, predicateConfig *extension.Pred return provider.new(predicateConfig) } +func validatePredicateArgs(name string, value interface{}, args []string) error { + if name != extension.PredicateDuration { + return nil + } + filterValue := strings.TrimSpace(strings.ToLower(fmt.Sprint(value))) + if filterValue == "" || filterValue == "" { + return nil + } + switch filterValue { + case "month", "thirty_days": + if len(args) < 7 || strings.TrimSpace(args[6]) == "" { + return errors.New("duration predicate requires MonthDayExpression argument for month/thirty_days") + } + } + return nil +} + func isCustomPredicate(keyName string) bool { return keyName == "handler" } @@ -168,6 +191,8 @@ func (p *predicateEvaluatorProvider) new(predicateConfig *extension.PredicateCon valueState: p.state, hasValueState: p.hasStateName, stateType: p.stateType, + name: predicateConfig.Name, + args: append([]string{}, predicateConfig.Args...), }, nil } From 709a0dfbb64d8e50da4ae3839c31fb17d197f4aa Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 4 May 2026 06:06:20 -0700 Subject: [PATCH 242/279] - added orig query parameters to cube request --- internal/translator/view.go | 21 +++- service/reader/service.go | 156 +++++++++++++++++++++++--- service/reader/service_warmup_test.go | 51 +++++++++ service/session/state.go | 1 - 4 files changed, 209 insertions(+), 20 deletions(-) create mode 100644 service/reader/service_warmup_test.go diff --git a/internal/translator/view.go b/internal/translator/view.go index f6601b9b1..de2243969 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -2,14 +2,13 @@ package translator import ( "fmt" + "path" + "strings" "github.com/viant/datly/internal/asset" "github.com/viant/datly/internal/inference" "github.com/viant/datly/internal/setter" "github.com/viant/datly/internal/translator/parser" - - "path" - "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/tagly/format/text" @@ -85,15 +84,27 @@ func (v *View) applyShorthands(viewlet *Viewlet) { } func (v *View) buildCacheWarmup(warmup map[string]interface{}, viewlet *Viewlet) *view.Warmup { - if warmup == nil || viewlet.Join == nil { + if warmup == nil { return nil } warmup = copyWarmup(warmup) - _, refColumn := inference.ExtractRelationColumns(viewlet.Join) + explicitIndex, _ := warmup["IndexColumn"] + delete(warmup, "IndexColumn") + var refColumn string + if viewlet.Join != nil { + _, refColumn = inference.ExtractRelationColumns(viewlet.Join) + } + result := &view.Warmup{ IndexColumn: refColumn, } + if explicit := strings.TrimSpace(fmt.Sprint(explicitIndex)); explicit != "" && explicit != "" { + result.IndexColumn = explicit + } + if result.IndexColumn == "" { + return nil + } multiSet := &view.CacheParameters{} for k, v := range warmup { diff --git a/service/reader/service.go b/service/reader/service.go index f8ca76675..a0074eddb 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -377,6 +377,10 @@ func (s *Service) buildParametrizedSQL(ctx context.Context, aView *view.View, st defer wg.Done() if (aView.Cache != nil && aView.Cache.Warmup != nil) || relation != nil { data, _ := session.ParentData() + if relation == nil && aView.Cache != nil && aView.Cache.Warmup != nil { + columnInMatcher, cacheErr = s.topLevelWarmupMatcher(ctx, aView, statelet, data.AsParam()) + return + } columnInMatcher, cacheErr = s.sqlBuilder.CacheSQLWithOptions(ctx, aView, statelet, batchData, relation, data.AsParam()) } }() @@ -391,6 +395,129 @@ func (s *Service) buildParametrizedSQL(ctx context.Context, aView *view.View, st return parametrizedSQL, columnInMatcher, cacheErr } +func normalizeWarmupName(input string) string { + input = strings.ToLower(strings.TrimSpace(input)) + input = strings.ReplaceAll(input, "_", "") + input = strings.ReplaceAll(input, "-", "") + input = strings.ReplaceAll(input, ".", "") + return input +} + +func cloneStructologyState(src *structology.State) *structology.State { + if src == nil { + return nil + } + cloned := src.Type().NewState() + dstStatePtr := reflect.ValueOf(cloned.StatePtr()) + srcType := src.Type().Type() + if dstStatePtr.IsValid() { + if srcType.Kind() == reflect.Ptr { + srcStatePtr := reflect.ValueOf(src.StatePtr()) + if srcStatePtr.IsValid() && srcStatePtr.Kind() == reflect.Ptr && !srcStatePtr.IsNil() { + dstStatePtr.Elem().Set(srcStatePtr.Elem()) + } + } else { + currentValue := reflect.NewAt(srcType, src.Pointer()).Elem() + dstStatePtr.Elem().Set(currentValue) + } + } + if holder := src.MarkerHolder(); holder != nil { + holderVal := reflect.ValueOf(holder) + if holderVal.IsValid() && holderVal.Kind() == reflect.Ptr && !holderVal.IsNil() { + holderCopy := reflect.New(holderVal.Elem().Type()) + holderCopy.Elem().Set(holderVal.Elem()) + if dstStatePtr.IsValid() && dstStatePtr.Kind() == reflect.Ptr && !dstStatePtr.IsNil() { + hasField := dstStatePtr.Elem().FieldByName("Has") + if hasField.IsValid() && hasField.CanSet() { + hasField.Set(holderCopy) + } + } + } + } + cloned.Sync() + return cloned +} + +func warmupParamValues(value interface{}) []interface{} { + if value == nil { + return nil + } + switch actual := value.(type) { + case []interface{}: + return actual + } + rType := reflect.TypeOf(value) + if rType.Kind() == reflect.Slice { + rValue := reflect.ValueOf(value) + ret := make([]interface{}, rValue.Len()) + for i := 0; i < rValue.Len(); i++ { + ret[i] = rValue.Index(i).Interface() + } + return ret + } + return []interface{}{value} +} + +func (s *Service) topLevelWarmupMatcher(ctx context.Context, aView *view.View, statelet *view.Statelet, parent *expand.ViewContext) (*cache.ParmetrizedQuery, error) { + if aView == nil || aView.Cache == nil || aView.Cache.Warmup == nil || statelet == nil || statelet.Template == nil { + return nil, nil + } + indexColumn := strings.TrimSpace(aView.Cache.Warmup.IndexColumn) + if indexColumn == "" { + return nil, nil + } + target := normalizeWarmupName(indexColumn) + var matchParam *state.Parameter + for _, candidate := range aView.Template.Parameters { + if candidate == nil { + continue + } + if normalizeWarmupName(candidate.Name) == target { + matchParam = candidate + break + } + } + if matchParam == nil { + return nil, nil + } + liveSelector, selErr := statelet.Template.Selector(matchParam.Name) + if selErr != nil || liveSelector == nil { + return nil, selErr + } + value := liveSelector.Value(statelet.Template.Pointer()) + values := warmupParamValues(value) + if len(values) == 0 { + return nil, nil + } + clonedTemplate := cloneStructologyState(statelet.Template) + if clonedTemplate == nil { + return nil, nil + } + if clonedSelector, err := clonedTemplate.Selector(matchParam.Name); err == nil && clonedSelector != nil { + zero := reflect.Zero(clonedSelector.Type()).Interface() + _ = clonedSelector.SetValue(clonedTemplate.Pointer(), zero) + } + if marker := clonedTemplate.Type().Marker(); marker != nil { + clonedTemplate.EnsureMarker() + if idx := marker.Index(matchParam.Name); idx != -1 { + _ = marker.Set(clonedTemplate.Pointer(), idx, false) + } + } + cloned := *statelet + cloned.Template = clonedTemplate + + matcher, err := s.sqlBuilder.CacheSQLWithOptions(ctx, aView, &cloned, nil, nil, parent) + if err != nil { + return nil, err + } + if matcher == nil { + return nil, nil + } + matcher.By = indexColumn + matcher.In = values + return matcher, nil +} + func (s *Service) BuildCriteria(ctx context.Context, value interface{}, options *codec.CriteriaBuilderOptions) (*codec.Criteria, error) { baseView := view.Context(ctx) aSchema := state.NewSchema(reflect.TypeOf(value)) @@ -684,20 +811,9 @@ func (s *Service) HandleSQLError(err error, session *Session, aView *view.View, } func NewExecutionInfo(index *cache.ParmetrizedQuery, cacheStats *cache.Stats, collector *view.Collector) (*response.SQLExecution, func()) { - var cache *response.CacheStats + var cacheInfo *response.CacheStats if cacheStats != nil { - cache = &response.CacheStats{ - Type: string(cacheStats.Type), - RecordsCounter: cacheStats.RecordsCounter, - Key: cacheStats.Key, - Dataset: cacheStats.Dataset, - Namespace: cacheStats.Namespace, - FoundWarmup: cacheStats.FoundWarmup, - FoundLazy: cacheStats.FoundLazy, - ErrorType: cacheStats.ErrorType, - ErrorCode: int(cacheStats.ErrorCode), - ExpiryTime: cacheStats.ExpiryTime, - } + cacheInfo = &response.CacheStats{} } var parentId string if parent := collector.Parent(); parent != nil { @@ -712,13 +828,25 @@ func NewExecutionInfo(index *cache.ParmetrizedQuery, cacheStats *cache.Stats, co EndTime: now, SQL: index.SQL, Args: index.Args, - CacheStats: cache, + CacheStats: cacheInfo, } return ret, func() { now := time.Now() ret.EndTime = now ret.Rows = collector.Len() + if cacheStats != nil && ret.CacheStats != nil { + ret.CacheStats.Type = string(cacheStats.Type) + ret.CacheStats.RecordsCounter = cacheStats.RecordsCounter + ret.CacheStats.Key = cacheStats.Key + ret.CacheStats.Dataset = cacheStats.Dataset + ret.CacheStats.Namespace = cacheStats.Namespace + ret.CacheStats.FoundWarmup = cacheStats.FoundWarmup + ret.CacheStats.FoundLazy = cacheStats.FoundLazy + ret.CacheStats.ErrorType = cacheStats.ErrorType + ret.CacheStats.ErrorCode = int(cacheStats.ErrorCode) + ret.CacheStats.ExpiryTime = cacheStats.ExpiryTime + } } } diff --git a/service/reader/service_warmup_test.go b/service/reader/service_warmup_test.go new file mode 100644 index 000000000..fb2c7a3aa --- /dev/null +++ b/service/reader/service_warmup_test.go @@ -0,0 +1,51 @@ +package reader + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/structology" +) + +type warmupCloneInputHas struct { + AdOrderID bool +} + +type warmupCloneInput struct { + AdOrderID int + Has *warmupCloneInputHas `setMarker:"true"` +} + +func TestCloneStructologyState_DeepCopiesValueAndMarker(t *testing.T) { + stateType := structology.NewStateType(reflect.TypeOf(warmupCloneInput{})) + original := stateType.NewState() + + original.EnsureMarker() + require.NoError(t, original.SetValue("AdOrderID", 2653813)) + + origSelector, err := original.Selector("AdOrderID") + require.NoError(t, err) + require.Equal(t, 2653813, origSelector.Value(original.Pointer())) + require.True(t, origSelector.Has(original.Pointer())) + + cloned := cloneStructologyState(original) + require.NotNil(t, cloned) + + cloneSelector, err := cloned.Selector("AdOrderID") + require.NoError(t, err) + require.Equal(t, 2653813, cloneSelector.Value(cloned.Pointer())) + + require.NoError(t, cloneSelector.SetValue(cloned.Pointer(), 0)) + cloned.EnsureMarker() + marker := cloned.Type().Marker() + require.NotNil(t, marker) + idx := marker.Index("AdOrderID") + require.NotEqual(t, -1, idx) + require.NoError(t, marker.Set(cloned.Pointer(), idx, false)) + + require.Equal(t, 2653813, origSelector.Value(original.Pointer())) + require.True(t, origSelector.Has(original.Pointer())) + require.Equal(t, 0, cloneSelector.Value(cloned.Pointer())) + require.False(t, cloneSelector.Has(cloned.Pointer())) +} diff --git a/service/session/state.go b/service/session/state.go index 9e182c61d..6911045d0 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -367,7 +367,6 @@ func (s *Session) populateParameter(ctx context.Context, parameter *state.Parame if err = parameterSelector.SetValue(aState.Pointer(), value); err != nil { return err } - if parameterSelector.Type().Kind() == reflect.Ptr { s.cache.put(parameter, parameterSelector.Value(aState.Pointer())) } From f3de5b5c9aa814ec3cb8f3ed2abf3939b8fd05e0 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 8 May 2026 13:20:44 -0700 Subject: [PATCH 243/279] - added orig query parameters to cube request --- service/operator/service.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/service/operator/service.go b/service/operator/service.go index ee9300503..09e9683a4 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -137,7 +137,29 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes if err != nil { return nil, err } - return aSession.NewSession(aComponent), nil + originalRequest, _ := aSession.HttpRequest(ctx, aSession.Clone()) + request, _ := http.NewRequest(route.Method, route.URL, nil) + if originalRequest != nil { + request.Header = originalRequest.Header + } + unmarshal := aComponent.UnmarshalFunc(request) + locatorOptions := append(aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal)) + childSession := session.New(aComponent.View, + session.WithAuth(aSession.Auth()), + session.WithLocatorOptions(locatorOptions...), + session.WithOperate(aSession.Options.Operate()), + session.WithTypes(&aComponent.Contract.Input.Type, &aComponent.Contract.Output.Type), + session.WithComponent(aComponent), + session.WithLogger(aSession.Logger()), + session.WithRegistry(aSession.Registry()), + ) + if tx := aSession.Options.SqlTx(); tx != nil { + childSession.Apply(session.WithSQLTx(tx)) + } + if err := childSession.InitKinds(state.KindComponent, state.KindHeader, state.KindRequestBody, state.KindForm, state.KindQuery); err != nil { + return nil, err + } + return childSession, nil } err = injectorFinalizer.Finalize(ctx, lookup) From 81706c4592fce34405cc34ed39d84514933ac054 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 11 May 2026 09:06:06 -0700 Subject: [PATCH 244/279] - patched cube --- service/reader/sql.go | 129 ++++++++++++++++++++++++--- service/reader/sql_groupable_test.go | 53 +++++++++++ 2 files changed, 170 insertions(+), 12 deletions(-) diff --git a/service/reader/sql.go b/service/reader/sql.go index f3bddf4ed..bbaf298d3 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "reflect" "strconv" "strings" @@ -324,18 +325,10 @@ func (b *Builder) rewriteGroupBy(SQL string, allColumns []*view.Column, projecte return SQL, err } - selectedPositions := projectedColumnPositions(allColumns, projectedColumns) - if len(selectedPositions) > 0 { - items := make(query.List, 0, len(selectedPositions)) - for _, position := range selectedPositions { - if position <= 0 || position > len(parsed.List) { - continue - } - items = append(items, parsed.List[position-1]) - } - if len(items) > 0 { - parsed.List = items - } + selectedItems, matchedColumns := projectedSelectItems(parsed.List, projectedColumns) + if len(selectedItems) > 0 { + parsed.List = selectedItems + projectedColumns = matchedColumns } positions := projectedGroupByPositions(parsed.List, projectedColumns) @@ -374,6 +367,118 @@ func projectedColumnPositions(allColumns []*view.Column, projectedColumns []*vie return result } +func projectedSelectItems(items query.List, projectedColumns []*view.Column) (query.List, []*view.Column) { + if len(items) == 0 || len(projectedColumns) == 0 { + return nil, nil + } + resultItems := make(query.List, 0, len(projectedColumns)) + resultColumns := make([]*view.Column, 0, len(projectedColumns)) + used := map[int]bool{} + for _, column := range projectedColumns { + position := projectedSelectItemPosition(items, column, used) + if position <= 0 || position > len(items) { + continue + } + used[position] = true + resultItems = append(resultItems, items[position-1]) + resultColumns = append(resultColumns, column) + } + return resultItems, resultColumns +} + +func projectedSelectItemPosition(items query.List, column *view.Column, used map[int]bool) int { + if column == nil { + return 0 + } + keys := projectedColumnLookupKeys(column) + if len(keys) == 0 { + return 0 + } + for i, item := range items { + position := i + 1 + if used[position] || item == nil { + continue + } + if selectItemMatchesColumn(item, keys) { + return position + } + } + return 0 +} + +func projectedColumnLookupKeys(column *view.Column) map[string]bool { + result := map[string]bool{} + appendKey := func(value string) { + value = normalizeSelectItemName(value) + if value == "" { + return + } + result[value] = true + } + appendKey(column.Name) + appendKey(column.DatabaseColumn) + appendKey(column.FieldName()) + if field := column.Field(); field != nil { + appendKey(field.Name) + if source := reflect.StructTag(field.Tag).Get("source"); source != "" { + appendKey(source) + } + } + if source := reflect.StructTag(column.Tag).Get("source"); source != "" { + appendKey(source) + } + return result +} + +func selectItemMatchesColumn(item *query.Item, keys map[string]bool) bool { + for _, candidate := range selectItemLookupNames(item) { + if keys[candidate] { + return true + } + } + return false +} + +func selectItemLookupNames(item *query.Item) []string { + if item == nil { + return nil + } + var result []string + appendKey := func(value string) { + value = normalizeSelectItemName(value) + if value == "" { + return + } + for _, existing := range result { + if existing == value { + return + } + } + result = append(result, value) + } + appendKey(item.Alias) + if item.Expr != nil { + exprText := strings.TrimSpace(sqlparser.Stringify(item.Expr)) + appendKey(exprText) + if idx := strings.LastIndex(exprText, "."); idx != -1 && idx+1 < len(exprText) { + appendKey(exprText[idx+1:]) + } + } + return result +} + +func normalizeSelectItemName(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + value = strings.Trim(value, "`") + if idx := strings.LastIndex(value, "."); idx != -1 && idx+1 < len(value) { + value = value[idx+1:] + } + return strings.ToLower(strings.TrimSpace(value)) +} + func filterGroupedOrderBy(orderBy query.List, items query.List) query.List { if len(orderBy) == 0 || len(items) == 0 { return orderBy diff --git a/service/reader/sql_groupable_test.go b/service/reader/sql_groupable_test.go index ca648cdbf..16e652c3d 100644 --- a/service/reader/sql_groupable_test.go +++ b/service/reader/sql_groupable_test.go @@ -175,6 +175,59 @@ func TestBuilder_rewriteGroupBy(t *testing.T) { "(9100 * APPROX_COUNT_DISTINCT(uid)) AS device_uniqs " + "FROM audience_event_v1 v)", }, + { + description: "rewrite grouped aggregates matches selected forecasting measure by alias not stale metadata position", + sql: "(SELECT account_id, country, " + + "(99 * SUM(avails)) + MOD(SUM(avails), 99) AS avails, " + + "(101 * SUM(IF(avails != 100, avails, 0))) AS bids " + + "FROM audience_event_v1 v GROUP BY 1, 2, 3, 4 LIMIT 200)", + allColumns: []*view.Column{ + {Name: "account_id", Groupable: true}, + {Name: "country", Groupable: true}, + {Name: "size", Groupable: true}, + {Name: "avails"}, + {Name: "bids"}, + }, + projected: []*view.Column{ + {Name: "avails"}, + }, + expected: "(SELECT (99 * SUM(avails)) + MOD(SUM(avails), 99) AS avails FROM audience_event_v1 v)", + }, + { + description: "rewrite grouped aggregates matches reordered forecasting measures by alias not metadata order", + sql: "(SELECT IFNULL(STRING_AGG(DISTINCT IAB[SAFE_OFFSET(0)], ', ' LIMIT 20), '') AS iab_cats, " + + "(99 * SUM(avails)) + MOD(SUM(avails), 99) AS avails, " + + "(9100 * APPROX_COUNT_DISTINCT(IF(avails = 100, COALESCE(alias_ip, IF(hhip_flag = 1, ip, NULL)), NULL))) AS hh_uniqs, " + + "(101 * SUM(IF(avails != 100, avails, 0))) AS bids, " + + "(9100 * APPROX_COUNT_DISTINCT(uid)) AS device_uniqs, " + + "AVG(v.clearing_price) AS min_clearing_price, " + + "MAX(v.clearing_price) AS max_clearing_price " + + "FROM audience_event_v1 v GROUP BY 1, 2, 3, 4, 5, 6, 7 LIMIT 200)", + allColumns: []*view.Column{ + {Name: "account_id", Groupable: true}, + {Name: "country", Groupable: true}, + {Name: "size", Groupable: true}, + {Name: "min_clearing_price"}, + {Name: "max_clearing_price"}, + {Name: "avails"}, + {Name: "hh_uniqs"}, + {Name: "bids"}, + {Name: "device_uniqs"}, + }, + projected: []*view.Column{ + {Name: "min_clearing_price"}, + {Name: "max_clearing_price"}, + {Name: "avails"}, + {Name: "hh_uniqs"}, + {Name: "device_uniqs"}, + }, + expected: "(SELECT (99 * SUM(avails)) + MOD(SUM(avails), 99) AS avails, " + + "(9100 * APPROX_COUNT_DISTINCT(IF(avails = 100, COALESCE(alias_ip, IF(hhip_flag = 1, ip, NULL)), NULL))) AS hh_uniqs, " + + "(9100 * APPROX_COUNT_DISTINCT(uid)) AS device_uniqs, " + + "AVG(v.clearing_price) AS min_clearing_price, " + + "MAX(v.clearing_price) AS max_clearing_price " + + "FROM audience_event_v1 v)", + }, { description: "rewrite grouped metrics query prunes unselected dimensions from select list", sql: "(SELECT p.event_date, p.agency_id, p.advertiser_id, p.campaign_id, p.ad_order_id, p.audience_id, p.deal_id, p.publisher_id, p.channel_id, p.country, p.site_type, SUM(p.bids) AS bids, SUM(p.impressions) AS impressions, SUM(p.clicks) AS clicks, SUM(p.conversions) AS conversions, SUM(p.total_spend) AS total_spend FROM `viant-mediator.forecaster.fact_perf_daily_mv` p WHERE p.event_date BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY) AND DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND (((p.agency_id = ?))) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 LIMIT 1000)", From b6bcfaa08641249a6d0ee41d198aabe8ef6826ad Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 11 May 2026 14:31:44 -0700 Subject: [PATCH 245/279] - patched duration predicate --- view/extension/predicates.go | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/view/extension/predicates.go b/view/extension/predicates.go index 626eda97c..da3af6ee4 100644 --- a/view/extension/predicates.go +++ b/view/extension/predicates.go @@ -199,16 +199,39 @@ func NewDurationPredicate() *Predicate { }, } clause := ` -#if($FilterValue == "hour" || $FilterValue == "HOUR") +#if($FilterValue == "hour") ${DayExpression} = ${CurrentDayExpression} AND ${HourExpression} = ${CurrentHourExpression} -#elseif($FilterValue == "day" || $FilterValue == "DAY" || $FilterValue == "today" || $FilterValue == "TODAY") +#elseif($FilterValue == "HOUR") + ${DayExpression} = ${CurrentDayExpression} + AND ${HourExpression} = ${CurrentHourExpression} +#elseif($FilterValue == "day") + ${DayExpression} = ${CurrentDayExpression} +#elseif($FilterValue == "DAY") ${DayExpression} = ${CurrentDayExpression} -#elseif($FilterValue == "yesterday" || $FilterValue == "YESTERDAY") +#elseif($FilterValue == "today") + ${DayExpression} = ${CurrentDayExpression} +#elseif($FilterValue == "TODAY") + ${DayExpression} = ${CurrentDayExpression} +#elseif($FilterValue == "yesterday") + ${DayExpression} = ${YesterdayDayExpression} +#elseif($FilterValue == "YESTERDAY") ${DayExpression} = ${YesterdayDayExpression} - #elseif($FilterValue == "week" || $FilterValue == "WEEK" || $FilterValue == "seven_days" || $FilterValue == "SEVEN_DAYS") +#elseif($FilterValue == "week") ${DayExpression} BETWEEN ${WeekDayExpression} AND ${CurrentDayExpression} - #elseif($FilterValue == "month" || $FilterValue == "MONTH" || $FilterValue == "thirty_days" || $FilterValue == "THIRTY_DAYS") +#elseif($FilterValue == "WEEK") + ${DayExpression} BETWEEN ${WeekDayExpression} AND ${CurrentDayExpression} +#elseif($FilterValue == "seven_days") + ${DayExpression} BETWEEN ${WeekDayExpression} AND ${CurrentDayExpression} +#elseif($FilterValue == "SEVEN_DAYS") + ${DayExpression} BETWEEN ${WeekDayExpression} AND ${CurrentDayExpression} +#elseif($FilterValue == "month") + ${DayExpression} BETWEEN ${MonthDayExpression} AND ${CurrentDayExpression} +#elseif($FilterValue == "MONTH") + ${DayExpression} BETWEEN ${MonthDayExpression} AND ${CurrentDayExpression} +#elseif($FilterValue == "thirty_days") + ${DayExpression} BETWEEN ${MonthDayExpression} AND ${CurrentDayExpression} +#elseif($FilterValue == "THIRTY_DAYS") ${DayExpression} BETWEEN ${MonthDayExpression} AND ${CurrentDayExpression} #end ` From 7780b9554c1113af923778a1004a3667d78ac660 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 15 May 2026 14:43:47 -0700 Subject: [PATCH 246/279] - extended cache warmup --- gateway/app/datly.go | 7 + gateway/route_warmup.go | 20 ++- gateway/route_warmup_test.go | 74 ++++++++ gateway/router.go | 16 +- gateway/warmup/cache.go | 76 +++++++- gateway/warmup/cache_test.go | 29 +++ internal/translator/function/cache_warmup.go | 139 ++++++++++++++ .../translator/function/cache_warmup_test.go | 64 +++++++ internal/translator/function/init.go | 1 + internal/translator/view.go | 10 ++ internal/translator/view_warmup_test.go | 26 +++ service/reader/service.go | 169 ++++++++++++++---- service/reader/service_warmup_test.go | 97 ++++++++++ view/cache.go | 22 ++- view/collector_placeholder_test.go | 46 +++++ view/extension/predicates_test.go | 16 ++ view/predicate_test.go | 44 +++++ view/relation_sql_tags.go | 120 +++++++++++++ warmup/cache.go | 155 ++++++++++++++-- warmup/cache_test.go | 39 ++++ 20 files changed, 1091 insertions(+), 79 deletions(-) create mode 100644 gateway/route_warmup_test.go create mode 100644 gateway/warmup/cache_test.go create mode 100644 internal/translator/function/cache_warmup.go create mode 100644 internal/translator/function/cache_warmup_test.go create mode 100644 internal/translator/view_warmup_test.go create mode 100644 view/collector_placeholder_test.go create mode 100644 view/extension/predicates_test.go create mode 100644 view/predicate_test.go create mode 100644 view/relation_sql_tags.go diff --git a/gateway/app/datly.go b/gateway/app/datly.go index 33dab76b9..37506f0a0 100644 --- a/gateway/app/datly.go +++ b/gateway/app/datly.go @@ -2,11 +2,13 @@ package main import ( _ "github.com/go-sql-driver/mysql" + "github.com/google/gops/agent" _ "github.com/lib/pq" _ "github.com/viant/afsc/gs" _ "github.com/viant/bigquery" "github.com/viant/datly/cmd/env" "github.com/viant/datly/gateway/runtime/standalone" + "log" "os" "strconv" "time" @@ -30,5 +32,10 @@ func init() { } func main() { + go func() { + if err := agent.Listen(agent.Options{}); err != nil { + log.Printf("[WARN] failed to start gops agent: %v", err) + } + }() standalone.RunApp(Version, os.Args[1:]) } diff --git a/gateway/route_warmup.go b/gateway/route_warmup.go index 536c228bf..19758f507 100644 --- a/gateway/route_warmup.go +++ b/gateway/route_warmup.go @@ -29,27 +29,33 @@ func (r *Router) NewWarmupRoute(URL string, providers ...*repository.Provider) * func (r *Router) handleCacheWarmup(ctx context.Context, writer http.ResponseWriter, provider []*repository.Provider) { statusCode, content := r.handleCacheWarmupWithErr(ctx, provider) + setContentType(writer, statusCode, "application/json") write(writer, statusCode, content) } func (r *Router) handleCacheWarmupWithErr(ctx context.Context, providers []*repository.Provider) (int, []byte) { - var views []*view.View - URIs := make([]string, len(providers)) - for i, provider := range providers { + viewsByURI := make(map[string][]*view.View, len(providers)) + URIs := make([]string, 0, len(providers)) + for _, provider := range providers { aComponent, err := provider.Component(ctx) if err != nil { return http.StatusInternalServerError, []byte(err.Error()) } + if aComponent == nil { + return http.StatusNotFound, []byte("component was not found") + } views, err := router.ExtractCacheableViews(ctx, aComponent) if err != nil { return http.StatusInternalServerError, []byte(err.Error()) } - views = append(views, views...) - URIs[i] = aComponent.URI + if _, ok := viewsByURI[aComponent.URI]; !ok { + URIs = append(URIs, aComponent.URI) + } + viewsByURI[aComponent.URI] = append(viewsByURI[aComponent.URI], views...) } - lookup := func(_ context.Context, _, _ string) ([]*view.View, error) { - return views, nil + lookup := func(_ context.Context, _, matchingURI string) ([]*view.View, error) { + return viewsByURI[matchingURI], nil } response := warmup.PreCache(ctx, lookup, URIs...) data, err := json.Marshal(response) diff --git a/gateway/route_warmup_test.go b/gateway/route_warmup_test.go new file mode 100644 index 000000000..6151da3e8 --- /dev/null +++ b/gateway/route_warmup_test.go @@ -0,0 +1,74 @@ +package gateway + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/gateway/runtime/meta" + "github.com/viant/datly/gateway/warmup" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/repository/path" + "github.com/viant/datly/repository/version" + "github.com/viant/datly/view" +) + +func TestRouterAppendCacheWarmupRoute_GET(t *testing.T) { + router := &Router{ + config: &Config{ + ExposableConfig: ExposableConfig{ + APIPrefix: "/v1/api", + Meta: meta.Config{CacheWarmURI: "/v1/api/cache/warmup"}, + }, + }, + } + aPath := &path.Path{Path: *contract.NewPath(http.MethodGet, "/v1/api/order")} + + routes := router.appendCacheWarmupRoute(nil, aPath, nil) + + require.Len(t, routes, 1) + require.Equal(t, RouteWarmupKind, routes[0].Kind) + require.Equal(t, http.MethodPost, routes[0].Path.Method) + require.Equal(t, "/v1/api/cache/warmup/order", routes[0].Path.URI) +} + +func TestRouterAppendCacheWarmupRoute_NonGET(t *testing.T) { + router := &Router{ + config: &Config{ + ExposableConfig: ExposableConfig{ + APIPrefix: "/v1/api", + Meta: meta.Config{CacheWarmURI: "/v1/api/cache/warmup"}, + }, + }, + } + aPath := &path.Path{Path: *contract.NewPath(http.MethodPost, "/v1/api/order")} + + routes := router.appendCacheWarmupRoute(nil, aPath, nil) + + require.Empty(t, routes) +} + +func TestRouterHandleCacheWarmupWithErr_NoCacheViews(t *testing.T) { + router := &Router{} + provider := repository.NewProvider( + *contract.NewPath(http.MethodGet, "/v1/api/order"), + &version.Control{}, + func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { + return &repository.Component{ + Path: *contract.NewPath(http.MethodGet, "/v1/api/order"), + View: &view.View{Name: "order"}, + }, nil + }, + ) + + statusCode, body := router.handleCacheWarmupWithErr(context.Background(), []*repository.Provider{provider}) + + require.Equal(t, http.StatusOK, statusCode) + response := &warmup.Response{} + require.NoError(t, json.Unmarshal(body, response)) + require.Equal(t, "ok", response.Status) + require.Empty(t, response.PreCached) +} diff --git a/gateway/router.go b/gateway/router.go index a63bf8a98..f76df5e29 100644 --- a/gateway/router.go +++ b/gateway/router.go @@ -402,10 +402,7 @@ func (r *Router) newMatcher(ctx context.Context) (*matcher.Matcher, []*contract. routes = append(routes, r.NewMetricRoute(r.metricURL(r.config.Meta.MetricURI, aPath.URI))) } - //TODO extend path.Path with cache info to pre exract cacheable view - //if views := router.ExtractCacheableViews(route); len(views) > 0 { - // routes = append(routes, r.NewWarmupRoute(r.routeURL(r.config.APIPrefix, r.config.Build.CacheWarmURI, route.URI), route)) - //} + routes = r.appendCacheWarmupRoute(routes, aPath, provider) } if len(apiKeys) > 0 { //update keys to all path derived routes for i := offset; i < len(routes); i++ { @@ -442,6 +439,17 @@ func (r *Router) newMatcher(ctx context.Context) (*matcher.Matcher, []*contract. } return matcher.NewMatcher(matchables), paths, nil } + +func (r *Router) appendCacheWarmupRoute(routes []*Route, aPath *path.Path, provider *repository.Provider) []*Route { + if aPath.Method != http.MethodGet { + return routes + } + if strings.TrimSpace(r.config.Meta.CacheWarmURI) == "" { + return routes + } + return append(routes, r.NewWarmupRoute(r.routeURL(r.config.Meta.CacheWarmURI, aPath.URI), provider)) +} + func (r *Router) NewContentRoute(aPath *path.Path) []*Route { if !strings.HasSuffix(aPath.Path.URI, "/") && !strings.HasSuffix(aPath.Path.URI, "*") { aPath.Path.URI += "/" diff --git a/gateway/warmup/cache.go b/gateway/warmup/cache.go index eeda083e8..476a5d1f6 100644 --- a/gateway/warmup/cache.go +++ b/gateway/warmup/cache.go @@ -2,16 +2,21 @@ package warmup import ( "context" + "fmt" "github.com/viant/datly/view" "github.com/viant/datly/warmup" "net/http" + "strings" "sync" "time" ) type PreCachables func(ctx context.Context, method, matchingURI string) ([]*view.View, error) type PreCached struct { + URI string View string + Column string + Params string Elapsed string TimeTaken time.Duration Rows int @@ -24,30 +29,50 @@ type Response struct { } func PreCache(ctx context.Context, lookup PreCachables, warmupURIs ...string) *Response { + started := time.Now() + fmt.Printf("[INFO] cache warmup request start start_time=%s uris=%v\n", started.Format(time.RFC3339), warmupURIs) group := sync.WaitGroup{} var err error var mux = sync.Mutex{} var response = &Response{Status: "ok"} + setErr := func(e error) { + if e == nil { + return + } + mux.Lock() + defer mux.Unlock() + err = e + } for _, URI := range warmupURIs { group.Add(1) go func(URI string) { defer group.Done() startTime := time.Now() + fmt.Printf("[INFO] cache warmup uri start uri=%s start_time=%s\n", URI, startTime.Format(time.RFC3339)) views, e := lookup(ctx, http.MethodGet, URI) if e != nil { - err = e + fmt.Printf("[INFO] cache warmup uri lookup error uri=%s elapsed=%s error=%v\n", URI, time.Since(startTime), e) + setErr(e) } - var added int - if added, e = warmup.PopulateCache(views); e != nil { - err = e + fmt.Printf("[INFO] cache warmup uri views uri=%s count=%d views=%s elapsed=%s\n", URI, len(views), viewNames(views), time.Since(startTime)) + var result *warmup.Result + if result, e = warmup.PopulateCacheWithDetails(views); e != nil { + fmt.Printf("[INFO] cache warmup uri populate error uri=%s elapsed=%s error=%v\n", URI, time.Since(startTime), e) + setErr(e) } elapsed := time.Now().Sub(startTime) - for _, v := range views { - mux.Lock() - response.PreCached = append(response.PreCached, &PreCached{View: v.Name, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: added}) - mux.Unlock() + rows := 0 + if result != nil { + rows = result.Rows + } + fmt.Printf("[INFO] cache warmup uri done uri=%s rows=%d elapsed=%s\n", URI, rows, elapsed) + if result == nil { + return } + mux.Lock() + appendPreCached(response, URI, result) + mux.Unlock() }(URI) } group.Wait() @@ -55,5 +80,40 @@ func PreCache(ctx context.Context, lookup PreCachables, warmupURIs ...string) *R response.Error = err.Error() response.Status = "error" } + fmt.Printf("[INFO] cache warmup request done status=%s elapsed=%s\n", response.Status, time.Since(started)) return response } + +func appendPreCached(response *Response, URI string, result *warmup.Result) { + if response == nil || result == nil { + return + } + for _, entry := range result.Entries { + if entry == nil { + continue + } + response.PreCached = append(response.PreCached, &PreCached{ + URI: URI, + View: entry.View, + Column: entry.Column, + Params: entry.Params, + Elapsed: entry.Elapsed, + TimeTaken: entry.TimeTaken, + Rows: entry.Rows, + }) + } +} + +func viewNames(views []*view.View) string { + if len(views) == 0 { + return "" + } + names := make([]string, 0, len(views)) + for _, candidate := range views { + if candidate == nil { + continue + } + names = append(names, candidate.Name) + } + return strings.Join(names, ",") +} diff --git a/gateway/warmup/cache_test.go b/gateway/warmup/cache_test.go new file mode 100644 index 000000000..2489056f1 --- /dev/null +++ b/gateway/warmup/cache_test.go @@ -0,0 +1,29 @@ +package warmup + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + datlywarmup "github.com/viant/datly/warmup" +) + +func TestAppendPreCachedUsesEntryRows(t *testing.T) { + response := &Response{} + result := &datlywarmup.Result{ + Rows: 30, + Entries: []*datlywarmup.EntryResult{ + {View: "periodSummary#", Column: "order_id", Params: "Period=today", Elapsed: "1s", TimeTaken: time.Second, Rows: 10}, + {View: "periodSummary#", Column: "order_id", Params: "Period=month", Elapsed: "2s", TimeTaken: 2 * time.Second, Rows: 20}, + }, + } + + appendPreCached(response, "/v1/api/cache/warmup/order", result) + + require.Len(t, response.PreCached, 2) + require.Equal(t, "Period=today", response.PreCached[0].Params) + require.Equal(t, 10, response.PreCached[0].Rows) + require.Equal(t, "Period=month", response.PreCached[1].Params) + require.Equal(t, 20, response.PreCached[1].Rows) + require.Equal(t, "/v1/api/cache/warmup/order", response.PreCached[1].URI) +} diff --git a/internal/translator/function/cache_warmup.go b/internal/translator/function/cache_warmup.go new file mode 100644 index 000000000..9835c0509 --- /dev/null +++ b/internal/translator/function/cache_warmup.go @@ -0,0 +1,139 @@ +package function + +import ( + "fmt" + "strings" + + "github.com/viant/datly/view" + "github.com/viant/sqlparser" +) + +type cacheWarmup struct{} + +func (c *cacheWarmup) Apply(args []string, column *sqlparser.Column, resource *view.Resource, aView *view.View) error { + if _, err := convertArguments(c, args); err != nil { + return err + } + if aView.Cache == nil { + return fmt.Errorf("cache_warmup requires cache to be configured first") + } + + warmup := &view.Warmup{IndexColumn: args[0]} + parameters := &view.CacheParameters{} + for _, raw := range args[1:] { + if connector, ok, err := parseWarmupConnector(raw); ok || err != nil { + if err != nil { + return err + } + warmup.Connector = connector + continue + } + if indexParameter, ok, err := parseWarmupIndexParameter(raw); ok || err != nil { + if err != nil { + return err + } + warmup.IndexParameter = indexParameter + continue + } + param, err := parseWarmupParam(raw) + if err != nil { + return err + } + parameters.Set = append(parameters.Set, param) + } + if len(parameters.Set) > 0 { + warmup.Cases = append(warmup.Cases, parameters) + } + aView.Cache.Warmup = warmup + return nil +} + +func parseWarmupConnector(raw string) (*view.Connector, bool, error) { + name, value, ok := splitWarmupOption(raw) + if !ok { + return nil, false, nil + } + switch strings.ToLower(name) { + case "connector": + default: + return nil, false, nil + } + if value == "" { + return nil, true, fmt.Errorf("warmup connector was empty") + } + if strings.Contains(value, ",") { + return nil, true, fmt.Errorf("warmup connector %q must be a single connector name", value) + } + return view.NewRefConnector(value), true, nil +} + +func parseWarmupIndexParameter(raw string) (string, bool, error) { + name, value, ok := splitWarmupOption(raw) + if !ok { + return "", false, nil + } + switch strings.ToLower(name) { + case "indexparameter", "index_param", "indexparam": + default: + return "", false, nil + } + if value == "" { + return "", true, fmt.Errorf("warmup index parameter was empty") + } + if strings.Contains(value, ",") { + return "", true, fmt.Errorf("warmup index parameter %q must be a single parameter name", value) + } + return value, true, nil +} + +func parseWarmupParam(raw string) (*view.ParamValue, error) { + name, rawValues, ok := splitWarmupOption(raw) + if !ok { + return nil, fmt.Errorf("invalid warmup parameter %q, expected name=value1,value2", raw) + } + if name == "" { + return nil, fmt.Errorf("warmup parameter name was empty") + } + + values := strings.Split(rawValues, ",") + result := &view.ParamValue{Name: name, Values: make([]interface{}, 0, len(values)), ExcludeDefault: true} + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + result.Values = append(result.Values, value) + } + if len(result.Values) == 0 { + return nil, fmt.Errorf("warmup parameter %q has no values", name) + } + return result, nil +} + +func splitWarmupOption(raw string) (string, string, bool) { + raw = strings.TrimSpace(raw) + parts := strings.SplitN(raw, "=", 2) + if len(parts) != 2 { + return "", "", false + } + return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), true +} + +func (c *cacheWarmup) Name() string { + return "cache_warmup" +} + +func (c *cacheWarmup) Description() string { + return "set view.Cache.Warmup connector and parameter permutations" +} + +func (c *cacheWarmup) Arguments() []*Argument { + return []*Argument{ + { + Name: "indexColumn", + Description: "cache warmup index column", + Required: true, + DataType: "string", + }, + } +} diff --git a/internal/translator/function/cache_warmup_test.go b/internal/translator/function/cache_warmup_test.go new file mode 100644 index 000000000..cdc1fd514 --- /dev/null +++ b/internal/translator/function/cache_warmup_test.go @@ -0,0 +1,64 @@ +package function + +import ( + "testing" + + "github.com/viant/datly/view" +) + +func TestCacheWarmupApply(t *testing.T) { + aView := &view.View{Cache: view.NewRefCache("aerospike")} + subject := &cacheWarmup{} + + err := subject.Apply([]string{ + "order_id", + "Connector=bq_metrics_prewarm", + "IndexParameter=OrderId", + "Period=today,yesterday", + "Granularity=hour,day", + }, nil, &view.Resource{}, aView) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if aView.Cache.Warmup == nil { + t.Fatalf("expected warmup to be set") + } + if aView.Cache.Warmup.IndexColumn != "order_id" { + t.Fatalf("unexpected index column: %v", aView.Cache.Warmup.IndexColumn) + } + if aView.Cache.Warmup.IndexParameter != "OrderId" { + t.Fatalf("unexpected index parameter: %v", aView.Cache.Warmup.IndexParameter) + } + if len(aView.Cache.Warmup.Cases) != 1 { + t.Fatalf("unexpected cases count: %v", len(aView.Cache.Warmup.Cases)) + } + if len(aView.Cache.Warmup.Cases[0].Set) != 2 { + t.Fatalf("unexpected warmup parameter count: %v", len(aView.Cache.Warmup.Cases[0].Set)) + } + if aView.Cache.Warmup.Connector == nil || aView.Cache.Warmup.Connector.Ref != "bq_metrics_prewarm" { + t.Fatalf("unexpected warmup connector: %#v", aView.Cache.Warmup.Connector) + } +} + +func TestCacheWarmupApplyRequiresCache(t *testing.T) { + err := (&cacheWarmup{}).Apply([]string{"order_id"}, nil, &view.Resource{}, &view.View{}) + if err == nil { + t.Fatalf("expected error") + } +} + +func TestCacheWarmupApplyRejectsEmptyConnector(t *testing.T) { + aView := &view.View{Cache: view.NewRefCache("aerospike")} + err := (&cacheWarmup{}).Apply([]string{"order_id", "Connector="}, nil, &view.Resource{}, aView) + if err == nil { + t.Fatalf("expected error") + } +} + +func TestCacheWarmupApplyRejectsEmptyIndexParameter(t *testing.T) { + aView := &view.View{Cache: view.NewRefCache("aerospike")} + err := (&cacheWarmup{}).Apply([]string{"order_id", "IndexParameter="}, nil, &view.Resource{}, aView) + if err == nil { + t.Fatalf("expected error") + } +} diff --git a/internal/translator/function/init.go b/internal/translator/function/init.go index 12442bb85..7b093f900 100644 --- a/internal/translator/function/init.go +++ b/internal/translator/function/init.go @@ -3,6 +3,7 @@ package function func init() { _registry.Register(&connector{}) _registry.Register(&cache{}) + _registry.Register(&cacheWarmup{}) _registry.Register(&limit{}) _registry.Register(&orderBy{}) _registry.Register(&allowedOrderByColumns{}) diff --git a/internal/translator/view.go b/internal/translator/view.go index de2243969..975df5581 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -91,6 +91,10 @@ func (v *View) buildCacheWarmup(warmup map[string]interface{}, viewlet *Viewlet) explicitIndex, _ := warmup["IndexColumn"] delete(warmup, "IndexColumn") + indexParameter, _ := warmup["IndexParameter"] + delete(warmup, "IndexParameter") + connector, _ := warmup["Connector"] + delete(warmup, "Connector") var refColumn string if viewlet.Join != nil { _, refColumn = inference.ExtractRelationColumns(viewlet.Join) @@ -105,6 +109,12 @@ func (v *View) buildCacheWarmup(warmup map[string]interface{}, viewlet *Viewlet) if result.IndexColumn == "" { return nil } + if parameterName := strings.TrimSpace(fmt.Sprint(indexParameter)); parameterName != "" && parameterName != "" { + result.IndexParameter = parameterName + } + if connectorName := strings.TrimSpace(fmt.Sprint(connector)); connectorName != "" && connectorName != "" { + result.Connector = view.NewRefConnector(connectorName) + } multiSet := &view.CacheParameters{} for k, v := range warmup { diff --git a/internal/translator/view_warmup_test.go b/internal/translator/view_warmup_test.go new file mode 100644 index 000000000..f345adada --- /dev/null +++ b/internal/translator/view_warmup_test.go @@ -0,0 +1,26 @@ +package translator + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestViewBuildCacheWarmup_RootViewUsesExplicitIndexColumn(t *testing.T) { + subject := &View{} + viewlet := &Viewlet{Name: "adOrderRoot"} + + warmup := subject.buildCacheWarmup(map[string]interface{}{ + "IndexColumn": "ad_order_id", + "IndexParameter": "AdOrderId", + "Connector": "bq_metrics_prewarm", + }, viewlet) + + require.NotNil(t, warmup) + require.Equal(t, "ad_order_id", warmup.IndexColumn) + require.Equal(t, "AdOrderId", warmup.IndexParameter) + require.NotNil(t, warmup.Connector) + require.Equal(t, "bq_metrics_prewarm", warmup.Connector.Ref) + require.Len(t, warmup.Cases, 1) + require.Len(t, warmup.Cases[0].Set, 0) +} diff --git a/service/reader/service.go b/service/reader/service.go index a0074eddb..d381f42b0 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -363,6 +363,7 @@ func (s *Service) querySummary(ctx context.Context, session *Session, aView *vie } finished := Now() aView.Logger.Log("reading view %v meta took %v, SQL: %v , Args: %v\n", aView.Name, finished.Sub(now).String(), SQL, args) + logCacheRead(aView, cacheStats, finished.Sub(now), collector.Len(), args) return execInfo, nil } @@ -377,11 +378,20 @@ func (s *Service) buildParametrizedSQL(ctx context.Context, aView *view.View, st defer wg.Done() if (aView.Cache != nil && aView.Cache.Warmup != nil) || relation != nil { data, _ := session.ParentData() - if relation == nil && aView.Cache != nil && aView.Cache.Warmup != nil { - columnInMatcher, cacheErr = s.topLevelWarmupMatcher(ctx, aView, statelet, data.AsParam()) + if aView.Cache != nil && aView.Cache.Warmup != nil { + if relation == nil { + columnInMatcher, cacheErr = s.topLevelWarmupMatcher(ctx, aView, statelet, data.AsParam()) + return + } + columnInMatcher, cacheErr = s.relationWarmupMatcher(ctx, aView, statelet, batchData, relation) + if cacheErr != nil || columnInMatcher != nil { + return + } + } + if relation != nil { + columnInMatcher, cacheErr = s.sqlBuilder.CacheSQLWithOptions(ctx, aView, statelet, batchData, relation, data.AsParam()) return } - columnInMatcher, cacheErr = s.sqlBuilder.CacheSQLWithOptions(ctx, aView, statelet, batchData, relation, data.AsParam()) } }() @@ -395,12 +405,44 @@ func (s *Service) buildParametrizedSQL(ctx context.Context, aView *view.View, st return parametrizedSQL, columnInMatcher, cacheErr } -func normalizeWarmupName(input string) string { - input = strings.ToLower(strings.TrimSpace(input)) - input = strings.ReplaceAll(input, "_", "") - input = strings.ReplaceAll(input, "-", "") - input = strings.ReplaceAll(input, ".", "") - return input +func (s *Service) relationWarmupMatcher(ctx context.Context, aView *view.View, statelet *view.Statelet, batchData *view.BatchData, relation *view.Relation) (*cache.ParmetrizedQuery, error) { + if aView == nil || aView.Cache == nil || aView.Cache.Warmup == nil || batchData == nil || relation == nil || relation.Of == nil || len(relation.Of.On) != 1 { + return nil, nil + } + indexColumn := strings.TrimSpace(aView.Cache.Warmup.IndexColumn) + if indexColumn == "" || len(batchData.ValuesBatch) == 0 || batchData.HasComposite() || len(batchData.ColumnNames) != 1 { + return nil, nil + } + if !matchesWarmupIndexColumn(indexColumn, relation.Of.On[0], batchData.ColumnNames[0]) { + return nil, nil + } + matcher, err := s.warmupMatcher(ctx, aView, statelet, nil) + if err != nil || matcher == nil { + return matcher, err + } + matcher.By = indexColumn + matcher.In = batchData.ValuesBatch + return matcher, nil +} + +func matchesWarmupIndexColumn(indexColumn string, link *view.Link, batchColumn string) bool { + if link == nil { + return false + } + relationColumn := strings.TrimSpace(link.Column) + if relationColumn == "" { + return false + } + return strings.EqualFold(normalizeWarmupColumnName(relationColumn), normalizeWarmupColumnName(indexColumn)) && + strings.EqualFold(normalizeWarmupColumnName(batchColumn), normalizeWarmupColumnName(indexColumn)) +} + +func normalizeWarmupColumnName(input string) string { + input = strings.TrimSpace(input) + if index := strings.LastIndex(input, "."); index != -1 { + input = input[index+1:] + } + return strings.TrimSpace(input) } func cloneStructologyState(src *structology.State) *structology.State { @@ -466,17 +508,7 @@ func (s *Service) topLevelWarmupMatcher(ctx context.Context, aView *view.View, s if indexColumn == "" { return nil, nil } - target := normalizeWarmupName(indexColumn) - var matchParam *state.Parameter - for _, candidate := range aView.Template.Parameters { - if candidate == nil { - continue - } - if normalizeWarmupName(candidate.Name) == target { - matchParam = candidate - break - } - } + matchParam := warmupIndexParameter(aView) if matchParam == nil { return nil, nil } @@ -489,33 +521,58 @@ func (s *Service) topLevelWarmupMatcher(ctx context.Context, aView *view.View, s if len(values) == 0 { return nil, nil } + matcher, err := s.warmupMatcher(ctx, aView, statelet, parent) + if err != nil || matcher == nil { + return matcher, err + } + matcher.By = indexColumn + matcher.In = values + return matcher, nil +} + +func (s *Service) warmupMatcher(ctx context.Context, aView *view.View, statelet *view.Statelet, parent *expand.ViewContext) (*cache.ParmetrizedQuery, error) { + if statelet == nil || statelet.Template == nil { + return nil, nil + } clonedTemplate := cloneStructologyState(statelet.Template) if clonedTemplate == nil { return nil, nil } - if clonedSelector, err := clonedTemplate.Selector(matchParam.Name); err == nil && clonedSelector != nil { - zero := reflect.Zero(clonedSelector.Type()).Interface() - _ = clonedSelector.SetValue(clonedTemplate.Pointer(), zero) - } - if marker := clonedTemplate.Type().Marker(); marker != nil { - clonedTemplate.EnsureMarker() - if idx := marker.Index(matchParam.Name); idx != -1 { - _ = marker.Set(clonedTemplate.Pointer(), idx, false) + if candidate := warmupIndexParameter(aView); candidate != nil { + if clonedSelector, err := clonedTemplate.Selector(candidate.Name); err == nil && clonedSelector != nil { + zero := reflect.Zero(clonedSelector.Type()).Interface() + _ = clonedSelector.SetValue(clonedTemplate.Pointer(), zero) + } + if marker := clonedTemplate.Type().Marker(); marker != nil { + clonedTemplate.EnsureMarker() + if idx := marker.Index(candidate.Name); idx != -1 { + _ = marker.Set(clonedTemplate.Pointer(), idx, false) + } } } cloned := *statelet cloned.Template = clonedTemplate - matcher, err := s.sqlBuilder.CacheSQLWithOptions(ctx, aView, &cloned, nil, nil, parent) - if err != nil { - return nil, err + return s.sqlBuilder.CacheSQLWithOptions(ctx, aView, &cloned, nil, nil, parent) +} + +func warmupIndexParameter(aView *view.View) *state.Parameter { + if aView == nil || aView.Cache == nil || aView.Cache.Warmup == nil || aView.Template == nil { + return nil } - if matcher == nil { - return nil, nil + parameterName := strings.TrimSpace(aView.Cache.Warmup.IndexParameter) + if parameterName == "" { + return nil } - matcher.By = indexColumn - matcher.In = values - return matcher, nil + for _, candidate := range aView.Template.Parameters { + if candidate == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(candidate.Name), parameterName) { + return candidate + } + } + return nil } func (s *Service) BuildCriteria(ctx context.Context, value interface{}, options *codec.CriteriaBuilderOptions) (*codec.Criteria, error) { @@ -697,6 +754,7 @@ BEGIN: end := time.Now() aView.Logger.ReadingData(end.Sub(begin), parametrizedSQL.SQL, *readData, parametrizedSQL.Args, err) + logCacheRead(aView, cacheStats, end.Sub(begin), *readData, parametrizedSQL.Args) if err != nil { stats.SetError(err) anExec, err := s.HandleSQLError(err, session, aView, parametrizedSQL, stats) @@ -810,6 +868,45 @@ func (s *Service) HandleSQLError(err error, session *Session, aView *view.View, return stats, fmt.Errorf("database error occured while fetching Data for view %v %w", aView.Name, err) } +func logCacheRead(aView *view.View, stats *cache.Stats, elapsed time.Duration, rows int, args []interface{}) { + if stats == nil { + return + } + fmt.Printf("[INFO] datly cache read view=%s source=%s type=%s found_warmup=%t found_lazy=%t records=%d rows=%d namespace=%s set=%s elapsed=%s args=%v\n", + aView.Name, + cacheReadSource(stats), + stats.Type, + stats.FoundWarmup, + stats.FoundLazy, + stats.RecordsCounter, + rows, + stats.Namespace, + stats.Dataset, + elapsed, + args) +} + +func cacheReadSource(stats *cache.Stats) string { + if stats.ErrorType != "" { + return "error" + } + switch stats.Type { + case cache.TypeReadMulti: + return "warmup" + case cache.TypeReadSingle: + return "lazy" + case cache.TypeWrite: + return "miss_write" + } + if stats.FoundWarmup { + return "warmup" + } + if stats.FoundLazy { + return "lazy" + } + return "miss" +} + func NewExecutionInfo(index *cache.ParmetrizedQuery, cacheStats *cache.Stats, collector *view.Collector) (*response.SQLExecution, func()) { var cacheInfo *response.CacheStats if cacheStats != nil { diff --git a/service/reader/service_warmup_test.go b/service/reader/service_warmup_test.go index fb2c7a3aa..15149f262 100644 --- a/service/reader/service_warmup_test.go +++ b/service/reader/service_warmup_test.go @@ -1,10 +1,13 @@ package reader import ( + "context" "reflect" "testing" "github.com/stretchr/testify/require" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" "github.com/viant/structology" ) @@ -49,3 +52,97 @@ func TestCloneStructologyState_DeepCopiesValueAndMarker(t *testing.T) { require.Equal(t, 0, cloneSelector.Value(cloned.Pointer())) require.False(t, cloneSelector.Has(cloned.Pointer())) } + +func TestRelationWarmupMatcherRequiresExactRelationKey(t *testing.T) { + aView := &view.View{ + Cache: &view.Cache{ + Warmup: &view.Warmup{IndexColumn: "order_id"}, + }, + } + relation := &view.Relation{ + Of: &view.ReferenceView{ + On: view.JoinOn(view.WithLink("CampaignId", "campaign_id")), + }, + } + batchData := &view.BatchData{ + ColumnNames: []string{"campaign_id"}, + ValuesBatch: []interface{}{ + 101, + }, + } + + matcher, err := (&Service{}).relationWarmupMatcher(context.Background(), aView, view.NewStatelet(), batchData, relation) + + require.NoError(t, err) + require.Nil(t, matcher) +} + +func TestMatchesWarmupIndexColumnUsesReferenceColumn(t *testing.T) { + link := view.WithLink("OrderId", "p.order_id") + + matched := matchesWarmupIndexColumn("order_id", link, "p.order_id") + + require.True(t, matched) +} + +func TestMatchesWarmupIndexColumnRejectsFieldOnlyMatch(t *testing.T) { + link := view.WithLink("OrderId", "campaign_id") + + matched := matchesWarmupIndexColumn("order_id", link, "campaign_id") + + require.False(t, matched) +} + +func TestMatchesWarmupIndexColumnRejectsCollapsedIdentifier(t *testing.T) { + link := view.WithLink("OrderId", "p.orderid") + + matched := matchesWarmupIndexColumn("order_id", link, "p.orderid") + + require.False(t, matched) +} + +func TestWarmupIndexParameterUsesExplicitParameter(t *testing.T) { + aView := &view.View{ + Cache: &view.Cache{ + Warmup: &view.Warmup{IndexColumn: "order_id", IndexParameter: "OrderId"}, + }, + Template: view.NewTemplate("", + view.WithTemplateParameters(state.NewParameter("OrderId", state.NewQueryLocation("order_id"))), + ), + } + + parameter := warmupIndexParameter(aView) + + require.NotNil(t, parameter) + require.Equal(t, "OrderId", parameter.Name) +} + +func TestWarmupIndexParameterDoesNotInferCamelCase(t *testing.T) { + aView := &view.View{ + Cache: &view.Cache{ + Warmup: &view.Warmup{IndexColumn: "order_id"}, + }, + Template: view.NewTemplate("", + view.WithTemplateParameters(state.NewParameter("OrderId", state.NewQueryLocation("order_id"))), + ), + } + + parameter := warmupIndexParameter(aView) + + require.Nil(t, parameter) +} + +func TestWarmupIndexParameterDoesNotFallbackToMatchingColumnName(t *testing.T) { + aView := &view.View{ + Cache: &view.Cache{ + Warmup: &view.Warmup{IndexColumn: "order_id"}, + }, + Template: view.NewTemplate("", + view.WithTemplateParameters(state.NewParameter("order_id", state.NewQueryLocation("order_id"))), + ), + } + + parameter := warmupIndexParameter(aView) + + require.Nil(t, parameter) +} diff --git a/view/cache.go b/view/cache.go index 628b717df..e7db66b14 100644 --- a/view/cache.go +++ b/view/cache.go @@ -49,10 +49,11 @@ type ( } Warmup struct { - IndexColumn string - IndexMeta bool `json:",omitempty"` - Connector *Connector `json:",omitempty"` - Cases []*CacheParameters + IndexColumn string + IndexParameter string `json:",omitempty" yaml:",omitempty"` + IndexMeta bool `json:",omitempty"` + Connector *Connector `json:",omitempty"` + Cases []*CacheParameters } CacheParameters struct { @@ -62,6 +63,8 @@ type ( ParamValue struct { Name string Values []interface{} + // ExcludeDefault keeps explicitly declared warmup cases from adding an extra nil/default selector. + ExcludeDefault bool `json:",omitempty" yaml:",omitempty"` _param *state.Parameter } @@ -71,6 +74,7 @@ type ( Column string MetaColumn string IndexMeta bool + Label string } CacheInputFn func() ([]*CacheInput, error) @@ -393,7 +397,7 @@ func (c *Cache) getParamValues(ctx context.Context, paramValue *ParamValue) ([]i result[i] = converted } - if !paramValue._param.IsRequired() { + if !paramValue._param.IsRequired() && !paramValue.ExcludeDefault { result = append(result, nil) } @@ -483,9 +487,11 @@ outer: for { selector := &Statelet{} selector.Init(c.owner) + debugParams := make([]string, 0, len(paramValues)) for i, possibleValues := range paramValues { actualValue := possibleValues[indexes[i]] + debugParams = append(debugParams, fmt.Sprintf("%s=%v", set.Set[i].Name, actualValue)) if actualValue == nil { continue } @@ -495,7 +501,11 @@ outer: } } - *selectors = append(*selectors, c.NewInput(selector)) + label := strings.Join(debugParams, ",") + input := c.NewInput(selector) + input.Label = label + *selectors = append(*selectors, input) + fmt.Printf("[INFO] cache warmup selector view=%s index_column=%s params=%s\n", c.owner.Name, c.Warmup.IndexColumn, label) for i := len(indexes) - 1; i >= 0; i-- { if indexes[i] < len(paramValues[i])-1 { diff --git a/view/collector_placeholder_test.go b/view/collector_placeholder_test.go new file mode 100644 index 000000000..2cf32145b --- /dev/null +++ b/view/collector_placeholder_test.go @@ -0,0 +1,46 @@ +package view + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/viant/datly/view/state" + "github.com/viant/xunsafe" +) + +type placeholderParentRow struct { + LinkedConversationID string +} + +func TestCollector_ParentPlaceholders_SkipsBlankStringKeys(t *testing.T) { + parentView := &View{Schema: state.NewSchema(reflect.TypeOf([]*placeholderParentRow{}))} + parentDest := []*placeholderParentRow{ + {LinkedConversationID: ""}, + {LinkedConversationID: "child-1"}, + {LinkedConversationID: " "}, + } + parentCollector := NewCollector(parentView.Schema.Slice(), parentView, &parentDest, nil, false) + + relation := &Relation{ + On: Links{ + &Link{ + Field: "LinkedConversationID", + Column: "LINKED_CONVERSATION_ID", + xField: xunsafe.FieldByName(reflect.TypeOf(placeholderParentRow{}), "LinkedConversationID"), + }, + }, + Of: &ReferenceView{ + On: Links{ + &Link{Field: "ID", Column: "ID"}, + }, + }, + } + + childCollector := &Collector{parent: parentCollector, relation: relation} + values, composite, columns := childCollector.ParentPlaceholders() + + assert.Equal(t, []interface{}{"child-1"}, values) + assert.Nil(t, composite) + assert.Equal(t, []string{"ID"}, columns) +} diff --git a/view/extension/predicates_test.go b/view/extension/predicates_test.go new file mode 100644 index 000000000..384e6798f --- /dev/null +++ b/view/extension/predicates_test.go @@ -0,0 +1,16 @@ +package extension + +import ( + "strings" + "testing" +) + +func TestNewDurationPredicate_DoesNotUseLogicalOrInVelty(t *testing.T) { + predicate := NewDurationPredicate() + if predicate == nil || predicate.Template == nil { + t.Fatalf("expected duration predicate template") + } + if strings.Contains(predicate.Template.Source, "||") { + t.Fatalf("expected duration predicate template to avoid logical OR, got:\n%s", predicate.Template.Source) + } +} diff --git a/view/predicate_test.go b/view/predicate_test.go new file mode 100644 index 000000000..1c266810d --- /dev/null +++ b/view/predicate_test.go @@ -0,0 +1,44 @@ +package view + +import "testing" + +func TestValidatePredicateArgs_Duration(t *testing.T) { + testCases := []struct { + name string + predicate string + value interface{} + args []string + wantErr bool + }{ + { + name: "thirty days requires seventh arg", + predicate: "duration", + value: "thirty_days", + args: []string{"d", "cd", "h", "ch", "yd", "wd"}, + wantErr: true, + }, + { + name: "thirty days with seventh arg", + predicate: "duration", + value: "thirty_days", + args: []string{"d", "cd", "h", "ch", "yd", "wd", "md"}, + wantErr: false, + }, + { + name: "week remains backward compatible with six args", + predicate: "duration", + value: "week", + args: []string{"d", "cd", "h", "ch", "yd", "wd"}, + wantErr: false, + }, + } + for _, testCase := range testCases { + err := validatePredicateArgs(testCase.predicate, testCase.value, testCase.args) + if testCase.wantErr && err == nil { + t.Fatalf("%s: expected error", testCase.name) + } + if !testCase.wantErr && err != nil { + t.Fatalf("%s: unexpected error: %v", testCase.name, err) + } + } +} diff --git a/view/relation_sql_tags.go b/view/relation_sql_tags.go new file mode 100644 index 000000000..3cf5fcc35 --- /dev/null +++ b/view/relation_sql_tags.go @@ -0,0 +1,120 @@ +package view + +import ( + "reflect" + "strings" +) + +func NormalizeRelationSQLTagURIsWithViews(typeDef string, views []*View) string { + if strings.TrimSpace(typeDef) == "" || len(views) == 0 { + return typeDef + } + uris := map[string]string{} + for _, item := range views { + if item == nil || item.Template == nil { + continue + } + uri := strings.TrimSpace(item.Template.SourceURL) + if uri == "" { + continue + } + if name := strings.TrimSpace(item.Name); name != "" { + uris[name] = uri + } + if item.Schema != nil { + if name := strings.TrimSpace(item.Schema.Name); strings.HasSuffix(name, "View") { + base := strings.TrimSuffix(name, "View") + if base != "" { + uris[base] = uri + } + } + } + } + for holder, uri := range uris { + typeDef = replaceRelationFieldSQLTagWithURI(typeDef, holder, uri) + } + return typeDef +} + +func NormalizeRelationStructFields(fields []reflect.StructField, parent *View) []reflect.StructField { + if len(fields) == 0 || parent == nil { + return fields + } + result := make([]reflect.StructField, len(fields)) + copy(result, fields) + uris := collectRelationSQLURIs(map[string]string{}, parent) + for i := range result { + uri := uris[result[i].Name] + if uri == "" { + continue + } + tag := replaceRelationFieldSQLTagWithURI(result[i].Name+" "+string(result[i].Tag), result[i].Name, uri) + tag = strings.TrimPrefix(tag, result[i].Name+" ") + result[i].Tag = reflect.StructTag(tag) + } + return result +} + +func collectRelationSQLURIs(dest map[string]string, parent *View) map[string]string { + for _, rel := range parent.With { + if rel == nil || rel.Of == nil { + continue + } + child := rel.Of.View + if child.Template != nil { + uri := strings.TrimSpace(child.Template.SourceURL) + holder := strings.TrimSpace(rel.Holder) + if holder == "" { + holder = strings.TrimSpace(child.Name) + } + if holder != "" && uri != "" { + dest[holder] = uri + } + } + collectRelationSQLURIs(dest, &child) + } + return dest +} + +func replaceRelationFieldSQLTagWithURI(typeDef string, fieldName string, uri string) string { + fieldName = strings.TrimSpace(fieldName) + uri = strings.TrimSpace(uri) + if strings.TrimSpace(typeDef) == "" || fieldName == "" || uri == "" { + return typeDef + } + search := fieldName + " " + start := 0 + for { + idx := strings.Index(typeDef[start:], search) + if idx == -1 { + return typeDef + } + idx += start + sqlIdx := strings.Index(typeDef[idx:], `sql:"`) + if sqlIdx == -1 { + return typeDef + } + sqlIdx += idx + valueStart := sqlIdx + len(`sql:"`) + valueEnd := findTagValueEnd(typeDef, valueStart) + if valueEnd == -1 { + return typeDef + } + return typeDef[:valueStart] + "uri=" + uri + typeDef[valueEnd:] + } +} + +func findTagValueEnd(tagValue string, start int) int { + escaped := false + for i := start; i < len(tagValue); i++ { + switch { + case escaped: + escaped = false + case tagValue[i] == '\\': + escaped = true + case tagValue[i] == '"': + return i + } + } + return -1 +} diff --git a/warmup/cache.go b/warmup/cache.go index 31303418d..392012bc8 100644 --- a/warmup/cache.go +++ b/warmup/cache.go @@ -8,7 +8,9 @@ import ( errUtils "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/sqlx/io/read/cache" + "strings" "sync" + "time" ) type ( @@ -24,10 +26,25 @@ type ( matcher *cache.ParmetrizedQuery view *view.View column string + label string } warmupEntryFn func() (*warmupEntry, error) notifierFn func() (int, error) + + EntryResult struct { + View string + Column string + Params string + Elapsed string + TimeTaken time.Duration + Rows int + } + + Result struct { + Rows int + Entries []*EntryResult + } ) func (c *matchersCollector) populate(ctx context.Context, collector chan warmupEntryFn, notifier chan notifierFn) { @@ -41,10 +58,13 @@ func (c *matchersCollector) populate(ctx context.Context, collector chan warmupE } func (c *matchersCollector) populateCacheCases(ctx context.Context, collector chan warmupEntryFn) (int, error) { + started := time.Now() cacheCases, err := c.view.Cache.GenerateCacheInput(ctx) if err != nil { + fmt.Printf("[INFO] cache warmup selector error view=%s cache=%s elapsed=%s error=%v\n", c.view.Name, cacheLabel(c.view), time.Since(started), err) return 0, err } + fmt.Printf("[INFO] cache warmup selector done view=%s cache=%s cases=%d elapsed=%s\n", c.view.Name, cacheLabel(c.view), len(cacheCases), time.Since(started)) for i := range cacheCases { go c.populateChan(ctx, c.view, collector, cacheCases[i]) @@ -73,6 +93,7 @@ func (c *matchersCollector) populateChan(ctx context.Context, aView *view.View, func (c *matchersCollector) createMetaWarmupEntry(ctx context.Context, aView *view.View, aChan chan warmupEntryFn, input *view.CacheInput) { cacheIndex, err := c.builder.CacheMetaSQL(ctx, aView, input.Selector, nil, nil, nil) if err != nil { + fmt.Printf("[INFO] cache warmup entry build error view=%s type=meta column=%s error=%v\n", aView.Name, input.MetaColumn, err) aChan <- func() (*warmupEntry, error) { return nil, err } @@ -84,12 +105,16 @@ func (c *matchersCollector) createMetaWarmupEntry(ctx context.Context, aView *vi matcher: cacheIndex, view: aView, column: input.MetaColumn, + label: input.Label, }, nil } } func (c *matchersCollector) createIndexWarmupEntry(ctx context.Context, aView *view.View, aChan chan warmupEntryFn, cacheInput *view.CacheInput) { build, err := c.builder.CacheSQL(ctx, c.view, cacheInput.Selector) + if err != nil { + fmt.Printf("[INFO] cache warmup entry build error view=%s type=index column=%s error=%v\n", aView.Name, cacheInput.Column, err) + } aChan <- func() (*warmupEntry, error) { if err != nil { return nil, err @@ -99,6 +124,7 @@ func (c *matchersCollector) createIndexWarmupEntry(ctx context.Context, aView *v matcher: build, view: aView, column: cacheInput.Column, + label: cacheInput.Label, }, err } } @@ -113,37 +139,44 @@ func populateCollector(ctx context.Context, aView *view.View, builder *reader.Bu }).populate(ctx, collector, notifier) } -func warmup(ctx context.Context, entries []*warmupEntry, notifier chan func() (int, error)) { +func warmup(ctx context.Context, entries []*warmupEntry, notifier chan func() (*EntryResult, error)) { for i := range entries { go readWithChan(ctx, entries[i], notifier) } } -func readWithChan(ctx context.Context, entry *warmupEntry, notifier chan func() (int, error)) { - indexed, err := readWithErr(ctx, entry) - notifier <- func() (int, error) { - return indexed, err +func readWithChan(ctx context.Context, entry *warmupEntry, notifier chan func() (*EntryResult, error)) { + result, err := readWithErr(ctx, entry) + notifier <- func() (*EntryResult, error) { + return result, err } } -func readWithErr(ctx context.Context, entry *warmupEntry) (int, error) { +func readWithErr(ctx context.Context, entry *warmupEntry) (*EntryResult, error) { + started := time.Now() + fmt.Printf("[INFO] cache warmup query start start_time=%s view=%s cache=%s db_connector=%s column=%s params=%s args=%v sql=%q\n", started.Format(time.RFC3339), entry.view.Name, cacheLabel(entry.view), warmupConnectorLabel(entry.view), entry.column, entry.label, entry.matcher.Args, truncateSQL(entry.matcher.SQL)) db, err := DB(entry) if err != nil { - return 0, err + fmt.Printf("[INFO] cache warmup query error view=%s column=%s params=%s elapsed=%s error=%v\n", entry.view.Name, entry.column, entry.label, time.Since(started), err) + return nil, err } service, err := entry.view.Cache.Service() if err != nil { - return 0, err + fmt.Printf("[INFO] cache warmup query error view=%s column=%s params=%s elapsed=%s error=%v\n", entry.view.Name, entry.column, entry.label, time.Since(started), err) + return nil, err } matcher := entry.matcher indexed, err := service.IndexBy(ctx, db, entry.column, matcher.SQL, matcher.Args) + elapsed := time.Since(started) if err != nil { - return indexed, fmt.Errorf("failed to index: %w, %v", err, matcher.SQL) + fmt.Printf("[INFO] cache warmup query error view=%s column=%s params=%s rows=%d elapsed=%s error=%v\n", entry.view.Name, entry.column, entry.label, indexed, elapsed, err) + return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: indexed}, fmt.Errorf("failed to index: %w, %v", err, matcher.SQL) } - return indexed, nil + fmt.Printf("[INFO] cache warmup query done view=%s cache=%s db_connector=%s column=%s params=%s rows=%d elapsed=%s\n", entry.view.Name, cacheLabel(entry.view), warmupConnectorLabel(entry.view), entry.column, entry.label, indexed, elapsed) + return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: indexed}, nil } func DB(entry *warmupEntry) (*sql.DB, error) { @@ -155,10 +188,21 @@ func DB(entry *warmupEntry) (*sql.DB, error) { } func PopulateCache(views []*view.View) (int, error) { + result, err := PopulateCacheWithDetails(views) + if result == nil { + return 0, err + } + return result.Rows, err +} + +func PopulateCacheWithDetails(views []*view.View) (*Result, error) { + started := time.Now() viewsWithCache := FilterCacheViews(views) + fmt.Printf("[INFO] cache warmup populate start start_time=%s views=%s cache_views=%s cache_count=%d\n", started.Format(time.RFC3339), namesOf(views), namesOf(viewsWithCache), len(viewsWithCache)) if len(viewsWithCache) == 0 { - return 0, nil + fmt.Printf("[INFO] cache warmup populate done rows=0 elapsed=%s\n", time.Since(started)) + return &Result{}, nil } collector := make(chan warmupEntryFn) @@ -187,8 +231,10 @@ func PopulateCache(views []*view.View) (int, error) { } if collectorSize == 0 { - return 0, nil + fmt.Printf("[INFO] cache warmup populate done rows=0 entries=0 elapsed=%s\n", time.Since(started)) + return &Result{}, nil } + fmt.Printf("[INFO] cache warmup entries expected entries=%d elapsed=%s\n", collectorSize, time.Since(started)) var errors []error var warmupEntries []*warmupEntry @@ -209,19 +255,24 @@ func PopulateCache(views []*view.View) (int, error) { close(collector) if err := errUtils.CombineErrors("errors while populating cache: ", errors); err != nil { - return 0, err + fmt.Printf("[INFO] cache warmup populate error entries=%d elapsed=%s error=%v\n", len(warmupEntries), time.Since(started), err) + return &Result{}, err } + fmt.Printf("[INFO] cache warmup entries built entries=%d elapsed=%s\n", len(warmupEntries), time.Since(started)) - notifierErr := make(chan func() (int, error)) + notifierErr := make(chan func() (*EntryResult, error)) warmup(ctx, warmupEntries, notifierErr) - indexed := 0 + result := &Result{} for i := 0; i < len(warmupEntries); i++ { select { case actual := <-notifierErr: if actual != nil { - currIndexed, err := actual() - indexed += currIndexed + entryResult, err := actual() + if entryResult != nil { + result.Rows += entryResult.Rows + result.Entries = append(result.Entries, entryResult) + } if err != nil { errors = append(errors, err) } @@ -230,7 +281,13 @@ func PopulateCache(views []*view.View) (int, error) { } close(notifier) - return indexed, errUtils.CombineErrors("errors while populating cache: ", errors) + err := errUtils.CombineErrors("errors while populating cache: ", errors) + if err != nil { + fmt.Printf("[INFO] cache warmup populate error rows=%d entries=%d elapsed=%s error=%v\n", result.Rows, len(warmupEntries), time.Since(started), err) + return result, err + } + fmt.Printf("[INFO] cache warmup populate done rows=%d entries=%d elapsed=%s\n", result.Rows, len(warmupEntries), time.Since(started)) + return result, nil } func FilterCacheViews(views []*view.View) []*view.View { @@ -244,3 +301,65 @@ func FilterCacheViews(views []*view.View) []*view.View { return viewsWithCache } + +func namesOf(views []*view.View) string { + if len(views) == 0 { + return "" + } + names := make([]string, 0, len(views)) + for _, candidate := range views { + if candidate == nil { + continue + } + names = append(names, candidate.Name) + } + return strings.Join(names, ",") +} + +func cacheLabel(aView *view.View) string { + if aView == nil || aView.Cache == nil { + return "" + } + if aView.Cache.Name != "" { + return aView.Cache.Name + } + return aView.Cache.Provider +} + +func warmupConnectorLabel(aView *view.View) string { + if aView == nil || aView.Cache == nil || aView.Cache.Warmup == nil || aView.Cache.Warmup.Connector == nil { + return viewConnectorLabel(aView) + } + return connectorLabel(aView.Cache.Warmup.Connector) +} + +func viewConnectorLabel(aView *view.View) string { + if aView == nil || aView.Connector == nil { + return "" + } + return connectorLabel(aView.Connector) +} + +func connectorLabel(connector *view.Connector) string { + if connector == nil { + return "" + } + if connector.Ref != "" { + return connector.Ref + } + if connector.Name != "" { + return connector.Name + } + if connector.Driver != "" { + return connector.Driver + } + return "" +} + +func truncateSQL(SQL string) string { + SQL = strings.Join(strings.Fields(SQL), " ") + if len(SQL) <= 512 { + return SQL + } + return SQL[:512] + "...(truncated)" +} diff --git a/warmup/cache_test.go b/warmup/cache_test.go index 9196529d6..836b5283d 100644 --- a/warmup/cache_test.go +++ b/warmup/cache_test.go @@ -85,6 +85,45 @@ func TestPopulateCache(t *testing.T) { } } +func TestWarmupConnectorLabelUsesExplicitWarmupConnector(t *testing.T) { + aView := &view.View{ + Connector: view.NewRefConnector("bq_metrics"), + Cache: &view.Cache{ + Warmup: &view.Warmup{ + Connector: view.NewRefConnector("bq_metrics_prewarm"), + }, + }, + } + + assert.Equal(t, "bq_metrics_prewarm", warmupConnectorLabel(aView)) +} + +func TestWarmupConnectorLabelFallsBackToViewConnector(t *testing.T) { + aView := &view.View{ + Connector: view.NewRefConnector("bq_metrics"), + Cache: &view.Cache{Warmup: &view.Warmup{}}, + } + + assert.Equal(t, "bq_metrics", warmupConnectorLabel(aView)) +} + +func TestDBUsesExplicitWarmupConnector(t *testing.T) { + entry := &warmupEntry{ + view: &view.View{ + Connector: view.NewConnector("runtime", "runtime_missing_driver", "runtime_dsn"), + Cache: &view.Cache{ + Warmup: &view.Warmup{ + Connector: view.NewConnector("prewarm", "prewarm_missing_driver", "prewarm_dsn"), + }, + }, + }, + } + + _, err := DB(entry) + + assert.ErrorContains(t, err, "prewarm_missing_driver") +} + func checkIfCached(t *testing.T, cache *view.Cache, ctx context.Context, testCase struct { description string URL string From cb9270a9a16d7c708a25bf930736f0a3bc793387 Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Fri, 22 May 2026 09:34:23 -0700 Subject: [PATCH 247/279] support xlsx from input --- gateway/router/route.go | 4 ++ repository/component.go | 12 ++++++ repository/contract/input.go | 21 +++++++++ service/session/stater.go | 83 ++++++++++++++++++++++++++++++++++++ shared/marshaller.go | 20 +++++++++ 5 files changed, 140 insertions(+) diff --git a/gateway/router/route.go b/gateway/router/route.go index 96d5874ce..19421b24f 100644 --- a/gateway/router/route.go +++ b/gateway/router/route.go @@ -79,6 +79,10 @@ func (r *Route) UnmarshalFunc(request *http.Request) shared.Unmarshal { contentType := request.Header.Get(HeaderContentType) setter.SetStringIfEmpty(&contentType, request.Header.Get(strings.ToLower(HeaderContentType))) switch contentType { + case content.XLSContentType: + return func(data []byte, dest interface{}) error { + return shared.DecodeXLS(request.Context(), data, dest) + } case content.XMLContentType: return r.Marshaller.XML.Unmarshal case content.CSVContentType: diff --git a/repository/component.go b/repository/component.go index 0aba879c7..f9cc7f9a7 100644 --- a/repository/component.go +++ b/repository/component.go @@ -417,6 +417,18 @@ func (c *Component) UnmarshalFor(opts ...UnmarshalOption) shared.Unmarshal { } switch contentType { + case content.XLSContentType: + if c.Content.Marshaller.XLS.CanUnmarshal() { + return c.Content.Marshaller.XLS.Unmarshal + } + req := options.request + return func(data []byte, dest interface{}) error { + ctx := context.Background() + if req != nil { + ctx = req.Context() + } + return shared.DecodeXLS(ctx, data, dest) + } case content.XMLContentType: return c.Content.Marshaller.XML.Unmarshal case content.CSVContentType: diff --git a/repository/contract/input.go b/repository/contract/input.go index 07819664d..c0ebf33c8 100644 --- a/repository/contract/input.go +++ b/repository/contract/input.go @@ -3,8 +3,10 @@ package contract import ( "context" "fmt" + "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/datly/view/state" + "reflect" ) type Input struct { @@ -41,6 +43,9 @@ func (i *Input) Init(ctx context.Context, aView *view.View) error { } } } + if i.Body.Schema == nil && len(i.Type.Parameters.FilterByKind(state.KindRequestBody)) == 0 && implementsXLSUnmarshaller(i.Type.Schema) { + i.Body.Schema = i.Type.Schema.Clone() + } pkg := pkgPath if i.Type.Schema != nil && i.Type.Package != "" { @@ -77,3 +82,19 @@ func (i *Input) Init(ctx context.Context, aView *view.View) error { return nil } + +var xlsUnmarshallerType = reflect.TypeOf((*shared.XLSUnmarshaller)(nil)).Elem() + +func implementsXLSUnmarshaller(schema *state.Schema) bool { + if schema == nil { + return false + } + rType := schema.Type() + if rType == nil { + return false + } + if rType.Implements(xlsUnmarshallerType) { + return true + } + return rType.Kind() != reflect.Ptr && reflect.PtrTo(rType).Implements(xlsUnmarshallerType) +} diff --git a/service/session/stater.go b/service/session/stater.go index 02b48002e..fcf7116c3 100644 --- a/service/session/stater.go +++ b/service/session/stater.go @@ -7,9 +7,12 @@ import ( "os" "reflect" "runtime/debug" + "strings" "embed" + "github.com/viant/datly/repository/content" + "github.com/viant/datly/shared" "github.com/viant/datly/utils/types" "github.com/viant/datly/view" "github.com/viant/datly/view/state" @@ -170,6 +173,10 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt options := s.Indirect(true, stateOptions...) options.scope = hOptions.Scope() + if err = s.populateTopLevelXLSBody(ctx, dest, stateType, options); err != nil { + return err + } + if err = s.SetState(ctx, stateType.Parameters, aState, options); err != nil { return err } @@ -180,6 +187,82 @@ func (s *Session) Bind(ctx context.Context, dest interface{}, opts ...hstate.Opt return err } +func (s *Session) populateTopLevelXLSBody(ctx context.Context, dest interface{}, stateType *state.Type, opts *Options) error { + if dest == nil || stateType == nil || opts == nil || opts.kindLocator == nil { + return nil + } + if len(stateType.Parameters.FilterByKind(state.KindRequestBody)) > 0 { + return nil + } + + destType := reflect.TypeOf(dest) + if !implementsSessionXLSUnmarshaller(destType) { + return nil + } + + request, err := s.HttpRequest(ctx, opts) + if err != nil || request == nil { + return err + } + if !isXLSContentType(request.Header.Get(content.HeaderContentType)) { + return nil + } + + bodyLocator, err := opts.kindLocator.Lookup(state.KindRequestBody) + if err != nil { + return nil + } + value, has, err := bodyLocator.Value(ctx, destType, "") + if err != nil || !has || value == nil { + return err + } + return assignBoundBody(dest, value) +} + +func assignBoundBody(dest interface{}, value interface{}) error { + dst := reflect.ValueOf(dest) + if dst.Kind() != reflect.Ptr || dst.IsNil() { + return fmt.Errorf("destination must be a non-nil pointer, but had %T", dest) + } + src := reflect.ValueOf(value) + if !src.IsValid() { + return nil + } + if src.Type() == dst.Type() { + dst.Elem().Set(src.Elem()) + return nil + } + if src.Type().AssignableTo(dst.Elem().Type()) { + dst.Elem().Set(src) + return nil + } + if src.Kind() == reflect.Ptr && !src.IsNil() && src.Elem().Type().AssignableTo(dst.Elem().Type()) { + dst.Elem().Set(src.Elem()) + return nil + } + return fmt.Errorf("unable to assign request body value of type %T into %T", value, dest) +} + +var sessionXLSUnmarshallerType = reflect.TypeOf((*shared.XLSUnmarshaller)(nil)).Elem() + +func implementsSessionXLSUnmarshaller(rType reflect.Type) bool { + if rType == nil { + return false + } + if rType.Implements(sessionXLSUnmarshallerType) { + return true + } + return rType.Kind() != reflect.Ptr && reflect.PtrTo(rType).Implements(sessionXLSUnmarshallerType) +} + +func isXLSContentType(contentType string) bool { + if contentType == "" { + return false + } + mediaType := strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0]) + return mediaType == content.XLSContentType +} + func (s *Session) handleInputState(ctx context.Context, hOptions *hstate.Options, embedFs *embed.FS) error { // Handle WithInput: preload cache from provided input data input := hOptions.Input() diff --git a/shared/marshaller.go b/shared/marshaller.go index b8fb7208d..6e82df2a6 100644 --- a/shared/marshaller.go +++ b/shared/marshaller.go @@ -1,7 +1,27 @@ package shared +import ( + "context" + "fmt" +) + // Unmarshal converts data to destination, destination has to be a pointer to desired output type type Unmarshal func(data []byte, destination interface{}) error // Marshal converts source to byte array type Marshal func(src interface{}) ([]byte, error) + +// XLSUnmarshaller decodes an XLS/XLSX request body into the receiver. +type XLSUnmarshaller interface { + UnmarshalXLS(ctx context.Context, data []byte) error +} + +// DecodeXLS dispatches to an XLS/XLSX-aware request-body decoder on dest. +func DecodeXLS(ctx context.Context, data []byte, dest interface{}) error { + switch actual := dest.(type) { + case XLSUnmarshaller: + return actual.UnmarshalXLS(ctx, data) + default: + return fmt.Errorf("xlsx request body is not supported for %T", dest) + } +} From 214f28521a2d30ed3d524e33b76d0358eed8cf34 Mon Sep 17 00:00:00 2001 From: ppoudyal Date: Wed, 27 May 2026 09:32:35 -0400 Subject: [PATCH 248/279] nested json body marshall/unmarshall in mcp support. --- gateway/mcp.go | 10 +++++-- gateway/mcp_report_test.go | 53 ++++++++++++++++++++++++++++++++++++ repository/report/build.go | 9 +++++- repository/report_handler.go | 1 + 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index a47a5989a..429f6636c 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -261,10 +261,16 @@ func (r *Router) applyParamToRequest(baseURL string, values url.Values, p *state values.Add(queryName, fmt.Sprintf("%v", value)) } case state.KindRequestBody: - if text, ok := value.(string); ok { + bodyValue := value + if p != nil && !p.IsAnonymous() { + if bodyName := strings.TrimSpace(p.In.Name); bodyName != "" { + bodyValue = map[string]interface{}{bodyName: value} + } + } + if text, ok := bodyValue.(string); ok { body = strings.NewReader(text) } else { - data, err := json.Marshal(value) + data, err := json.Marshal(bodyValue) if err != nil { return baseURL, body, jsonrpc.NewInvalidParamsError("failed to marshal request body", nil) } diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go index 2ca1534c1..7ba841376 100644 --- a/gateway/mcp_report_test.go +++ b/gateway/mcp_report_test.go @@ -297,6 +297,59 @@ func TestRouter_mcpToolCallHandler_PassesAuthorizationToReportRoute(t *testing.T }`, actualBody) } +func TestRouter_mcpToolCallHandler_WrapsNamedBodyParameter(t *testing.T) { + bodyType := reflect.StructOf([]reflect.StructField{ + {Name: "AudienceId", Type: reflect.TypeOf(0), Tag: `json:"audience_id,omitempty"`}, + {Name: "Mode", Type: reflect.TypeOf(""), Tag: `json:"mode,omitempty"`}, + {Name: "ApplyStatus", Type: reflect.TypeOf(""), Tag: `json:"apply_status,omitempty"`}, + }) + bodyParam := state.NewParameter("Recommendation", state.NewBodyLocation("recommendation"), state.WithParameterSchema(state.NewSchema(bodyType))) + component := &repository.Component{ + Path: contract.Path{Method: http.MethodPatch, URI: "/v1/api/steward/recommendation"}, + Contract: contract.Contract{ + Input: contract.Input{ + Type: state.Type{Parameters: state.Parameters{bodyParam}}, + }, + }, + } + + var actualBody string + route := &Route{ + Path: &contract.Path{Method: http.MethodPatch, URI: "/v1/api/steward/recommendation"}, + Handler: func(ctx context.Context, response http.ResponseWriter, req *http.Request) { + if req.Body != nil { + payload, _ := io.ReadAll(req.Body) + actualBody = string(payload) + } + response.WriteHeader(http.StatusOK) + _, _ = response.Write([]byte(`{"ok":true}`)) + }, + } + + handler := (&Router{}).mcpToolCallHandler(component, route) + result, rpcErr := handler(context.Background(), &schema.CallToolRequest{ + Params: schema.CallToolRequestParams{ + Arguments: map[string]interface{}{ + "Recommendation": map[string]interface{}{ + "audience_id": 7193466, + "mode": "ADD", + "apply_status": "APPROVED", + }, + }, + }, + }) + + require.Nil(t, rpcErr) + require.NotNil(t, result) + assert.JSONEq(t, `{ + "recommendation": { + "audience_id": 7193466, + "mode": "ADD", + "apply_status": "APPROVED" + } + }`, actualBody) +} + func TestRouter_mcpToolCallHandler_MapsComponentAndSelectorArgumentsToHTTPQuery(t *testing.T) { component := &repository.Component{ Path: contract.Path{Method: http.MethodGet, URI: "/v1/api/steward/metadata/ad_profile"}, diff --git a/repository/report/build.go b/repository/report/build.go index 6d054e6a0..c8288bd42 100644 --- a/repository/report/build.go +++ b/repository/report/build.go @@ -7,6 +7,7 @@ import ( "reflect" "strconv" "strings" + "time" "github.com/viant/datly/view" "github.com/viant/datly/view/state" @@ -259,7 +260,11 @@ func filterStructType(filters []*Filter) reflect.Type { for _, filter := range filters { rType := reflect.TypeOf("") if schemaType := filter.SchemaType(); schemaType != nil { - rType = schemaType + if schemaType == timeType || (schemaType.Kind() == reflect.Ptr && schemaType.Elem() == timeType) { + rType = reflect.TypeOf("") + } else { + rType = schemaType + } } structFields = append(structFields, reflect.StructField{ Name: filter.FieldName, @@ -270,6 +275,8 @@ func filterStructType(filters []*Filter) reflect.Type { return reflect.StructOf(structFields) } +var timeType = reflect.TypeOf(time.Time{}) + func buildTag(jsonName, description string) reflect.StructTag { result := fmt.Sprintf(`json:"%s,omitempty"`, jsonName) if description = strings.TrimSpace(description); description != "" { diff --git a/repository/report_handler.go b/repository/report_handler.go index 8bc7cc77c..f625487c3 100644 --- a/repository/report_handler.go +++ b/repository/report_handler.go @@ -66,6 +66,7 @@ func (r *cubeHandler) Exec(ctx context.Context, session xhandler.Session) (inter } func (r *cubeHandler) reportInput(ctx context.Context, request *http.Request) (interface{}, error) { + input := ctx.Value(xhandler.InputKey) if request != nil && request.Body != nil && r.BodyType != nil { payload, err := io.ReadAll(request.Body) From a1caf9d655fca42b7968b76be289071f70f035fe Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 12 Jun 2026 12:28:56 -0700 Subject: [PATCH 249/279] - extended mcp integration --- gateway/mcp.go | 8 ++-- gateway/mcp_report_test.go | 40 ++++++++++++++++- gateway/route.go | 6 ++- gateway/router/handler.go | 6 ++- mcp/extension/handler.go | 71 ++++++++++++++++++++++++++++++ mcp/extension/handler_test.go | 83 +++++++++++++++++++++++++++++++++-- service/operator/service.go | 32 +++++++++++++- view/state/hook.go | 43 ++++++++++++++++++ 8 files changed, 277 insertions(+), 12 deletions(-) diff --git a/gateway/mcp.go b/gateway/mcp.go index a47a5989a..32c210d6a 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -97,7 +97,7 @@ func (r *Router) mcpToolCallHandler(component *repository.Component, aRoute *Rou } // 4) Build HTTP request and route - httpReq, rpcErr := r.newToolHTTPRequest(aRoute.Path.Method, finalURL, body) + httpReq, rpcErr := r.newToolHTTPRequest(ctx, aRoute.Path.Method, finalURL, body) if rpcErr != nil { return nil, rpcErr } @@ -303,8 +303,8 @@ func selectorPublicParamName(p *state.Parameter) (string, bool) { } // newToolHTTPRequest constructs an HTTP request for routed tool invocation. -func (r *Router) newToolHTTPRequest(method, URL string, body io.Reader) (*http.Request, *jsonrpc.Error) { - httpRequest, err := http.NewRequest(method, URL, body) +func (r *Router) newToolHTTPRequest(ctx context.Context, method, URL string, body io.Reader) (*http.Request, *jsonrpc.Error) { + httpRequest, err := http.NewRequestWithContext(ctx, method, URL, body) if err != nil { return nil, jsonrpc.NewInvalidRequest(err.Error(), nil) } @@ -801,7 +801,7 @@ func (r *Router) handleMcpRead(ctx context.Context, params *schema.ReadResourceR } responseWriter := proxy.NewWriter() - httpRequest, err := http.NewRequest(http.MethodGet, URL, nil) + httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, URL, nil) if err != nil { return nil, jsonrpc.NewInvalidRequest(err.Error(), nil) } diff --git a/gateway/mcp_report_test.go b/gateway/mcp_report_test.go index 2ca1534c1..d7c617f4e 100644 --- a/gateway/mcp_report_test.go +++ b/gateway/mcp_report_test.go @@ -33,6 +33,14 @@ import ( type repositoryReportTestResource struct{} +type gatewayTestMCPContext struct { + Name string +} + +func (g *gatewayTestMCPContext) Client() state.MCPClient { + return nil +} + func (r *repositoryReportTestResource) LookupParameter(name string) (*state.Parameter, error) { return nil, nil } @@ -341,12 +349,42 @@ func TestRouter_mcpToolCallHandler_MapsComponentAndSelectorArgumentsToHTTPQuery( } func TestRouter_newToolHTTPRequest_SetsJSONContentTypeForBody(t *testing.T) { - req, rpcErr := (&Router{}).newToolHTTPRequest(http.MethodPost, "http://localhost/v1/api/dev/vendors-grouping/report", strings.NewReader(`{"dimensions":{"accountId":true}}`)) + req, rpcErr := (&Router{}).newToolHTTPRequest(context.Background(), http.MethodPost, "http://localhost/v1/api/dev/vendors-grouping/report", strings.NewReader(`{"dimensions":{"accountId":true}}`)) require.Nil(t, rpcErr) require.NotNil(t, req) assert.Equal(t, "application/json", req.Header.Get("Content-Type")) } +func TestRouter_mcpToolCallHandler_PropagatesMCPContextToRoute(t *testing.T) { + component := &repository.Component{ + Path: contract.Path{Method: http.MethodGet, URI: "/v1/api/test"}, + View: &view.View{}, + } + + var ctxHasMCP bool + var reqCtxHasMCP bool + route := &Route{ + Path: &contract.Path{Method: http.MethodGet, URI: "/v1/api/test"}, + Handler: func(ctx context.Context, response http.ResponseWriter, req *http.Request) { + _, ctxHasMCP = state.LookupMCPContext(ctx) + _, reqCtxHasMCP = state.LookupMCPContext(req.Context()) + response.WriteHeader(http.StatusOK) + _, _ = response.Write([]byte(`{"ok":true}`)) + }, + } + + mcpCtx := &gatewayTestMCPContext{Name: "gateway-mcp"} + handler := (&Router{}).mcpToolCallHandler(component, route) + result, rpcErr := handler(state.WithMCPContext(context.Background(), mcpCtx), &schema.CallToolRequest{ + Params: schema.CallToolRequestParams{}, + }) + + require.Nil(t, rpcErr) + require.NotNil(t, result) + assert.True(t, ctxHasMCP) + assert.True(t, reqCtxHasMCP) +} + func TestRouter_buildToolsIntegration_RegistersCubeTool(t *testing.T) { bodyType := reflect.StructOf([]reflect.StructField{ { diff --git a/gateway/route.go b/gateway/route.go index 9f5cf6eed..8ac9d53aa 100644 --- a/gateway/route.go +++ b/gateway/route.go @@ -47,9 +47,13 @@ func (r *Route) Handle(res http.ResponseWriter, req *http.Request) int { if !r.CanHandle(req) { write(res, http.StatusForbidden, nil) } - ctx := context.Background() + ctx := req.Context() + if ctx == nil { + ctx = context.Background() + } execContext := exec.NewContext(req.Method, req.RequestURI, req.Header, r.Version) ctx = vcontext.WithValue(ctx, exec.ContextKey, execContext) + req = req.WithContext(ctx) var onDone func(time.Time, ...interface{}) int64 = nil var start time.Time if r.Counter != nil { diff --git a/gateway/router/handler.go b/gateway/router/handler.go index dbc40f3da..5c3132992 100644 --- a/gateway/router/handler.go +++ b/gateway/router/handler.go @@ -172,9 +172,13 @@ func (r *Handler) Serve(serverPath string) error { } func (r *Handler) ServeHTTP(writer http.ResponseWriter, req *http.Request) { - ctx := context.Background() + ctx := req.Context() + if ctx == nil { + ctx = context.Background() + } execContext := exec.NewContext(req.Method, req.RequestURI, req.Header, r.Version) ctx = vcontext.WithValue(ctx, exec.ContextKey, execContext) + req = req.WithContext(ctx) r.HandleRequest(ctx, writer, req) if execContext.StatusCode == 0 { execContext.StatusCode = http.StatusOK diff --git a/mcp/extension/handler.go b/mcp/extension/handler.go index 43919fc93..ca4cd7e28 100644 --- a/mcp/extension/handler.go +++ b/mcp/extension/handler.go @@ -2,6 +2,9 @@ package extension import ( "context" + + "github.com/viant/datly/view/state" + "github.com/viant/jsonrpc" "github.com/viant/jsonrpc/transport" "github.com/viant/mcp-protocol/client" "github.com/viant/mcp-protocol/logger" @@ -15,6 +18,66 @@ type ( } ) +type mcpContext struct { + client state.MCPClient +} + +func (m *mcpContext) Client() state.MCPClient { + if m == nil { + return nil + } + return m.client +} + +type mcpClient struct { + operations client.Operations +} + +func (m *mcpClient) CanElicit() bool { + return m != nil && m.operations != nil && m.operations.Implements(schema.MethodElicitationCreate) +} + +func (m *mcpClient) CanGenerateContent() bool { + return m != nil && m.operations != nil && m.operations.Implements(schema.MethodSamplingCreateMessage) +} + +func (m *mcpClient) Elicit(ctx context.Context, params *schema.ElicitRequestParams) (*schema.ElicitResult, error) { + if m == nil || m.operations == nil { + return nil, jsonrpc.NewInternalError("mcp client unavailable", nil) + } + request := &schema.ElicitRequest{Method: schema.MethodElicitationCreate} + if params != nil { + request.Params = *params + } + result, err := m.operations.Elicit(ctx, &jsonrpc.TypedRequest[*schema.ElicitRequest]{Request: request}) + if err != nil { + return nil, err + } + return result, nil +} + +func (m *mcpClient) GenerateContent(ctx context.Context, params *schema.CreateMessageRequestParams) (*schema.CreateMessageResult, error) { + if m == nil || m.operations == nil { + return nil, jsonrpc.NewInternalError("mcp client unavailable", nil) + } + request := &schema.CreateMessageRequest{Method: schema.MethodSamplingCreateMessage} + if params != nil { + request.Params = *params + } + result, err := m.operations.CreateMessage(ctx, &jsonrpc.TypedRequest[*schema.CreateMessageRequest]{Request: request}) + if err != nil { + return nil, err + } + return result, nil +} + +func (i *Handler) withMCPContext(ctx context.Context) context.Context { + if i == nil || i.DefaultHandler == nil || i.DefaultHandler.Client == nil { + return ctx + } + return state.WithMCPContext(ctx, &mcpContext{client: &mcpClient{operations: i.DefaultHandler.Client}}) +} + // Implements checks if the method is implemented func (i *Handler) Implements(method string) bool { switch method { @@ -30,6 +93,14 @@ func (i *Handler) Implements(method string) bool { return false } +func (i *Handler) ReadResource(ctx context.Context, request *jsonrpc.TypedRequest[*schema.ReadResourceRequest]) (*schema.ReadResourceResult, *jsonrpc.Error) { + return i.DefaultHandler.ReadResource(i.withMCPContext(ctx), request) +} + +func (i *Handler) CallTool(ctx context.Context, request *jsonrpc.TypedRequest[*schema.CallToolRequest]) (*schema.CallToolResult, *jsonrpc.Error) { + return i.DefaultHandler.CallTool(i.withMCPContext(ctx), request) +} + // New creates a new implementer func New(registry *server.Registry) server.NewHandler { return func(_ context.Context, notifier transport.Notifier, logger logger.Logger, client client.Operations) (server.Handler, error) { diff --git a/mcp/extension/handler_test.go b/mcp/extension/handler_test.go index 0a5410767..63fef2238 100644 --- a/mcp/extension/handler_test.go +++ b/mcp/extension/handler_test.go @@ -1,12 +1,87 @@ package extension import ( - "fmt" + "context" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/view/state" + "github.com/viant/jsonrpc" + "github.com/viant/jsonrpc/transport" + pclient "github.com/viant/mcp-protocol/client" + "github.com/viant/mcp-protocol/schema" + serverproto "github.com/viant/mcp-protocol/server" ) -func TestNew(t *testing.T) { +type fakeClientOps struct{} + +func (f *fakeClientOps) Notify(ctx context.Context, notification *jsonrpc.Notification) error { + return nil +} + +func (f *fakeClientOps) NextRequestID() jsonrpc.RequestId { + return 1 +} + +func (f *fakeClientOps) LastRequestID() jsonrpc.RequestId { + return 1 +} + +func (f *fakeClientOps) ListRoots(ctx context.Context, request *jsonrpc.TypedRequest[*schema.ListRootsRequest]) (*schema.ListRootsResult, *jsonrpc.Error) { + return &schema.ListRootsResult{}, nil +} + +func (f *fakeClientOps) CreateMessage(ctx context.Context, request *jsonrpc.TypedRequest[*schema.CreateMessageRequest]) (*schema.CreateMessageResult, *jsonrpc.Error) { + return &schema.CreateMessageResult{}, nil +} + +func (f *fakeClientOps) Elicit(ctx context.Context, request *jsonrpc.TypedRequest[*schema.ElicitRequest]) (*schema.ElicitResult, *jsonrpc.Error) { + return &schema.ElicitResult{}, nil +} + +func (f *fakeClientOps) Implements(method string) bool { + switch method { + case schema.MethodElicitationCreate, schema.MethodSamplingCreateMessage: + return true + default: + return false + } +} + +func (f *fakeClientOps) Init(ctx context.Context, capabilities *schema.ClientCapabilities) {} + +var _ pclient.Operations = (*fakeClientOps)(nil) +var _ transport.Notifier = (*fakeClientOps)(nil) +var _ transport.Sequencer = (*fakeClientOps)(nil) + +func TestHandler_CallTool_InjectsMCPContext(t *testing.T) { + registry := serverproto.NewRegistry() + registry.RegisterTool(&serverproto.ToolEntry{ + Metadata: schema.Tool{Name: "test"}, + Handler: func(ctx context.Context, request *schema.CallToolRequest) (*schema.CallToolResult, *jsonrpc.Error) { + mcp, ok := state.LookupMCPContext(ctx) + require.True(t, ok) + require.NotNil(t, mcp) + require.NotNil(t, mcp.Client()) + assert.True(t, mcp.Client().CanElicit()) + assert.True(t, mcp.Client().CanGenerateContent()) + return &schema.CallToolResult{}, nil + }, + }) + + newHandler := New(registry) + actual, err := newHandler(context.Background(), nil, nil, &fakeClientOps{}) + require.NoError(t, err) + + typed := &jsonrpc.TypedRequest[*schema.CallToolRequest]{ + Request: &schema.CallToolRequest{ + Method: schema.MethodToolsCall, + Params: schema.CallToolRequestParams{Name: "test"}, + }, + } - i := Handler{} - fmt.Println(i) + result, rpcErr := actual.CallTool(context.Background(), typed) + require.Nil(t, rpcErr) + require.NotNil(t, result) } diff --git a/service/operator/service.go b/service/operator/service.go index 09e9683a4..d5df2c212 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -163,6 +163,12 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes } err = injectorFinalizer.Finalize(ctx, lookup) + if err != nil { + return ret, err + } + if err = finalizeMCPOutput(ctx, ret); err != nil { + return ret, err + } return ret, err } if finalizer, ok := ret.(state.FinalizerWithError); ok { @@ -173,7 +179,13 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes } return ret, err } - return ret, finalizeErr + if finalizeErr != nil { + return ret, finalizeErr + } + if err = finalizeMCPOutput(ctx, ret); err != nil { + return ret, err + } + return ret, nil } if err != nil { return ret, err @@ -181,9 +193,27 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes if finalizer, ok := ret.(state.Finalizer); ok { err = finalizer.Finalize(ctx) } + if err != nil { + return ret, err + } + if err = finalizeMCPOutput(ctx, ret); err != nil { + return ret, err + } return ret, err } +func finalizeMCPOutput(ctx context.Context, ret interface{}) error { + finalizer, ok := ret.(state.MCPFinalizer) + if !ok { + return nil + } + mcp, ok := state.LookupMCPContext(ctx) + if !ok { + return nil + } + return finalizer.FinalizeMCP(ctx, mcp) +} + func (s *Service) EnsureContext(ctx context.Context, aSession *session.Session, aComponent *repository.Component) (context.Context, error) { ctx = vcontext.WithValue(ctx, codec.CriteriaBuilderKey, reader.New()) diff --git a/view/state/hook.go b/view/state/hook.go index 144dd93ae..f94d57323 100644 --- a/view/state/hook.go +++ b/view/state/hook.go @@ -3,6 +3,7 @@ package state import ( "context" + "github.com/viant/mcp-protocol/schema" "github.com/viant/xdatly/handler/http" "github.com/viant/xdatly/handler/state" ) @@ -25,3 +26,45 @@ type FinalizerWithError interface { type InjectorFinalizer interface { Finalize(ctx context.Context, getInjector func(ctx context.Context, path http.Route) (state.Injector, error)) error } + +// MCPClient exposes the MCP client-side capabilities that a server-side output +// may use during MCP-specific finalization. +type MCPClient interface { + CanElicit() bool + CanGenerateContent() bool + Elicit(ctx context.Context, params *schema.ElicitRequestParams) (*schema.ElicitResult, error) + GenerateContent(ctx context.Context, params *schema.CreateMessageRequestParams) (*schema.CreateMessageResult, error) +} + +// MCPContext carries MCP-specific runtime capabilities for output +// finalization. The concrete implementation is supplied by the MCP host +// runtime. +type MCPContext interface { + Client() MCPClient +} + +// MCPFinalizer is an MCP-aware finalizer that runs only when MCP context is +// available on the current request path. +type MCPFinalizer interface { + FinalizeMCP(ctx context.Context, mcp MCPContext) error +} + +type mcpContextKey struct{} + +// WithMCPContext attaches MCP-specific runtime context to ctx. +func WithMCPContext(ctx context.Context, mcp MCPContext) context.Context { + return context.WithValue(ctx, mcpContextKey{}, mcp) +} + +// LookupMCPContext extracts MCP-specific runtime context from ctx. +func LookupMCPContext(ctx context.Context) (MCPContext, bool) { + if ctx == nil { + return nil, false + } + value := ctx.Value(mcpContextKey{}) + if value == nil { + return nil, false + } + mcp, ok := value.(MCPContext) + return mcp, ok && mcp != nil +} From 049c3e416cb99b258d837fde898915688d0f6cb9 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 12 Jun 2026 12:42:16 -0700 Subject: [PATCH 250/279] - extended mcp integration --- service/operator/service.go | 42 ++++++++++++++++++++++++++++++++----- view/state/hook.go | 10 ++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/service/operator/service.go b/service/operator/service.go index d5df2c212..bd250879b 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -166,7 +166,7 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes if err != nil { return ret, err } - if err = finalizeMCPOutput(ctx, ret); err != nil { + if err = s.finalizeMCPOutput(ctx, ret, aSession); err != nil { return ret, err } return ret, err @@ -182,7 +182,7 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes if finalizeErr != nil { return ret, finalizeErr } - if err = finalizeMCPOutput(ctx, ret); err != nil { + if err = s.finalizeMCPOutput(ctx, ret, aSession); err != nil { return ret, err } return ret, nil @@ -196,13 +196,13 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes if err != nil { return ret, err } - if err = finalizeMCPOutput(ctx, ret); err != nil { + if err = s.finalizeMCPOutput(ctx, ret, aSession); err != nil { return ret, err } return ret, err } -func finalizeMCPOutput(ctx context.Context, ret interface{}) error { +func (s *Service) finalizeMCPOutput(ctx context.Context, ret interface{}, aSession *session.Session) error { finalizer, ok := ret.(state.MCPFinalizer) if !ok { return nil @@ -211,7 +211,39 @@ func finalizeMCPOutput(ctx context.Context, ret interface{}) error { if !ok { return nil } - return finalizer.FinalizeMCP(ctx, mcp) + getBinder := func(ctx context.Context, route xhttp.Route) (xhandler.Session, error) { + if aSession == nil || aSession.Registry() == nil { + return nil, fmt.Errorf("session registry unavailable") + } + aComponent, err := aSession.Registry().Lookup(ctx, contract.NewPath(route.Method, route.URL)) + if err != nil { + return nil, err + } + originalRequest, _ := aSession.HttpRequest(ctx, aSession.Clone()) + request, _ := http.NewRequest(route.Method, route.URL, nil) + if originalRequest != nil { + request.Header = originalRequest.Header + } + unmarshal := aComponent.UnmarshalFunc(request) + locatorOptions := append(aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal)) + childSession := session.New(aComponent.View, + session.WithAuth(aSession.Auth()), + session.WithLocatorOptions(locatorOptions...), + session.WithOperate(aSession.Options.Operate()), + session.WithTypes(&aComponent.Contract.Input.Type, &aComponent.Contract.Output.Type), + session.WithComponent(aComponent), + session.WithLogger(aSession.Logger()), + session.WithRegistry(aSession.Registry()), + ) + if tx := aSession.Options.SqlTx(); tx != nil { + childSession.Apply(session.WithSQLTx(tx)) + } + if err := childSession.InitKinds(state.KindComponent, state.KindHeader, state.KindRequestBody, state.KindForm, state.KindQuery); err != nil { + return nil, err + } + return s.HandlerSession(ctx, aComponent, childSession) + } + return finalizer.FinalizeMCP(ctx, mcp, getBinder) } func (s *Service) EnsureContext(ctx context.Context, aSession *session.Session, aComponent *repository.Component) (context.Context, error) { diff --git a/view/state/hook.go b/view/state/hook.go index f94d57323..88ab09a31 100644 --- a/view/state/hook.go +++ b/view/state/hook.go @@ -4,8 +4,9 @@ import ( "context" "github.com/viant/mcp-protocol/schema" + xhandler "github.com/viant/xdatly/handler" "github.com/viant/xdatly/handler/http" - "github.com/viant/xdatly/handler/state" + hstate "github.com/viant/xdatly/handler/state" ) // Initializer is an interface that should be implemented by any type that needs to be initialized @@ -24,7 +25,7 @@ type FinalizerWithError interface { } type InjectorFinalizer interface { - Finalize(ctx context.Context, getInjector func(ctx context.Context, path http.Route) (state.Injector, error)) error + Finalize(ctx context.Context, getInjector func(ctx context.Context, path http.Route) (hstate.Injector, error)) error } // MCPClient exposes the MCP client-side capabilities that a server-side output @@ -43,10 +44,13 @@ type MCPContext interface { Client() MCPClient } +// BinderProvider resolves a handler session for an HTTP route on demand. +type BinderProvider func(ctx context.Context, path http.Route) (xhandler.Session, error) + // MCPFinalizer is an MCP-aware finalizer that runs only when MCP context is // available on the current request path. type MCPFinalizer interface { - FinalizeMCP(ctx context.Context, mcp MCPContext) error + FinalizeMCP(ctx context.Context, mcp MCPContext, getBinder BinderProvider) error } type mcpContextKey struct{} From f4a9219587fad293e394729c4e8e90da2df461a3 Mon Sep 17 00:00:00 2001 From: adranwit Date: Fri, 12 Jun 2026 13:28:51 -0700 Subject: [PATCH 251/279] - improved cache warmup --- gateway/route_warmup.go | 8 +- gateway/route_warmup_test.go | 24 ++++ gateway/router/handler.go | 9 ++ gateway/warmup/cache.go | 34 ++--- gateway/warmup/cache_test.go | 7 +- internal/translator/function/cache_warmup.go | 35 ++++++ .../translator/function/cache_warmup_test.go | 12 ++ service/operator/reader.go | 2 +- service/operator/service.go | 1 + service/session/option.go | 11 ++ view/cache.go | 93 +++++++++++++- warmup/cache.go | 117 +++++++++++++---- warmup/cache_test.go | 118 ++++++++++++++++++ 13 files changed, 424 insertions(+), 47 deletions(-) diff --git a/gateway/route_warmup.go b/gateway/route_warmup.go index 19758f507..1e58fbd14 100644 --- a/gateway/route_warmup.go +++ b/gateway/route_warmup.go @@ -34,17 +34,19 @@ func (r *Router) handleCacheWarmup(ctx context.Context, writer http.ResponseWrit } func (r *Router) handleCacheWarmupWithErr(ctx context.Context, providers []*repository.Provider) (int, []byte) { + // HTTP-triggered warmup should survive client/LB timeout cancellation. + warmupCtx := context.Background() viewsByURI := make(map[string][]*view.View, len(providers)) URIs := make([]string, 0, len(providers)) for _, provider := range providers { - aComponent, err := provider.Component(ctx) + aComponent, err := provider.Component(warmupCtx) if err != nil { return http.StatusInternalServerError, []byte(err.Error()) } if aComponent == nil { return http.StatusNotFound, []byte("component was not found") } - views, err := router.ExtractCacheableViews(ctx, aComponent) + views, err := router.ExtractCacheableViews(warmupCtx, aComponent) if err != nil { return http.StatusInternalServerError, []byte(err.Error()) } @@ -57,7 +59,7 @@ func (r *Router) handleCacheWarmupWithErr(ctx context.Context, providers []*repo lookup := func(_ context.Context, _, matchingURI string) ([]*view.View, error) { return viewsByURI[matchingURI], nil } - response := warmup.PreCache(ctx, lookup, URIs...) + response := warmup.PreCache(warmupCtx, lookup, URIs...) data, err := json.Marshal(response) if err != nil { return http.StatusInternalServerError, []byte(err.Error()) diff --git a/gateway/route_warmup_test.go b/gateway/route_warmup_test.go index 6151da3e8..4fad3f078 100644 --- a/gateway/route_warmup_test.go +++ b/gateway/route_warmup_test.go @@ -3,6 +3,7 @@ package gateway import ( "context" "encoding/json" + "fmt" "net/http" "testing" @@ -72,3 +73,26 @@ func TestRouterHandleCacheWarmupWithErr_NoCacheViews(t *testing.T) { require.Equal(t, "ok", response.Status) require.Empty(t, response.PreCached) } + +func TestRouterHandleCacheWarmupWithErr_DetachesRequestContext(t *testing.T) { + router := &Router{} + provider := repository.NewProvider( + *contract.NewPath(http.MethodGet, "/v1/api/order"), + &version.Control{}, + func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("warmup context should not be request-canceled: %w", err) + } + return &repository.Component{ + Path: *contract.NewPath(http.MethodGet, "/v1/api/order"), + View: &view.View{Name: "order"}, + }, nil + }, + ) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + statusCode, body := router.handleCacheWarmupWithErr(ctx, []*repository.Provider{provider}) + + require.Equal(t, http.StatusOK, statusCode, string(body)) +} diff --git a/gateway/router/handler.go b/gateway/router/handler.go index 5c3132992..cc2f447ae 100644 --- a/gateway/router/handler.go +++ b/gateway/router/handler.go @@ -418,6 +418,7 @@ func (r *Handler) handleComponent(ctx context.Context, request *http.Request, aC session.WithAuth(r.auth), session.WithLogger(r.logger), session.WithComponent(aComponent), + session.WithCacheDisabled(isCacheDisabled(request)), session.WithLocatorOptions(locatorOptions...), session.WithRegistry(r.registry), @@ -493,6 +494,14 @@ func (r *Handler) handleComponent(ctx context.Context, request *http.Request, aC return r.marshalComponentOutput(output, aComponent, options) } +func isCacheDisabled(request *http.Request) bool { + if request == nil { + return false + } + return request.Header.Get(httputils.DatlyRequestDisableCacheHeader) != "" || + request.Header.Get(strings.ToLower(httputils.DatlyRequestDisableCacheHeader)) != "" +} + func createRequest(ctx context.Context, redirect *session.Redirect) (*http.Request, error) { var err error request := redirect.Request diff --git a/gateway/warmup/cache.go b/gateway/warmup/cache.go index 476a5d1f6..86f32ac59 100644 --- a/gateway/warmup/cache.go +++ b/gateway/warmup/cache.go @@ -13,13 +13,15 @@ import ( type PreCachables func(ctx context.Context, method, matchingURI string) ([]*view.View, error) type PreCached struct { - URI string - View string - Column string - Params string - Elapsed string - TimeTaken time.Duration - Rows int + URI string + View string + Column string + Params string + CacheKey string + FieldNames string `json:",omitempty"` + Elapsed string + TimeTaken time.Duration + Rows int } type Response struct { @@ -57,7 +59,7 @@ func PreCache(ctx context.Context, lookup PreCachables, warmupURIs ...string) *R } fmt.Printf("[INFO] cache warmup uri views uri=%s count=%d views=%s elapsed=%s\n", URI, len(views), viewNames(views), time.Since(startTime)) var result *warmup.Result - if result, e = warmup.PopulateCacheWithDetails(views); e != nil { + if result, e = warmup.PopulateCacheWithDetailsContext(ctx, views); e != nil { fmt.Printf("[INFO] cache warmup uri populate error uri=%s elapsed=%s error=%v\n", URI, time.Since(startTime), e) setErr(e) } @@ -93,13 +95,15 @@ func appendPreCached(response *Response, URI string, result *warmup.Result) { continue } response.PreCached = append(response.PreCached, &PreCached{ - URI: URI, - View: entry.View, - Column: entry.Column, - Params: entry.Params, - Elapsed: entry.Elapsed, - TimeTaken: entry.TimeTaken, - Rows: entry.Rows, + URI: URI, + View: entry.View, + Column: entry.Column, + Params: entry.Params, + CacheKey: entry.CacheKey, + FieldNames: entry.FieldNames, + Elapsed: entry.Elapsed, + TimeTaken: entry.TimeTaken, + Rows: entry.Rows, }) } } diff --git a/gateway/warmup/cache_test.go b/gateway/warmup/cache_test.go index 2489056f1..8f793a83f 100644 --- a/gateway/warmup/cache_test.go +++ b/gateway/warmup/cache_test.go @@ -13,8 +13,8 @@ func TestAppendPreCachedUsesEntryRows(t *testing.T) { result := &datlywarmup.Result{ Rows: 30, Entries: []*datlywarmup.EntryResult{ - {View: "periodSummary#", Column: "order_id", Params: "Period=today", Elapsed: "1s", TimeTaken: time.Second, Rows: 10}, - {View: "periodSummary#", Column: "order_id", Params: "Period=month", Elapsed: "2s", TimeTaken: 2 * time.Second, Rows: 20}, + {View: "periodSummary#", Column: "order_id", Params: "Period=today", CacheKey: "cache://today", Elapsed: "1s", TimeTaken: time.Second, Rows: 10}, + {View: "periodSummary#", Column: "order_id", Params: "Period=month", CacheKey: "cache://month", FieldNames: "OrderId,Spend", Elapsed: "2s", TimeTaken: 2 * time.Second, Rows: 20}, }, } @@ -22,8 +22,11 @@ func TestAppendPreCachedUsesEntryRows(t *testing.T) { require.Len(t, response.PreCached, 2) require.Equal(t, "Period=today", response.PreCached[0].Params) + require.Equal(t, "cache://today", response.PreCached[0].CacheKey) require.Equal(t, 10, response.PreCached[0].Rows) require.Equal(t, "Period=month", response.PreCached[1].Params) + require.Equal(t, "cache://month", response.PreCached[1].CacheKey) + require.Equal(t, "OrderId,Spend", response.PreCached[1].FieldNames) require.Equal(t, 20, response.PreCached[1].Rows) require.Equal(t, "/v1/api/cache/warmup/order", response.PreCached[1].URI) } diff --git a/internal/translator/function/cache_warmup.go b/internal/translator/function/cache_warmup.go index 9835c0509..74c2f1813 100644 --- a/internal/translator/function/cache_warmup.go +++ b/internal/translator/function/cache_warmup.go @@ -35,6 +35,13 @@ func (c *cacheWarmup) Apply(args []string, column *sqlparser.Column, resource *v warmup.IndexParameter = indexParameter continue } + if fieldNames, ok, err := parseWarmupFieldNames(raw); ok || err != nil { + if err != nil { + return err + } + warmup.FieldNames = fieldNames + continue + } param, err := parseWarmupParam(raw) if err != nil { return err @@ -86,6 +93,34 @@ func parseWarmupIndexParameter(raw string) (string, bool, error) { return value, true, nil } +func parseWarmupFieldNames(raw string) ([]string, bool, error) { + name, value, ok := splitWarmupOption(raw) + if !ok { + return nil, false, nil + } + switch strings.ToLower(name) { + case "fieldnames", "field_names", "fields", "filednames": + default: + return nil, false, nil + } + if value == "" { + return nil, true, fmt.Errorf("warmup fieldNames was empty") + } + values := strings.Split(value, ",") + result := make([]string, 0, len(values)) + for _, candidate := range values { + candidate = strings.TrimSpace(candidate) + if candidate == "" { + continue + } + result = append(result, candidate) + } + if len(result) == 0 { + return nil, true, fmt.Errorf("warmup fieldNames has no values") + } + return result, true, nil +} + func parseWarmupParam(raw string) (*view.ParamValue, error) { name, rawValues, ok := splitWarmupOption(raw) if !ok { diff --git a/internal/translator/function/cache_warmup_test.go b/internal/translator/function/cache_warmup_test.go index cdc1fd514..220e89cc3 100644 --- a/internal/translator/function/cache_warmup_test.go +++ b/internal/translator/function/cache_warmup_test.go @@ -14,6 +14,7 @@ func TestCacheWarmupApply(t *testing.T) { "order_id", "Connector=bq_metrics_prewarm", "IndexParameter=OrderId", + "FieldNames=Id,Name", "Period=today,yesterday", "Granularity=hour,day", }, nil, &view.Resource{}, aView) @@ -29,6 +30,9 @@ func TestCacheWarmupApply(t *testing.T) { if aView.Cache.Warmup.IndexParameter != "OrderId" { t.Fatalf("unexpected index parameter: %v", aView.Cache.Warmup.IndexParameter) } + if len(aView.Cache.Warmup.FieldNames) != 2 || aView.Cache.Warmup.FieldNames[0] != "Id" || aView.Cache.Warmup.FieldNames[1] != "Name" { + t.Fatalf("unexpected field names: %#v", aView.Cache.Warmup.FieldNames) + } if len(aView.Cache.Warmup.Cases) != 1 { t.Fatalf("unexpected cases count: %v", len(aView.Cache.Warmup.Cases)) } @@ -62,3 +66,11 @@ func TestCacheWarmupApplyRejectsEmptyIndexParameter(t *testing.T) { t.Fatalf("expected error") } } + +func TestCacheWarmupApplyRejectsEmptyFieldNames(t *testing.T) { + aView := &view.View{Cache: view.NewRefCache("aerospike")} + err := (&cacheWarmup{}).Apply([]string{"order_id", "FieldNames="}, nil, &view.Resource{}, aView) + if err == nil { + t.Fatalf("expected error") + } +} diff --git a/service/operator/reader.go b/service/operator/reader.go index 5a29d9eeb..68a84a2e3 100644 --- a/service/operator/reader.go +++ b/service/operator/reader.go @@ -39,7 +39,7 @@ func (s *Service) runQuery(ctx context.Context, component *repository.Component, readerHandler := handler.New(component.Output.Type.Type(), &component.Output.Type) var options = []reader.Option{ - reader.WithCacheDisabled(false), + reader.WithCacheDisabled(aSession.Options.CacheDisabled()), } startTime := time.Now() s.adjustAsyncOptions(ctx, aSession, component.View, &options) diff --git a/service/operator/service.go b/service/operator/service.go index bd250879b..583e0fd9f 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -148,6 +148,7 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes session.WithAuth(aSession.Auth()), session.WithLocatorOptions(locatorOptions...), session.WithOperate(aSession.Options.Operate()), + session.WithCacheDisabled(aSession.Options.CacheDisabled()), session.WithTypes(&aComponent.Contract.Input.Type, &aComponent.Contract.Output.Type), session.WithComponent(aComponent), session.WithLogger(aSession.Logger()), diff --git a/service/session/option.go b/service/session/option.go index 3568b7b35..40ee8c6c5 100644 --- a/service/session/option.go +++ b/service/session/option.go @@ -35,6 +35,7 @@ type ( embeddedFS *embed.FS auth *auth.Service preseedCache bool + cacheDisabled bool sqlTx *sql.Tx } @@ -54,6 +55,10 @@ func (o *Options) SqlTx() *sql.Tx { return o.sqlTx } +func (o *Options) CacheDisabled() bool { + return o.cacheDisabled +} + func (o *Options) HasInputParameters() bool { if o.locatorOpt == nil { return false @@ -177,6 +182,12 @@ func WithPreseedCache(flag bool) Option { } } +func WithCacheDisabled(flag bool) Option { + return func(s *Options) { + s.cacheDisabled = flag + } +} + func WithComponent(component *repository.Component) Option { return func(s *Options) { s.component = component diff --git a/view/cache.go b/view/cache.go index e7db66b14..557c97f7c 100644 --- a/view/cache.go +++ b/view/cache.go @@ -52,12 +52,14 @@ type ( IndexColumn string IndexParameter string `json:",omitempty" yaml:",omitempty"` IndexMeta bool `json:",omitempty"` + FieldNames []string `json:",omitempty" yaml:",omitempty"` Connector *Connector `json:",omitempty"` Cases []*CacheParameters } CacheParameters struct { - Set []*ParamValue + Set []*ParamValue + FieldNames []string `json:",omitempty" yaml:",omitempty"` } ParamValue struct { @@ -75,6 +77,7 @@ type ( MetaColumn string IndexMeta bool Label string + FieldNames []string } CacheInputFn func() ([]*CacheInput, error) @@ -431,6 +434,15 @@ func (c *Cache) initWarmup(ctx context.Context, resource *Resource) error { } } + if err := c.validateWarmupFieldNames(c.Warmup.FieldNames); err != nil { + return err + } + for _, dataset := range c.Warmup.Cases { + if err := c.validateWarmupFieldNames(dataset.FieldNames); err != nil { + return err + } + } + return nil } @@ -480,6 +492,9 @@ func (c *Cache) appendSelectors(set *CacheParameters, paramValues [][]interface{ indexes := make([]int, len(paramValues)) if len(indexes) == 0 { + input := c.newInput(NewStatelet(), set) + *selectors = append(*selectors, input) + fmt.Printf("[INFO] cache warmup selector view=%s index_column=%s params= field_names=%s\n", c.owner.Name, c.Warmup.IndexColumn, strings.Join(input.FieldNames, ",")) return nil } @@ -502,10 +517,10 @@ outer: } label := strings.Join(debugParams, ",") - input := c.NewInput(selector) + input := c.newInput(selector, set) input.Label = label *selectors = append(*selectors, input) - fmt.Printf("[INFO] cache warmup selector view=%s index_column=%s params=%s\n", c.owner.Name, c.Warmup.IndexColumn, label) + fmt.Printf("[INFO] cache warmup selector view=%s index_column=%s params=%s field_names=%s\n", c.owner.Name, c.Warmup.IndexColumn, label, strings.Join(input.FieldNames, ",")) for i := len(indexes) - 1; i >= 0; i-- { if indexes[i] < len(paramValues[i])-1 { @@ -525,11 +540,83 @@ outer: } func (c *Cache) NewInput(selector *Statelet) *CacheInput { + return c.newInput(selector, nil) +} + +func (c *Cache) newInput(selector *Statelet, set *CacheParameters) *CacheInput { + fieldNames := c.fieldNamesFor(set) + c.applyWarmupFieldNames(selector, fieldNames) return &CacheInput{ Selector: selector, Column: c.Warmup.IndexColumn, MetaColumn: c.Warmup.IndexColumn, IndexMeta: (c.Warmup.IndexMeta || c.Warmup.IndexColumn != "") && c.owner.Template.Summary != nil, + FieldNames: append([]string(nil), fieldNames...), + } +} + +func (c *Cache) fieldNamesFor(set *CacheParameters) []string { + if set != nil && len(set.FieldNames) > 0 { + return set.FieldNames + } + if c.Warmup == nil { + return nil + } + return c.Warmup.FieldNames +} + +func (c *Cache) validateWarmupFieldNames(fieldNames []string) error { + if len(fieldNames) == 0 { + return nil + } + viewName := "" + if c.owner != nil { + viewName = c.owner.Name + } + if c.owner == nil || c.owner.Selector == nil || c.owner.Selector.Constraints == nil || !c.owner.Selector.Constraints.Projection { + return fmt.Errorf("warmup fieldNames require projection selector on view %v", viewName) + } + for _, fieldName := range fieldNames { + fieldName = strings.TrimSpace(fieldName) + if fieldName == "" { + return fmt.Errorf("warmup fieldNames contains empty field on view %v", viewName) + } + if _, ok := c.owner.ColumnByName(fieldName); !ok { + return fmt.Errorf("not found warmup fieldName %v at View %v", fieldName, viewName) + } + } + return nil +} + +func (c *Cache) applyWarmupFieldNames(selector *Statelet, fieldNames []string) { + if selector == nil || c.owner == nil || len(fieldNames) == 0 { + return + } + if selector._columnNames == nil { + selector._columnNames = map[string]bool{} + } + for _, fieldName := range fieldNames { + fieldName = strings.TrimSpace(fieldName) + if fieldName == "" { + continue + } + column, ok := c.owner.ColumnByName(fieldName) + if !ok { + continue + } + columnName := column.Name + outputName := column.FieldName() + if outputName == "" { + outputName = columnName + } + if selector.Has(columnName) || selector.Has(outputName) { + continue + } + selector._columnNames[columnName] = true + selector._columnNames[strings.ToLower(columnName)] = true + selector._columnNames[outputName] = true + selector.Columns = append(selector.Columns, columnName) + selector.Fields = append(selector.Fields, outputName) } } diff --git a/warmup/cache.go b/warmup/cache.go index 392012bc8..9185a1618 100644 --- a/warmup/cache.go +++ b/warmup/cache.go @@ -8,11 +8,14 @@ import ( errUtils "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/sqlx/io/read/cache" + cachehash "github.com/viant/sqlx/io/read/cache/hash" "strings" "sync" "time" ) +const maxWarmupConcurrency = 20 + type ( matchersCollector struct { size int @@ -27,18 +30,22 @@ type ( view *view.View column string label string + fields string + key string } warmupEntryFn func() (*warmupEntry, error) notifierFn func() (int, error) EntryResult struct { - View string - Column string - Params string - Elapsed string - TimeTaken time.Duration - Rows int + View string + Column string + Params string + CacheKey string + FieldNames string + Elapsed string + TimeTaken time.Duration + Rows int } Result struct { @@ -93,7 +100,15 @@ func (c *matchersCollector) populateChan(ctx context.Context, aView *view.View, func (c *matchersCollector) createMetaWarmupEntry(ctx context.Context, aView *view.View, aChan chan warmupEntryFn, input *view.CacheInput) { cacheIndex, err := c.builder.CacheMetaSQL(ctx, aView, input.Selector, nil, nil, nil) if err != nil { - fmt.Printf("[INFO] cache warmup entry build error view=%s type=meta column=%s error=%v\n", aView.Name, input.MetaColumn, err) + fmt.Printf("[INFO] cache warmup entry build error view=%s type=meta column=%s field_names=%s error=%v\n", aView.Name, input.MetaColumn, strings.Join(input.FieldNames, ","), err) + aChan <- func() (*warmupEntry, error) { + return nil, err + } + return + } + cacheKey, err := warmupCacheKey(cacheIndex) + if err != nil { + fmt.Printf("[INFO] cache warmup entry build error view=%s type=meta column=%s field_names=%s error=%v\n", aView.Name, input.MetaColumn, strings.Join(input.FieldNames, ","), err) aChan <- func() (*warmupEntry, error) { return nil, err } @@ -106,6 +121,8 @@ func (c *matchersCollector) createMetaWarmupEntry(ctx context.Context, aView *vi view: aView, column: input.MetaColumn, label: input.Label, + fields: strings.Join(input.FieldNames, ","), + key: cacheKey, }, nil } } @@ -113,19 +130,30 @@ func (c *matchersCollector) createMetaWarmupEntry(ctx context.Context, aView *vi func (c *matchersCollector) createIndexWarmupEntry(ctx context.Context, aView *view.View, aChan chan warmupEntryFn, cacheInput *view.CacheInput) { build, err := c.builder.CacheSQL(ctx, c.view, cacheInput.Selector) if err != nil { - fmt.Printf("[INFO] cache warmup entry build error view=%s type=index column=%s error=%v\n", aView.Name, cacheInput.Column, err) + fmt.Printf("[INFO] cache warmup entry build error view=%s type=index column=%s field_names=%s error=%v\n", aView.Name, cacheInput.Column, strings.Join(cacheInput.FieldNames, ","), err) + aChan <- func() (*warmupEntry, error) { + return nil, err + } + return } - aChan <- func() (*warmupEntry, error) { - if err != nil { + cacheKey, err := warmupCacheKey(build) + if err != nil { + fmt.Printf("[INFO] cache warmup entry build error view=%s type=index column=%s field_names=%s error=%v\n", aView.Name, cacheInput.Column, strings.Join(cacheInput.FieldNames, ","), err) + aChan <- func() (*warmupEntry, error) { return nil, err } + return + } + aChan <- func() (*warmupEntry, error) { return &warmupEntry{ matcher: build, view: aView, column: cacheInput.Column, label: cacheInput.Label, - }, err + fields: strings.Join(cacheInput.FieldNames, ","), + key: cacheKey, + }, nil } } @@ -140,13 +168,42 @@ func populateCollector(ctx context.Context, aView *view.View, builder *reader.Bu } func warmup(ctx context.Context, entries []*warmupEntry, notifier chan func() (*EntryResult, error)) { - for i := range entries { - go readWithChan(ctx, entries[i], notifier) + warmupWithLimit(ctx, entries, notifier, maxWarmupConcurrency, readWithErr) +} + +type warmupReadFn func(context.Context, *warmupEntry) (*EntryResult, error) + +func warmupWithLimit(ctx context.Context, entries []*warmupEntry, notifier chan func() (*EntryResult, error), limit int, read warmupReadFn) { + if len(entries) == 0 { + return + } + if limit <= 0 || limit > len(entries) { + limit = len(entries) + } + fmt.Printf("[INFO] cache warmup workers start entries=%d concurrency=%d\n", len(entries), limit) + jobs := make(chan *warmupEntry) + wg := sync.WaitGroup{} + wg.Add(limit) + for i := 0; i < limit; i++ { + go func() { + defer wg.Done() + for entry := range jobs { + readWithChan(ctx, entry, notifier, read) + } + }() } + go func() { + for _, entry := range entries { + jobs <- entry + } + close(jobs) + wg.Wait() + close(notifier) + }() } -func readWithChan(ctx context.Context, entry *warmupEntry, notifier chan func() (*EntryResult, error)) { - result, err := readWithErr(ctx, entry) +func readWithChan(ctx context.Context, entry *warmupEntry, notifier chan func() (*EntryResult, error), read warmupReadFn) { + result, err := read(ctx, entry) notifier <- func() (*EntryResult, error) { return result, err } @@ -154,16 +211,16 @@ func readWithChan(ctx context.Context, entry *warmupEntry, notifier chan func() func readWithErr(ctx context.Context, entry *warmupEntry) (*EntryResult, error) { started := time.Now() - fmt.Printf("[INFO] cache warmup query start start_time=%s view=%s cache=%s db_connector=%s column=%s params=%s args=%v sql=%q\n", started.Format(time.RFC3339), entry.view.Name, cacheLabel(entry.view), warmupConnectorLabel(entry.view), entry.column, entry.label, entry.matcher.Args, truncateSQL(entry.matcher.SQL)) + fmt.Printf("[INFO] cache warmup query start start_time=%s view=%s cache=%s cache_key=%s db_connector=%s column=%s params=%s field_names=%s args=%v sql=%q\n", started.Format(time.RFC3339), entry.view.Name, cacheLabel(entry.view), entry.key, warmupConnectorLabel(entry.view), entry.column, entry.label, entry.fields, entry.matcher.Args, truncateSQL(entry.matcher.SQL)) db, err := DB(entry) if err != nil { - fmt.Printf("[INFO] cache warmup query error view=%s column=%s params=%s elapsed=%s error=%v\n", entry.view.Name, entry.column, entry.label, time.Since(started), err) + fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, time.Since(started), err) return nil, err } service, err := entry.view.Cache.Service() if err != nil { - fmt.Printf("[INFO] cache warmup query error view=%s column=%s params=%s elapsed=%s error=%v\n", entry.view.Name, entry.column, entry.label, time.Since(started), err) + fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, time.Since(started), err) return nil, err } @@ -171,12 +228,23 @@ func readWithErr(ctx context.Context, entry *warmupEntry) (*EntryResult, error) indexed, err := service.IndexBy(ctx, db, entry.column, matcher.SQL, matcher.Args) elapsed := time.Since(started) if err != nil { - fmt.Printf("[INFO] cache warmup query error view=%s column=%s params=%s rows=%d elapsed=%s error=%v\n", entry.view.Name, entry.column, entry.label, indexed, elapsed, err) - return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: indexed}, fmt.Errorf("failed to index: %w, %v", err, matcher.SQL) + fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s rows=%d elapsed=%s cache_write=error error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, indexed, elapsed, err) + return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, CacheKey: entry.key, FieldNames: entry.fields, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: indexed}, fmt.Errorf("failed to index: %w, %v", err, matcher.SQL) } - fmt.Printf("[INFO] cache warmup query done view=%s cache=%s db_connector=%s column=%s params=%s rows=%d elapsed=%s\n", entry.view.Name, cacheLabel(entry.view), warmupConnectorLabel(entry.view), entry.column, entry.label, indexed, elapsed) - return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: indexed}, nil + fmt.Printf("[INFO] cache warmup query done view=%s cache=%s cache_key=%s db_connector=%s column=%s params=%s field_names=%s rows=%d elapsed=%s cache_write=success\n", entry.view.Name, cacheLabel(entry.view), entry.key, warmupConnectorLabel(entry.view), entry.column, entry.label, entry.fields, indexed, elapsed) + return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, CacheKey: entry.key, FieldNames: entry.fields, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: indexed}, nil +} + +func warmupCacheKey(query *cache.ParmetrizedQuery) (string, error) { + if query == nil { + return "", fmt.Errorf("warmup cache key query was nil") + } + args := query.Args + if args == nil { + args = []interface{}{} + } + return cachehash.GenerateURL(query.SQL, "", "", args) } func DB(entry *warmupEntry) (*sql.DB, error) { @@ -196,6 +264,10 @@ func PopulateCache(views []*view.View) (int, error) { } func PopulateCacheWithDetails(views []*view.View) (*Result, error) { + return PopulateCacheWithDetailsContext(context.Background(), views) +} + +func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (*Result, error) { started := time.Now() viewsWithCache := FilterCacheViews(views) fmt.Printf("[INFO] cache warmup populate start start_time=%s views=%s cache_views=%s cache_count=%d\n", started.Format(time.RFC3339), namesOf(views), namesOf(viewsWithCache), len(viewsWithCache)) @@ -207,7 +279,6 @@ func PopulateCacheWithDetails(views []*view.View) (*Result, error) { collector := make(chan warmupEntryFn) notifier := make(chan notifierFn) - ctx := context.Background() builder := reader.NewBuilder() for i := range viewsWithCache { diff --git a/warmup/cache_test.go b/warmup/cache_test.go index 836b5283d..6eb6c3527 100644 --- a/warmup/cache_test.go +++ b/warmup/cache_test.go @@ -4,12 +4,16 @@ import ( "context" "os" "path" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/viant/datly/internal/tests" "github.com/viant/datly/service/reader" "github.com/viant/datly/view" + sqlcache "github.com/viant/sqlx/io/read/cache" ) func TestPopulateCache(t *testing.T) { @@ -124,6 +128,120 @@ func TestDBUsesExplicitWarmupConnector(t *testing.T) { assert.ErrorContains(t, err, "prewarm_missing_driver") } +func TestWarmupWithLimitCapsConcurrency(t *testing.T) { + entries := make([]*warmupEntry, 50) + for i := range entries { + entries[i] = &warmupEntry{} + } + var active int64 + var maxActive int64 + read := func(ctx context.Context, entry *warmupEntry) (*EntryResult, error) { + current := atomic.AddInt64(&active, 1) + for { + max := atomic.LoadInt64(&maxActive) + if current <= max || atomic.CompareAndSwapInt64(&maxActive, max, current) { + break + } + } + time.Sleep(5 * time.Millisecond) + atomic.AddInt64(&active, -1) + return &EntryResult{Rows: 1}, nil + } + + notifier := make(chan func() (*EntryResult, error)) + warmupWithLimit(context.Background(), entries, notifier, maxWarmupConcurrency, read) + + total := 0 + for i := 0; i < len(entries); i++ { + actual := <-notifier + result, err := actual() + assert.Nil(t, err) + total += result.Rows + } + + assert.Equal(t, len(entries), total) + assert.LessOrEqual(t, atomic.LoadInt64(&maxActive), int64(maxWarmupConcurrency)) +} + +func TestWarmupCacheKeyNormalizesNilArgs(t *testing.T) { + nilArgsKey, err := warmupCacheKey(&sqlcache.ParmetrizedQuery{SQL: "SELECT * FROM events", Args: nil}) + assert.Nil(t, err) + + emptyArgsKey, err := warmupCacheKey(&sqlcache.ParmetrizedQuery{SQL: "SELECT * FROM events", Args: []interface{}{}}) + assert.Nil(t, err) + + assert.Equal(t, emptyArgsKey, nilArgsKey) + + _, err = warmupCacheKey(nil) + assert.ErrorContains(t, err, "query was nil") +} + +func TestWarmupFieldNamesAffectGeneratedCacheKey(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + - Name: quantity + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + Selector: + Constraints: + Projection: true + Template: + Source: SELECT * FROM EVENTS +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + aView := resource.Views[0] + + input, err := aView.Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.NotEmpty(t, input) + + builder := reader.NewBuilder() + fullQuery, err := builder.CacheSQL(context.Background(), aView, input[0].Selector) + require.NoError(t, err) + fullKey, err := warmupCacheKey(fullQuery) + require.NoError(t, err) + + aView.Cache.Warmup.FieldNames = []string{"Quantity"} + fieldInput, err := aView.Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.NotEmpty(t, fieldInput) + assert.Equal(t, []string{"Quantity"}, fieldInput[0].FieldNames) + + fieldQuery, err := builder.CacheSQL(context.Background(), aView, fieldInput[0].Selector) + require.NoError(t, err) + fieldKey, err := warmupCacheKey(fieldQuery) + require.NoError(t, err) + + assert.NotEqual(t, fullQuery.SQL, fieldQuery.SQL) + assert.NotEqual(t, fullKey, fieldKey) + assert.Contains(t, fieldQuery.SQL, "quantity") +} + func checkIfCached(t *testing.T, cache *view.Cache, ctx context.Context, testCase struct { description string URL string From 07cbe781f4421a7a8832558be5c8c27d77ebe71a Mon Sep 17 00:00:00 2001 From: vc42 Date: Tue, 30 Jun 2026 12:23:13 -0400 Subject: [PATCH 252/279] Fix JWT RSA auth resource parsing --- cmd/command/transcribe.go | 23 +++++++++++++--- cmd/command/transcribe_auth_test.go | 36 +++++++++++++++++++++++++ internal/translator/oauth.go | 2 +- internal/translator/oauth_test.go | 42 +++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 cmd/command/transcribe_auth_test.go create mode 100644 internal/translator/oauth_test.go diff --git a/cmd/command/transcribe.go b/cmd/command/transcribe.go index 8eec70e0a..78d73e4be 100644 --- a/cmd/command/transcribe.go +++ b/cmd/command/transcribe.go @@ -174,8 +174,12 @@ func applyAuth(cfg *gateway.Config, auth *options.Auth) { return } if strings.TrimSpace(auth.RSA) != "" { - cfg.JWTValidator = &verifier.Config{RSA: getScyResources(auth.RSA)} - cfg.JwtSigner = &signer.Config{RSA: getScyResource(strings.Split(auth.RSA, ";")[0])} + publicRes, privateRes := splitAuthResourcePair(auth.RSA) + cfg.JWTValidator = &verifier.Config{RSA: getScyResources(publicRes)} + if privateRes == "" { + privateRes = publicRes + } + cfg.JwtSigner = &signer.Config{RSA: getScyResource(privateRes)} } if strings.TrimSpace(auth.HMAC) != "" { cfg.JWTValidator = &verifier.Config{HMAC: getScyResource(auth.HMAC)} @@ -195,7 +199,7 @@ func getScyResource(location string) *scy.Resource { func getScyResources(location string) []*scy.Resource { var result []*scy.Resource - for _, item := range strings.Split(location, "-") { + for _, item := range strings.Split(location, ";") { item = strings.TrimSpace(item) if item == "" { continue @@ -205,6 +209,19 @@ func getScyResources(location string) []*scy.Resource { return result } +func splitAuthResourcePair(location string) (string, string) { + location = strings.TrimSpace(location) + if location == "" { + return "", "" + } + parts := strings.SplitN(location, ";", 2) + publicRes := strings.TrimSpace(parts[0]) + if len(parts) == 1 { + return publicRes, "" + } + return publicRes, strings.TrimSpace(parts[1]) +} + func existingBootstrapSources(ctx context.Context, fs afs.Service, cfgURL string) []string { data, err := fs.DownloadWithURL(ctx, cfgURL) if err != nil { diff --git a/cmd/command/transcribe_auth_test.go b/cmd/command/transcribe_auth_test.go new file mode 100644 index 000000000..92e2afee1 --- /dev/null +++ b/cmd/command/transcribe_auth_test.go @@ -0,0 +1,36 @@ +package command + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/gateway" +) + +func TestApplyAuth_RSAUsesSemicolonSeparatedKeys(t *testing.T) { + cfg := &gateway.Config{} + auth := &options.Auth{ + RSA: "./github.com/viant-internal/public.pem|pubKey;./github.com/viant-internal/private.pem|privKey", + } + + applyAuth(cfg, auth) + + require.NotNil(t, cfg.JWTValidator) + require.Len(t, cfg.JWTValidator.RSA, 1) + assert.True(t, strings.HasSuffix(cfg.JWTValidator.RSA[0].URL, "/github.com/viant-internal/public.pem"), cfg.JWTValidator.RSA[0].URL) + assert.Equal(t, "pubKey", cfg.JWTValidator.RSA[0].Key) + require.NotNil(t, cfg.JwtSigner) + require.NotNil(t, cfg.JwtSigner.RSA) + assert.True(t, strings.HasSuffix(cfg.JwtSigner.RSA.URL, "/github.com/viant-internal/private.pem"), cfg.JwtSigner.RSA.URL) + assert.Equal(t, "privKey", cfg.JwtSigner.RSA.Key) +} + +func TestGetScyResources_PreservesHyphenatedPaths(t *testing.T) { + resources := getScyResources("./github.com/viant-internal/public.pem|pubKey") + require.Len(t, resources, 1) + assert.True(t, strings.HasSuffix(resources[0].URL, "/github.com/viant-internal/public.pem"), resources[0].URL) + assert.Equal(t, "pubKey", resources[0].Key) +} diff --git a/internal/translator/oauth.go b/internal/translator/oauth.go index aa512075b..04e3abc90 100644 --- a/internal/translator/oauth.go +++ b/internal/translator/oauth.go @@ -88,7 +88,7 @@ func getScyResource(location string) *scy.Resource { func getScyResources(location string) []*scy.Resource { var result []*scy.Resource - for _, location := range strings.Split(location, "-") { + for _, location := range strings.Split(location, ";") { if strings.TrimSpace(location) == "" { continue } diff --git a/internal/translator/oauth_test.go b/internal/translator/oauth_test.go new file mode 100644 index 000000000..3d66ba829 --- /dev/null +++ b/internal/translator/oauth_test.go @@ -0,0 +1,42 @@ +package translator + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/cmd/options" + "github.com/viant/datly/gateway" + "github.com/viant/datly/gateway/runtime/standalone" +) + +func TestConfig_updateAuth_RSAUsesSemicolonSeparatedKeys(t *testing.T) { + cfg := &Config{ + repository: &options.Repository{ + Auth: options.Auth{ + RSA: "./github.com/viant-internal/public.pem|pubKey;./github.com/viant-internal/private.pem|privKey", + }, + }, + Config: &standalone.Config{Config: &gateway.Config{}}, + } + + err := cfg.updateAuth(context.Background()) + require.NoError(t, err) + require.NotNil(t, cfg.Config.Config.JWTValidator) + require.Len(t, cfg.Config.Config.JWTValidator.RSA, 1) + assert.True(t, strings.HasSuffix(cfg.Config.Config.JWTValidator.RSA[0].URL, "/github.com/viant-internal/public.pem"), cfg.Config.Config.JWTValidator.RSA[0].URL) + assert.Equal(t, "pubKey", cfg.Config.Config.JWTValidator.RSA[0].Key) + require.NotNil(t, cfg.Config.Config.JwtSigner) + require.NotNil(t, cfg.Config.Config.JwtSigner.RSA) + assert.True(t, strings.HasSuffix(cfg.Config.Config.JwtSigner.RSA.URL, "/github.com/viant-internal/private.pem"), cfg.Config.Config.JwtSigner.RSA.URL) + assert.Equal(t, "privKey", cfg.Config.Config.JwtSigner.RSA.Key) +} + +func TestGetScyResources_PreservesHyphenatedPaths(t *testing.T) { + resources := getScyResources("./github.com/viant-internal/public.pem|pubKey") + require.Len(t, resources, 1) + assert.True(t, strings.HasSuffix(resources[0].URL, "/github.com/viant-internal/public.pem"), resources[0].URL) + assert.Equal(t, "pubKey", resources[0].Key) +} From e0ff74b5f18475cd30416bfeb10fb891bbd701f6 Mon Sep 17 00:00:00 2001 From: vcarey Date: Thu, 2 Jul 2026 10:54:03 -0400 Subject: [PATCH 253/279] Improve cache warmup token and limit handling --- internal/translator/function/cache_warmup.go | 31 ++ .../translator/function/cache_warmup_test.go | 20 + internal/translator/view.go | 36 +- internal/translator/view_warmup_test.go | 32 +- service/reader/sql.go | 3 + service/reader/sql_test.go | 427 +----------------- view/cache.go | 95 +++- view/cache_warmup_test.go | 127 ++++++ view/state.go | 10 +- warmup/cache_test.go | 100 ++++ 10 files changed, 441 insertions(+), 440 deletions(-) create mode 100644 view/cache_warmup_test.go diff --git a/internal/translator/function/cache_warmup.go b/internal/translator/function/cache_warmup.go index 74c2f1813..c5759f0fc 100644 --- a/internal/translator/function/cache_warmup.go +++ b/internal/translator/function/cache_warmup.go @@ -2,6 +2,7 @@ package function import ( "fmt" + "strconv" "strings" "github.com/viant/datly/view" @@ -42,6 +43,13 @@ func (c *cacheWarmup) Apply(args []string, column *sqlparser.Column, resource *v warmup.FieldNames = fieldNames continue } + if limit, ok, err := parseWarmupLimit(raw); ok || err != nil { + if err != nil { + return err + } + warmup.Limit = limit + continue + } param, err := parseWarmupParam(raw) if err != nil { return err @@ -121,6 +129,29 @@ func parseWarmupFieldNames(raw string) ([]string, bool, error) { return result, true, nil } +func parseWarmupLimit(raw string) (*int, bool, error) { + name, value, ok := splitWarmupOption(raw) + if !ok { + return nil, false, nil + } + switch strings.ToLower(name) { + case "limit": + default: + return nil, false, nil + } + if value == "" { + return nil, true, fmt.Errorf("warmup limit was empty") + } + limit, err := strconv.Atoi(value) + if err != nil { + return nil, true, fmt.Errorf("warmup limit %q was invalid: %w", value, err) + } + if limit < 0 { + return nil, true, fmt.Errorf("warmup limit %q must be zero or greater", value) + } + return &limit, true, nil +} + func parseWarmupParam(raw string) (*view.ParamValue, error) { name, rawValues, ok := splitWarmupOption(raw) if !ok { diff --git a/internal/translator/function/cache_warmup_test.go b/internal/translator/function/cache_warmup_test.go index 220e89cc3..a75b5c7f8 100644 --- a/internal/translator/function/cache_warmup_test.go +++ b/internal/translator/function/cache_warmup_test.go @@ -14,6 +14,7 @@ func TestCacheWarmupApply(t *testing.T) { "order_id", "Connector=bq_metrics_prewarm", "IndexParameter=OrderId", + "Limit=0", "FieldNames=Id,Name", "Period=today,yesterday", "Granularity=hour,day", @@ -30,6 +31,9 @@ func TestCacheWarmupApply(t *testing.T) { if aView.Cache.Warmup.IndexParameter != "OrderId" { t.Fatalf("unexpected index parameter: %v", aView.Cache.Warmup.IndexParameter) } + if aView.Cache.Warmup.Limit == nil || *aView.Cache.Warmup.Limit != 0 { + t.Fatalf("unexpected warmup limit: %#v", aView.Cache.Warmup.Limit) + } if len(aView.Cache.Warmup.FieldNames) != 2 || aView.Cache.Warmup.FieldNames[0] != "Id" || aView.Cache.Warmup.FieldNames[1] != "Name" { t.Fatalf("unexpected field names: %#v", aView.Cache.Warmup.FieldNames) } @@ -74,3 +78,19 @@ func TestCacheWarmupApplyRejectsEmptyFieldNames(t *testing.T) { t.Fatalf("expected error") } } + +func TestCacheWarmupApplyRejectsInvalidLimit(t *testing.T) { + aView := &view.View{Cache: view.NewRefCache("aerospike")} + err := (&cacheWarmup{}).Apply([]string{"order_id", "Limit=invalid"}, nil, &view.Resource{}, aView) + if err == nil { + t.Fatalf("expected error") + } +} + +func TestCacheWarmupApplyRejectsNegativeLimit(t *testing.T) { + aView := &view.View{Cache: view.NewRefCache("aerospike")} + err := (&cacheWarmup{}).Apply([]string{"order_id", "Limit=-1"}, nil, &view.Resource{}, aView) + if err == nil { + t.Fatalf("expected error") + } +} diff --git a/internal/translator/view.go b/internal/translator/view.go index 975df5581..b2967515d 100644 --- a/internal/translator/view.go +++ b/internal/translator/view.go @@ -3,6 +3,7 @@ package translator import ( "fmt" "path" + "strconv" "strings" "github.com/viant/datly/internal/asset" @@ -43,11 +44,10 @@ func (v *View) applyHintSettings(namespace *Viewlet) error { return fmt.Errorf("invalid view %v hint, %w, %s", v, err, viewJSONHint) } - v.applyShorthands(namespace) - return nil + return v.applyShorthands(namespace) } -func (v *View) applyShorthands(viewlet *Viewlet) { +func (v *View) applyShorthands(viewlet *Viewlet) error { if v.Self != nil { v.SelfReference = v.Self } @@ -79,13 +79,18 @@ func (v *View) applyShorthands(viewlet *Viewlet) { } if len(v.Warmup) > 0 { - v.View.Cache.Warmup = v.buildCacheWarmup(v.Warmup, viewlet) + warmup, err := v.buildCacheWarmup(v.Warmup, viewlet) + if err != nil { + return err + } + v.View.Cache.Warmup = warmup } + return nil } -func (v *View) buildCacheWarmup(warmup map[string]interface{}, viewlet *Viewlet) *view.Warmup { +func (v *View) buildCacheWarmup(warmup map[string]interface{}, viewlet *Viewlet) (*view.Warmup, error) { if warmup == nil { - return nil + return nil, nil } warmup = copyWarmup(warmup) @@ -93,6 +98,8 @@ func (v *View) buildCacheWarmup(warmup map[string]interface{}, viewlet *Viewlet) delete(warmup, "IndexColumn") indexParameter, _ := warmup["IndexParameter"] delete(warmup, "IndexParameter") + limit, hasLimit := warmup["Limit"] + delete(warmup, "Limit") connector, _ := warmup["Connector"] delete(warmup, "Connector") var refColumn string @@ -107,11 +114,24 @@ func (v *View) buildCacheWarmup(warmup map[string]interface{}, viewlet *Viewlet) result.IndexColumn = explicit } if result.IndexColumn == "" { - return nil + return nil, nil } if parameterName := strings.TrimSpace(fmt.Sprint(indexParameter)); parameterName != "" && parameterName != "" { result.IndexParameter = parameterName } + if hasLimit { + limitValue := strings.TrimSpace(fmt.Sprint(limit)) + if limitValue != "" && limitValue != "" { + parsed, err := strconv.Atoi(limitValue) + if err != nil { + return nil, fmt.Errorf("invalid warmup limit %q: %w", limitValue, err) + } + if parsed < 0 { + return nil, fmt.Errorf("invalid warmup limit %q: must be zero or greater", limitValue) + } + result.Limit = &parsed + } + } if connectorName := strings.TrimSpace(fmt.Sprint(connector)); connectorName != "" && connectorName != "" { result.Connector = view.NewRefConnector(connectorName) } @@ -127,7 +147,7 @@ func (v *View) buildCacheWarmup(warmup map[string]interface{}, viewlet *Viewlet) } result.Cases = append(result.Cases, multiSet) - return result + return result, nil } func copyWarmup(warmup map[string]interface{}) map[string]interface{} { diff --git a/internal/translator/view_warmup_test.go b/internal/translator/view_warmup_test.go index f345adada..085c14ea2 100644 --- a/internal/translator/view_warmup_test.go +++ b/internal/translator/view_warmup_test.go @@ -10,17 +10,47 @@ func TestViewBuildCacheWarmup_RootViewUsesExplicitIndexColumn(t *testing.T) { subject := &View{} viewlet := &Viewlet{Name: "adOrderRoot"} - warmup := subject.buildCacheWarmup(map[string]interface{}{ + warmup, err := subject.buildCacheWarmup(map[string]interface{}{ "IndexColumn": "ad_order_id", "IndexParameter": "AdOrderId", + "Limit": "0", "Connector": "bq_metrics_prewarm", }, viewlet) + require.NoError(t, err) require.NotNil(t, warmup) require.Equal(t, "ad_order_id", warmup.IndexColumn) require.Equal(t, "AdOrderId", warmup.IndexParameter) + require.NotNil(t, warmup.Limit) + require.Equal(t, 0, *warmup.Limit) require.NotNil(t, warmup.Connector) require.Equal(t, "bq_metrics_prewarm", warmup.Connector.Ref) require.Len(t, warmup.Cases, 1) require.Len(t, warmup.Cases[0].Set, 0) } + +func TestViewBuildCacheWarmup_InvalidLimitReturnsError(t *testing.T) { + subject := &View{} + viewlet := &Viewlet{Name: "adOrderRoot"} + + warmup, err := subject.buildCacheWarmup(map[string]interface{}{ + "IndexColumn": "ad_order_id", + "Limit": "invalid", + }, viewlet) + + require.Nil(t, warmup) + require.Error(t, err) +} + +func TestViewBuildCacheWarmup_NegativeLimitReturnsError(t *testing.T) { + subject := &View{} + viewlet := &Viewlet{Name: "adOrderRoot"} + + warmup, err := subject.buildCacheWarmup(map[string]interface{}{ + "IndexColumn": "ad_order_id", + "Limit": "-1", + }, viewlet) + + require.Nil(t, warmup) + require.Error(t, err) +} diff --git a/service/reader/sql.go b/service/reader/sql.go index bbaf298d3..be0d86f65 100644 --- a/service/reader/sql.go +++ b/service/reader/sql.go @@ -911,6 +911,9 @@ func (b *Builder) lookupRelationColumn(aView *view.View, relation *view.Relation } func actualLimit(aView *view.View, selector *view.Statelet) int { + if selector.WarmupNoLimit { + return 0 + } if selector.Limit != 0 { return selector.Limit } diff --git a/service/reader/sql_test.go b/service/reader/sql_test.go index 4d90b967f..0d2692b33 100644 --- a/service/reader/sql_test.go +++ b/service/reader/sql_test.go @@ -1,428 +1,17 @@ package reader -/* import ( - "context" - "fmt" + "testing" + "github.com/stretchr/testify/assert" - "github.com/viant/assertly" - "github.com/viant/datly/internal/tests" "github.com/viant/datly/view" - "github.com/viant/datly/view/state" - "github.com/viant/dsunit" - "github.com/viant/toolbox" - "path" - "reflect" - "strings" - "testing" ) -func TestBuilder_Build(t *testing.T) { - testLocation := toolbox.CallerDirectory(3) - - type Params struct { - EventId int - } - - type PresenceMap struct { - } - - useCases := []struct { - batchData *view.BatchData - view *view.View - relation *view.Relation - selector *view.Statelet - placeholders []interface{} - description string - output string - dataset string - }{ - { - dataset: "dataset001_events/", - description: `select statement`, - output: `SELECT t.ID, t.Price FROM events AS t`, - view: &view.View{ - Columns: []*view.Column{ - { - Name: "ID", - DataType: "Int", - }, - { - Name: "Price", - DataType: "Float", - }, - }, - Name: "events", - Table: "events", - Template: &view.Template{ - Output: state.NewSchema(reflect.TypeOf(Params{})), - PresenceSchema: state.NewSchema(reflect.TypeOf(PresenceMap{})), - }, - }, - batchData: &view.BatchData{}, - selector: &view.Statelet{InputParameters: view.ParamState{ - Values: Params{}, - Has: PresenceMap{}, - }}, - }, - { - dataset: "dataset001_events/", - description: `select statement with offset and limit`, - output: `SELECT t.ID, t.Price FROM events AS t LIMIT 10 OFFSET 5`, - view: &view.View{ - Columns: []*view.Column{ - { - Name: "ID", - DataType: "Int", - }, - { - Name: "Price", - DataType: "Float", - }, - }, - Name: "events", - Selector: &view.Config{ - Limit: 10, - }, - Table: "events", - Template: &view.Template{ - Output: state.NewSchema(reflect.TypeOf(Params{})), - PresenceSchema: state.NewSchema(reflect.TypeOf(PresenceMap{})), - }, - }, - batchData: &view.BatchData{}, - selector: &view.Statelet{ - InputParameters: view.ParamState{ - Values: Params{}, - Has: PresenceMap{}, - }, - Offset: 5, - }, - }, - { - dataset: "dataset001_events/", - description: `select statement with $PAGINATION`, - output: `SELECT t.ID, t.Price FROM (SELECT * FROM EVENTS LIMIT 10 OFFSET 5 ) AS t`, - view: &view.View{ - Columns: []*view.Column{ - { - Name: "ID", - DataType: "Int", - }, - { - Name: "Price", - DataType: "Float", - }, - }, - Name: "events", - Selector: &view.Config{ - Limit: 10, - }, - From: "SELECT * FROM EVENTS $PAGINATION", - Table: "events", - Template: &view.Template{ - Output: state.NewSchema(reflect.TypeOf(Params{})), - PresenceSchema: state.NewSchema(reflect.TypeOf(PresenceMap{})), - }, - }, - batchData: &view.BatchData{}, - selector: &view.Statelet{ - InputParameters: view.ParamState{ - Values: Params{}, - Has: PresenceMap{}, - }, - Offset: 5, - }, - }, - { - dataset: "dataset001_events/", - description: `select statement with View Criteria`, - output: `SELECT t.ID, t.Price FROM (SELECT * FROM EVENTS ) AS t`, - view: &view.View{ - Columns: []*view.Column{ - { - Name: "ID", - DataType: "Int", - }, - { - Name: "Price", - DataType: "Float", - }, - }, - Name: "events", - From: "SELECT * FROM EVENTS $PAGINATION", - Table: "Events", - Template: &view.Template{ - Output: state.NewSchema(reflect.TypeOf(Params{})), - PresenceSchema: state.NewSchema(reflect.TypeOf(PresenceMap{})), - }, - }, - batchData: &view.BatchData{}, - selector: &view.Statelet{ - InputParameters: view.ParamState{ - Values: Params{}, - Has: PresenceMap{}, - }, - }, - }, - { - dataset: "dataset001_events/", - description: `select statement with $WHERE_CRITERIA`, - output: `SELECT t.ID, t.Price FROM (SELECT * FROM EVENTS ) AS t`, - view: &view.View{ - Columns: []*view.Column{ - { - Name: "ID", - DataType: "Int", - }, - { - Name: "Price", - DataType: "Float", - }, - }, - Name: "events", - From: "SELECT * FROM EVENTS $WHERE_CRITERIA", - Table: "Events", - Template: &view.Template{ - Output: state.NewSchema(reflect.TypeOf(Params{})), - PresenceSchema: state.NewSchema(reflect.TypeOf(PresenceMap{})), - }, - }, - batchData: &view.BatchData{}, - selector: &view.Statelet{ - InputParameters: view.ParamState{ - Values: Params{}, - Has: PresenceMap{}, - }, - }, - }, - { - dataset: "dataset001_events/", - description: `select statement with parameters`, - output: `SELECT t.ID, t.Price FROM (SELECT * FROM EVENTS WHERE ID = ? ) AS t`, - view: &view.View{ - Columns: []*view.Column{ - { - Name: "ID", - DataType: "Int", - }, - { - Name: "Price", - DataType: "Float", - }, - }, - Name: "events", - From: "SELECT * FROM EVENTS WHERE ID = $EventId", - Table: "Events", - Template: &view.Template{ - Output: state.NewSchema(reflect.TypeOf(Params{})), - InputParameters: []*state.Parameter{ - { - Name: "EventId", - In: &state.State{ - Kind: state.KindPath, - Name: "eventId", - }, - Output: &state.Output{ - DataType: "int", - }, - }, - }, - }, - }, - placeholders: []interface{}{10}, - batchData: &view.BatchData{}, - selector: &view.Statelet{ - InputParameters: view.ParamState{ - Values: Params{EventId: 10}, - Has: PresenceMap{}, - }, - }, - }, - { - dataset: "dataset001_events/", - description: `select statement with $AND_COLUMN_IN`, - output: `SELECT t.ID, t.Price FROM (SELECT * FROM EVENTS ev WHERE ev.ID = ? AND ( ev.user_id IN (?, ?, ?, ?)) ) AS t`, - placeholders: []interface{}{10, 4, 5, 9, 2}, - relation: &view.Relation{ColumnNamespace: "ev", Of: &view.ReferenceView{Column: "ID"}}, - view: &view.View{ - Columns: []*view.Column{ - { - Name: "ID", - DataType: "Int", - }, - { - Name: "Price", - DataType: "Float", - }, - }, - Name: "events", - From: "SELECT * FROM EVENTS ev WHERE ev.ID = $EventId $AND_COLUMN_IN", - Table: "Events", - Template: &view.Template{ - Output: state.NewSchema(reflect.TypeOf(Params{})), - InputParameters: []*state.Parameter{ - { - Name: "EventId", - In: &state.State{ - Kind: state.KindPath, - Name: "eventId", - }, - Output: &state.Output{ - DataType: "int", - }, - }, - }, - }, - }, - batchData: &view.BatchData{ - ColumnNames: "user_id", - ValuesBatch: []interface{}{4, 5, 9, 2}, - }, - selector: &view.Statelet{ - InputParameters: view.ParamState{ - Values: Params{EventId: 10}, - Has: PresenceMap{}, - }, - }, - }, - { - dataset: "dataset001_events/", - description: `select statement without $COLUMN_IN`, - output: `SELECT t.ID, t.Price FROM (SELECT * FROM EVENTS ev WHERE ev.ID = ? AND ( ev.user_id IN (?, ?, ?, ?)) ) AS t`, - placeholders: []interface{}{10, 4, 5, 9, 2}, - relation: &view.Relation{ColumnNamespace: "ev", Of: &view.ReferenceView{Column: "ID"}}, - view: &view.View{ - Columns: []*view.Column{ - { - Name: "ID", - DataType: "Int", - }, - { - Name: "Price", - DataType: "Float", - }, - }, - Name: "events", - From: "SELECT * FROM EVENTS ev WHERE ev.ID = $EventId", - Table: "Events", - Template: &view.Template{ - Output: state.NewSchema(reflect.TypeOf(Params{})), - InputParameters: []*state.Parameter{ - { - Name: "EventId", - In: &state.State{ - Kind: state.KindPath, - Name: "eventId", - }, - Output: &state.Output{ - DataType: "int", - }, - }, - }, - }, - }, - batchData: &view.BatchData{ - ColumnNames: "user_id", - ValuesBatch: []interface{}{4, 5, 9, 2}, - }, - selector: &view.Statelet{ - InputParameters: view.ParamState{ - Values: Params{EventId: 10}, - Has: PresenceMap{}, - }, - }, - }, - { - dataset: "dataset001_events/", - description: `select statement | selectors`, - output: `SELECT t.ID, t.Price FROM (SELECT * FROM EVENTS ORDER BY Price LIMIT 100 OFFSET 10) AS t WHERE price > 10`, - view: &view.View{ - Columns: []*view.Column{ - { - Name: "ID", - DataType: "Int", - }, - { - Name: "Price", - DataType: "Float", - }, - }, - Name: "events", - From: "SELECT * FROM EVENTS", - Table: "Events", - Template: &view.Template{ - Output: state.NewSchema(reflect.TypeOf(Params{})), - InputParameters: []*state.Parameter{ - { - Name: "EventId", - In: &state.State{ - Kind: state.KindPath, - Name: "eventId", - }, - Output: &state.Output{ - DataType: "int", - }, - }, - }, - }, - }, - selector: &view.Statelet{ - OrderBy: "price", - Criteria: "price > 10", - Limit: 100, - Offset: 10, - InputParameters: view.ParamState{ - Values: Params{}, - Has: PresenceMap{}, - }, - }, - }, - } - - //for index, useCase := range useCases[len(useCases)-1:] { - for index, useCase := range useCases { - tests.LogHeader(fmt.Sprintf("Running testcase nr: %v | %v \n", index, useCase.description)) - resourcePath := path.Join(testLocation, "testdata", "datasets", useCase.dataset, "populate") - if initDb(t, path.Join(testLocation, "testdata", "db_config.yaml"), resourcePath, "db") { - return - } - - useCase.view.Connector = &view.Connector{ - Name: "db", - DSN: "./testdata/db/db.db", - Driver: "sqlite3", - } - - if !assert.Nil(t, useCase.view.init(context.TODO(), view.EmptyResource()), useCase.description) { - continue - } +func TestActualLimit_PrefersSelectorNoLimit(t *testing.T) { + aView := &view.View{Selector: &view.Config{Limit: 1}} + selector := &view.Statelet{} + selector.WarmupNoLimit = true + selector.Limit = 0 - builder := NewBuilder() - - useCase.selector.init(useCase.view) - matcher, err := builder.Build(useCase.view, useCase.selector, useCase.batchData, useCase.relation, nil, nil, nil) - - assert.Nil(t, err, useCase.description) - assertly.AssertValues(t, useCase.placeholders, matcher.Args, useCase.description) - assert.Equal(t, useCase.output, strings.TrimSpace(matcher.SQL), useCase.description) - } -} - -func initDb(t *testing.T, configPath, datasetPath, dataStore string) bool { - datasetPath = datasetPath + "_" + dataStore - if !dsunit.InitFromURL(t, configPath) { - return true - } - - initDataset := dsunit.NewDatasetResource(dataStore, datasetPath, "", "") - request := dsunit.NewPrepareRequest(initDataset) - if !dsunit.Prepare(t, request) { - return true - } - - return false + assert.Equal(t, 0, actualLimit(aView, selector)) } - - -*/ diff --git a/view/cache.go b/view/cache.go index 557c97f7c..83075319a 100644 --- a/view/cache.go +++ b/view/cache.go @@ -12,7 +12,9 @@ import ( "github.com/viant/sqlx/io/read/cache" "github.com/viant/sqlx/io/read/cache/aerospike" "github.com/viant/sqlx/io/read/cache/afs" + "github.com/viant/tagly/format" rdata "github.com/viant/toolbox/data" + "reflect" "strconv" "strings" "sync" @@ -52,6 +54,7 @@ type ( IndexColumn string IndexParameter string `json:",omitempty" yaml:",omitempty"` IndexMeta bool `json:",omitempty"` + Limit *int `json:",omitempty" yaml:",omitempty"` FieldNames []string `json:",omitempty" yaml:",omitempty"` Connector *Connector `json:",omitempty"` Cases []*CacheParameters @@ -68,7 +71,9 @@ type ( // ExcludeDefault keeps explicitly declared warmup cases from adding an extra nil/default selector. ExcludeDefault bool `json:",omitempty" yaml:",omitempty"` - _param *state.Parameter + _param *state.Parameter + _location *time.Location + _locationInit bool } CacheInput struct { @@ -89,6 +94,8 @@ const ( aerospikeType = "aerospike" ) +var warmupNow = time.Now + func (c Caches) Has(name string) bool { for _, candidate := range c { if candidate.Name == name { @@ -392,6 +399,7 @@ func (c *Cache) generateDatasetSelectorsErr(ctx context.Context, set *CacheParam func (c *Cache) getParamValues(ctx context.Context, paramValue *ParamValue) ([]interface{}, error) { result := make([]interface{}, len(paramValue.Values), len(paramValue.Values)+1) for i, value := range paramValue.Values { + value = resolveWarmupValue(value, paramValue._param, paramValue._location) marshal := fmt.Sprintf("%v", value) converted, _, err := converter.Convert(marshal, paramValue._param.Schema.Type(), false, paramValue._param.DateFormat) if err != nil { @@ -407,6 +415,65 @@ func (c *Cache) getParamValues(ctx context.Context, paramValue *ParamValue) ([]i return result, nil } +func resolveWarmupValue(value interface{}, param *state.Parameter, location *time.Location) interface{} { + raw, ok := value.(string) + if !ok { + return value + } + raw = strings.TrimSpace(raw) + if raw == "" || !strings.HasPrefix(raw, "@") { + return value + } + now := warmupReferenceTime(location) + switch strings.ToLower(raw) { + case "@today": + return formatWarmupDate(now, param) + case "@yesterday": + return formatWarmupDate(now.AddDate(0, 0, -1), param) + default: + return value + } +} + +func warmupReferenceTime(location *time.Location) time.Time { + now := warmupNow() + if location == nil { + return now.UTC() + } + return now.In(location) +} + +func warmupLocation(param *state.Parameter) (*time.Location, error) { + if param == nil || strings.TrimSpace(param.Tag) == "" { + return nil, nil + } + parsed, err := format.Parse(reflect.StructTag(param.Tag)) + if err != nil { + return nil, fmt.Errorf("invalid warmup format tag on parameter %s: %w", param.Name, err) + } + if parsed == nil || strings.TrimSpace(parsed.Timezone) == "" { + return nil, nil + } + switch timezone := strings.TrimSpace(parsed.Timezone); timezone { + case "UTC", "utc": + return time.UTC, nil + default: + location, loadErr := time.LoadLocation(timezone) + if loadErr != nil { + return nil, fmt.Errorf("invalid warmup timezone %q on parameter %s: %w", timezone, param.Name, loadErr) + } + return location, nil + } +} + +func formatWarmupDate(value time.Time, param *state.Parameter) string { + layout := "2006-01-02" + if param != nil && strings.TrimSpace(param.DateFormat) != "" { + layout = param.DateFormat + } + return value.Format(layout) +} + func (c *Cache) initWarmup(ctx context.Context, resource *Resource) error { if c.owner == nil || c.Warmup == nil { return nil @@ -447,16 +514,24 @@ func (c *Cache) initWarmup(ctx context.Context, resource *Resource) error { } func (c *Cache) ensureParam(paramValue *ParamValue) error { - if paramValue._param != nil { - return nil + param := paramValue._param + if param == nil { + var err error + param, err = c.owner.Template._parametersIndex.Lookup(paramValue.Name) + if err != nil { + return err + } + paramValue._param = param } - param, err := c.owner.Template._parametersIndex.Lookup(paramValue.Name) - if err != nil { - return err + if !paramValue._locationInit { + location, err := warmupLocation(param) + if err != nil { + return err + } + paramValue._location = location + paramValue._locationInit = true } - - paramValue._param = param return nil } @@ -545,6 +620,10 @@ func (c *Cache) NewInput(selector *Statelet) *CacheInput { func (c *Cache) newInput(selector *Statelet, set *CacheParameters) *CacheInput { fieldNames := c.fieldNamesFor(set) + if selector != nil && c.Warmup != nil && c.Warmup.Limit != nil { + selector.Limit = *c.Warmup.Limit + selector.WarmupNoLimit = *c.Warmup.Limit == 0 + } c.applyWarmupFieldNames(selector, fieldNames) return &CacheInput{ Selector: selector, diff --git a/view/cache_warmup_test.go b/view/cache_warmup_test.go new file mode 100644 index 000000000..91cf58b9f --- /dev/null +++ b/view/cache_warmup_test.go @@ -0,0 +1,127 @@ +package view + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/viant/datly/view/state" +) + +func TestResolveWarmupValue_RelativeDates(t *testing.T) { + prevNow := warmupNow + warmupNow = func() time.Time { + return time.Date(2026, time.June, 30, 23, 30, 0, 0, time.UTC) + } + defer func() { + warmupNow = prevNow + }() + + param := &state.Parameter{DateFormat: "2006-01-02"} + + today := resolveWarmupValue("@today", param, nil) + yesterday := resolveWarmupValue("@yesterday", param, nil) + + assert.Regexp(t, `^\d{4}-\d{2}-\d{2}$`, today) + assert.Regexp(t, `^\d{4}-\d{2}-\d{2}$`, yesterday) + assert.NotEqual(t, today, yesterday) +} + +func TestResolveWarmupValue_PreservesLiteralValues(t *testing.T) { + param := &state.Parameter{DateFormat: "2006-01-02"} + + assert.Equal(t, "today", resolveWarmupValue("today", param, nil)) + assert.Equal(t, "@unsupported", resolveWarmupValue("@unsupported", param, nil)) + assert.Equal(t, 7, resolveWarmupValue(7, param, nil)) +} + +func TestResolveWarmupValue_UsesParameterTimezone(t *testing.T) { + prevNow := warmupNow + warmupNow = func() time.Time { + return time.Date(2026, time.July, 1, 0, 30, 0, 0, time.UTC) + } + defer func() { + warmupNow = prevNow + }() + + param := &state.Parameter{ + DateFormat: "2006-01-02", + Tag: `format:"dateFormat=2006-01-02,tz=America/New_York"`, + } + + location, err := warmupLocation(param) + assert.NoError(t, err) + assert.NotNil(t, location) + assert.Equal(t, "2026-06-30", resolveWarmupValue("@today", param, location)) + assert.Equal(t, "2026-06-29", resolveWarmupValue("@yesterday", param, location)) +} + +func TestWarmupLocation_RejectsInvalidTimezone(t *testing.T) { + param := &state.Parameter{ + Name: "From", + Tag: `format:"dateFormat=2006-01-02,tz=America/NewYork"`, + } + + location, err := warmupLocation(param) + assert.Nil(t, location) + assert.Error(t, err) +} + +func TestWarmupLocation_RejectsInvalidFormatTag(t *testing.T) { + param := &state.Parameter{ + Name: "From", + Tag: `format:"bogus=value"`, + } + + location, err := warmupLocation(param) + assert.Nil(t, location) + assert.Error(t, err) +} + +func TestInitWarmup_RejectsInvalidTimezoneOnGeneratedOptionalParam(t *testing.T) { + param := state.NewParameter("From", state.NewQueryLocation("from")) + param.Tag = `format:"dateFormat=2006-01-02,tz=America/NewYork"` + aView := &View{ + Name: "events", + Columns: []*Column{ + {Name: "event_type_id"}, + }, + Template: NewTemplate("", WithTemplateParameters(param)), + Cache: &Cache{ + Warmup: &Warmup{IndexColumn: "event_type_id"}, + }, + } + aView.Template._parametersIndex = aView.Template.Parameters.Index() + aView.indexColumns() + aView.Cache.owner = aView + + err := aView.Cache.initWarmup(context.Background(), EmptyResource()) + + assert.Error(t, err) +} + +func TestInitWarmup_SetsTimezoneOnGeneratedOptionalParam(t *testing.T) { + param := state.NewParameter("From", state.NewQueryLocation("from")) + param.Tag = `format:"dateFormat=2006-01-02,tz=America/New_York"` + aView := &View{ + Name: "events", + Columns: []*Column{ + {Name: "event_type_id"}, + }, + Template: NewTemplate("", WithTemplateParameters(param)), + Cache: &Cache{ + Warmup: &Warmup{IndexColumn: "event_type_id"}, + }, + } + aView.Template._parametersIndex = aView.Template.Parameters.Index() + aView.indexColumns() + aView.Cache.owner = aView + + err := aView.Cache.initWarmup(context.Background(), EmptyResource()) + + assert.NoError(t, err) + if assert.Len(t, aView.Cache.Warmup.Cases, 1) && assert.Len(t, aView.Cache.Warmup.Cases[0].Set, 1) { + assert.NotNil(t, aView.Cache.Warmup.Cases[0].Set[0]._location) + } +} diff --git a/view/state.go b/view/state.go index 370b9d386..48761aa7e 100644 --- a/view/state.go +++ b/view/state.go @@ -22,10 +22,11 @@ type ( Template *structology.State state.QuerySelector QuerySettings - filtersMu sync.Mutex - initialized bool - _columnNames map[string]bool - result *cache.ParmetrizedQuery + filtersMu sync.Mutex + initialized bool + WarmupNoLimit bool + _columnNames map[string]bool + result *cache.ParmetrizedQuery predicate.Filters Ignore bool } @@ -166,6 +167,7 @@ func (s *Statelet) CloneForSummary() *Statelet { QuerySelector: s.QuerySelector, QuerySettings: s.QuerySettings, initialized: s.initialized, + WarmupNoLimit: s.WarmupNoLimit, result: s.result, Ignore: s.Ignore, } diff --git a/warmup/cache_test.go b/warmup/cache_test.go index 6eb6c3527..aba4b86c4 100644 --- a/warmup/cache_test.go +++ b/warmup/cache_test.go @@ -242,6 +242,106 @@ Views: assert.Contains(t, fieldQuery.SQL, "quantity") } +func TestGenerateCacheInput_AppliesWarmupLimitOverride(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + - Name: quantity + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + Limit: 25 + Selector: + Constraints: + Limit: true + Limit: 1 + Template: + Source: SELECT * FROM EVENTS +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + aView := resource.Views[0] + + input, err := aView.Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.Len(t, input, 1) + require.NotNil(t, input[0].Selector) + assert.Equal(t, 25, input[0].Selector.Limit) + assert.False(t, input[0].Selector.WarmupNoLimit) +} + +func TestGenerateCacheInput_ZeroWarmupLimitSetsNoLimit(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + - Name: quantity + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + Limit: 0 + Selector: + Constraints: + Limit: true + Limit: 1 + Template: + Source: SELECT * FROM EVENTS +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + aView := resource.Views[0] + + input, err := aView.Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.Len(t, input, 1) + require.NotNil(t, input[0].Selector) + assert.Equal(t, 0, input[0].Selector.Limit) + assert.True(t, input[0].Selector.WarmupNoLimit) +} + func checkIfCached(t *testing.T, cache *view.Cache, ctx context.Context, testCase struct { description string URL string From 98ec04e20945c803c8cc6c093168db383e177683 Mon Sep 17 00:00:00 2001 From: vcarey Date: Wed, 8 Jul 2026 16:15:43 -0400 Subject: [PATCH 254/279] Improve warmup metrics and case limits --- gateway/route.go | 1 + gateway/route_metrics_test.go | 56 +++ gateway/warmup/cache.go | 210 ++++++++++- gateway/warmup/cache_test.go | 180 ++++++++++ go.mod | 2 +- go.sum | 4 +- internal/translator/function/cache_warmup.go | 32 +- .../translator/function/cache_warmup_test.go | 12 + service/reader/service.go | 27 ++ service/reader/service_metrics_test.go | 90 +++++ view/cache.go | 131 +++++-- warmup/cache.go | 216 +++++++++-- warmup/cache_test.go | 340 ++++++++++++++++++ 13 files changed, 1226 insertions(+), 75 deletions(-) create mode 100644 gateway/route_metrics_test.go create mode 100644 service/reader/service_metrics_test.go diff --git a/gateway/route.go b/gateway/route.go index 8ac9d53aa..7c28c396b 100644 --- a/gateway/route.go +++ b/gateway/route.go @@ -71,6 +71,7 @@ func (r *Route) Handle(res http.ResponseWriter, req *http.Request) int { if statusCode == 0 { statusCode = http.StatusOK } + r.Counter.IncrementValue("Request") // Increment error/success buckets if statusCode >= 200 && statusCode < 300 { r.Counter.IncrementValue("Success") diff --git a/gateway/route_metrics_test.go b/gateway/route_metrics_test.go new file mode 100644 index 000000000..c3b193aa2 --- /dev/null +++ b/gateway/route_metrics_test.go @@ -0,0 +1,56 @@ +package gateway + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/logger" + "github.com/viant/gmetric/counter" + "github.com/viant/xdatly/handler/exec" +) + +type routeTestCounter struct { + values map[interface{}]int +} + +func newRouteTestCounter() *routeTestCounter { + return &routeTestCounter{values: map[interface{}]int{}} +} + +func (c *routeTestCounter) Begin(started time.Time) counter.OnDone { + return func(time.Time, ...interface{}) int64 { return 0 } +} + +func (c *routeTestCounter) DecrementValue(value interface{}) int64 { + c.values[value]-- + return int64(c.values[value]) +} + +func (c *routeTestCounter) IncrementValue(value interface{}) int64 { + c.values[value]++ + return int64(c.values[value]) +} + +func TestRouteHandleIncrementsRequestBucket(t *testing.T) { + counter := newRouteTestCounter() + route := &Route{ + Counter: logger.NewCounter(counter), + Handler: func(ctx context.Context, response http.ResponseWriter, req *http.Request) { + ctx.Value(exec.ContextKey).(*exec.Context).StatusCode = http.StatusCreated + response.WriteHeader(http.StatusCreated) + }, + } + + req := httptest.NewRequest(http.MethodGet, "/v1/api/test", nil) + recorder := httptest.NewRecorder() + status := route.Handle(recorder, req) + + require.Equal(t, http.StatusCreated, status) + require.Equal(t, 1, counter.values["Request"]) + require.Equal(t, 1, counter.values["Success"]) + require.Equal(t, 1, counter.values["status:2xx"]) +} diff --git a/gateway/warmup/cache.go b/gateway/warmup/cache.go index 86f32ac59..4ac420ea4 100644 --- a/gateway/warmup/cache.go +++ b/gateway/warmup/cache.go @@ -5,10 +5,26 @@ import ( "fmt" "github.com/viant/datly/view" "github.com/viant/datly/warmup" + "github.com/viant/gmetric" + "github.com/viant/gmetric/counter/base" "net/http" + "path" + "sort" "strings" "sync" "time" + + "github.com/viant/afs/url" +) + +const ( + warmupRunOKKey = "run.ok" + warmupRunErrorKey = "run.error" + warmupCasesCompletedKey = "cases.completed" + warmupCasesFailedKey = "cases.failed" + warmupRowsKey = "rows" + warmupMetricFallbackPkg = "datly" + warmupMetricRecentBuckets = 2 ) type PreCachables func(ctx context.Context, method, matchingURI string) ([]*view.View, error) @@ -22,11 +38,27 @@ type PreCached struct { Elapsed string TimeTaken time.Duration Rows int + Error string `json:"error,omitempty"` +} + +type Summary struct { + CompletedCases int `json:"completedCases"` + FailedCases int `json:"failedCases"` + WarmedRows int `json:"warmedRows"` +} + +type viewSummary struct { + View string + CompletedCases int + FailedCases int + WarmedRows int + Elapsed time.Duration } type Response struct { Error string `json:"error,omitempty"` Status string `json:"status"` + Summary *Summary `json:"summary,omitempty"` PreCached []*PreCached `json:"preCached"` } @@ -34,16 +66,16 @@ func PreCache(ctx context.Context, lookup PreCachables, warmupURIs ...string) *R started := time.Now() fmt.Printf("[INFO] cache warmup request start start_time=%s uris=%v\n", started.Format(time.RFC3339), warmupURIs) group := sync.WaitGroup{} - var err error var mux = sync.Mutex{} var response = &Response{Status: "ok"} + var errors []string setErr := func(e error) { if e == nil { return } mux.Lock() defer mux.Unlock() - err = e + errors = append(errors, e.Error()) } for _, URI := range warmupURIs { @@ -56,6 +88,15 @@ func PreCache(ctx context.Context, lookup PreCachables, warmupURIs ...string) *R if e != nil { fmt.Printf("[INFO] cache warmup uri lookup error uri=%s elapsed=%s error=%v\n", URI, time.Since(startTime), e) setErr(e) + mux.Lock() + response.PreCached = append(response.PreCached, &PreCached{ + URI: URI, + Elapsed: time.Since(startTime).String(), + TimeTaken: time.Since(startTime), + Error: e.Error(), + }) + mux.Unlock() + return } fmt.Printf("[INFO] cache warmup uri views uri=%s count=%d views=%s elapsed=%s\n", URI, len(views), viewNames(views), time.Since(startTime)) var result *warmup.Result @@ -72,14 +113,16 @@ func PreCache(ctx context.Context, lookup PreCachables, warmupURIs ...string) *R if result == nil { return } + logViewSummaries(URI, views, result) mux.Lock() appendPreCached(response, URI, result) mux.Unlock() }(URI) } group.Wait() - if err != nil { - response.Error = err.Error() + response.Summary = summarize(response.PreCached) + if len(errors) > 0 { + response.Error = joinErrors(errors) response.Status = "error" } fmt.Printf("[INFO] cache warmup request done status=%s elapsed=%s\n", response.Status, time.Since(started)) @@ -104,10 +147,169 @@ func appendPreCached(response *Response, URI string, result *warmup.Result) { Elapsed: entry.Elapsed, TimeTaken: entry.TimeTaken, Rows: entry.Rows, + Error: entry.Error, }) } } +func summarize(entries []*PreCached) *Summary { + summary := &Summary{} + if len(entries) == 0 { + return summary + } + for _, entry := range entries { + if entry == nil { + continue + } + if entry.Error != "" { + summary.FailedCases++ + continue + } + summary.CompletedCases++ + summary.WarmedRows += entry.Rows + } + return summary +} + +func summarizeByView(entries []*warmup.EntryResult) []*viewSummary { + if len(entries) == 0 { + return nil + } + index := map[string]*viewSummary{} + for _, entry := range entries { + if entry == nil || entry.View == "" { + continue + } + current := index[entry.View] + if current == nil { + current = &viewSummary{View: entry.View} + index[entry.View] = current + } + current.Elapsed += entry.TimeTaken + if entry.Error != "" { + current.FailedCases++ + continue + } + current.CompletedCases++ + current.WarmedRows += entry.Rows + } + result := make([]*viewSummary, 0, len(index)) + for _, item := range index { + result = append(result, item) + } + sort.Slice(result, func(i, j int) bool { + return result[i].View < result[j].View + }) + return result +} + +func logViewSummaries(uri string, views []*view.View, result *warmup.Result) { + if result == nil { + return + } + viewsIndex := indexViewsByName(views) + for _, summary := range summarizeByView(result.Entries) { + fmt.Printf("[INFO] cache warmup view summary uri=%s view=%s completed_cases=%d failed_cases=%d warmed_rows=%d elapsed=%s\n", + uri, + summary.View, + summary.CompletedCases, + summary.FailedCases, + summary.WarmedRows, + summary.Elapsed) + recordWarmupViewMetrics(viewsIndex[summary.View], summary) + } +} + +func joinErrors(values []string) string { + if len(values) == 0 { + return "" + } + sorted := append([]string{}, values...) + sort.Strings(sorted) + return strings.Join(sorted, "; ") +} + +func indexViewsByName(views []*view.View) map[string]*view.View { + if len(views) == 0 { + return nil + } + result := make(map[string]*view.View, len(views)) + for _, candidate := range views { + if candidate == nil || candidate.Name == "" { + continue + } + result[candidate.Name] = candidate + } + return result +} + +func recordWarmupViewMetrics(aView *view.View, summary *viewSummary) { + if aView == nil || summary == nil { + return + } + operation := warmupMetricOperation(aView) + if operation == nil { + return + } + end := time.Now() + started := end.Add(-summary.Elapsed) + runStatus := warmupRunOKKey + if summary.FailedCases > 0 { + runStatus = warmupRunErrorKey + } + operation.Begin(started)(end, runStatus) + if summary.CompletedCases > 0 { + operation.IncrementValueBy(warmupCasesCompletedKey, int64(summary.CompletedCases)) + } + if summary.FailedCases > 0 { + operation.IncrementValueBy(warmupCasesFailedKey, int64(summary.FailedCases)) + } + if summary.WarmedRows > 0 { + operation.IncrementValueBy(warmupRowsKey, int64(summary.WarmedRows)) + } +} + +func warmupMetricOperation(aView *view.View) *gmetric.Operation { + if aView == nil { + return nil + } + resource := aView.GetResource() + if resource == nil || resource.Metrics == nil || resource.Metrics.Service == nil { + return nil + } + metricName := warmupMetricName(aView) + if counter := resource.Metrics.Service.LookupOperation(metricName); counter != nil { + return counter + } + pkg := warmupMetricPackage(aView) + title := aView.Name + " warmup" + return resource.Metrics.Service.MultiOperationCounter(pkg, metricName, title, time.Millisecond, time.Minute, warmupMetricRecentBuckets, base.NewProvider( + warmupRunOKKey, + warmupRunErrorKey, + warmupCasesCompletedKey, + warmupCasesFailedKey, + warmupRowsKey, + )) +} + +func warmupMetricName(aView *view.View) string { + name := warmupMetricPackage(aView) + "." + aView.Name + ".warmup" + return strings.ReplaceAll(name, "/", ".") +} + +func warmupMetricPackage(aView *view.View) string { + resource := aView.GetResource() + if resource == nil { + return warmupMetricFallbackPkg + } + sourceURL := url.Path(resource.SourceURL) + parent, _ := path.Split(sourceURL) + if idx := strings.Index(parent, "/routes/"); idx != -1 { + return strings.Trim(parent[idx+len("/routes/"):], "/") + } + return warmupMetricFallbackPkg +} + func viewNames(views []*view.View) string { if len(views) == 0 { return "" diff --git a/gateway/warmup/cache_test.go b/gateway/warmup/cache_test.go index 8f793a83f..94574492d 100644 --- a/gateway/warmup/cache_test.go +++ b/gateway/warmup/cache_test.go @@ -1,11 +1,19 @@ package warmup import ( + "context" + "fmt" + "os" + "path" "testing" "time" + _ "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/require" + "github.com/viant/datly/view" datlywarmup "github.com/viant/datly/warmup" + "github.com/viant/gmetric" + "github.com/viant/gmetric/stat" ) func TestAppendPreCachedUsesEntryRows(t *testing.T) { @@ -30,3 +38,175 @@ func TestAppendPreCachedUsesEntryRows(t *testing.T) { require.Equal(t, 20, response.PreCached[1].Rows) require.Equal(t, "/v1/api/cache/warmup/order", response.PreCached[1].URI) } + +func TestAppendPreCachedPreservesEntryErrors(t *testing.T) { + response := &Response{} + result := &datlywarmup.Result{ + Entries: []*datlywarmup.EntryResult{ + {View: "diagnostics", Column: "ad_order_id", Params: "From=2026-07-02", CacheKey: "cache://today", Elapsed: "250ms", TimeTaken: 250 * time.Millisecond, Rows: 7, Error: "failed to index"}, + }, + } + + appendPreCached(response, "/v1/api/cache/warmup/diagnostics", result) + + require.Len(t, response.PreCached, 1) + require.Equal(t, "failed to index", response.PreCached[0].Error) +} + +func TestSummarize(t *testing.T) { + summary := summarize([]*PreCached{ + {Rows: 10, TimeTaken: 100 * time.Millisecond}, + {Rows: 20, TimeTaken: 200 * time.Millisecond}, + {Rows: 99, TimeTaken: 300 * time.Millisecond, Error: "failed to index"}, + {Rows: 30, TimeTaken: 400 * time.Millisecond}, + }) + + require.NotNil(t, summary) + require.Equal(t, 3, summary.CompletedCases) + require.Equal(t, 1, summary.FailedCases) + require.Equal(t, 60, summary.WarmedRows) +} + +func TestSummarizeEmpty(t *testing.T) { + summary := summarize(nil) + + require.NotNil(t, summary) + require.Zero(t, summary.CompletedCases) + require.Zero(t, summary.FailedCases) + require.Zero(t, summary.WarmedRows) +} + +func TestSummarizeByView(t *testing.T) { + summaries := summarizeByView([]*datlywarmup.EntryResult{ + {View: "periodSummary#", Rows: 10, TimeTaken: time.Second}, + {View: "periodSummary#", Rows: 99, TimeTaken: 2 * time.Second, Error: "failed"}, + {View: "timeline#", Rows: 20, TimeTaken: 3 * time.Second}, + {View: "periodSummary#", Rows: 30, TimeTaken: 4 * time.Second}, + }) + + require.Len(t, summaries, 2) + require.Equal(t, "periodSummary#", summaries[0].View) + require.Equal(t, 2, summaries[0].CompletedCases) + require.Equal(t, 1, summaries[0].FailedCases) + require.Equal(t, 40, summaries[0].WarmedRows) + require.Equal(t, 7*time.Second, summaries[0].Elapsed) + require.Equal(t, "timeline#", summaries[1].View) + require.Equal(t, 1, summaries[1].CompletedCases) + require.Equal(t, 0, summaries[1].FailedCases) + require.Equal(t, 20, summaries[1].WarmedRows) + require.Equal(t, 3*time.Second, summaries[1].Elapsed) +} + +func TestPreCacheFailureAccounting(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Views: + - Name: diagnostics + Connector: + Ref: db + Columns: + - Name: ad_order_id + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: ad_order_id + Cases: + - Set: + - Name: AD_ORDER_ID + Values: [ abc ] + Template: + Source: 'SELECT * FROM DIAGNOSTICS WHERE ad_order_id = $AD_ORDER_ID' + Parameters: + - Name: AD_ORDER_ID + In: + Kind: query + Name: ad_order_id + Schema: + DataType: int + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ':memory:' +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + + response := PreCache(context.Background(), func(ctx context.Context, method, matchingURI string) ([]*view.View, error) { + return resource.Views, nil + }, "/v1/api/cache/warmup/diagnostics") + + require.Equal(t, "error", response.Status) + require.NotNil(t, response.Summary) + require.Equal(t, 0, response.Summary.CompletedCases) + require.Equal(t, 1, response.Summary.FailedCases) + require.Zero(t, response.Summary.WarmedRows) + require.Len(t, response.PreCached, 1) +} + +func TestPreCacheLookupFailureAccounting(t *testing.T) { + response := PreCache(context.Background(), func(ctx context.Context, method, matchingURI string) ([]*view.View, error) { + return nil, fmt.Errorf("lookup failed") + }, "/v1/api/cache/warmup/diagnostics") + + require.Equal(t, "error", response.Status) + require.NotNil(t, response.Summary) + require.Equal(t, 0, response.Summary.CompletedCases) + require.Equal(t, 1, response.Summary.FailedCases) + require.Zero(t, response.Summary.WarmedRows) + require.Len(t, response.PreCached, 1) + require.Equal(t, "lookup failed", response.PreCached[0].Error) +} + +func TestPreCacheAggregatesErrorsDeterministically(t *testing.T) { + response := PreCache(context.Background(), func(ctx context.Context, method, matchingURI string) ([]*view.View, error) { + switch matchingURI { + case "/b": + return nil, fmt.Errorf("lookup b") + case "/a": + return nil, fmt.Errorf("lookup a") + default: + return nil, nil + } + }, "/b", "/a") + + require.Equal(t, "error", response.Status) + require.Equal(t, "lookup a; lookup b", response.Error) + require.NotNil(t, response.Summary) + require.Equal(t, 0, response.Summary.CompletedCases) + require.Equal(t, 2, response.Summary.FailedCases) +} + +func TestRecordWarmupViewMetrics(t *testing.T) { + metrics := gmetric.New() + resource := view.EmptyResource() + resource.SourceURL = "/tmp/routes/steward/performance/line.yaml" + resource.Metrics = &view.Metrics{Service: metrics} + aView := &view.View{Name: "linePeriodSummary#"} + aView.SetResource(resource) + + recordWarmupViewMetrics(aView, &viewSummary{ + View: aView.Name, + CompletedCases: 2, + FailedCases: 1, + WarmedRows: 40, + Elapsed: 1500 * time.Millisecond, + }) + + metricName := warmupMetricName(aView) + require.Equal(t, int64(1), metrics.LookupOperationCumulativeMetric(metricName, stat.CounterValueKey)) + require.Equal(t, int64(1), metrics.LookupOperationCumulativeMetric(metricName, warmupRunErrorKey)) + require.Equal(t, int64(2), metrics.LookupOperationCumulativeMetric(metricName, warmupCasesCompletedKey)) + require.Equal(t, int64(1), metrics.LookupOperationCumulativeMetric(metricName, warmupCasesFailedKey)) + require.Equal(t, int64(40), metrics.LookupOperationCumulativeMetric(metricName, warmupRowsKey)) + require.GreaterOrEqual(t, metrics.LookupOperationCumulativeMetric(metricName, stat.CounterTimeTakenKey), int64(1500)) +} diff --git a/go.mod b/go.mod index 83185bbe3..55c079567 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 - github.com/viant/sqlx v0.22.1-0.20260326175456-cec446e28372 + github.com/viant/sqlx v0.23.1-0.20260708200154-5839ab951f68 github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 diff --git a/go.sum b/go.sum index 31f7ee131..053931bc7 100644 --- a/go.sum +++ b/go.sum @@ -1196,8 +1196,8 @@ github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQnwf69oc86xW1e4wEEzKUg5OC6tOYegE= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.22.1-0.20260326175456-cec446e28372 h1:5qW+4AbQ8YA0MsyoUx3uaNgTHl52F0JBoC2vsdwKXIM= -github.com/viant/sqlx v0.22.1-0.20260326175456-cec446e28372/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= +github.com/viant/sqlx v0.23.1-0.20260708200154-5839ab951f68 h1:cEhgA76FQhhtl9VegrvGdoYYVGrk2ImKg9We0DkwlYg= +github.com/viant/sqlx v0.23.1-0.20260708200154-5839ab951f68/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= github.com/viant/structology v0.9.0/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= diff --git a/internal/translator/function/cache_warmup.go b/internal/translator/function/cache_warmup.go index c5759f0fc..df4f72937 100644 --- a/internal/translator/function/cache_warmup.go +++ b/internal/translator/function/cache_warmup.go @@ -50,6 +50,13 @@ func (c *cacheWarmup) Apply(args []string, column *sqlparser.Column, resource *v warmup.Limit = limit continue } + if maxCases, ok, err := parseWarmupMaxCases(raw); ok || err != nil { + if err != nil { + return err + } + warmup.MaxCases = maxCases + continue + } param, err := parseWarmupParam(raw) if err != nil { return err @@ -130,24 +137,37 @@ func parseWarmupFieldNames(raw string) ([]string, bool, error) { } func parseWarmupLimit(raw string) (*int, bool, error) { + return parseWarmupNonNegativeInt(raw, []string{"limit"}, "warmup limit") +} + +func parseWarmupMaxCases(raw string) (*int, bool, error) { + return parseWarmupNonNegativeInt(raw, []string{"maxcases", "max_cases"}, "warmup maxCases") +} + +func parseWarmupNonNegativeInt(raw string, options []string, description string) (*int, bool, error) { name, value, ok := splitWarmupOption(raw) if !ok { return nil, false, nil } - switch strings.ToLower(name) { - case "limit": - default: + matched := false + for _, option := range options { + if strings.EqualFold(name, option) { + matched = true + break + } + } + if !matched { return nil, false, nil } if value == "" { - return nil, true, fmt.Errorf("warmup limit was empty") + return nil, true, fmt.Errorf("%s was empty", description) } limit, err := strconv.Atoi(value) if err != nil { - return nil, true, fmt.Errorf("warmup limit %q was invalid: %w", value, err) + return nil, true, fmt.Errorf("%s %q was invalid: %w", description, value, err) } if limit < 0 { - return nil, true, fmt.Errorf("warmup limit %q must be zero or greater", value) + return nil, true, fmt.Errorf("%s %q must be zero or greater", description, value) } return &limit, true, nil } diff --git a/internal/translator/function/cache_warmup_test.go b/internal/translator/function/cache_warmup_test.go index a75b5c7f8..4e09ed588 100644 --- a/internal/translator/function/cache_warmup_test.go +++ b/internal/translator/function/cache_warmup_test.go @@ -15,6 +15,7 @@ func TestCacheWarmupApply(t *testing.T) { "Connector=bq_metrics_prewarm", "IndexParameter=OrderId", "Limit=0", + "MaxCases=3", "FieldNames=Id,Name", "Period=today,yesterday", "Granularity=hour,day", @@ -34,6 +35,9 @@ func TestCacheWarmupApply(t *testing.T) { if aView.Cache.Warmup.Limit == nil || *aView.Cache.Warmup.Limit != 0 { t.Fatalf("unexpected warmup limit: %#v", aView.Cache.Warmup.Limit) } + if aView.Cache.Warmup.MaxCases == nil || *aView.Cache.Warmup.MaxCases != 3 { + t.Fatalf("unexpected warmup maxCases: %#v", aView.Cache.Warmup.MaxCases) + } if len(aView.Cache.Warmup.FieldNames) != 2 || aView.Cache.Warmup.FieldNames[0] != "Id" || aView.Cache.Warmup.FieldNames[1] != "Name" { t.Fatalf("unexpected field names: %#v", aView.Cache.Warmup.FieldNames) } @@ -94,3 +98,11 @@ func TestCacheWarmupApplyRejectsNegativeLimit(t *testing.T) { t.Fatalf("expected error") } } + +func TestCacheWarmupApplyRejectsNegativeMaxCases(t *testing.T) { + aView := &view.View{Cache: view.NewRefCache("aerospike")} + err := (&cacheWarmup{}).Apply([]string{"order_id", "MaxCases=-1"}, nil, &view.Resource{}, aView) + if err == nil { + t.Fatalf("expected error") + } +} diff --git a/service/reader/service.go b/service/reader/service.go index d381f42b0..2b4f045da 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -872,6 +872,7 @@ func logCacheRead(aView *view.View, stats *cache.Stats, elapsed time.Duration, r if stats == nil { return } + recordCacheReadMetrics(aView, stats) fmt.Printf("[INFO] datly cache read view=%s source=%s type=%s found_warmup=%t found_lazy=%t records=%d rows=%d namespace=%s set=%s elapsed=%s args=%v\n", aView.Name, cacheReadSource(stats), @@ -886,6 +887,32 @@ func logCacheRead(aView *view.View, stats *cache.Stats, elapsed time.Duration, r args) } +func recordCacheReadMetrics(aView *view.View, stats *cache.Stats) { + if aView == nil || aView.Counter == nil || stats == nil { + return + } + if stats.ErrorType != "" { + aView.Counter.IncrementValue("cache:error") + return + } + if stats.FoundWarmup { + aView.Counter.IncrementValue("cache:hit") + aView.Counter.IncrementValue("cache:warmup_hit") + return + } + if stats.FoundLazy { + aView.Counter.IncrementValue("cache:hit") + aView.Counter.IncrementValue("cache:lazy_hit") + return + } + if stats.Type == cache.TypeWrite { + aView.Counter.IncrementValue("cache:miss") + aView.Counter.IncrementValue("cache:miss_write") + return + } + aView.Counter.IncrementValue("cache:miss") +} + func cacheReadSource(stats *cache.Stats) string { if stats.ErrorType != "" { return "error" diff --git a/service/reader/service_metrics_test.go b/service/reader/service_metrics_test.go new file mode 100644 index 000000000..d9bd3f4db --- /dev/null +++ b/service/reader/service_metrics_test.go @@ -0,0 +1,90 @@ +package reader + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/logger" + "github.com/viant/datly/view" + "github.com/viant/gmetric/counter" + "github.com/viant/sqlx/io/read/cache" +) + +type metricsTestCounter struct { + values map[interface{}]int +} + +func newMetricsTestCounter() *metricsTestCounter { + return &metricsTestCounter{values: map[interface{}]int{}} +} + +func (c *metricsTestCounter) Begin(started time.Time) counter.OnDone { + return func(time.Time, ...interface{}) int64 { return 0 } +} + +func (c *metricsTestCounter) DecrementValue(value interface{}) int64 { + c.values[value]-- + return int64(c.values[value]) +} + +func (c *metricsTestCounter) IncrementValue(value interface{}) int64 { + c.values[value]++ + return int64(c.values[value]) +} + +func TestRecordCacheReadMetrics(t *testing.T) { + testCases := []struct { + description string + stats *cache.Stats + expected []string + }{ + { + description: "warmup hit", + stats: &cache.Stats{Type: cache.TypeReadMulti, FoundWarmup: true}, + expected: []string{"cache:hit", "cache:warmup_hit"}, + }, + { + description: "lazy hit", + stats: &cache.Stats{Type: cache.TypeReadSingle, FoundLazy: true}, + expected: []string{"cache:hit", "cache:lazy_hit"}, + }, + { + description: "warmup probe miss", + stats: &cache.Stats{Type: cache.TypeReadMulti}, + expected: []string{"cache:miss"}, + }, + { + description: "lazy probe miss", + stats: &cache.Stats{Type: cache.TypeReadSingle}, + expected: []string{"cache:miss"}, + }, + { + description: "miss with write", + stats: &cache.Stats{Type: cache.TypeWrite}, + expected: []string{"cache:miss", "cache:miss_write"}, + }, + { + description: "miss", + stats: &cache.Stats{}, + expected: []string{"cache:miss"}, + }, + { + description: "error", + stats: &cache.Stats{ErrorType: "backend"}, + expected: []string{"cache:error"}, + }, + } + + for _, testCase := range testCases { + counter := newMetricsTestCounter() + aView := &view.View{Counter: logger.NewCounter(counter)} + + recordCacheReadMetrics(aView, testCase.stats) + + for _, metric := range testCase.expected { + require.Equalf(t, 1, counter.values[metric], testCase.description) + } + require.Lenf(t, counter.values, len(testCase.expected), testCase.description) + } +} diff --git a/view/cache.go b/view/cache.go index 83075319a..6c3c1a1b0 100644 --- a/view/cache.go +++ b/view/cache.go @@ -55,6 +55,7 @@ type ( IndexParameter string `json:",omitempty" yaml:",omitempty"` IndexMeta bool `json:",omitempty"` Limit *int `json:",omitempty" yaml:",omitempty"` + MaxCases *int `json:",omitempty" yaml:",omitempty"` FieldNames []string `json:",omitempty" yaml:",omitempty"` Connector *Connector `json:",omitempty"` Cases []*CacheParameters @@ -86,6 +87,12 @@ type ( } CacheInputFn func() ([]*CacheInput, error) + + cacheParamValuesResult struct { + index int + values [][]interface{} + err error + } ) const ( @@ -254,6 +261,13 @@ func (c *Cache) expandLocation(aView *View) (string, error) { return expanded, nil } +func (c *Cache) ExpandedLocation(aView *View) (string, error) { + if c == nil { + return "", nil + } + return c.expandLocation(aView) +} + func (c *Cache) Service() (cache.Cache, error) { return c.newCache() } @@ -338,45 +352,52 @@ func (c *Cache) inherit(source *Cache) error { } func (c *Cache) GenerateCacheInput(ctx context.Context) ([]*CacheInput, error) { - var cacheInputPermutations []*CacheInput - chanSize := len(c.Warmup.Cases) - selectorChan := make(chan CacheInputFn, chanSize) - if chanSize == 0 { - close(selectorChan) - return []*CacheInput{ - c.NewInput(NewStatelet()), - }, nil + if len(c.Warmup.Cases) == 0 { + input := c.NewInput(NewStatelet()) + if c.maxCasesExceeded(0, 0, input) { + if maxCases := c.maxCases(); maxCases > 0 { + fmt.Printf("[INFO] cache warmup selector cap view=%s max_cases=%d selected_entries=0 selected_selectors=0\n", c.owner.Name, maxCases) + } + return []*CacheInput{}, nil + } + return []*CacheInput{input}, nil } - for i := range c.Warmup.Cases { - go c.generateDatasetSelectorsChan(ctx, selectorChan, c.Warmup.Cases[i]) + paramValues := make([][][]interface{}, len(c.Warmup.Cases)) + results := make(chan cacheParamValuesResult, len(c.Warmup.Cases)) + for i, dataSet := range c.Warmup.Cases { + go func(index int, set *CacheParameters) { + values, err := c.generateDatasetParamValues(ctx, set) + results <- cacheParamValuesResult{index: index, values: values, err: err} + }(i, dataSet) + } + for i := 0; i < len(c.Warmup.Cases); i++ { + result := <-results + if result.err != nil { + return nil, result.err + } + paramValues[result.index] = result.values } - counter := 0 - for selectorFn := range selectorChan { - selectors, err := selectorFn() + var cacheInputPermutations []*CacheInput + selectedEntries := 0 + for i, dataSet := range c.Warmup.Cases { + selectors, err := c.generateDatasetSelectors(dataSet, paramValues[i], selectedEntries) if err != nil { return nil, err } - cacheInputPermutations = append(cacheInputPermutations, selectors...) - counter++ - if counter == chanSize { - close(selectorChan) + selectedEntries += c.cacheInputEntryCount(selectors...) + if maxCases := c.maxCases(); maxCases > 0 && selectedEntries >= maxCases { + fmt.Printf("[INFO] cache warmup selector cap view=%s max_cases=%d selected_entries=%d selected_selectors=%d\n", c.owner.Name, maxCases, selectedEntries, len(cacheInputPermutations)) + break } } return cacheInputPermutations, nil } -func (c *Cache) generateDatasetSelectorsChan(ctx context.Context, selectorChan chan CacheInputFn, dataSet *CacheParameters) { - selectors, err := c.generateDatasetSelectorsErr(ctx, dataSet) - selectorChan <- func() ([]*CacheInput, error) { - return selectors, err - } -} - -func (c *Cache) generateDatasetSelectorsErr(ctx context.Context, set *CacheParameters) ([]*CacheInput, error) { +func (c *Cache) generateDatasetParamValues(ctx context.Context, set *CacheParameters) ([][]interface{}, error) { var availableValues [][]interface{} for i := range set.Set { @@ -388,8 +409,12 @@ func (c *Cache) generateDatasetSelectorsErr(ctx context.Context, set *CacheParam availableValues = append(availableValues, paramValues) } + return availableValues, nil +} + +func (c *Cache) generateDatasetSelectors(set *CacheParameters, availableValues [][]interface{}, selectedEntries int) ([]*CacheInput, error) { var result []*CacheInput - if err := c.appendSelectors(set, availableValues, &result); err != nil { + if err := c.appendSelectors(set, availableValues, &result, selectedEntries); err != nil { return nil, err } @@ -504,6 +529,9 @@ func (c *Cache) initWarmup(ctx context.Context, resource *Resource) error { if err := c.validateWarmupFieldNames(c.Warmup.FieldNames); err != nil { return err } + if err := c.validateWarmupBudget("maxCases", c.Warmup.MaxCases); err != nil { + return err + } for _, dataset := range c.Warmup.Cases { if err := c.validateWarmupFieldNames(dataset.FieldNames); err != nil { return err @@ -558,7 +586,7 @@ func (c *Cache) addNonRequiredWarmupIfNeeded() { }) } -func (c *Cache) appendSelectors(set *CacheParameters, paramValues [][]interface{}, selectors *[]*CacheInput) error { +func (c *Cache) appendSelectors(set *CacheParameters, paramValues [][]interface{}, selectors *[]*CacheInput, selectedEntries int) error { for i, value := range paramValues { if len(value) == 0 { return fmt.Errorf("parameter %v is required but there was no data", set.Set[i].Name) @@ -566,9 +594,14 @@ func (c *Cache) appendSelectors(set *CacheParameters, paramValues [][]interface{ } indexes := make([]int, len(paramValues)) + generatedEntries := 0 if len(indexes) == 0 { input := c.newInput(NewStatelet(), set) + if c.maxCasesExceeded(selectedEntries, generatedEntries, input) { + return nil + } *selectors = append(*selectors, input) + generatedEntries += c.cacheInputEntryCount(input) fmt.Printf("[INFO] cache warmup selector view=%s index_column=%s params= field_names=%s\n", c.owner.Name, c.Warmup.IndexColumn, strings.Join(input.FieldNames, ",")) return nil } @@ -594,7 +627,11 @@ outer: label := strings.Join(debugParams, ",") input := c.newInput(selector, set) input.Label = label + if c.maxCasesExceeded(selectedEntries, generatedEntries, input) { + return nil + } *selectors = append(*selectors, input) + generatedEntries += c.cacheInputEntryCount(input) fmt.Printf("[INFO] cache warmup selector view=%s index_column=%s params=%s field_names=%s\n", c.owner.Name, c.Warmup.IndexColumn, label, strings.Join(input.FieldNames, ",")) for i := len(indexes) - 1; i >= 0; i-- { @@ -644,6 +681,32 @@ func (c *Cache) fieldNamesFor(set *CacheParameters) []string { return c.Warmup.FieldNames } +func (c *Cache) maxCases() int { + if c == nil || c.Warmup == nil || c.Warmup.MaxCases == nil || *c.Warmup.MaxCases <= 0 { + return 0 + } + return *c.Warmup.MaxCases +} + +func (c *Cache) maxCasesExceeded(selectedEntries, generatedEntries int, input *CacheInput) bool { + maxCases := c.maxCases() + return maxCases > 0 && selectedEntries+generatedEntries+c.cacheInputEntryCount(input) > maxCases +} + +func (c *Cache) cacheInputEntryCount(inputs ...*CacheInput) int { + result := 0 + for _, input := range inputs { + if input == nil { + continue + } + result++ + if input.IndexMeta { + result++ + } + } + return result +} + func (c *Cache) validateWarmupFieldNames(fieldNames []string) error { if len(fieldNames) == 0 { return nil @@ -667,6 +730,20 @@ func (c *Cache) validateWarmupFieldNames(fieldNames []string) error { return nil } +func (c *Cache) validateWarmupBudget(name string, value *int) error { + if value == nil { + return nil + } + if *value < 0 { + viewName := "" + if c.owner != nil { + viewName = c.owner.Name + } + return fmt.Errorf("warmup %s must be zero or greater on view %v", name, viewName) + } + return nil +} + func (c *Cache) applyWarmupFieldNames(selector *Statelet, fieldNames []string) { if selector == nil || c.owner == nil || len(fieldNames) == 0 { return diff --git a/warmup/cache.go b/warmup/cache.go index 9185a1618..e61f37794 100644 --- a/warmup/cache.go +++ b/warmup/cache.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "github.com/viant/afs/url" "github.com/viant/datly/service/reader" errUtils "github.com/viant/datly/shared" "github.com/viant/datly/view" @@ -14,7 +15,9 @@ import ( "time" ) -const maxWarmupConcurrency = 20 +const ( + maxWarmupConcurrency = 20 +) type ( matchersCollector struct { @@ -35,7 +38,7 @@ type ( } warmupEntryFn func() (*warmupEntry, error) - notifierFn func() (int, error) + notifierFn func() (int, *EntryResult, error) EntryResult struct { View string @@ -46,6 +49,7 @@ type ( Elapsed string TimeTaken time.Duration Rows int + Error string `json:",omitempty"` } Result struct { @@ -58,8 +62,11 @@ func (c *matchersCollector) populate(ctx context.Context, collector chan warmupE go func() { size, err := c.populateCacheCases(ctx, collector) - notifier <- func() (int, error) { - return size, err + notifier <- func() (int, *EntryResult, error) { + if err == nil { + return size, nil, nil + } + return size, failedEntryResult(&warmupEntry{view: c.view}, 0, 0, err), err } }() } @@ -102,7 +109,12 @@ func (c *matchersCollector) createMetaWarmupEntry(ctx context.Context, aView *vi if err != nil { fmt.Printf("[INFO] cache warmup entry build error view=%s type=meta column=%s field_names=%s error=%v\n", aView.Name, input.MetaColumn, strings.Join(input.FieldNames, ","), err) aChan <- func() (*warmupEntry, error) { - return nil, err + return &warmupEntry{ + view: aView, + column: input.MetaColumn, + label: input.Label, + fields: strings.Join(input.FieldNames, ","), + }, err } return } @@ -110,7 +122,12 @@ func (c *matchersCollector) createMetaWarmupEntry(ctx context.Context, aView *vi if err != nil { fmt.Printf("[INFO] cache warmup entry build error view=%s type=meta column=%s field_names=%s error=%v\n", aView.Name, input.MetaColumn, strings.Join(input.FieldNames, ","), err) aChan <- func() (*warmupEntry, error) { - return nil, err + return &warmupEntry{ + view: aView, + column: input.MetaColumn, + label: input.Label, + fields: strings.Join(input.FieldNames, ","), + }, err } return } @@ -132,7 +149,12 @@ func (c *matchersCollector) createIndexWarmupEntry(ctx context.Context, aView *v if err != nil { fmt.Printf("[INFO] cache warmup entry build error view=%s type=index column=%s field_names=%s error=%v\n", aView.Name, cacheInput.Column, strings.Join(cacheInput.FieldNames, ","), err) aChan <- func() (*warmupEntry, error) { - return nil, err + return &warmupEntry{ + view: aView, + column: cacheInput.Column, + label: cacheInput.Label, + fields: strings.Join(cacheInput.FieldNames, ","), + }, err } return } @@ -140,7 +162,12 @@ func (c *matchersCollector) createIndexWarmupEntry(ctx context.Context, aView *v if err != nil { fmt.Printf("[INFO] cache warmup entry build error view=%s type=index column=%s field_names=%s error=%v\n", aView.Name, cacheInput.Column, strings.Join(cacheInput.FieldNames, ","), err) aChan <- func() (*warmupEntry, error) { - return nil, err + return &warmupEntry{ + view: aView, + column: cacheInput.Column, + label: cacheInput.Label, + fields: strings.Join(cacheInput.FieldNames, ","), + }, err } return } @@ -214,28 +241,61 @@ func readWithErr(ctx context.Context, entry *warmupEntry) (*EntryResult, error) fmt.Printf("[INFO] cache warmup query start start_time=%s view=%s cache=%s cache_key=%s db_connector=%s column=%s params=%s field_names=%s args=%v sql=%q\n", started.Format(time.RFC3339), entry.view.Name, cacheLabel(entry.view), entry.key, warmupConnectorLabel(entry.view), entry.column, entry.label, entry.fields, entry.matcher.Args, truncateSQL(entry.matcher.SQL)) db, err := DB(entry) if err != nil { - fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, time.Since(started), err) - return nil, err + elapsed := time.Since(started) + fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, elapsed, err) + return failedEntryResult(entry, elapsed, 0, err), err } service, err := entry.view.Cache.Service() if err != nil { - fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, time.Since(started), err) - return nil, err + elapsed := time.Since(started) + fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, elapsed, err) + return failedEntryResult(entry, elapsed, 0, err), err } matcher := entry.matcher - indexed, err := service.IndexBy(ctx, db, entry.column, matcher.SQL, matcher.Args) + indexed, err := service.IndexBy(indexProgressContext(ctx, entry), db, entry.column, matcher.SQL, matcher.Args) elapsed := time.Since(started) if err != nil { fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s rows=%d elapsed=%s cache_write=error error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, indexed, elapsed, err) - return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, CacheKey: entry.key, FieldNames: entry.fields, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: indexed}, fmt.Errorf("failed to index: %w, %v", err, matcher.SQL) + indexErr := fmt.Errorf("failed to index: %w", err) + return failedEntryResult(entry, elapsed, indexed, indexErr), indexErr } fmt.Printf("[INFO] cache warmup query done view=%s cache=%s cache_key=%s db_connector=%s column=%s params=%s field_names=%s rows=%d elapsed=%s cache_write=success\n", entry.view.Name, cacheLabel(entry.view), entry.key, warmupConnectorLabel(entry.view), entry.column, entry.label, entry.fields, indexed, elapsed) return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, CacheKey: entry.key, FieldNames: entry.fields, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: indexed}, nil } +func failedEntryResult(entry *warmupEntry, elapsed time.Duration, rows int, err error) *EntryResult { + result := &EntryResult{ + Elapsed: elapsed.String(), + TimeTaken: elapsed, + Rows: rows, + } + if entry != nil { + if entry.view != nil { + result.View = entry.view.Name + } + result.Column = entry.column + result.Params = entry.label + result.CacheKey = entry.key + result.FieldNames = entry.fields + } + if err != nil { + result.Error = err.Error() + } + return result +} + +func firstError(errors []error) error { + for _, err := range errors { + if err != nil { + return err + } + } + return nil +} + func warmupCacheKey(query *cache.ParmetrizedQuery) (string, error) { if query == nil { return "", fmt.Errorf("warmup cache key query was nil") @@ -271,10 +331,11 @@ func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (* started := time.Now() viewsWithCache := FilterCacheViews(views) fmt.Printf("[INFO] cache warmup populate start start_time=%s views=%s cache_views=%s cache_count=%d\n", started.Format(time.RFC3339), namesOf(views), namesOf(viewsWithCache), len(viewsWithCache)) + result := &Result{} if len(viewsWithCache) == 0 { fmt.Printf("[INFO] cache warmup populate done rows=0 elapsed=%s\n", time.Since(started)) - return &Result{}, nil + return result, nil } collector := make(chan warmupEntryFn) @@ -287,14 +348,19 @@ func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (* counter := 0 collectorSize := 0 + var errors []error for counter < len(viewsWithCache) { select { case fn := <-notifier: - chunkSize, err := fn() + chunkSize, entryResult, err := fn() collectorSize += chunkSize + if entryResult != nil { + result.Entries = append(result.Entries, entryResult) + } if err != nil { fmt.Printf("encounter err while creating selectors: %v\n", err.Error()) + errors = append(errors, err) } counter++ @@ -303,17 +369,21 @@ func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (* if collectorSize == 0 { fmt.Printf("[INFO] cache warmup populate done rows=0 entries=0 elapsed=%s\n", time.Since(started)) - return &Result{}, nil + err := errUtils.CombineErrors("errors while populating cache: ", errors) + if err != nil { + return result, err + } + return result, nil } fmt.Printf("[INFO] cache warmup entries expected entries=%d elapsed=%s\n", collectorSize, time.Since(started)) - var errors []error var warmupEntries []*warmupEntry var collectorsCounter int for fn := range collector { entry, err := fn() if err != nil { errors = append(errors, err) + result.Entries = append(result.Entries, failedEntryResult(entry, 0, 0, err)) } else { warmupEntries = append(warmupEntries, entry) } @@ -326,35 +396,32 @@ func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (* close(collector) if err := errUtils.CombineErrors("errors while populating cache: ", errors); err != nil { - fmt.Printf("[INFO] cache warmup populate error entries=%d elapsed=%s error=%v\n", len(warmupEntries), time.Since(started), err) - return &Result{}, err + fmt.Printf("[INFO] cache warmup populate error entries=%d failures=%d elapsed=%s first_error=%v\n", len(warmupEntries), len(errors), time.Since(started), firstError(errors)) + return result, err } fmt.Printf("[INFO] cache warmup entries built entries=%d elapsed=%s\n", len(warmupEntries), time.Since(started)) notifierErr := make(chan func() (*EntryResult, error)) warmup(ctx, warmupEntries, notifierErr) - result := &Result{} - for i := 0; i < len(warmupEntries); i++ { - select { - case actual := <-notifierErr: - if actual != nil { - entryResult, err := actual() - if entryResult != nil { - result.Rows += entryResult.Rows - result.Entries = append(result.Entries, entryResult) - } - if err != nil { - errors = append(errors, err) - } - } + for actual := range notifierErr { + if actual == nil { + continue + } + entryResult, err := actual() + if entryResult != nil { + result.Entries = append(result.Entries, entryResult) + result.Rows += entryResult.Rows + } + if err != nil { + errors = append(errors, err) } } close(notifier) err := errUtils.CombineErrors("errors while populating cache: ", errors) if err != nil { - fmt.Printf("[INFO] cache warmup populate error rows=%d entries=%d elapsed=%s error=%v\n", result.Rows, len(warmupEntries), time.Since(started), err) + fmt.Printf("[INFO] cache warmup populate error rows=%d entries=%d failures=%d elapsed=%s first_error=%v\n", result.Rows, len(warmupEntries), len(errors), time.Since(started), firstError(errors)) return result, err } fmt.Printf("[INFO] cache warmup populate done rows=%d entries=%d elapsed=%s\n", result.Rows, len(warmupEntries), time.Since(started)) @@ -434,3 +501,82 @@ func truncateSQL(SQL string) string { } return SQL[:512] + "...(truncated)" } + +func indexProgressContext(ctx context.Context, entry *warmupEntry) context.Context { + if entry == nil { + return ctx + } + ctx = cache.WithIndexProgress(ctx, &cache.IndexProgress{ + View: warmupViewName(entry), + Dataset: warmupDatasetName(entry), + Case: entry.label, + }) + return cache.WithIndexProgressCallback(ctx, logIndexProgress) +} + +func logIndexProgress(event *cache.IndexProgressEvent) { + if event == nil { + return + } + if event.Done { + fmt.Printf("[INFO] aerospike cache index read done%s column=%s rows=%d elapsed=%s\n", formatIndexProgressEvent(event), event.Column, event.Rows, event.Elapsed) + return + } + fmt.Printf("[INFO] aerospike cache index progress%s column=%s rows=%d elapsed=%s\n", formatIndexProgressEvent(event), event.Column, event.Rows, event.Elapsed) +} + +func formatIndexProgressEvent(event *cache.IndexProgressEvent) string { + if event == nil { + return "" + } + parts := make([]string, 0, 3) + if event.View != "" { + parts = append(parts, " view="+event.View) + } + if event.Dataset != "" { + parts = append(parts, " dataset="+event.Dataset) + } + if event.Case != "" { + parts = append(parts, " case="+event.Case) + } + return strings.Join(parts, "") +} + +func warmupViewName(entry *warmupEntry) string { + if entry == nil || entry.view == nil { + return "" + } + return entry.view.Name +} + +func warmupDatasetName(entry *warmupEntry) string { + if entry == nil || entry.view == nil || entry.view.Cache == nil { + return "" + } + location := strings.TrimSpace(entry.view.Name) + if entry.view.Template != nil && entry.view.Selector != nil { + expandedLocation, err := entry.view.Cache.ExpandedLocation(entry.view) + if err == nil && strings.TrimSpace(expandedLocation) != "" { + location = strings.TrimSpace(expandedLocation) + } + } + + namespace := warmupCacheNamespace(entry.view.Cache.Provider) + if namespace == "" { + return location + } + if location == "" { + return namespace + } + return namespace + "/" + location +} + +func warmupCacheNamespace(provider string) string { + provider = strings.TrimSpace(provider) + if provider == "" { + return "" + } + scheme := url.Scheme(provider, "") + _, namespace := url.Split(provider, scheme) + return strings.TrimSpace(namespace) +} diff --git a/warmup/cache_test.go b/warmup/cache_test.go index aba4b86c4..9b1f7c797 100644 --- a/warmup/cache_test.go +++ b/warmup/cache_test.go @@ -2,6 +2,7 @@ package warmup import ( "context" + "database/sql" "os" "path" "sync/atomic" @@ -176,6 +177,57 @@ func TestWarmupCacheKeyNormalizesNilArgs(t *testing.T) { assert.ErrorContains(t, err, "query was nil") } +func TestIndexProgressContext(t *testing.T) { + aView := &view.View{ + Name: "performanceTimeline", + Template: &view.Template{}, + Selector: &view.Config{}, + Cache: &view.Cache{ + Location: "${View.Name}_dataset", + Provider: "aerospike://127.0.0.1:3000/ns_memory", + }, + } + entry := &warmupEntry{ + view: aView, + label: "Period=today,Granularity=hour", + } + + ctx := indexProgressContext(context.Background(), entry) + progress, ok := sqlcache.IndexProgressFromContext(ctx) + require.True(t, ok) + require.NotNil(t, progress) + require.Equal(t, "performanceTimeline", progress.View) + require.Equal(t, "ns_memory/performanceTimeline_dataset", progress.Dataset) + require.Equal(t, "Period=today,Granularity=hour", progress.Case) + + callback, ok := sqlcache.IndexProgressCallbackFromContext(ctx) + require.True(t, ok) + require.NotNil(t, callback) + + var actual *sqlcache.IndexProgressEvent + ctx = sqlcache.WithIndexProgressCallback(ctx, func(event *sqlcache.IndexProgressEvent) { + if event == nil { + return + } + cloned := *event + actual = &cloned + }) + sqlcache.EmitIndexProgress(ctx, &sqlcache.IndexProgressEvent{ + Column: "order_id", + Rows: 42, + Elapsed: 3 * time.Second, + Done: true, + }) + require.NotNil(t, actual) + require.Equal(t, "performanceTimeline", actual.View) + require.Equal(t, "ns_memory/performanceTimeline_dataset", actual.Dataset) + require.Equal(t, "Period=today,Granularity=hour", actual.Case) + require.Equal(t, "order_id", actual.Column) + require.Equal(t, 42, actual.Rows) + require.Equal(t, 3*time.Second, actual.Elapsed) + require.True(t, actual.Done) +} + func TestWarmupFieldNamesAffectGeneratedCacheKey(t *testing.T) { resourcePath := path.Join(t.TempDir(), "resource.yaml") require.NoError(t, os.WriteFile(resourcePath, []byte(` @@ -342,6 +394,294 @@ Views: assert.True(t, input[0].Selector.WarmupNoLimit) } +func TestGenerateCacheInput_AppliesMaxCases(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + MaxCases: 2 + Cases: + - Set: + - Name: EventTypeId + Values: [1, 2, 3, 4] + Template: + Source: SELECT * FROM EVENTS WHERE event_type_id = $EventTypeId + Parameters: + - Name: EventTypeId + Required: true + In: + Kind: query + Name: event_type_id + Schema: + DataType: int +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + + input, err := resource.Views[0].Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.Len(t, input, 2) + require.Equal(t, "EventTypeId=1", input[0].Label) + require.Equal(t, "EventTypeId=2", input[1].Label) +} + +func TestGenerateCacheInput_AppliesMaxCasesAcrossDatasets(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + MaxCases: 3 + Cases: + - Set: + - Name: EventTypeId + Values: [1, 2] + - Set: + - Name: EventTypeId + Values: [3, 4] + Template: + Source: SELECT * FROM EVENTS WHERE event_type_id = $EventTypeId + Parameters: + - Name: EventTypeId + Required: true + In: + Kind: query + Name: event_type_id + Schema: + DataType: int +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + + input, err := resource.Views[0].Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.Len(t, input, 3) + require.Equal(t, "EventTypeId=1", input[0].Label) + require.Equal(t, "EventTypeId=2", input[1].Label) + require.Equal(t, "EventTypeId=3", input[2].Label) +} + +func TestGenerateCacheInput_MaxCasesZeroMeansUnlimited(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + MaxCases: 0 + Cases: + - Set: + - Name: EventTypeId + Values: [1, 2, 3, 4] + Template: + Source: SELECT * FROM EVENTS WHERE event_type_id = $EventTypeId + Parameters: + - Name: EventTypeId + Required: true + In: + Kind: query + Name: event_type_id + Schema: + DataType: int +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + + input, err := resource.Views[0].Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.Len(t, input, 4) +} + +func TestGenerateCacheInput_MaxCasesCountsIndexMetaExecutions(t *testing.T) { + dbPath := path.Join(t.TempDir(), "events.db") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + _, err = db.Exec(`CREATE TABLE EVENTS (event_type_id INTEGER)`) + require.NoError(t, err) + + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: "`+dbPath+`" + +Views: + - Name: events + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + MaxCases: 3 + Cases: + - Set: + - Name: EventTypeId + Values: [1, 2, 3] + Selector: + Constraints: + Projection: true + Template: + Summary: + Name: EventsMeta + Source: 'SELECT COUNT(*) AS TOTAL_RECORDS, event_type_id FROM ($View.Expand($criteria)) GROUP BY event_type_id' + Source: SELECT * FROM EVENTS + Parameters: + - Name: EventTypeId + Required: true + In: + Kind: query + Name: event_type_id + Schema: + DataType: int +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + + input, err := resource.Views[0].Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.Len(t, input, 1) + require.True(t, input[0].IndexMeta) + require.Equal(t, "EventTypeId=1", input[0].Label) +} + +func TestGenerateCacheInput_MaxCasesAppliesWithoutExplicitCases(t *testing.T) { + dbPath := path.Join(t.TempDir(), "events.db") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + _, err = db.Exec(`CREATE TABLE EVENTS (event_type_id INTEGER)`) + require.NoError(t, err) + + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: "`+dbPath+`" + +Views: + - Name: events + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + MaxCases: 1 + Selector: + Constraints: + Projection: true + Template: + Summary: + Name: EventsMeta + Source: 'SELECT COUNT(*) AS TOTAL_RECORDS, event_type_id FROM ($View.Expand($criteria)) GROUP BY event_type_id' + Source: SELECT * FROM EVENTS +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + + input, err := resource.Views[0].Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.Empty(t, input) +} + func checkIfCached(t *testing.T, cache *view.Cache, ctx context.Context, testCase struct { description string URL string From 4f24a328c0f59ea8b41505499f59b6778ec71e14 Mon Sep 17 00:00:00 2001 From: vcarey Date: Thu, 9 Jul 2026 17:44:24 -0400 Subject: [PATCH 255/279] Expose metrics and add request trace correlation --- gateway/route.go | 2 + gateway/route_metric.go | 28 ++++++ gateway/route_metric_provider.go | 26 ++++++ gateway/route_metric_provider_test.go | 36 ++++++++ gateway/route_metric_test.go | 23 +++++ gateway/route_metrics.go | 14 ++- gateway/router.go | 3 + gateway/router/handler.go | 2 + gateway/warmup/cache.go | 22 ++--- internal/gmetricx/counter.go | 120 +++++++++++++++++++++++++ internal/gmetricx/service.go | 18 ++++ internal/gmetricx/service_test.go | 65 ++++++++++++++ internal/requesttrace/context.go | 29 ++++++ service/operator/service.go | 2 + service/reader/service.go | 37 ++++++-- service/reader/service_metrics_test.go | 63 ++++++++++++- view/view.go | 18 ++-- view/view_metric_provider.go | 94 +++++++++++++++++++ view/view_metric_provider_test.go | 44 +++++++++ 19 files changed, 607 insertions(+), 39 deletions(-) create mode 100644 gateway/route_metric_provider.go create mode 100644 gateway/route_metric_provider_test.go create mode 100644 gateway/route_metric_test.go create mode 100644 internal/gmetricx/counter.go create mode 100644 internal/gmetricx/service.go create mode 100644 internal/gmetricx/service_test.go create mode 100644 internal/requesttrace/context.go create mode 100644 view/view_metric_provider.go create mode 100644 view/view_metric_provider_test.go diff --git a/gateway/route.go b/gateway/route.go index 7c28c396b..e2e8a31b9 100644 --- a/gateway/route.go +++ b/gateway/route.go @@ -8,6 +8,7 @@ import ( "time" "github.com/viant/afs/url" + "github.com/viant/datly/internal/requesttrace" "github.com/viant/datly/gateway/router" "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" @@ -53,6 +54,7 @@ func (r *Route) Handle(res http.ResponseWriter, req *http.Request) int { } execContext := exec.NewContext(req.Method, req.RequestURI, req.Header, r.Version) ctx = vcontext.WithValue(ctx, exec.ContextKey, execContext) + ctx = requesttrace.Ensure(ctx, execContext.TraceID) req = req.WithContext(ctx) var onDone func(time.Time, ...interface{}) int64 = nil var start time.Time diff --git a/gateway/route_metric.go b/gateway/route_metric.go index e7bff88ad..164bb01cc 100644 --- a/gateway/route_metric.go +++ b/gateway/route_metric.go @@ -27,6 +27,34 @@ func (r *Router) NewMetricRoute(URI string) *Route { } } +func (r *Router) NewGlobalMetricRoutes(URI string) []*Route { + if !strings.HasSuffix(URI, "/") { + URI += "/" + } + handler := func(ctx context.Context, response http.ResponseWriter, req *http.Request) { + r.handleMetrics(response, req, URI) + } + paths := []string{ + URI + "operations", + URI + "operation/{name}", + URI + "operation/{name}/cumulative/{metric}", + URI + "operation/{name}/recent/{metric}", + URI + "operation/{name}/recent", + URI + "counters", + URI + "counter/{name}", + } + routes := make([]*Route, 0, len(paths)) + for _, pathURI := range paths { + routes = append(routes, &Route{ + Path: contract.NewPath(http.MethodGet, pathURI), + Handler: handler, + Config: r.config.Logging, + Version: r.config.Version, + }) + } + return routes +} + func (r *Router) handleMetrics(writer http.ResponseWriter, req *http.Request, URI string) { gmetric.NewHandler(URI, r.metrics).ServeHTTP(writer, req) } diff --git a/gateway/route_metric_provider.go b/gateway/route_metric_provider.go new file mode 100644 index 000000000..4c914ba5d --- /dev/null +++ b/gateway/route_metric_provider.go @@ -0,0 +1,26 @@ +package gateway + +import ( + "github.com/viant/gmetric/counter" + "github.com/viant/gmetric/counter/base" +) + +const ( + routeRequestMetric = "Request" + routeSuccessMetric = "Success" + routeErrorMetric = "Error" + routeStatus2xxMetric = "status:2xx" + routeStatus4xxMetric = "status:4xx" + routeStatus5xxMetric = "status:5xx" +) + +func newRouteMetricProvider() counter.Provider { + return base.NewProvider( + routeRequestMetric, + routeSuccessMetric, + routeErrorMetric, + routeStatus2xxMetric, + routeStatus4xxMetric, + routeStatus5xxMetric, + ) +} diff --git a/gateway/route_metric_provider_test.go b/gateway/route_metric_provider_test.go new file mode 100644 index 000000000..1f09d468f --- /dev/null +++ b/gateway/route_metric_provider_test.go @@ -0,0 +1,36 @@ +package gateway + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/viant/gmetric" +) + +func TestRouteMetricProvider_ExportsRouteCounters(t *testing.T) { + metrics := gmetric.New() + counter := metrics.MultiOperationCounter("steward/metadata", "steward.metadata.signalPerformance.request", "signal performance request", time.Millisecond, time.Minute, 2, newRouteMetricProvider()) + + counter.IncrementValue(routeRequestMetric) + counter.IncrementValue(routeSuccessMetric) + counter.IncrementValue(routeStatus2xxMetric) + counter.IncrementValue(routeErrorMetric) + counter.IncrementValue(routeStatus4xxMetric) + counter.IncrementValue(routeStatus5xxMetric) + + operation := metrics.LookupOperation("steward.metadata.signalPerformance.request") + require.NotNil(t, operation) + + values := map[string]int64{} + for _, item := range operation.Counters { + values[item.Value] = item.Count + } + + require.Equal(t, int64(1), values[routeRequestMetric]) + require.Equal(t, int64(1), values[routeSuccessMetric]) + require.Equal(t, int64(1), values[routeStatus2xxMetric]) + require.Equal(t, int64(1), values[routeErrorMetric]) + require.Equal(t, int64(1), values[routeStatus4xxMetric]) + require.Equal(t, int64(1), values[routeStatus5xxMetric]) +} diff --git a/gateway/route_metric_test.go b/gateway/route_metric_test.go new file mode 100644 index 000000000..2dbdce845 --- /dev/null +++ b/gateway/route_metric_test.go @@ -0,0 +1,23 @@ +package gateway + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRouterNewGlobalMetricRoutes(t *testing.T) { + router := &Router{config: &Config{}} + + routes := router.NewGlobalMetricRoutes("/v1/api/meta/metric") + + require.Len(t, routes, 7) + require.Equal(t, "/v1/api/meta/metric/operations", routes[0].Path.URI) + require.Equal(t, "/v1/api/meta/metric/operation/{name}", routes[1].Path.URI) + require.Equal(t, "/v1/api/meta/metric/operation/{name}/cumulative/{metric}", routes[2].Path.URI) + require.Equal(t, "/v1/api/meta/metric/operation/{name}/recent/{metric}", routes[3].Path.URI) + require.Equal(t, "/v1/api/meta/metric/operation/{name}/recent", routes[4].Path.URI) + require.Equal(t, "/v1/api/meta/metric/counters", routes[5].Path.URI) + require.Equal(t, "/v1/api/meta/metric/counter/{name}", routes[6].Path.URI) + require.Nil(t, routes[0].ApiKeys) +} diff --git a/gateway/route_metrics.go b/gateway/route_metrics.go index 213f6c4a3..7eb09d9e7 100644 --- a/gateway/route_metrics.go +++ b/gateway/route_metrics.go @@ -6,9 +6,10 @@ import ( "strings" "time" + "github.com/viant/datly/internal/gmetricx" dlogger "github.com/viant/datly/logger" "github.com/viant/datly/repository" - gprovider "github.com/viant/gmetric/provider" + "github.com/viant/gmetric" ) // ensureRouteCounter pre-registers a per-route counter and returns a logger-compatible adapter. @@ -45,13 +46,10 @@ func (r *Router) ensureRouteCounter(ctx context.Context, prov *repository.Provid } metricName = strings.ReplaceAll(metricName, "/", ".") - cnt := r.metrics.LookupOperation(metricName) - if cnt == nil { - // Title: human-friendly - title := v.Name + " request" - cnt = r.metrics.MultiOperationCounter(pkg, metricName, title, time.Millisecond, time.Minute, 2, gprovider.NewBasic()) - } - return dlogger.NewCounter(cnt) + title := v.Name + " request" + return dlogger.NewCounter(gmetricx.NewCounter(r.metrics, metricName, func() *gmetric.Operation { + return r.metrics.MultiOperationCounter(pkg, metricName, title, time.Millisecond, time.Minute, 2, newRouteMetricProvider()) + })) } // normalizeURI replaces path parameters like {id} with a constant token to limit cardinality. diff --git a/gateway/router.go b/gateway/router.go index f76df5e29..473b010ab 100644 --- a/gateway/router.go +++ b/gateway/router.go @@ -432,6 +432,9 @@ func (r *Router) newMatcher(ctx context.Context) (*matcher.Matcher, []*contract. routes, r.NewConfigRoute(), ) + if strings.TrimSpace(r.config.Meta.MetricURI) != "" { + routes = append(routes, r.NewGlobalMetricRoutes(r.config.Meta.MetricURI)...) + } matchables := make([]matcher.Matchable, 0, len(routes)) for _, route := range routes { diff --git a/gateway/router/handler.go b/gateway/router/handler.go index cc2f447ae..2f6380378 100644 --- a/gateway/router/handler.go +++ b/gateway/router/handler.go @@ -11,6 +11,7 @@ import ( "github.com/viant/afs/option" acontent "github.com/viant/afs/option/content" "github.com/viant/afs/url" + "github.com/viant/datly/internal/requesttrace" "github.com/viant/datly/gateway/router/openapi" "github.com/viant/datly/gateway/router/status" "github.com/viant/datly/repository" @@ -178,6 +179,7 @@ func (r *Handler) ServeHTTP(writer http.ResponseWriter, req *http.Request) { } execContext := exec.NewContext(req.Method, req.RequestURI, req.Header, r.Version) ctx = vcontext.WithValue(ctx, exec.ContextKey, execContext) + ctx = requesttrace.Ensure(ctx, execContext.TraceID) req = req.WithContext(ctx) r.HandleRequest(ctx, writer, req) if execContext.StatusCode == 0 { diff --git a/gateway/warmup/cache.go b/gateway/warmup/cache.go index 4ac420ea4..bb4215f43 100644 --- a/gateway/warmup/cache.go +++ b/gateway/warmup/cache.go @@ -3,6 +3,7 @@ package warmup import ( "context" "fmt" + "github.com/viant/datly/internal/gmetricx" "github.com/viant/datly/view" "github.com/viant/datly/warmup" "github.com/viant/gmetric" @@ -269,7 +270,7 @@ func recordWarmupViewMetrics(aView *view.View, summary *viewSummary) { } } -func warmupMetricOperation(aView *view.View) *gmetric.Operation { +func warmupMetricOperation(aView *view.View) *gmetricx.OperationRef { if aView == nil { return nil } @@ -278,18 +279,17 @@ func warmupMetricOperation(aView *view.View) *gmetric.Operation { return nil } metricName := warmupMetricName(aView) - if counter := resource.Metrics.Service.LookupOperation(metricName); counter != nil { - return counter - } pkg := warmupMetricPackage(aView) title := aView.Name + " warmup" - return resource.Metrics.Service.MultiOperationCounter(pkg, metricName, title, time.Millisecond, time.Minute, warmupMetricRecentBuckets, base.NewProvider( - warmupRunOKKey, - warmupRunErrorKey, - warmupCasesCompletedKey, - warmupCasesFailedKey, - warmupRowsKey, - )) + return gmetricx.NewOperationRef(resource.Metrics.Service, metricName, func() *gmetric.Operation { + return resource.Metrics.Service.MultiOperationCounter(pkg, metricName, title, time.Millisecond, time.Minute, warmupMetricRecentBuckets, base.NewProvider( + warmupRunOKKey, + warmupRunErrorKey, + warmupCasesCompletedKey, + warmupCasesFailedKey, + warmupRowsKey, + )) + }) } func warmupMetricName(aView *view.View) string { diff --git a/internal/gmetricx/counter.go b/internal/gmetricx/counter.go new file mode 100644 index 000000000..5cc3f1762 --- /dev/null +++ b/internal/gmetricx/counter.go @@ -0,0 +1,120 @@ +package gmetricx + +import ( + "sync" + "time" + + "github.com/viant/datly/logger" + "github.com/viant/gmetric" + "github.com/viant/gmetric/counter" +) + +var serviceLocks sync.Map + +type OperationRef struct { + service *gmetric.Service + name string + create func() *gmetric.Operation +} + +type operationCounter struct { + ref *OperationRef +} + +func NewCounter(service *gmetric.Service, name string, create func() *gmetric.Operation) logger.Counter { + return &operationCounter{ + ref: NewOperationRef(service, name, create), + } +} + +func NewOperationRef(service *gmetric.Service, name string, create func() *gmetric.Operation) *OperationRef { + return &OperationRef{ + service: service, + name: name, + create: create, + } +} + +func (c *operationCounter) Begin(started time.Time) counter.OnDone { + if c == nil || c.ref == nil { + return func(time.Time, ...interface{}) int64 { return 0 } + } + return c.ref.Begin(started) +} + +func (c *operationCounter) DecrementValue(value interface{}) int64 { + if c == nil || c.ref == nil { + return 0 + } + return c.ref.DecrementValue(value) +} + +func (c *operationCounter) IncrementValue(value interface{}) int64 { + if c == nil || c.ref == nil { + return 0 + } + return c.ref.IncrementValue(value) +} + +func (r *OperationRef) Begin(started time.Time) counter.OnDone { + return func(end time.Time, values ...interface{}) int64 { + return withOperation(r, func(op *gmetric.Operation) int64 { + return op.Begin(started)(end, values...) + }) + } +} + +func (r *OperationRef) DecrementValue(value interface{}) int64 { + return withOperation(r, func(op *gmetric.Operation) int64 { + return op.DecrementValue(value) + }) +} + +func (r *OperationRef) IncrementValue(value interface{}) int64 { + return withOperation(r, func(op *gmetric.Operation) int64 { + return op.IncrementValue(value) + }) +} + +func (r *OperationRef) IncrementValueBy(value interface{}, delta int64) int64 { + return withOperation(r, func(op *gmetric.Operation) int64 { + return op.IncrementValueBy(value, delta) + }) +} + +func withOperation(ref *OperationRef, fn func(*gmetric.Operation) int64) int64 { + if ref == nil || ref.service == nil { + return 0 + } + mux := serviceLock(ref.service) + mux.Lock() + defer mux.Unlock() + + op := lookupOperationUnlocked(ref.service, ref.name) + if op == nil && ref.create != nil { + op = ref.create() + } + if op == nil { + return 0 + } + return fn(op) +} + +func serviceLock(service *gmetric.Service) *sync.Mutex { + if actual, ok := serviceLocks.Load(service); ok { + return actual.(*sync.Mutex) + } + mux := &sync.Mutex{} + actual, _ := serviceLocks.LoadOrStore(service, mux) + return actual.(*sync.Mutex) +} + +func lookupOperationUnlocked(service *gmetric.Service, name string) *gmetric.Operation { + operations := service.OperationCounters() + for i := range operations { + if operations[i].Name == name { + return &operations[i] + } + } + return nil +} diff --git a/internal/gmetricx/service.go b/internal/gmetricx/service.go new file mode 100644 index 000000000..687ef5eb7 --- /dev/null +++ b/internal/gmetricx/service.go @@ -0,0 +1,18 @@ +package gmetricx + +import "github.com/viant/gmetric" + +// LookupOperation returns a snapshot of the named operation under the gmetricx service lock. +func LookupOperation(service *gmetric.Service, name string) *gmetric.Operation { + if service == nil { + return nil + } + mux := serviceLock(service) + mux.Lock() + defer mux.Unlock() + if op := lookupOperationUnlocked(service, name); op != nil { + snapshot := *op + return &snapshot + } + return nil +} diff --git a/internal/gmetricx/service_test.go b/internal/gmetricx/service_test.go new file mode 100644 index 000000000..89fbe0022 --- /dev/null +++ b/internal/gmetricx/service_test.go @@ -0,0 +1,65 @@ +package gmetricx + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/logger" + "github.com/viant/gmetric" + gprovider "github.com/viant/gmetric/provider" +) + +func TestLookupOperationReturnsOperationSnapshot(t *testing.T) { + metrics := gmetric.New() + counterName := "steward.metadata.signalPerformance" + resolver := logger.NewCounter(NewCounter(metrics, counterName, func() *gmetric.Operation { + return metrics.MultiOperationCounter("steward/metadata", counterName, "signal performance", time.Millisecond, time.Minute, 2, gprovider.NewBasic()) + })) + resolver.IncrementValue("pending") + + operation := LookupOperation(metrics, counterName) + require.NotNil(t, operation) + require.Equal(t, counterName, operation.Name) + require.Equal(t, int64(1), operation.Counters[1].Count) +} + +func TestNewCounterSurvivesOperationSliceGrowth(t *testing.T) { + metrics := gmetric.New() + counterName := "steward.metadata.signalPerformance" + resolver := logger.NewCounter(NewCounter(metrics, counterName, func() *gmetric.Operation { + return metrics.MultiOperationCounter("steward/metadata", counterName, "signal performance", time.Millisecond, time.Minute, 2, gprovider.NewBasic()) + })) + + resolver.IncrementValue("pending") + + for i := 0; i < 32; i++ { + metrics.MultiOperationCounter("steward/metadata", fmt.Sprintf("%s.extra.%d", counterName, i), "extra", time.Millisecond, time.Minute, 2, gprovider.NewBasic()) + } + + resolver.IncrementValue("pending") + + require.Equal(t, int64(2), metrics.LookupOperationCumulativeMetric(counterName, "pending")) +} + +func TestNewCounterBeginSurvivesOperationSliceGrowth(t *testing.T) { + metrics := gmetric.New() + counterName := "steward.metadata.signalPerformance" + resolver := logger.NewCounter(NewCounter(metrics, counterName, func() *gmetric.Operation { + return metrics.MultiOperationCounter("steward/metadata", counterName, "signal performance", time.Millisecond, time.Minute, 2, gprovider.NewBasic()) + })) + + started := time.Now().Add(-10 * time.Millisecond) + onDone := resolver.Begin(started) + + for i := 0; i < 32; i++ { + metrics.MultiOperationCounter("steward/metadata", fmt.Sprintf("%s.begin.extra.%d", counterName, i), "extra", time.Millisecond, time.Minute, 2, gprovider.NewBasic()) + } + + onDone(time.Now(), "pending") + + require.Equal(t, int64(1), metrics.LookupOperationCumulativeMetric(counterName, "count")) + require.Equal(t, int64(1), metrics.LookupOperationCumulativeMetric(counterName, "pending")) + require.GreaterOrEqual(t, metrics.LookupOperationCumulativeMetric(counterName, "timeTaken"), int64(1)) +} diff --git a/internal/requesttrace/context.go b/internal/requesttrace/context.go new file mode 100644 index 000000000..75a5e023d --- /dev/null +++ b/internal/requesttrace/context.go @@ -0,0 +1,29 @@ +package requesttrace + +import "context" + +type contextKey struct{} + +// Ensure stores the root request trace ID on the context if it is not already set. +func Ensure(ctx context.Context, traceID string) context.Context { + if ctx == nil || traceID == "" { + return ctx + } + if Current(ctx) != "" { + return ctx + } + return context.WithValue(ctx, contextKey{}, traceID) +} + +// Current returns the root request trace ID stored on the context. +func Current(ctx context.Context) string { + if ctx == nil { + return "" + } + value := ctx.Value(contextKey{}) + if value == nil { + return "" + } + traceID, _ := value.(string) + return traceID +} diff --git a/service/operator/service.go b/service/operator/service.go index 583e0fd9f..dc5865ec5 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -12,6 +12,7 @@ import ( "github.com/viant/afs" "github.com/viant/afs/file" + "github.com/viant/datly/internal/requesttrace" "github.com/viant/datly/repository" rasync "github.com/viant/datly/repository/async" "github.com/viant/datly/repository/content" @@ -262,6 +263,7 @@ func (s *Service) EnsureContext(ctx context.Context, aSession *session.Session, } else { info = infoValue.(*exec.Context) } + ctx = requesttrace.Ensure(ctx, info.TraceID) provider := ctx.Value(hstate.DBProviderKey) if provider == nil { if aView := aComponent.View; aView != nil { diff --git a/service/reader/service.go b/service/reader/service.go index 2b4f045da..1034f0764 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -12,6 +12,7 @@ import ( "unsafe" "github.com/google/uuid" + "github.com/viant/datly/internal/requesttrace" "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/shared" "github.com/viant/datly/view" @@ -96,12 +97,24 @@ func (s *Service) afterRead(ctx context.Context, aSession *Session, collector *v Rows: collector.Len(), } aSession.AddMetric(metrics) + status := Success if err != nil { - aSession.View.Counter.IncrementValue(Error) - } else { - aSession.View.Counter.IncrementValue(Success) + status = Error + } + statusText := "ok" + if err != nil { + statusText = "error" + } + onFinish(end, status) + if aSession.DryRun { + aSession.View.Counter.IncrementValue(status) } - onFinish(end) + fmt.Printf("[INFO] datly view read reqTraceId=%s view=%s rows=%d elapsed=%s status=%s\n", + reqTraceID(ctx), + viewName, + collector.Len(), + elapsed, + statusText) if value := ctx.Value(exec.ContextKey); value != nil { if exeCtx := value.(*exec.Context); exeCtx != nil { exeCtx.AppendMetrics(metrics) @@ -363,7 +376,7 @@ func (s *Service) querySummary(ctx context.Context, session *Session, aView *vie } finished := Now() aView.Logger.Log("reading view %v meta took %v, SQL: %v , Args: %v\n", aView.Name, finished.Sub(now).String(), SQL, args) - logCacheRead(aView, cacheStats, finished.Sub(now), collector.Len(), args) + logCacheRead(ctx, aView, cacheStats, finished.Sub(now), collector.Len(), args) return execInfo, nil } @@ -754,7 +767,7 @@ BEGIN: end := time.Now() aView.Logger.ReadingData(end.Sub(begin), parametrizedSQL.SQL, *readData, parametrizedSQL.Args, err) - logCacheRead(aView, cacheStats, end.Sub(begin), *readData, parametrizedSQL.Args) + logCacheRead(ctx, aView, cacheStats, end.Sub(begin), *readData, parametrizedSQL.Args) if err != nil { stats.SetError(err) anExec, err := s.HandleSQLError(err, session, aView, parametrizedSQL, stats) @@ -868,12 +881,13 @@ func (s *Service) HandleSQLError(err error, session *Session, aView *view.View, return stats, fmt.Errorf("database error occured while fetching Data for view %v %w", aView.Name, err) } -func logCacheRead(aView *view.View, stats *cache.Stats, elapsed time.Duration, rows int, args []interface{}) { +func logCacheRead(ctx context.Context, aView *view.View, stats *cache.Stats, elapsed time.Duration, rows int, args []interface{}) { if stats == nil { return } recordCacheReadMetrics(aView, stats) - fmt.Printf("[INFO] datly cache read view=%s source=%s type=%s found_warmup=%t found_lazy=%t records=%d rows=%d namespace=%s set=%s elapsed=%s args=%v\n", + fmt.Printf("[INFO] datly cache read reqTraceId=%s view=%s source=%s type=%s found_warmup=%t found_lazy=%t records=%d rows=%d namespace=%s set=%s elapsed=%s args=%v\n", + reqTraceID(ctx), aView.Name, cacheReadSource(stats), stats.Type, @@ -887,6 +901,13 @@ func logCacheRead(aView *view.View, stats *cache.Stats, elapsed time.Duration, r args) } +func reqTraceID(ctx context.Context) string { + if traceID := requesttrace.Current(ctx); traceID != "" { + return traceID + } + return "unknown" +} + func recordCacheReadMetrics(aView *view.View, stats *cache.Stats) { if aView == nil || aView.Counter == nil || stats == nil { return diff --git a/service/reader/service_metrics_test.go b/service/reader/service_metrics_test.go index d9bd3f4db..7d65633a9 100644 --- a/service/reader/service_metrics_test.go +++ b/service/reader/service_metrics_test.go @@ -1,14 +1,19 @@ package reader import ( + "context" + "reflect" "testing" "time" "github.com/stretchr/testify/require" + "github.com/viant/datly/internal/requesttrace" "github.com/viant/datly/logger" "github.com/viant/datly/view" + "github.com/viant/datly/view/state" "github.com/viant/gmetric/counter" "github.com/viant/sqlx/io/read/cache" + "github.com/viant/xunsafe" ) type metricsTestCounter struct { @@ -20,7 +25,12 @@ func newMetricsTestCounter() *metricsTestCounter { } func (c *metricsTestCounter) Begin(started time.Time) counter.OnDone { - return func(time.Time, ...interface{}) int64 { return 0 } + return func(_ time.Time, values ...interface{}) int64 { + for _, value := range values { + c.values[value]++ + } + return 0 + } } func (c *metricsTestCounter) DecrementValue(value interface{}) int64 { @@ -88,3 +98,54 @@ func TestRecordCacheReadMetrics(t *testing.T) { require.Lenf(t, counter.values, len(testCase.expected), testCase.description) } } + +type metricsTestRow struct { + ID int +} + +func TestAfterReadRecordsLifecycleStatusViaOnFinish(t *testing.T) { + testCases := []struct { + name string + err error + dryRun bool + expected interface{} + }{ + {name: "success", expected: Success}, + {name: "error", err: context.Canceled, expected: Error}, + {name: "dry run success", dryRun: true, expected: Success}, + {name: "dry run error", dryRun: true, err: context.Canceled, expected: Error}, + } + + for _, testCase := range testCases { + counter := newMetricsTestCounter() + aView := &view.View{ + Name: "signalPerformance", + Schema: state.NewSchema(reflect.TypeOf(&metricsTestRow{})), + Counter: logger.NewCounter(counter), + } + dest := make([]*metricsTestRow, 0) + collector := view.NewCollector(xunsafe.NewSlice(reflect.TypeOf(dest)), aView, &dest, nil, false) + session := &Session{View: aView, DryRun: testCase.dryRun} + + onFinish := aView.Counter.Begin(time.Now()) + if testCase.dryRun { + onFinish = nopCounterDone + } + (&Service{}).afterRead(context.Background(), session, collector, ptrTime(time.Now().Add(-time.Millisecond)), nil, testCase.err, onFinish) + + require.Equal(t, 1, counter.values[testCase.expected], testCase.name) + } +} + +func ptrTime(value time.Time) *time.Time { + return &value +} + +func TestReqTraceID(t *testing.T) { + require.Equal(t, "unknown", reqTraceID(nil)) + require.Equal(t, "unknown", reqTraceID(context.Background())) + + ctx := requesttrace.Ensure(context.Background(), "trace-123") + + require.Equal(t, "trace-123", reqTraceID(ctx)) +} diff --git a/view/view.go b/view/view.go index 86f13b389..b614fbd9b 100644 --- a/view/view.go +++ b/view/view.go @@ -13,6 +13,7 @@ import ( "github.com/viant/afs/url" "github.com/viant/datly/gateway/router/marshal" "github.com/viant/datly/internal/setter" + "github.com/viant/datly/internal/gmetricx" "github.com/viant/datly/logger" expand2 "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/shared" @@ -22,7 +23,7 @@ import ( "github.com/viant/datly/view/keywords" "github.com/viant/datly/view/state" "github.com/viant/datly/view/tags" - "github.com/viant/gmetric/provider" + "github.com/viant/gmetric" "github.com/viant/sqlx" "github.com/viant/sqlx/io" "github.com/viant/structology" @@ -853,7 +854,6 @@ func (v *View) ensureCounter() { if v.Counter != nil { return } - var counter logger.Counter if metric := v._resource.Metrics; metric != nil { name := v.Name @@ -863,16 +863,12 @@ func (v *View) ensureCounter() { metricName = metric.Method + ":" + metricName } metricName = strings.ReplaceAll(metricName, "/", ".") - cnt := metric.Service.LookupOperation(metricName) - - if cnt == nil { - counter = metric.Service.MultiOperationCounter(pkg, metricName, name+" performance", time.Millisecond, time.Minute, 2, provider.NewBasic()) - } else { - counter = cnt - } + v.Counter = logger.NewCounter(gmetricx.NewCounter(metric.Service, metricName, func() *gmetric.Operation { + return metric.Service.MultiOperationCounter(pkg, metricName, name+" performance", time.Millisecond, time.Minute, 2, newViewMetricProvider()) + })) + return } - - v.Counter = logger.NewCounter(counter) + v.Counter = logger.NewCounter(nil) } diff --git a/view/view_metric_provider.go b/view/view_metric_provider.go new file mode 100644 index 000000000..dde9112c5 --- /dev/null +++ b/view/view_metric_provider.go @@ -0,0 +1,94 @@ +package view + +import ( + "reflect" + + "github.com/viant/gmetric/counter" + "github.com/viant/gmetric/stat" +) + +const ( + successMetric = "Success" + errorMetric = "Error" + pendingMetric = "Pending" + cacheHitMetric = "cache:hit" + cacheWarmupHitMetric = "cache:warmup_hit" + cacheLazyHitMetric = "cache:lazy_hit" + cacheMissMetric = "cache:miss" + cacheMissWriteMetric = "cache:miss_write" + cacheErrorMetric = "cache:error" +) + +type viewMetricProvider struct{} + +var viewMetricKeys = []string{ + successMetric, + errorMetric, + pendingMetric, + stat.ErrorKey, + stat.Pending, + cacheHitMetric, + cacheWarmupHitMetric, + cacheLazyHitMetric, + cacheMissMetric, + cacheMissWriteMetric, + cacheErrorMetric, +} + +func newViewMetricProvider() counter.Provider { + return &viewMetricProvider{} +} + +func (p *viewMetricProvider) Keys() []string { + return viewMetricKeys +} + +func (p *viewMetricProvider) Map(value interface{}) int { + if value == nil { + return -1 + } + if _, ok := value.(error); ok { + return 1 + } + text, ok := metricText(value) + if !ok { + return -1 + } + switch text { + case successMetric: + return 0 + case errorMetric: + return 1 + case pendingMetric: + return 2 + case stat.ErrorKey: + return 3 + case stat.Pending: + return 4 + case cacheHitMetric: + return 5 + case cacheWarmupHitMetric: + return 6 + case cacheLazyHitMetric: + return 7 + case cacheMissMetric: + return 8 + case cacheMissWriteMetric: + return 9 + case cacheErrorMetric: + return 10 + default: + return -1 + } +} + +func metricText(value interface{}) (string, bool) { + if text, ok := value.(string); ok { + return text, true + } + rv := reflect.ValueOf(value) + if !rv.IsValid() || rv.Kind() != reflect.String { + return "", false + } + return rv.String(), true +} diff --git a/view/view_metric_provider_test.go b/view/view_metric_provider_test.go new file mode 100644 index 000000000..25060faea --- /dev/null +++ b/view/view_metric_provider_test.go @@ -0,0 +1,44 @@ +package view + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/viant/gmetric" +) + +type namedMetric string + +func TestViewMetricProvider_ExportsViewCounters(t *testing.T) { + metrics := gmetric.New() + counter := metrics.MultiOperationCounter("steward/metadata", "steward.metadata.softIneligibilities", "softIneligibilities performance", time.Millisecond, time.Minute, 2, newViewMetricProvider()) + + counter.IncrementValue(namedMetric(successMetric)) + counter.IncrementValue(namedMetric(errorMetric)) + counter.IncrementValue(namedMetric(pendingMetric)) + counter.IncrementValue(cacheHitMetric) + counter.IncrementValue(cacheWarmupHitMetric) + counter.IncrementValue(cacheLazyHitMetric) + counter.IncrementValue(cacheMissMetric) + counter.IncrementValue(cacheMissWriteMetric) + counter.IncrementValue(cacheErrorMetric) + + operation := metrics.LookupOperation("steward.metadata.softIneligibilities") + require.NotNil(t, operation) + + values := map[string]int64{} + for _, item := range operation.Counters { + values[item.Value] = item.Count + } + + require.Equal(t, int64(1), values[successMetric]) + require.Equal(t, int64(1), values[errorMetric]) + require.Equal(t, int64(1), values[pendingMetric]) + require.Equal(t, int64(1), values[cacheHitMetric]) + require.Equal(t, int64(1), values[cacheWarmupHitMetric]) + require.Equal(t, int64(1), values[cacheLazyHitMetric]) + require.Equal(t, int64(1), values[cacheMissMetric]) + require.Equal(t, int64(1), values[cacheMissWriteMetric]) + require.Equal(t, int64(1), values[cacheErrorMetric]) +} From d7fdb8b49b3f15c0f79c509225f77c774e848745 Mon Sep 17 00:00:00 2001 From: vcarey Date: Fri, 10 Jul 2026 16:26:42 -0400 Subject: [PATCH 256/279] Bump sqlx for cache logging updates --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 55c079567..0a1fbef8f 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 - github.com/viant/sqlx v0.23.1-0.20260708200154-5839ab951f68 + github.com/viant/sqlx v0.23.1-0.20260710202202-aa9e291febba github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 diff --git a/go.sum b/go.sum index 053931bc7..c29551e91 100644 --- a/go.sum +++ b/go.sum @@ -1198,6 +1198,8 @@ github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQ github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.23.1-0.20260708200154-5839ab951f68 h1:cEhgA76FQhhtl9VegrvGdoYYVGrk2ImKg9We0DkwlYg= github.com/viant/sqlx v0.23.1-0.20260708200154-5839ab951f68/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= +github.com/viant/sqlx v0.23.1-0.20260710202202-aa9e291febba h1:IJ7y+N3lXrsnndKzFhIoB8wjo8xduKqZ8+6snBxufYs= +github.com/viant/sqlx v0.23.1-0.20260710202202-aa9e291febba/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= github.com/viant/structology v0.9.0/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= From 4663163764d99ab9c0b2494326306053a0009bf0 Mon Sep 17 00:00:00 2001 From: vcarey Date: Sun, 12 Jul 2026 15:18:53 -0400 Subject: [PATCH 257/279] Fix relation warmup cache identity --- go.mod | 2 +- go.sum | 6 +-- service/reader/service.go | 74 ++++++++++++++++++++++++- service/reader/service_warmup_test.go | 42 +++++++++++++++ view/cache.go | 77 +++++++++++++++++++++++++++ view/connector.go | 29 ++++++++++ view/view.go | 5 +- warmup/cache.go | 17 ++++-- 8 files changed, 237 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 0a1fbef8f..8149c87e3 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.24.0 - github.com/viant/sqlx v0.23.1-0.20260710202202-aa9e291febba + github.com/viant/sqlx v0.23.1-0.20260712191511-2534f58bdccc github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 diff --git a/go.sum b/go.sum index c29551e91..b1438f25b 100644 --- a/go.sum +++ b/go.sum @@ -1196,10 +1196,8 @@ github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQnwf69oc86xW1e4wEEzKUg5OC6tOYegE= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.23.1-0.20260708200154-5839ab951f68 h1:cEhgA76FQhhtl9VegrvGdoYYVGrk2ImKg9We0DkwlYg= -github.com/viant/sqlx v0.23.1-0.20260708200154-5839ab951f68/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= -github.com/viant/sqlx v0.23.1-0.20260710202202-aa9e291febba h1:IJ7y+N3lXrsnndKzFhIoB8wjo8xduKqZ8+6snBxufYs= -github.com/viant/sqlx v0.23.1-0.20260710202202-aa9e291febba/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= +github.com/viant/sqlx v0.23.1-0.20260712191511-2534f58bdccc h1:uxPdh1l7dBvMUqJT2aMdPU9ubz3ErgQSmFoaDDe8row= +github.com/viant/sqlx v0.23.1-0.20260712191511-2534f58bdccc/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= github.com/viant/structology v0.9.0/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= diff --git a/service/reader/service.go b/service/reader/service.go index 1034f0764..85c02a18b 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "reflect" + "regexp" "strings" "sync" "sync/atomic" @@ -415,9 +416,23 @@ func (s *Service) buildParametrizedSQL(ctx context.Context, aView *view.View, st return nil, nil, err } wg.Wait() + applyWarmupIdentity(parametrizedSQL, columnInMatcher) return parametrizedSQL, columnInMatcher, cacheErr } +func applyWarmupIdentity(target *cache.ParmetrizedQuery, identity *cache.ParmetrizedQuery) { + if target == nil || identity == nil { + return + } + if identity.IdentitySQL != "" { + target.IdentitySQL = identity.IdentitySQL + target.IdentityArgs = append([]interface{}{}, identity.IdentityArgs...) + return + } + target.IdentitySQL = identity.SQL + target.IdentityArgs = append([]interface{}{}, identity.Args...) +} + func (s *Service) relationWarmupMatcher(ctx context.Context, aView *view.View, statelet *view.Statelet, batchData *view.BatchData, relation *view.Relation) (*cache.ParmetrizedQuery, error) { if aView == nil || aView.Cache == nil || aView.Cache.Warmup == nil || batchData == nil || relation == nil || relation.Of == nil || len(relation.Of.On) != 1 { return nil, nil @@ -426,18 +441,73 @@ func (s *Service) relationWarmupMatcher(ctx context.Context, aView *view.View, s if indexColumn == "" || len(batchData.ValuesBatch) == 0 || batchData.HasComposite() || len(batchData.ColumnNames) != 1 { return nil, nil } - if !matchesWarmupIndexColumn(indexColumn, relation.Of.On[0], batchData.ColumnNames[0]) { + if !matchesWarmupIndex(aView, indexColumn, relation.Of.On[0], batchData.ColumnNames[0]) { return nil, nil } matcher, err := s.warmupMatcher(ctx, aView, statelet, nil) if err != nil || matcher == nil { return matcher, err } - matcher.By = indexColumn + matcher.By = warmupMarkerColumn(indexColumn, relation, batchData) matcher.In = batchData.ValuesBatch return matcher, nil } +func warmupMarkerColumn(indexColumn string, relation *view.Relation, batchData *view.BatchData) string { + if column := normalizeWarmupColumnName(indexColumn); column != "" { + return column + } + if batchData != nil && len(batchData.ColumnNames) > 0 { + if column := normalizeWarmupColumnName(batchData.ColumnNames[0]); column != "" { + return column + } + } + if relation != nil && relation.Of != nil && len(relation.Of.On) > 0 && relation.Of.On[0] != nil { + if column := normalizeWarmupColumnName(relation.Of.On[0].Column); column != "" { + return column + } + } + return normalizeWarmupColumnName(indexColumn) +} + +func matchesWarmupIndex(aView *view.View, indexColumn string, link *view.Link, batchColumn string) bool { + if matchesWarmupIndexColumn(indexColumn, link, batchColumn) { + return true + } + if aView == nil || link == nil { + return false + } + if !strings.EqualFold(normalizeWarmupColumnName(batchColumn), normalizeWarmupColumnName(link.Column)) { + return false + } + warmupField := warmupIndexFieldName(indexColumn) + if warmupField == "" { + return false + } + return strings.EqualFold(strings.TrimSpace(link.Field), strings.TrimSpace(warmupField)) +} + +var warmupFieldAliasPattern = regexp.MustCompile(`[^a-zA-Z0-9]+`) + +func warmupIndexFieldName(indexColumn string) string { + normalized := strings.TrimSpace(normalizeWarmupColumnName(indexColumn)) + if normalized == "" { + return "" + } + parts := warmupFieldAliasPattern.Split(strings.ToLower(normalized), -1) + builder := strings.Builder{} + for _, part := range parts { + if part == "" { + continue + } + builder.WriteString(strings.ToUpper(part[:1])) + if len(part) > 1 { + builder.WriteString(part[1:]) + } + } + return builder.String() +} + func matchesWarmupIndexColumn(indexColumn string, link *view.Link, batchColumn string) bool { if link == nil { return false diff --git a/service/reader/service_warmup_test.go b/service/reader/service_warmup_test.go index 15149f262..1b5ffb1b7 100644 --- a/service/reader/service_warmup_test.go +++ b/service/reader/service_warmup_test.go @@ -101,6 +101,48 @@ func TestMatchesWarmupIndexColumnRejectsCollapsedIdentifier(t *testing.T) { require.False(t, matched) } +func TestWarmupMarkerColumnPrefersBatchColumnToken(t *testing.T) { + relation := &view.Relation{ + Of: &view.ReferenceView{ + On: view.JoinOn(view.WithLink("CampaignId", "t.campaign_id")), + }, + } + batchData := &view.BatchData{ + ColumnNames: []string{"Campaign_Id"}, + } + + actual := warmupMarkerColumn("CampaignId", relation, batchData) + + require.Equal(t, "CampaignId", actual) +} + +func TestWarmupMarkerColumnFallsBackToRelationColumnToken(t *testing.T) { + relation := &view.Relation{ + Of: &view.ReferenceView{ + On: view.JoinOn(view.WithLink("CampaignId", "t.campaign_id")), + }, + } + + actual := warmupMarkerColumn("CampaignId", relation, nil) + + require.Equal(t, "CampaignId", actual) +} + +func TestWarmupMarkerColumnFallsBackToConfiguredIndexColumn(t *testing.T) { + actual := warmupMarkerColumn("t.campaign_id", nil, nil) + + require.Equal(t, "campaign_id", actual) +} + +func TestMatchesWarmupIndexAcceptsWarmupAliasForRelationField(t *testing.T) { + aView := &view.View{} + link := view.WithLink("CampaignId", "ID") + + matched := matchesWarmupIndex(aView, "CAMPAIGN_ID", link, "ID") + + require.True(t, matched) +} + func TestWarmupIndexParameterUsesExplicitParameter(t *testing.T) { aView := &view.View{ Cache: &view.Cache{ diff --git a/view/cache.go b/view/cache.go index 6c3c1a1b0..3c59bbab7 100644 --- a/view/cache.go +++ b/view/cache.go @@ -351,6 +351,83 @@ func (c *Cache) inherit(source *Cache) error { return nil } +func (c *Cache) cloneForInheritance() *Cache { + if c == nil { + return nil + } + + cloned := &Cache{ + Reference: c.Reference, + Name: c.Name, + Location: c.Location, + Provider: c.Provider, + TimeToLiveMs: c.TimeToLiveMs, + PartSize: c.PartSize, + AerospikeConfig: c.AerospikeConfig, + Warmup: c.Warmup.clone(), + } + + return cloned +} + +func (w *Warmup) clone() *Warmup { + if w == nil { + return nil + } + + cloned := &Warmup{ + IndexColumn: w.IndexColumn, + IndexParameter: w.IndexParameter, + IndexMeta: w.IndexMeta, + FieldNames: append([]string(nil), w.FieldNames...), + Cases: make([]*CacheParameters, 0, len(w.Cases)), + } + if w.Limit != nil { + limit := *w.Limit + cloned.Limit = &limit + } + if w.MaxCases != nil { + maxCases := *w.MaxCases + cloned.MaxCases = &maxCases + } + cloned.Connector = w.Connector.clone() + for _, item := range w.Cases { + cloned.Cases = append(cloned.Cases, item.clone()) + } + + return cloned +} + +func (c *CacheParameters) clone() *CacheParameters { + if c == nil { + return nil + } + + cloned := &CacheParameters{ + FieldNames: append([]string(nil), c.FieldNames...), + Set: make([]*ParamValue, 0, len(c.Set)), + } + for _, item := range c.Set { + cloned.Set = append(cloned.Set, item.clone()) + } + + return cloned +} + +func (p *ParamValue) clone() *ParamValue { + if p == nil { + return nil + } + + cloned := &ParamValue{ + Name: p.Name, + ExcludeDefault: p.ExcludeDefault, + } + cloned.Values = append([]interface{}(nil), p.Values...) + + return cloned +} + func (c *Cache) GenerateCacheInput(ctx context.Context) ([]*CacheInput, error) { if len(c.Warmup.Cases) == 0 { input := c.NewInput(NewStatelet()) diff --git a/view/connector.go b/view/connector.go index 7da951f7f..7eba5380d 100644 --- a/view/connector.go +++ b/view/connector.go @@ -234,6 +234,35 @@ func (c *Connection) inherit(connector *Connection) { } } +func (c *Connector) clone() *Connector { + if c == nil { + return nil + } + + cloned := &Connector{ + Connection: *c.Connection.clone(), + Connections: make([]*Connection, 0, len(c.Connections)), + } + for _, connection := range c.Connections { + cloned.Connections = append(cloned.Connections, connection.clone()) + } + + return cloned +} + +func (c *Connection) clone() *Connection { + if c == nil { + return nil + } + + cloned := &Connection{ + DBConfig: c.DBConfig, + } + cloned.DSN = c.getDSN() + + return cloned +} + func (c *Connection) setDriverOptions(secret *scy.Secret) { if secret == nil || c._initialized { return diff --git a/view/view.go b/view/view.go index b614fbd9b..dc1b8f479 100644 --- a/view/view.go +++ b/view/view.go @@ -12,8 +12,8 @@ import ( "github.com/viant/afs/url" "github.com/viant/datly/gateway/router/marshal" - "github.com/viant/datly/internal/setter" "github.com/viant/datly/internal/gmetricx" + "github.com/viant/datly/internal/setter" "github.com/viant/datly/logger" expand2 "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/shared" @@ -1184,8 +1184,7 @@ func (v *View) inherit(view *View) error { } if v.Cache == nil && view.Cache != nil { - shallowCopy := *view.Cache - v.Cache = &shallowCopy + v.Cache = view.Cache.cloneForInheritance() } if v.ColumnsConfig == nil { diff --git a/warmup/cache.go b/warmup/cache.go index e61f37794..a3a3ec38c 100644 --- a/warmup/cache.go +++ b/warmup/cache.go @@ -254,7 +254,7 @@ func readWithErr(ctx context.Context, entry *warmupEntry) (*EntryResult, error) } matcher := entry.matcher - indexed, err := service.IndexBy(indexProgressContext(ctx, entry), db, entry.column, matcher.SQL, matcher.Args) + indexed, err := service.IndexBy(indexProgressContext(ctx, entry), db, entry.column, matcher.SQL, matcher.Args, matcher) elapsed := time.Since(started) if err != nil { fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s rows=%d elapsed=%s cache_write=error error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, indexed, elapsed, err) @@ -300,11 +300,18 @@ func warmupCacheKey(query *cache.ParmetrizedQuery) (string, error) { if query == nil { return "", fmt.Errorf("warmup cache key query was nil") } - args := query.Args - if args == nil { - args = []interface{}{} + return warmupIdentityURL(query) +} + +func warmupIdentityURL(query *cache.ParmetrizedQuery) (string, error) { + if query == nil { + return "", fmt.Errorf("warmup identity query was nil") + } + SQL, _, argsMarshal, err := query.WarmupIdentity() + if err != nil { + return "", err } - return cachehash.GenerateURL(query.SQL, "", "", args) + return cachehash.GenerateWithMarshal(SQL, "", "", argsMarshal) } func DB(entry *warmupEntry) (*sql.DB, error) { From 577230d5fbb25d8fd67d75607c8b876feec59e56 Mon Sep 17 00:00:00 2001 From: vcarey Date: Sun, 12 Jul 2026 15:22:11 -0400 Subject: [PATCH 258/279] Add cache inheritance regression tests --- view/cache_clone_test.go | 148 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 view/cache_clone_test.go diff --git a/view/cache_clone_test.go b/view/cache_clone_test.go new file mode 100644 index 000000000..ddbe156af --- /dev/null +++ b/view/cache_clone_test.go @@ -0,0 +1,148 @@ +package view + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/shared" + "github.com/viant/sqlx/io/read/cache" +) + +func TestCacheCloneForInheritance_StripsRuntimeState(t *testing.T) { + parent := &View{Name: "parent"} + limit := 25 + + source := &Cache{ + Reference: shared.Reference{Ref: "cacheRef"}, + Name: "cacheName", + Location: "records/${View.Name}", + Provider: "aerospike://127.0.0.1:3000/debrief", + TimeToLiveMs: 60000, + PartSize: 1024, + AerospikeConfig: AerospikeConfig{MaxRetries: 3, TotalTimeoutInMs: 1000}, + owner: parent, + _initialized: true, + newCache: func() (cache.Cache, error) { return nil, nil }, + Warmup: &Warmup{ + IndexColumn: "campaign_id", + Limit: &limit, + Connector: NewConnector("warmup", "sqlite3", "dsn"), + }, + } + + cloned := source.cloneForInheritance() + + require.NotNil(t, cloned) + require.NotSame(t, source, cloned) + require.Nil(t, cloned.owner) + require.False(t, cloned._initialized) + require.Nil(t, cloned.newCache) + require.Equal(t, source.Reference, cloned.Reference) + require.Equal(t, source.Name, cloned.Name) + require.Equal(t, source.Location, cloned.Location) + require.Equal(t, source.Provider, cloned.Provider) + require.Equal(t, source.TimeToLiveMs, cloned.TimeToLiveMs) + require.Equal(t, source.PartSize, cloned.PartSize) + require.Equal(t, source.AerospikeConfig, cloned.AerospikeConfig) +} + +func TestCacheCloneForInheritance_DeepCopiesWarmupAndConnectorConfig(t *testing.T) { + limit := 25 + maxCases := 50 + source := &Cache{ + Warmup: &Warmup{ + IndexColumn: "campaign_id", + IndexParameter: "CampaignID", + IndexMeta: true, + Limit: &limit, + MaxCases: &maxCases, + FieldNames: []string{"campaign_id", "status"}, + Connector: &Connector{ + Connection: Connection{ + DBConfig: DBConfig{ + Name: "warmup", + Driver: "sqlite3", + }, + _dsn: "resolved-dsn", + _initialized: true, + }, + Connections: []*Connection{ + { + DBConfig: DBConfig{ + Name: "replica", + Driver: "sqlite3", + }, + _dsn: "replica-dsn", + _initialized: true, + }, + }, + _initialized: true, + }, + Cases: []*CacheParameters{ + { + FieldNames: []string{"campaign_id"}, + Set: []*ParamValue{ + { + Name: "CampaignID", + Values: []interface{}{"A", "B"}, + ExcludeDefault: true, + }, + }, + }, + }, + }, + } + + cloned := source.cloneForInheritance() + + require.NotNil(t, cloned.Warmup) + require.NotSame(t, source.Warmup, cloned.Warmup) + require.NotSame(t, source.Warmup.Connector, cloned.Warmup.Connector) + require.NotSame(t, source.Warmup.Cases[0], cloned.Warmup.Cases[0]) + require.NotSame(t, source.Warmup.Cases[0].Set[0], cloned.Warmup.Cases[0].Set[0]) + require.NotSame(t, source.Warmup.Connector.Connections[0], cloned.Warmup.Connector.Connections[0]) + require.Equal(t, "resolved-dsn", cloned.Warmup.Connector.getDSN()) + require.Equal(t, "replica-dsn", cloned.Warmup.Connector.Connections[0].getDSN()) + + cloned.Warmup.FieldNames[0] = "mutated" + cloned.Warmup.Cases[0].FieldNames[0] = "mutated_case" + cloned.Warmup.Cases[0].Set[0].Values[0] = "mutated_value" + cloned.Warmup.Connector.Connection.DSN = "mutated-dsn" + cloned.Warmup.Connector.Connections[0].DSN = "mutated-replica-dsn" + + require.Equal(t, "campaign_id", source.Warmup.FieldNames[0]) + require.Equal(t, "campaign_id", source.Warmup.Cases[0].FieldNames[0]) + require.Equal(t, "A", source.Warmup.Cases[0].Set[0].Values[0]) + require.Equal(t, "resolved-dsn", source.Warmup.Connector.getDSN()) + require.Equal(t, "replica-dsn", source.Warmup.Connector.Connections[0].getDSN()) +} + +func TestViewInherit_ClonesCacheConfiguration(t *testing.T) { + limit := 25 + parent := &View{ + Cache: &Cache{ + Provider: "aerospike://127.0.0.1:3000/debrief", + owner: &View{Name: "parent"}, + newCache: func() (cache.Cache, error) { return nil, nil }, + _initialized: true, + Warmup: &Warmup{ + Limit: &limit, + FieldNames: []string{"campaign_id"}, + }, + }, + } + child := &View{} + + err := child.inherit(parent) + + require.NoError(t, err) + require.NotNil(t, child.Cache) + require.NotSame(t, parent.Cache, child.Cache) + require.NotSame(t, parent.Cache.Warmup, child.Cache.Warmup) + require.Nil(t, child.Cache.owner) + require.Nil(t, child.Cache.newCache) + require.False(t, child.Cache._initialized) + + child.Cache.Warmup.FieldNames[0] = "mutated" + require.Equal(t, "campaign_id", parent.Cache.Warmup.FieldNames[0]) +} From b9ea7414fc24f6137c4388e859a2839959d0c35a Mon Sep 17 00:00:00 2001 From: vcarey Date: Sun, 12 Jul 2026 17:43:00 -0400 Subject: [PATCH 259/279] Fix top-level warmup parameter matching --- service/reader/service.go | 30 +++++++++++++++++++++++++- service/reader/service_warmup_test.go | 31 +++++++++++++++++++++------ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/service/reader/service.go b/service/reader/service.go index 85c02a18b..526f81e7b 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -651,13 +651,41 @@ func warmupIndexParameter(aView *view.View) *state.Parameter { if candidate == nil { continue } - if strings.EqualFold(strings.TrimSpace(candidate.Name), parameterName) { + if matchesWarmupParameter(candidate, parameterName) { return candidate } } return nil } +func matchesWarmupParameter(candidate *state.Parameter, configured string) bool { + if candidate == nil { + return false + } + configured = strings.TrimSpace(configured) + if configured == "" { + return false + } + if strings.EqualFold(strings.TrimSpace(candidate.Name), configured) { + return true + } + if candidate.In != nil && strings.EqualFold(strings.TrimSpace(candidate.In.Name), configured) { + return true + } + + fieldName := warmupIndexFieldName(configured) + if fieldName == "" { + return false + } + if strings.EqualFold(strings.TrimSpace(candidate.Name), fieldName) { + return true + } + if strings.EqualFold(strings.TrimSpace(candidate.Name), fieldName+"s") { + return true + } + return false +} + func (s *Service) BuildCriteria(ctx context.Context, value interface{}, options *codec.CriteriaBuilderOptions) (*codec.Criteria, error) { baseView := view.Context(ctx) aSchema := state.NewSchema(reflect.TypeOf(value)) diff --git a/service/reader/service_warmup_test.go b/service/reader/service_warmup_test.go index 1b5ffb1b7..53ca54d05 100644 --- a/service/reader/service_warmup_test.go +++ b/service/reader/service_warmup_test.go @@ -159,28 +159,45 @@ func TestWarmupIndexParameterUsesExplicitParameter(t *testing.T) { require.Equal(t, "OrderId", parameter.Name) } -func TestWarmupIndexParameterDoesNotInferCamelCase(t *testing.T) { +func TestWarmupIndexParameterMatchesCanonicalQueryLocation(t *testing.T) { aView := &view.View{ Cache: &view.Cache{ - Warmup: &view.Warmup{IndexColumn: "order_id"}, + Warmup: &view.Warmup{IndexColumn: "ad_order_id", IndexParameter: "order_id"}, }, Template: view.NewTemplate("", - view.WithTemplateParameters(state.NewParameter("OrderId", state.NewQueryLocation("order_id"))), + view.WithTemplateParameters(state.NewParameter("OrderIds", state.NewQueryLocation("order_id"))), ), } parameter := warmupIndexParameter(aView) - require.Nil(t, parameter) + require.NotNil(t, parameter) + require.Equal(t, "OrderIds", parameter.Name) } -func TestWarmupIndexParameterDoesNotFallbackToMatchingColumnName(t *testing.T) { +func TestWarmupIndexParameterMatchesFieldAliasPlural(t *testing.T) { aView := &view.View{ Cache: &view.Cache{ - Warmup: &view.Warmup{IndexColumn: "order_id"}, + Warmup: &view.Warmup{IndexColumn: "ad_order_id", IndexParameter: "order_id"}, + }, + Template: view.NewTemplate("", + view.WithTemplateParameters(state.NewParameter("OrderIds", nil)), + ), + } + + parameter := warmupIndexParameter(aView) + + require.NotNil(t, parameter) + require.Equal(t, "OrderIds", parameter.Name) +} + +func TestWarmupIndexParameterDoesNotMatchUnrelatedParameter(t *testing.T) { + aView := &view.View{ + Cache: &view.Cache{ + Warmup: &view.Warmup{IndexColumn: "ad_order_id", IndexParameter: "order_id"}, }, Template: view.NewTemplate("", - view.WithTemplateParameters(state.NewParameter("order_id", state.NewQueryLocation("order_id"))), + view.WithTemplateParameters(state.NewParameter("CampaignIds", state.NewQueryLocation("campaign_id"))), ), } From c7499ac23f7dffd37a1f4868376a661bfab64529 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 13 Jul 2026 10:40:27 +0200 Subject: [PATCH 260/279] - added 404 a default response for root or unconfigred paths --- gateway/config.go | 29 +++++++++++++++++------------ gateway/router.go | 7 +++++++ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/gateway/config.go b/gateway/config.go index 58f8d2726..dfc84072c 100644 --- a/gateway/config.go +++ b/gateway/config.go @@ -27,18 +27,19 @@ type ( } ExposableConfig struct { - APIPrefix string //like /v1/api/ - RouteURL string - GoBootstrap *GoBootstrap - DQLBootstrap *DQLBootstrap - ContentURL string - PluginsURL string - DependencyURL string - JobURL string - FailedJobURL string - MaxJobs int - UseCacheFS bool - SyncFrequencyMs int + APIPrefix string //like /v1/api/ + RouteURL string + ShowAvailableRoutes *bool + GoBootstrap *GoBootstrap + DQLBootstrap *DQLBootstrap + ContentURL string + PluginsURL string + DependencyURL string + JobURL string + FailedJobURL string + MaxJobs int + UseCacheFS bool + SyncFrequencyMs int config.Config Logging logging.Config Meta meta.Config @@ -144,6 +145,10 @@ func (c *Config) Discovery() bool { return c.AutoDiscovery == nil || *c.AutoDiscovery } +func (c *Config) ShouldShowAvailableRoutes() bool { + return c != nil && c.ShowAvailableRoutes != nil && *c.ShowAvailableRoutes +} + func (c *Config) Init(ctx context.Context) error { if c.SyncFrequencyMs == 0 { c.SyncFrequencyMs = 2000 diff --git a/gateway/router.go b/gateway/router.go index 473b010ab..6af07904f 100644 --- a/gateway/router.go +++ b/gateway/router.go @@ -287,6 +287,13 @@ func (r *Router) PreCacheables(ctx context.Context, method string, uri string) ( } func (r *Router) availableRoutesErr(statusCode int, err error) error { + if r == nil || r.config == nil || !r.config.ShouldShowAvailableRoutes() { + return &HttpError{ + Code: statusCode, + Err: err, + } + } + return &HttpError{ Code: statusCode, Err: &AvailableRoutesError{ From c01460eb727a1cdbf80d74deed40f0818e8ccd77 Mon Sep 17 00:00:00 2001 From: vcarey Date: Wed, 15 Jul 2026 14:35:46 -0400 Subject: [PATCH 261/279] move warmup identity to matcher --- service/reader/service.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/service/reader/service.go b/service/reader/service.go index 526f81e7b..f635064c8 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -416,21 +416,16 @@ func (s *Service) buildParametrizedSQL(ctx context.Context, aView *view.View, st return nil, nil, err } wg.Wait() - applyWarmupIdentity(parametrizedSQL, columnInMatcher) + ensureWarmupIdentity(columnInMatcher) return parametrizedSQL, columnInMatcher, cacheErr } -func applyWarmupIdentity(target *cache.ParmetrizedQuery, identity *cache.ParmetrizedQuery) { - if target == nil || identity == nil { +func ensureWarmupIdentity(matcher *cache.ParmetrizedQuery) { + if matcher == nil || matcher.IdentitySQL != "" { return } - if identity.IdentitySQL != "" { - target.IdentitySQL = identity.IdentitySQL - target.IdentityArgs = append([]interface{}{}, identity.IdentityArgs...) - return - } - target.IdentitySQL = identity.SQL - target.IdentityArgs = append([]interface{}{}, identity.Args...) + matcher.IdentitySQL = matcher.SQL + matcher.IdentityArgs = append([]interface{}{}, matcher.Args...) } func (s *Service) relationWarmupMatcher(ctx context.Context, aView *view.View, statelet *view.Statelet, batchData *view.BatchData, relation *view.Relation) (*cache.ParmetrizedQuery, error) { From 84614e881b87103fabfcaabff81c23b3203b954d Mon Sep 17 00:00:00 2001 From: arao Date: Mon, 20 Jul 2026 22:36:09 -0700 Subject: [PATCH 262/279] ENG-00001: adding openAPI + swagger --- gateway/route_doc.go | 53 ++++++++ gateway/route_openapi.go | 66 ++++++++++ gateway/route_openapi_test.go | 71 +++++++++++ gateway/router.go | 13 ++ gateway/router/openapi/schema.go | 83 +++++++++++- gateway/router/openapi/schema_build.go | 108 ++++++++++++++-- .../openapi/schema_build_helpers_test.go | 24 ++++ gateway/router/openapi/schema_helpers_test.go | 119 ++++++++++++++++++ gateway/runtime/meta/config.go | 6 + 9 files changed, 526 insertions(+), 17 deletions(-) create mode 100644 gateway/route_doc.go create mode 100644 gateway/route_openapi_test.go diff --git a/gateway/route_doc.go b/gateway/route_doc.go new file mode 100644 index 000000000..558791cf3 --- /dev/null +++ b/gateway/route_doc.go @@ -0,0 +1,53 @@ +package gateway + +import ( + "context" + "fmt" + "github.com/viant/datly/repository/contract" + "net/http" +) + +// swaggerUITemplate renders a minimal Swagger UI page loaded from a public CDN. +// %s is replaced with the URL of the aggregate OpenAPI spec (JSON). +const swaggerUITemplate = ` + + + + Datly API + + + + + +
+ + + + +` + +// NewOpenAPIDocRoute serves an interactive Swagger UI page that renders the +// aggregate OpenAPI spec located at specURL. +func (r *Router) NewOpenAPIDocRoute(URL string, specURL string) *Route { + page := []byte(fmt.Sprintf(swaggerUITemplate, specURL)) + return &Route{ + Path: contract.NewPath(http.MethodGet, URL), + Handler: func(ctx context.Context, response http.ResponseWriter, req *http.Request) { + setContentType(response, http.StatusOK, "text/html") + write(response, http.StatusOK, page) + }, + Kind: RouteOpenAPIKind, + Config: r.config.Logging, + Version: r.config.Version, + } +} diff --git a/gateway/route_openapi.go b/gateway/route_openapi.go index d2f971c2c..51ae8e1db 100644 --- a/gateway/route_openapi.go +++ b/gateway/route_openapi.go @@ -2,11 +2,13 @@ package gateway import ( "context" + "encoding/json" "github.com/viant/datly/gateway/router/openapi" "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" "gopkg.in/yaml.v3" "net/http" + "strings" ) func (r *Router) NewOpenAPIRoute(URL string, components *repository.Service, providers ...*repository.Provider) *Route { @@ -44,3 +46,67 @@ func (r *Router) generateOpenAPI(ctx context.Context, components *repository.Ser return http.StatusOK, specMarshal } + +// NewOpenAPIAggregateRoute builds a route that serves a single OpenAPI 3.0.1 spec +// covering all supplied (public) providers. It defaults to JSON and can return YAML +// when the request asks for it via ?format=yaml or an Accept header containing yaml. +func (r *Router) NewOpenAPIAggregateRoute(URL string, components *repository.Service, providers ...*repository.Provider) *Route { + return &Route{ + Path: contract.NewPath(http.MethodGet, URL), + Providers: providers, + Handler: func(ctx context.Context, response http.ResponseWriter, req *http.Request) { + r.handleOpenAPIAggregate(ctx, components, response, req, providers) + }, + Kind: RouteOpenAPIKind, + Config: r.config.Logging, + Version: r.config.Version, + NewMultiRoute: func(routes []*contract.Path) *Route { + return r.NewOpenAPIAggregateRoute("", components, providers...) + }, + } +} + +func (r *Router) handleOpenAPIAggregate(ctx context.Context, components *repository.Service, res http.ResponseWriter, request *http.Request, providers []*repository.Provider) { + asYAML := wantsYAML(request) + statusCode, content, contentType := r.generateOpenAPIWithFormat(ctx, components, providers, asYAML) + setContentType(res, statusCode, contentType) + write(res, statusCode, content) +} + +func (r *Router) generateOpenAPIWithFormat(ctx context.Context, components *repository.Service, providers []*repository.Provider, asYAML bool) (int, []byte, string) { + spec, err := openapi.GenerateOpenAPI3Spec(ctx, components, r.OpenAPIInfo, providers...) + if err != nil { + return http.StatusInternalServerError, []byte(err.Error()), "text/plain" + } + + if asYAML { + specMarshal, err := yaml.Marshal(spec) + if err != nil { + return http.StatusInternalServerError, []byte(err.Error()), "text/plain" + } + return http.StatusOK, specMarshal, "text/yaml" + } + + specMarshal, err := json.MarshalIndent(spec, "", " ") + if err != nil { + return http.StatusInternalServerError, []byte(err.Error()), "text/plain" + } + return http.StatusOK, specMarshal, "application/json" +} + +func wantsYAML(request *http.Request) bool { + if request == nil { + return false + } + switch strings.ToLower(strings.TrimSpace(request.URL.Query().Get("format"))) { + case "yaml", "yml": + return true + case "json": + return false + } + accept := strings.ToLower(request.Header.Get("Accept")) + if strings.Contains(accept, "yaml") { + return true + } + return false +} diff --git a/gateway/route_openapi_test.go b/gateway/route_openapi_test.go new file mode 100644 index 000000000..f6a14cb5c --- /dev/null +++ b/gateway/route_openapi_test.go @@ -0,0 +1,71 @@ +package gateway + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRouterNewOpenAPIAggregateRoute(t *testing.T) { + router := &Router{config: &Config{}} + + route := router.NewOpenAPIAggregateRoute("/v1/api/meta/openapi", nil) + + require.NotNil(t, route) + require.Equal(t, "/v1/api/meta/openapi", route.Path.URI) + require.Equal(t, http.MethodGet, route.Path.Method) + require.Equal(t, RouteOpenAPIKind, route.Kind) + require.NotNil(t, route.Handler) +} + +func TestRouterNewOpenAPIDocRoute(t *testing.T) { + router := &Router{config: &Config{}} + + route := router.NewOpenAPIDocRoute("/v1/api/meta/doc", "/v1/api/meta/openapi") + + require.NotNil(t, route) + require.Equal(t, "/v1/api/meta/doc", route.Path.URI) + require.Equal(t, http.MethodGet, route.Path.Method) + require.NotNil(t, route.Handler) + + recorder := httptest.NewRecorder() + route.Handler(nil, recorder, nil) + + require.Equal(t, http.StatusOK, recorder.Code) + require.Contains(t, recorder.Header().Get("Content-Type"), "text/html") + body := recorder.Body.String() + require.Contains(t, body, "swagger-ui") + require.Contains(t, body, `"/v1/api/meta/openapi"`) +} + +func TestWantsYAML(t *testing.T) { + testCases := []struct { + description string + rawQuery string + accept string + expect bool + }{ + {description: "default is json", expect: false}, + {description: "format=yaml", rawQuery: "format=yaml", expect: true}, + {description: "format=yml", rawQuery: "format=yml", expect: true}, + {description: "format=json overrides accept", rawQuery: "format=json", accept: "application/yaml", expect: false}, + {description: "accept yaml", accept: "application/yaml", expect: true}, + {description: "accept json", accept: "application/json", expect: false}, + } + + for _, testCase := range testCases { + request := &http.Request{ + URL: &url.URL{RawQuery: testCase.rawQuery}, + Header: http.Header{}, + } + if testCase.accept != "" { + request.Header.Set("Accept", testCase.accept) + } + require.Equalf(t, testCase.expect, wantsYAML(request), testCase.description) + } + + require.False(t, wantsYAML(nil)) +} diff --git a/gateway/router.go b/gateway/router.go index 6af07904f..c520fe902 100644 --- a/gateway/router.go +++ b/gateway/router.go @@ -83,6 +83,10 @@ func NewRouter(ctx context.Context, components *repository.Service, config *Conf apiKeyMatcher: newApiKeyMatcher(config.APIKeys), mcpRegistry: mcpRegistry, logger: logging.New(logging.INFO, nil), + OpenAPIInfo: openapi3.Info{ + Title: "Datly API", + Version: config.Version, + }, } return r, r.init(ctx) } @@ -333,6 +337,7 @@ func (r *Router) newMatcher(ctx context.Context) (*matcher.Matcher, []*contract. unique := map[string]bool{} var openAPIs = map[string][]*repository.Provider{} + var allProviders []*repository.Provider var optionsPaths = map[string][]*path.Path{} for _, anItem := range container.Items { for _, aPath := range anItem.Paths { @@ -400,6 +405,7 @@ func (r *Router) newMatcher(ctx context.Context) (*matcher.Matcher, []*contract. routes = append(routes, r.NewViewMetaHandler(r.routeURL(r.config.Meta.ViewURI, aPath.URI), provider)) key := r.routeURL(r.config.Meta.OpenApiURI, aPath.URI) openAPIs[key] = append(openAPIs[key], provider) + allProviders = append(allProviders, provider) if !unique[aPath.URI] { unique[aPath.URI] = true @@ -426,6 +432,13 @@ func (r *Router) newMatcher(ctx context.Context) (*matcher.Matcher, []*contract. routes = append(routes, r.NewOpenAPIRoute(key, r.repository, providers...)) } + if strings.TrimSpace(r.config.Meta.OpenApiURI) != "" { + routes = append(routes, r.NewOpenAPIAggregateRoute(r.config.Meta.OpenApiURI, r.repository, allProviders...)) + if strings.TrimSpace(r.config.Meta.DocURI) != "" { + routes = append(routes, r.NewOpenAPIDocRoute(r.config.Meta.DocURI, r.config.Meta.OpenApiURI)) + } + } + for uri, paths := range optionsPaths { routes = append(routes, r.NewOptionsRoute(uri, paths)) diff --git a/gateway/router/openapi/schema.go b/gateway/router/openapi/schema.go index 3eef32319..f4a023520 100644 --- a/gateway/router/openapi/schema.go +++ b/gateway/router/openapi/schema.go @@ -13,6 +13,7 @@ import ( "github.com/viant/xdatly/docs" "github.com/viant/xreflect" "reflect" + "strings" "sync" ) @@ -49,6 +50,12 @@ type ( index map[string]int generatedSchemas map[string]*openapi3.Schema visitingTypes map[string]int + // typeNameByKey maps a unique type identity (import path + name) to the + // schema name assigned to it, and keyByName is the reverse mapping used + // to detect and disambiguate collisions between same-named types from + // different packages within a single (aggregate) spec. + typeNameByKey map[string]string + keyByName map[string]string } ) @@ -92,6 +99,8 @@ func NewContainer() *SchemaContainer { index: map[string]int{}, generatedSchemas: map[string]*openapi3.Schema{}, visitingTypes: map[string]int{}, + typeNameByKey: map[string]string{}, + keyByName: map[string]string{}, } } @@ -110,14 +119,27 @@ func NewComponentSchema(components *repository.Service, component *repository.Co } func (c *ComponentSchema) RequestBody(ctx context.Context) (*Schema, error) { - inputType := c.component.Input.Type + // Generate the request body schema from the actual body type (what the + // server unmarshals the payload into) rather than the whole input state, + // which may resolve to an opaque scalar for named body parameters. + bodyType := c.component.Input.Body + if bodyType.Schema == nil || bodyType.Schema.Type() == nil { + bodyType = c.component.Input.Type + } - name := inputType.SimpleTypeName() + // A unique, deterministic name is required because the aggregate spec shares + // one schema container across all routes; a constant fallback would make + // every request body collide on a single "RequestBody" schema. + name := bodyType.SimpleTypeName() if name == "" { - name = "Input" + if base := bodyTypeName(bodyType.Schema.Type()); base != "" { + name = base + "RequestBody" + } else { + name = c.pathDerivedName("RequestBody") + } } - result, err := c.TypedSchema(ctx, inputType, name, c.component.IOConfig(), true) + result, err := c.TypedSchema(ctx, bodyType, name, c.component.IOConfig(), true) if err != nil { return nil, err } @@ -126,11 +148,58 @@ func (c *ComponentSchema) RequestBody(ctx context.Context) (*Schema, error) { return result, nil } +// bodyTypeName derives a representative type name from a (possibly wrapped) +// request body type, e.g. struct{ Data []*patch.Campaign } -> "Campaign". +func bodyTypeName(rType reflect.Type) string { + if rType == nil { + return "" + } + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + switch rType.Kind() { + case reflect.Slice, reflect.Array: + return bodyTypeName(rType.Elem()) + case reflect.Struct: + if rType.Name() != "" { + return rType.Name() + } + for i := 0; i < rType.NumField(); i++ { + if rType.Field(i).PkgPath != "" { + continue + } + if name := bodyTypeName(rType.Field(i).Type); name != "" { + return name + } + } + } + return "" +} + +// pathDerivedName builds a unique schema name from the route URI, used as a last +// resort when a body type has no resolvable name. +func (c *ComponentSchema) pathDerivedName(suffix string) string { + uri := strings.Trim(c.component.Path.URI, "/") + base := state.SanitizeTypeName(strings.ReplaceAll(uri, "/", "_")) + if base == "" { + return suffix + } + return base + suffix +} + func (c *ComponentSchema) ResponseBody(ctx context.Context) (*Schema, error) { name := c.component.Output.Type.SimpleTypeName() if name == "" { - name = "Output" + var base string + if c.component.Output.Type.Schema != nil { + base = bodyTypeName(c.component.Output.Type.Schema.Type()) + } + if base != "" { + name = base + "Output" + } else { + name = c.pathDerivedName("Output") + } } schema, err := c.TypedSchema(ctx, c.component.Output.Type, name, c.component.IOConfig(), false) if err != nil { @@ -265,7 +334,9 @@ func (c *ComponentSchema) GenerateSchema(ctx context.Context, schema *Schema) (* ReadOnly: schema.tag.ReadOnly, MaxItems: schema.tag.MaxItems, Default: schema.tag.Default, - Example: schema.tag.Example, + } + if schema.tag.Example != "" { + result.Example = schema.tag.Example } if err := c.schemas.addToSchema(ctx, c, result, schema); err != nil { diff --git a/gateway/router/openapi/schema_build.go b/gateway/router/openapi/schema_build.go index e51b27450..4e54d2514 100644 --- a/gateway/router/openapi/schema_build.go +++ b/gateway/router/openapi/schema_build.go @@ -12,6 +12,7 @@ import ( "os" "reflect" "sort" + "strconv" "strings" "time" @@ -142,6 +143,9 @@ func (c *SchemaContainer) addDefaultSchema(ctx context.Context, component *Compo } dst.Type = apiType dst.Format = format + if dst.Example == nil { + dst.Example = exampleValueForType(rType, "") + } return nil } } @@ -314,9 +318,13 @@ func (c *SchemaContainer) createSchema(ctx context.Context, componentSchema *Com return nil, err } - if fieldSchema.tag.TypeName != "" { - if _, ok := c.generatedSchemas[fieldSchema.tag.TypeName]; ok { - return c.SchemaRef(fieldSchema.tag.TypeName, description), nil + // Resolve a schema name that is unique per type identity so same-named types + // from different packages do not collide in a shared (aggregate) spec. + schemaName := c.resolveSchemaName(fieldSchema.tag.TypeName, fieldSchema.rType) + + if schemaName != "" { + if _, ok := c.generatedSchemas[schemaName]; ok { + return c.SchemaRef(schemaName, description), nil } } @@ -325,32 +333,88 @@ func (c *SchemaContainer) createSchema(ctx context.Context, componentSchema *Com Type: apiType, Format: format, Description: description, - Example: example, + Example: exampleValueForType(fieldSchema.rType, example), }, nil } // Mark named schemas as in-progress before generation so recursive graphs // (for example polymorphic self references) resolve to $ref instead of looping. - if fieldSchema.tag.TypeName != "" { - c.generatedSchemas[fieldSchema.tag.TypeName] = nil + if schemaName != "" { + c.generatedSchemas[schemaName] = nil } schema, err := componentSchema.GenerateSchema(ctx, fieldSchema) if err != nil { - if fieldSchema.tag.TypeName != "" { - delete(c.generatedSchemas, fieldSchema.tag.TypeName) + if schemaName != "" { + delete(c.generatedSchemas, schemaName) } return nil, err } - if fieldSchema.tag.TypeName != "" { - c.generatedSchemas[fieldSchema.tag.TypeName] = schema + if schemaName != "" { + c.generatedSchemas[schemaName] = schema c.schemas = append(c.schemas, schema) - schema = c.SchemaRef(fieldSchema.tag.TypeName, description) + schema = c.SchemaRef(schemaName, description) } return schema, nil } +// schemaTypeKey returns a globally-unique identity for a type. It recurses +// through pointers, slices, arrays and maps and uses the full import path for +// named types so that composite types whose reflect.String() collapses to the +// same short package name (e.g. []*campaign/patch.Campaign vs +// []*adorder/patch.Campaign, both "[]*patch.Campaign") do not collide. +func schemaTypeKey(rType reflect.Type) string { + if rType == nil { + return "" + } + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + switch rType.Kind() { + case reflect.Slice: + return "[]" + schemaTypeKey(rType.Elem()) + case reflect.Array: + return "[" + strconv.Itoa(rType.Len()) + "]" + schemaTypeKey(rType.Elem()) + case reflect.Map: + return "map[" + schemaTypeKey(rType.Key()) + "]" + schemaTypeKey(rType.Elem()) + } + if rType.Name() != "" { + if pkg := rType.PkgPath(); pkg != "" { + return pkg + "." + rType.Name() + } + return rType.Name() + } + return rType.String() +} + +// resolveSchemaName maps a requested type name to a container-unique schema name +// keyed by the type's true identity. The first type to claim a name keeps it; +// subsequent distinct types with the same requested name get a numeric suffix. +func (c *SchemaContainer) resolveSchemaName(typeName string, rType reflect.Type) string { + if typeName == "" { + return "" + } + key := schemaTypeKey(rType) + if key == "" { + return typeName + } + if name, ok := c.typeNameByKey[key]; ok { + return name + } + name := typeName + for i := 2; ; i++ { + boundKey, taken := c.keyByName[name] + if !taken || boundKey == key { + break + } + name = typeName + strconv.Itoa(i) + } + c.typeNameByKey[key] = name + c.keyByName[name] = key + return name +} + func (c *SchemaContainer) SchemaRef(schemaName string, description string) *openapi3.Schema { return &openapi3.Schema{ Ref: "#/components/schemas/" + schemaName, @@ -472,6 +536,28 @@ func applySchemaExample(dst *openapi3.Schema, schema *Schema) { } } +// exampleValueForType returns a sample value for a primitive rType. When an +// explicit example is provided it is used as-is; otherwise a type-appropriate +// default is returned so tools such as Swagger UI render meaningful sample +// values (e.g. "string", 0, false) instead of empty double quotes. +func exampleValueForType(rType reflect.Type, provided string) interface{} { + if provided != "" { + return provided + } + switch dereferenceType(rType).Kind() { + case reflect.String: + return "string" + case reflect.Bool: + return false + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return 0 + case reflect.Float32, reflect.Float64: + return 0.0 + } + return nil +} + func addTimeSchema(dst *openapi3.Schema, schema *Schema) { dst.Type = stringOutput timeLayout := schema.tag._tag.TimeLayout diff --git a/gateway/router/openapi/schema_build_helpers_test.go b/gateway/router/openapi/schema_build_helpers_test.go index 0fc1516ef..fe50d4f7c 100644 --- a/gateway/router/openapi/schema_build_helpers_test.go +++ b/gateway/router/openapi/schema_build_helpers_test.go @@ -56,6 +56,30 @@ func TestSchemaBuildHelpers_Table(t *testing.T) { } }) + t.Run("example value for type", func(t *testing.T) { + cases := []struct { + name string + rType reflect.Type + provided string + expect interface{} + }{ + {name: "string default", rType: reflect.TypeOf(""), expect: "string"}, + {name: "bool default", rType: reflect.TypeOf(false), expect: false}, + {name: "int default", rType: reflect.TypeOf(0), expect: 0}, + {name: "int64 default", rType: reflect.TypeOf(int64(0)), expect: 0}, + {name: "float default", rType: reflect.TypeOf(0.0), expect: 0.0}, + {name: "pointer int default", rType: reflect.TypeOf(new(int)), expect: 0}, + {name: "provided overrides", rType: reflect.TypeOf(""), provided: "abc", expect: "abc"}, + {name: "provided on int", rType: reflect.TypeOf(0), provided: "42", expect: "42"}, + {name: "unsupported returns nil", rType: reflect.TypeOf(struct{}{}), expect: nil}, + } + for _, tc := range cases { + if got := exampleValueForType(tc.rType, tc.provided); got != tc.expect { + t.Fatalf("%s: expected %v (%T), got %v (%T)", tc.name, tc.expect, tc.expect, got, got) + } + } + }) + t.Run("root table", func(t *testing.T) { queryComp := &ComponentSchema{component: &repository.Component{View: &view.View{Mode: view.ModeQuery, Table: "users"}}} if got := rootTable(queryComp); got != "users" { diff --git a/gateway/router/openapi/schema_helpers_test.go b/gateway/router/openapi/schema_helpers_test.go index ddde60523..384342e6a 100644 --- a/gateway/router/openapi/schema_helpers_test.go +++ b/gateway/router/openapi/schema_helpers_test.go @@ -81,6 +81,125 @@ func TestSchemaField(t *testing.T) { } } +type sampleBody struct { + Name string + Age int + Score float64 + Active bool + Comment string `json:"comment" example:"looks good"` +} + +func TestGeneratedPrimitiveExamples(t *testing.T) { + container := NewContainer() + component := &ComponentSchema{component: &repository.Component{View: &view.View{}}, schemas: container} + + generated, err := container.createSchema(context.Background(), component, &Schema{ + rType: reflect.TypeOf(sampleBody{}), + tag: Tag{TypeName: "SampleBody"}, + isInput: true, + ioConfig: component.component.IOConfig(), + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Named types resolve to a $ref; find the concrete schema in the container. + var body *openapi3.Schema + for _, s := range container.schemas { + if s != nil && len(s.Properties) > 0 { + body = s + break + } + } + if body == nil { + if generated != nil && len(generated.Properties) > 0 { + body = generated + } else { + t.Fatalf("expected a generated object schema with properties") + } + } + + expect := map[string]interface{}{ + "Name": "string", + "Age": 0, + "Score": 0.0, + "Active": false, + "comment": "looks good", + } + for name, want := range expect { + prop, ok := body.Properties[name] + if !ok { + t.Fatalf("missing property %q; have %v", name, keysOf(body.Properties)) + } + if prop.Example != want { + t.Fatalf("property %q: expected example %v (%T), got %v (%T)", name, want, want, prop.Example, prop.Example) + } + } +} + +func keysOf(m openapi3.Schemas) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +type namedCampaign struct { + ID int + Name string +} + +func TestBodyTypeName(t *testing.T) { + cases := []struct { + name string + rType reflect.Type + expect string + }{ + {name: "named struct", rType: reflect.TypeOf(namedCampaign{}), expect: "namedCampaign"}, + {name: "pointer struct", rType: reflect.TypeOf(&namedCampaign{}), expect: "namedCampaign"}, + {name: "slice of pointer", rType: reflect.TypeOf([]*namedCampaign{}), expect: "namedCampaign"}, + {name: "wrapped named body", rType: reflect.TypeOf(struct{ Data []*namedCampaign }{}), expect: "namedCampaign"}, + {name: "anonymous scalar wrapper", rType: reflect.TypeOf(struct{ Data string }{}), expect: ""}, + {name: "nil", rType: nil, expect: ""}, + } + for _, tc := range cases { + if got := bodyTypeName(tc.rType); got != tc.expect { + t.Fatalf("%s: expected %q, got %q", tc.name, tc.expect, got) + } + } +} + +func TestResolveSchemaName(t *testing.T) { + container := NewContainer() + + type outputA struct{ A int } + type outputB struct{ B int } + + // Same requested name "Output" for two distinct types must disambiguate. + n1 := container.resolveSchemaName("Output", reflect.TypeOf(outputA{})) + n2 := container.resolveSchemaName("Output", reflect.TypeOf(outputB{})) + if n1 != "Output" { + t.Fatalf("expected first type to keep name Output, got %q", n1) + } + if n2 == n1 { + t.Fatalf("expected distinct name for second type, got %q for both", n2) + } + + // Idempotent: same type resolves to the same name. + if again := container.resolveSchemaName("Output", reflect.TypeOf(outputA{})); again != n1 { + t.Fatalf("expected stable name %q, got %q", n1, again) + } + if againB := container.resolveSchemaName("Output", reflect.TypeOf(&outputB{})); againB != n2 { + t.Fatalf("expected stable name %q for pointer of same type, got %q", n2, againB) + } + + // Empty type name yields empty (inline schema). + if got := container.resolveSchemaName("", reflect.TypeOf(outputA{})); got != "" { + t.Fatalf("expected empty name, got %q", got) + } +} + func TestContainsAny(t *testing.T) { tests := []struct { name string diff --git a/gateway/runtime/meta/config.go b/gateway/runtime/meta/config.go index aa93d534b..5f76516c3 100644 --- a/gateway/runtime/meta/config.go +++ b/gateway/runtime/meta/config.go @@ -11,6 +11,8 @@ const ( ViewURI = "/v1/api/meta/view" //OpenApiURI represents default config openapi URIPrefix OpenApiURI = "/v1/api/meta/openapi" + //DocURI represents default Swagger UI documentation URI + DocURI = "/v1/api/meta/doc" //CacheWarmupURI URIPrefix default value CacheWarmupURI = "/v1/api/cache/warmup" //StructURI URIPrefix that generates a Golang struct representation @@ -29,6 +31,7 @@ type Config struct { StatusURI string ViewURI string OpenApiURI string + DocURI string CacheWarmURI string StructURI string StateURI string @@ -55,6 +58,9 @@ func (m *Config) Init() { if m.OpenApiURI == "" { m.OpenApiURI = OpenApiURI } + if m.DocURI == "" { + m.DocURI = DocURI + } if m.CacheWarmURI == "" { m.CacheWarmURI = CacheWarmupURI } From 93430fbe571fd2aa1cc4d31b3708e2e5dad50ece Mon Sep 17 00:00:00 2001 From: arao Date: Tue, 21 Jul 2026 11:29:24 -0700 Subject: [PATCH 263/279] ENG-00001: adding jwt authorize --- gateway/router/openapi/generator_operation.go | 11 +++++- gateway/router/openapi/generator_test.go | 27 +++++++++++++ gateway/router/openapi/openapi3.go | 39 +++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/gateway/router/openapi/generator_operation.go b/gateway/router/openapi/generator_operation.go index 704ffcfb2..a89ffff6a 100644 --- a/gateway/router/openapi/generator_operation.go +++ b/gateway/router/openapi/generator_operation.go @@ -10,6 +10,8 @@ import ( ) func (g *generator) generateOperation(ctx context.Context, component *ComponentSchema) (*openapi.Operation, error) { + g.authSchemeName = "" + body, err := g.requestBody(ctx, component) if err != nil { return nil, err @@ -25,11 +27,16 @@ func (g *generator) generateOperation(ctx context.Context, component *ComponentS return nil, err } - return &openapi.Operation{ + operation := &openapi.Operation{ Parameters: dedupe(parameters), RequestBody: body, Responses: responses, - }, nil + } + if g.authSchemeName != "" { + security := openapi.SecurityRequirements{openapi.SecurityRequirement{g.authSchemeName: []string{}}} + operation.Security = &security + } + return operation, nil } func (g *generator) operationParameters(ctx context.Context, component *ComponentSchema) ([]*openapi.Parameter, error) { diff --git a/gateway/router/openapi/generator_test.go b/gateway/router/openapi/generator_test.go index 6a3756740..3ef861f82 100644 --- a/gateway/router/openapi/generator_test.go +++ b/gateway/router/openapi/generator_test.go @@ -262,6 +262,33 @@ func TestGeneratorHelpersMore_Table(t *testing.T) { } }) + t.Run("authorization header becomes security scheme", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + securitySchemes: map[string]*openapi3.SecurityScheme{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + param := &state.Parameter{Name: "Jwt", In: &state.Location{Kind: state.KindHeader, Name: "Authorization"}, Schema: state.NewSchema(reflect.TypeOf(""))} + + converted, ok, err := g.convertParam(context.Background(), cSchema, param, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ok || len(converted) != 0 { + t.Fatalf("expected Authorization header to be skipped as a parameter, got ok=%v n=%d", ok, len(converted)) + } + if g.authSchemeName != bearerAuthSchemeName { + t.Fatalf("expected auth scheme %q, got %q", bearerAuthSchemeName, g.authSchemeName) + } + scheme := g.securitySchemes[bearerAuthSchemeName] + if scheme == nil || scheme.Type != "http" || scheme.Scheme != "bearer" || scheme.BearerFormat != "JWT" { + t.Fatalf("unexpected security scheme: %+v", scheme) + } + }) + t.Run("convert param kind whitelist", func(t *testing.T) { testCases := []struct { name string diff --git a/gateway/router/openapi/openapi3.go b/gateway/router/openapi/openapi3.go index a92a38894..8bad9074d 100644 --- a/gateway/router/openapi/openapi3.go +++ b/gateway/router/openapi/openapi3.go @@ -12,6 +12,7 @@ import ( "github.com/viant/xdatly/handler/response" "net/http" "reflect" + "strings" ) const ( @@ -41,6 +42,11 @@ type ( _schemasIndex map[string]*openapi.Schema commonParameters openapi.ParametersMap _parametersIndex map[string]*openapi.Parameter + securitySchemes map[string]*openapi.SecurityScheme + // authSchemeName is set transiently while building a single operation + // when it declares an Authorization header, so the operation can require + // the corresponding security scheme. + authSchemeName string } paramLocation struct { @@ -77,6 +83,9 @@ func (g *generator) GenerateSpec(ctx context.Context, repoComponents *repository components.Schemas = schemas.generatedSchemas components.Parameters = g.commonParameters + if len(g.securitySchemes) > 0 { + components.SecuritySchemes = g.securitySchemes + } return &openapi.OpenAPI{ OpenAPI: "3.0.1", @@ -91,9 +100,32 @@ func GenerateOpenAPI3Spec(ctx context.Context, components *repository.Service, i _schemasIndex: map[string]*openapi.Schema{}, commonParameters: map[string]*openapi.Parameter{}, _parametersIndex: map[string]*openapi.Parameter{}, + securitySchemes: map[string]*openapi.SecurityScheme{}, }).GenerateSpec(ctx, components, info, providers...) } +// bearerAuthSchemeName is the name of the JWT bearer security scheme emitted +// for routes that declare an Authorization header. +const bearerAuthSchemeName = "BearerAuth" + +// ensureBearerScheme registers (once) an HTTP bearer JWT security scheme and +// returns its name. datly's JWT codec accepts both a raw token and a +// "Bearer " value, so the bearer scheme is safe. +func (g *generator) ensureBearerScheme() string { + if g.securitySchemes == nil { + g.securitySchemes = map[string]*openapi.SecurityScheme{} + } + if _, ok := g.securitySchemes[bearerAuthSchemeName]; !ok { + g.securitySchemes[bearerAuthSchemeName] = &openapi.SecurityScheme{ + Type: "http", + Scheme: "bearer", + BearerFormat: "JWT", + Description: "JWT bearer token sent in the Authorization header. Paste the token only; the 'Bearer ' prefix is added automatically.", + } + } + return bearerAuthSchemeName +} + func dedupe(parameters []*openapi.Parameter) openapi.Parameters { index := map[paramLocation]bool{} var result []*openapi.Parameter @@ -228,6 +260,13 @@ func (g *generator) convertParam(ctx context.Context, component *ComponentSchema return result, true, nil } + // Represent the Authorization header as a security scheme (Swagger UI + // "Authorize" button) instead of a plain header parameter. + if param.In.Kind == state.KindHeader && strings.EqualFold(param.In.Name, "Authorization") { + g.authSchemeName = g.ensureBearerScheme() + return nil, false, nil + } + if !isOpenAPIParameterKind(param.In.Kind) { return nil, false, nil } From ca3534cc9d0e2d32f141884d78890a0c1ee23fce Mon Sep 17 00:00:00 2001 From: arao Date: Tue, 21 Jul 2026 12:27:44 -0700 Subject: [PATCH 264/279] ENG-00001: adding jwt authorize 3 --- gateway/router/openapi/generator_test.go | 27 +++++++++++++++++++++++- gateway/router/openapi/openapi3.go | 9 ++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/gateway/router/openapi/generator_test.go b/gateway/router/openapi/generator_test.go index 3ef861f82..7a68b68b2 100644 --- a/gateway/router/openapi/generator_test.go +++ b/gateway/router/openapi/generator_test.go @@ -262,6 +262,31 @@ func TestGeneratorHelpersMore_Table(t *testing.T) { } }) + t.Run("path parameter is emitted and required", func(t *testing.T) { + g := &generator{ + _parametersIndex: map[string]*openapi3.Parameter{}, + commonParameters: map[string]*openapi3.Parameter{}, + } + comp := newTestComponent(t) + comp.View = &view.View{Template: &view.Template{}, Selector: &view.Config{}} + cSchema := &ComponentSchema{component: comp, schemas: NewContainer()} + param := &state.Parameter{Name: "ID", In: &state.Location{Kind: state.KindPath, Name: "id"}, Schema: state.NewSchema(reflect.TypeOf(1))} + + converted, ok, err := g.convertParam(context.Background(), cSchema, param, "") + if err != nil || !ok || len(converted) != 1 { + t.Fatalf("unexpected convert result: ok=%v err=%v n=%d", ok, err, len(converted)) + } + if converted[0].In != "path" { + t.Fatalf("expected in=path, got %q", converted[0].In) + } + if converted[0].Name != "id" { + t.Fatalf("expected name=id, got %q", converted[0].Name) + } + if !converted[0].Required { + t.Fatalf("expected path parameter to be required") + } + }) + t.Run("authorization header becomes security scheme", func(t *testing.T) { g := &generator{ _parametersIndex: map[string]*openapi3.Parameter{}, @@ -298,8 +323,8 @@ func TestGeneratorHelpersMore_Table(t *testing.T) { {name: "header", kind: state.KindHeader, expectKeep: true}, {name: "query", kind: state.KindQuery, expectKeep: true}, {name: "form", kind: state.KindForm, expectKeep: true}, + {name: "path", kind: state.KindPath, expectKeep: true}, {name: "body skipped in parameter list", kind: state.KindRequestBody, expectKeep: false}, - {name: "path skipped", kind: state.KindPath, expectKeep: false}, {name: "cookie skipped", kind: state.KindCookie, expectKeep: false}, {name: "state skipped", kind: state.KindState, expectKeep: false}, } diff --git a/gateway/router/openapi/openapi3.go b/gateway/router/openapi/openapi3.go index 8bad9074d..6cd7de67e 100644 --- a/gateway/router/openapi/openapi3.go +++ b/gateway/router/openapi/openapi3.go @@ -57,7 +57,7 @@ type ( func isRequestDerivedInputKind(kind state.Kind) bool { switch kind { - case state.KindHeader, state.KindRequestBody, state.KindQuery, state.KindForm: + case state.KindHeader, state.KindRequestBody, state.KindQuery, state.KindForm, state.KindPath: return true default: return false @@ -66,7 +66,7 @@ func isRequestDerivedInputKind(kind state.Kind) bool { func isOpenAPIParameterKind(kind state.Kind) bool { switch kind { - case state.KindHeader, state.KindQuery, state.KindForm: + case state.KindHeader, state.KindQuery, state.KindForm, state.KindPath: return true default: return false @@ -318,8 +318,9 @@ func (g *generator) convertParam(ctx context.Context, component *ComponentSchema In: string(param.In.Kind), Description: description, Style: param.Style, - Required: param.IsRequired(), - Schema: schema, + // OpenAPI requires path parameters to always be required. + Required: param.IsRequired() || param.In.Kind == state.KindPath, + Schema: schema, } g._parametersIndex[param.Name] = convertedParam From 80a507327b7435aed3eb587ab75984f4cfd8d50b Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 22 Jul 2026 00:52:17 +0200 Subject: [PATCH 265/279] - refactored global transaction handling --- gateway/router_not_found_test.go | 53 ++ go.mod | 2 +- go.sum | 2 + repository/locator/component/component.go | 13 +- .../locator/component/component_uow_test.go | 86 ++ service/executor/expand/data_unit.go | 20 +- service/executor/expand/parent.go | 16 + service/executor/expand/sql.go | 105 ++- service/executor/expand/sql_test.go | 47 ++ service/executor/handler/executor.go | 247 +++++- service/executor/handler/locator/handler.go | 14 + service/executor/handler/sqlx.go | 42 +- .../handler/transaction_scope_test.go | 108 +++ service/executor/sequencer/service.go | 23 +- service/executor/sequencer/service_test.go | 37 + service/executor/service.go | 44 +- service/executor/service_execution_test.go | 61 ++ service/executor/uow/scope.go | 748 ++++++++++++++++++ service/executor/uow/scope_test.go | 664 ++++++++++++++++ service/operator/executor.go | 8 + service/operator/invocation_injector.go | 28 + service/operator/service.go | 65 +- service/session/jwt_codec_test.go | 57 ++ service/session/state.go | 21 +- view/extension/codec/jwt.go | 77 +- view/extension/codec/jwt_test.go | 29 + view/state/kind/locator/repeated.go | 4 +- view/state/kind/locator/repeated_uow_test.go | 37 + 28 files changed, 2590 insertions(+), 68 deletions(-) create mode 100644 gateway/router_not_found_test.go create mode 100644 repository/locator/component/component_uow_test.go create mode 100644 service/executor/expand/sql_test.go create mode 100644 service/executor/handler/transaction_scope_test.go create mode 100644 service/executor/service_execution_test.go create mode 100644 service/executor/uow/scope.go create mode 100644 service/executor/uow/scope_test.go create mode 100644 service/operator/invocation_injector.go create mode 100644 service/session/jwt_codec_test.go create mode 100644 view/extension/codec/jwt_test.go create mode 100644 view/state/kind/locator/repeated_uow_test.go diff --git a/gateway/router_not_found_test.go b/gateway/router_not_found_test.go new file mode 100644 index 000000000..839c817a6 --- /dev/null +++ b/gateway/router_not_found_test.go @@ -0,0 +1,53 @@ +package gateway + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/contract" +) + +func TestRouterAvailableRoutesErr_DefaultHidesRoutes(t *testing.T) { + router := &Router{ + config: &Config{}, + paths: []*contract.Path{ + contract.NewPath(http.MethodGet, "/v1/api/orders"), + }, + } + recorder := httptest.NewRecorder() + err := router.availableRoutesErr(http.StatusNotFound, fmt.Errorf("not found route with Method: GET and URL: /")) + + router.handleErrorCode(recorder, http.StatusNotFound, err) + + require.Equal(t, http.StatusNotFound, recorder.Code) + require.Equal(t, "not found route with Method: GET and URL: /", recorder.Body.String()) +} + +func TestRouterAvailableRoutesErr_ShowRoutesWhenConfigured(t *testing.T) { + showAvailableRoutes := true + router := &Router{ + config: &Config{ + ExposableConfig: ExposableConfig{ + ShowAvailableRoutes: &showAvailableRoutes, + }, + }, + paths: []*contract.Path{ + contract.NewPath(http.MethodGet, "/v1/api/orders"), + }, + } + recorder := httptest.NewRecorder() + err := router.availableRoutesErr(http.StatusNotFound, fmt.Errorf("not found route with Method: GET and URL: /")) + + router.handleErrorCode(recorder, http.StatusNotFound, err) + + require.Equal(t, http.StatusNotFound, recorder.Code) + actual := &AvailableRoutesError{} + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), actual)) + require.Equal(t, "not found route with Method: GET and URL: /", actual.Message) + require.Len(t, actual.Paths, 1) + require.Equal(t, "/v1/api/orders", actual.Paths[0].URI) +} diff --git a/go.mod b/go.mod index 8149c87e3..fe9188b85 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/viant/godiff v0.4.1 github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 - github.com/viant/scy v0.24.0 + github.com/viant/scy v0.33.1 github.com/viant/sqlx v0.23.1-0.20260712191511-2534f58bdccc github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 diff --git a/go.sum b/go.sum index b1438f25b..7525af70e 100644 --- a/go.sum +++ b/go.sum @@ -1194,6 +1194,8 @@ github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= github.com/viant/pgo v0.11.0/go.mod h1:MFzHmkRFZlciugEgUvpl/3grK789PBSH4dUVSLOSo+Q= github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= +github.com/viant/scy v0.33.1 h1:jlSgOxwsLvY1/YvAd6y5shvig/6hfhX+/sHVKmh4B5s= +github.com/viant/scy v0.33.1/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQnwf69oc86xW1e4wEEzKUg5OC6tOYegE= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.23.1-0.20260712191511-2534f58bdccc h1:uxPdh1l7dBvMUqJT2aMdPU9ubz3ErgQSmFoaDDe8row= diff --git a/repository/locator/component/component.go b/repository/locator/component/component.go index ffb308c2e..86c9f38f0 100644 --- a/repository/locator/component/component.go +++ b/repository/locator/component/component.go @@ -9,6 +9,7 @@ import ( "reflect" "github.com/viant/datly/repository/contract" + "github.com/viant/datly/service/executor/uow" "github.com/viant/datly/shared" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind" @@ -37,6 +38,11 @@ func (l *componentLocator) Names() []string { } func (l *componentLocator) Value(ctx context.Context, _ reflect.Type, name string) (interface{}, bool, error) { + order := uow.BindingOrder(ctx) + if _, _, scoped := uow.FromContext(ctx); scoped && order == "" { + return nil, false, fmt.Errorf("component binding %q has no reserved declaration slot", name) + } + ctx = uow.PrepareChild(ctx, uow.RelationBinding, order) method, URI := shared.ExtractPath(name) request, err := l.getRequest() if err != nil { @@ -90,19 +96,12 @@ func tryExtractResponseStatus(value interface{}) (*response.Status, bool) { return (*response.Status)(uPtr), true } -// TODO passed locator options to dispatcher so that this wil not be nil -var dispatcher contract.Dispatcher - // newComponentLocator returns component locator func newComponentLocator(opts ...locator.Option) (kind.Locator, error) { options := locator.NewOptions(opts) - if options.Dispatcher == nil { - options.Dispatcher = dispatcher - } if options.Dispatcher == nil { return nil, fmt.Errorf("dispatcher was empty") } - dispatcher = options.Dispatcher ret := &componentLocator{ custom: options.Custom, dispatch: options.Dispatcher, diff --git a/repository/locator/component/component_uow_test.go b/repository/locator/component/component_uow_test.go new file mode 100644 index 000000000..f92e17cd6 --- /dev/null +++ b/repository/locator/component/component_uow_test.go @@ -0,0 +1,86 @@ +package component + +import ( + "context" + "database/sql" + "net/http" + "reflect" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/service/executor/uow" + "github.com/viant/datly/view/state/kind/locator" +) + +type componentTestOperation string + +func (o componentTestOperation) TableName() string { return string(o) } + +type componentScopeDispatcher struct { + db *sql.DB + order *[]string +} + +func (d *componentScopeDispatcher) Dispatch(ctx context.Context, path *contract.Path, _ ...contract.Option) (interface{}, error) { + ctx, _, frame, _, err := uow.Enter(ctx, path.Method+" "+path.URI) + if err != nil { + return nil, err + } + defer frame.Seal() + buffer := frame.NewBuffer(func(context.Context) (*sql.DB, error) { return d.db, nil }, nil, + func(_ context.Context, _ *sql.Tx, value any) error { + *d.order = append(*d.order, string(value.(componentTestOperation))) + return nil + }) + if err = buffer.Append(componentTestOperation(path.URI)); err != nil { + return nil, err + } + return struct{}{}, nil +} + +func TestComponentLocatorCreatesOrderedBindingFrames(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := uow.NewRoot(context.Background(), "root") + var order []string + dispatcher := &componentScopeDispatcher{db: db, order: &order} + request, _ := http.NewRequest(http.MethodGet, "/", nil) + componentLocator := &componentLocator{ + dispatch: dispatcher, + getRequest: func() (*http.Request, error) { + return request, nil + }, + } + for _, binding := range []struct { + order string + name string + }{{"00000001", "GET:/second"}, {"00000000", "GET:/first"}} { + bindingCtx := uow.WithBindingOrder(ctx, binding.order) + if _, found, err := componentLocator.Value(bindingCtx, reflect.TypeOf(""), binding.name); err != nil || !found { + t.Fatalf("Value(%s) found=%v err=%v", binding.name, found, err) + } + } + rootBuffer := root.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(_ context.Context, _ *sql.Tx, value any) error { + order = append(order, string(value.(componentTestOperation))) + return nil + }) + if err := rootBuffer.Append(componentTestOperation("root")); err != nil { + t.Fatal(err) + } + root.Seal() + if err := scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + want := []string{"root", "/first", "/second"} + if !reflect.DeepEqual(order, want) { + t.Fatalf("order=%v want=%v", order, want) + } +} + +func TestComponentLocatorRequiresInvocationDispatcher(t *testing.T) { + if _, err := newComponentLocator(locator.WithConstants(nil)); err == nil { + t.Fatal("expected missing dispatcher error") + } +} diff --git a/service/executor/expand/data_unit.go b/service/executor/expand/data_unit.go index 7f0358b2e..2f39ac0b2 100644 --- a/service/executor/expand/data_unit.go +++ b/service/executor/expand/data_unit.go @@ -2,6 +2,7 @@ package expand import ( "context" + "database/sql" "fmt" "os" "reflect" @@ -30,6 +31,7 @@ type ( sqlxValidator *validator.Service `velty:"-"` sliceIndex map[reflect.Type]*xunsafe.Slice `velty:"-"` ctx context.Context `velty:"-"` + transactionRunner func(func(*sql.Tx) error) error `velty:"-"` EvalLock sync.Mutex } @@ -75,8 +77,22 @@ func (c *DataUnit) Allocate(tableName string, dest interface{}, selector string) return "", fmt.Errorf("error occurred while connecting to DB") } - service := sequencer.New(context.Background(), db) - return "", service.Next(tableName, dest, selector) + ctx := context.Background() + if c.ctx != nil { + ctx = c.ctx + } + if c.transactionRunner != nil { + return "", c.transactionRunner(func(tx *sql.Tx) error { + return sequencer.New(ctx, db, tx).Next(tableName, dest, selector) + }) + } + return "", sequencer.New(ctx, db).Next(tableName, dest, selector) +} + +// SetTransactionRunner binds sequencing to the same serialized transaction as DML. +func (c *DataUnit) SetTransactionRunner(ctx context.Context, runner func(func(*sql.Tx) error) error) { + c.ctx = ctx + c.transactionRunner = runner } func (c *DataUnit) AsBinding(value interface{}) (string, error) { diff --git a/service/executor/expand/parent.go b/service/executor/expand/parent.go index 0a51b93e9..61d71cb12 100644 --- a/service/executor/expand/parent.go +++ b/service/executor/expand/parent.go @@ -102,6 +102,22 @@ func (e *Executable) MarkAsExecuted() { e.executed = true } +// TableName returns the executable's mutation table. +func (e *Executable) TableName() string { + if e == nil { + return "" + } + return e.Table +} + +// BatchKey groups only contiguous compatible mutations. +func (e *Executable) BatchKey() string { + if e == nil { + return "" + } + return e.ExecType.String() + ":" + e.Table +} + func (e *MockExpander) ParentJoinOn(column string, prepend ...string) (string, error) { return "", nil } diff --git a/service/executor/expand/sql.go b/service/executor/expand/sql.go index a062e9d9e..a1fe8cf3b 100644 --- a/service/executor/expand/sql.go +++ b/service/executor/expand/sql.go @@ -3,6 +3,7 @@ package expand import ( "github.com/google/uuid" "strings" + "sync" ) type ( @@ -11,14 +12,27 @@ type ( Index ExecutablesIndex Markers map[string]int currIndex int + mu sync.RWMutex + onAppend func(interface{}) } SQLStatment struct { - SQL string - Args []interface{} + SQL string + Args []interface{} + executed bool } ) +// TableName identifies raw SQL as having no table-specific flush target. +func (s *SQLStatment) TableName() string { return "" } + +func (s *SQLStatment) Executed() bool { return s != nil && s.executed } +func (s *SQLStatment) MarkAsExecuted() { + if s != nil { + s.executed = true + } +} + func NewStmtHolder() *Statements { return &Statements{ Executable: nil, @@ -60,8 +74,7 @@ func (s *Statements) appendExecutable(tableName string, data interface{}, execTy IsLast: true, } - s.Index.UpdateLastExecutable(execType, tableName, executable) - s.Executable = append(s.Executable, executable) + s.append(executable, func() { s.Index.UpdateLastExecutable(execType, tableName, executable) }) } func (s *Statements) Delete(name string, data interface{}) { @@ -69,16 +82,47 @@ func (s *Statements) Delete(name string, data interface{}) { } func (s *Statements) Execute(SQLStmt *SQLStatment) { - s.Executable = append(s.Executable, SQLStmt) + s.append(SQLStmt, nil) +} + +// SetAppendObserver observes newly buffered statements in authored order. +func (s *Statements) SetAppendObserver(observer func(interface{})) { + s.mu.Lock() + s.onAppend = observer + s.mu.Unlock() +} + +// Snapshot returns a stable copy of all statements. +func (s *Statements) Snapshot() []interface{} { + s.mu.RLock() + defer s.mu.RUnlock() + return append([]interface{}(nil), s.Executable...) +} + +func (s *Statements) append(value interface{}, updateIndex func()) { + s.mu.Lock() + if updateIndex != nil { + updateIndex() + } + s.Executable = append(s.Executable, value) + observer := s.onAppend + s.mu.Unlock() + if observer != nil { + observer(value) + } } func (s *Statements) generateMarker() string { + s.mu.Lock() + defer s.mu.Unlock() marker := uuid.New().String() s.Markers[marker] = len(s.Executable) - 1 return marker } func (s *Statements) LookupExecutable(sql string) (*Executable, bool) { + s.mu.RLock() + defer s.mu.RUnlock() sql = strings.TrimSpace(sql) i, ok := s.Markers[sql] if !ok { @@ -90,6 +134,8 @@ func (s *Statements) LookupExecutable(sql string) (*Executable, bool) { } func (s *Statements) FilterByTableName(name string) []interface{} { + s.mu.RLock() + defer s.mu.RUnlock() var result []interface{} for _, executable := range s.Executable { switch actual := executable.(type) { @@ -103,7 +149,56 @@ func (s *Statements) FilterByTableName(name string) []interface{} { return result } +// CausalPrefixByTableName returns all pending authored statements through the +// last pending mutation of name. Raw SQL and other-table mutations before it +// are causal predecessors and therefore cannot be skipped. +func (s *Statements) CausalPrefixByTableName(name string) []interface{} { + s.mu.RLock() + defer s.mu.RUnlock() + if name == "" { + result := make([]interface{}, 0, len(s.Executable)) + for _, candidate := range s.Executable { + switch actual := candidate.(type) { + case *Executable: + if !actual.Executed() { + result = append(result, actual) + } + case *SQLStatment: + if !actual.Executed() { + result = append(result, actual) + } + } + } + return result + } + last := -1 + for i, candidate := range s.Executable { + if executable, ok := candidate.(*Executable); ok && !executable.Executed() && strings.EqualFold(executable.Table, name) { + last = i + } + } + if last < 0 { + return nil + } + result := make([]interface{}, 0, last+1) + for _, candidate := range s.Executable[:last+1] { + switch actual := candidate.(type) { + case *Executable: + if !actual.Executed() { + result = append(result, actual) + } + case *SQLStatment: + if !actual.Executed() { + result = append(result, actual) + } + } + } + return result +} + func (s *Statements) NextNonExecuted() (*Executable, bool) { + s.mu.Lock() + defer s.mu.Unlock() if s.currIndex >= len(s.Executable) { return nil, false } diff --git a/service/executor/expand/sql_test.go b/service/executor/expand/sql_test.go new file mode 100644 index 000000000..73ce1f317 --- /dev/null +++ b/service/executor/expand/sql_test.go @@ -0,0 +1,47 @@ +package expand + +import ( + "reflect" + "testing" +) + +func TestStatementsCausalPrefixIncludesRawAndEarlierTables(t *testing.T) { + statements := NewStmtHolder() + first := &Executable{Table: "parent", ExecType: ExecTypeInsert} + raw := &SQLStatment{SQL: "PRAGMA foreign_keys = ON"} + target := &Executable{Table: "child", ExecType: ExecTypeInsert} + statements.append(first, nil) + statements.Execute(raw) + statements.append(target, nil) + actual := statements.CausalPrefixByTableName("child") + want := []interface{}{first, raw, target} + if !reflect.DeepEqual(actual, want) { + t.Fatalf("prefix=%v want %v", actual, want) + } + first.MarkAsExecuted() + raw.MarkAsExecuted() + actual = statements.CausalPrefixByTableName("child") + if want = []interface{}{target}; !reflect.DeepEqual(actual, want) { + t.Fatalf("pending prefix=%v want %v", actual, want) + } +} + +func TestStatementsCausalPrefixDoesNotFlushWithoutTarget(t *testing.T) { + statements := NewStmtHolder() + statements.Execute(&SQLStatment{SQL: "SELECT 1"}) + if actual := statements.CausalPrefixByTableName("missing"); len(actual) != 0 { + t.Fatalf("prefix=%v", actual) + } +} + +func TestStatementsEmptyTargetReturnsAllPending(t *testing.T) { + statements := NewStmtHolder() + raw := &SQLStatment{SQL: "SELECT 1"} + mutation := &Executable{Table: "audit", ExecType: ExecTypeInsert} + statements.Execute(raw) + statements.append(mutation, nil) + want := []interface{}{raw, mutation} + if actual := statements.CausalPrefixByTableName(""); !reflect.DeepEqual(actual, want) { + t.Fatalf("prefix=%v want %v", actual, want) + } +} diff --git a/service/executor/handler/executor.go b/service/executor/handler/executor.go index 70f1746ba..b6e34b990 100644 --- a/service/executor/handler/executor.go +++ b/service/executor/handler/executor.go @@ -3,14 +3,17 @@ package handler import ( "context" "database/sql" + "errors" "fmt" "net/http" + "sync" "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" executor "github.com/viant/datly/service/executor" expand "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/service/executor/extension" + "github.com/viant/datly/service/executor/uow" session "github.com/viant/datly/service/session" "github.com/viant/datly/view" "github.com/viant/datly/view/state" @@ -36,7 +39,17 @@ type ( connectors view.Connectors dataUnit *expand.DataUnit dataUnits map[string]*expand.DataUnit + unitsByDB map[*sql.DB]*expand.DataUnit + unitTx map[*expand.DataUnit]*sql.Tx + unitMu sync.Mutex + buffers map[*expand.DataUnit]*uow.Buffer + scope *uow.Scope + frame *uow.Frame + bufferErr error + bufferMu sync.Mutex + ctx context.Context tx *sql.Tx + txOwned bool response http.ResponseWriter } @@ -77,6 +90,9 @@ func NewExecutor(aView *view.View, aSession *session.Session, opts ...Option) *E } func (e *Executor) Session(ctx context.Context) (*executor.Session, error) { + if err := e.ensureUnitOfWork(ctx); err != nil { + return nil, err + } if e.executorSession != nil { return e.executorSession, nil } @@ -118,6 +134,9 @@ func (e *Executor) NewHandlerSession(ctx context.Context, opts ...Option) (handl } func (e *Executor) HandlerSession(ctx context.Context, opts ...Option) (*extension.Session, error) { + if err := e.ensureUnitOfWork(ctx); err != nil { + return nil, err + } if e.handlerSession != nil { return e.handlerSession, nil } @@ -139,7 +158,7 @@ func (e *Executor) newSession(aSession *session.Session, opts ...Option) *extens res := e.view.GetResource() sess := extension.NewSession( extension.WithTemplateFlush(func(ctx context.Context) error { - return e.Execute(ctx) + return e.flushTemplate(ctx) }), extension.WithStater(aSession), extension.WithRedirect(e.redirect), @@ -172,6 +191,24 @@ func (e *Executor) newSqlService(options *sqlx.Options) (sqlx.Sqlx, error) { if options.WithTx == nil && e.tx != nil { options.WithTx = e.tx } + if e.scope != nil && options.WithTx != nil { + db, dbErr := unit.MetaSource.Db() + if dbErr != nil { + return nil, dbErr + } + if dbErr = e.scope.AdoptTransaction(db, options.WithTx); dbErr != nil { + return nil, dbErr + } + } + e.unitMu.Lock() + if e.unitTx == nil { + e.unitTx = map[*expand.DataUnit]*sql.Tx{} + } + if options.WithTx != nil { + e.unitTx[unit] = options.WithTx + } + buffer := e.buffers[unit] + e.unitMu.Unlock() return &Service{ txNotifier: txStartedNotifier, dataUnit: unit, @@ -179,17 +216,32 @@ func (e *Executor) newSqlService(options *sqlx.Options) (sqlx.Sqlx, error) { validator: e.newValidator(), connectors: e.connectors, mainConnector: e.view.Connector, + buffer: buffer, + tx: options.WithTx, }, nil } func (e *Executor) getDataUnit(options *sqlx.Options) (*expand.DataUnit, error) { + e.unitMu.Lock() + defer e.unitMu.Unlock() e.ensureConnectors() if (options.WithDb == nil && options.WithTx == nil) && options.WithConnector == e.view.Connector.Name { return e.dataUnit, nil } if options.WithDb != nil { - return expand.NewDataUnit(&DBProvider{db: options.WithDb}), nil + if unit := e.unitsByDB[options.WithDb]; unit != nil { + return unit, nil + } + unit := expand.NewDataUnit(&DBProvider{db: options.WithDb}) + if e.unitsByDB == nil { + e.unitsByDB = map[*sql.DB]*expand.DataUnit{} + } + e.unitsByDB[options.WithDb] = unit + if err := e.attachBuffer(unit, func(context.Context) (*sql.DB, error) { return options.WithDb, nil }, options.WithTx); err != nil { + return nil, err + } + return unit, nil } if options.WithConnector != "" { @@ -221,6 +273,9 @@ func (e *Executor) getDataUnit(options *sqlx.Options) (*expand.DataUnit, error) unit := expand.NewDataUnit(&DBProvider{db: db}) e.dataUnits[options.WithConnector] = unit + if err := e.attachBuffer(unit, func(context.Context) (*sql.DB, error) { return db, nil }, options.WithTx); err != nil { + return nil, err + } return unit, nil } @@ -243,25 +298,65 @@ func (e *Executor) Execute(ctx context.Context) error { return nil } e.executed = true + if e.scope != nil { + return e.getBufferErr() + } service := executor.New() var dbOptions []executor.DBOption if e.tx != nil { dbOptions = append(dbOptions, executor.WithTx(e.tx)) } - err := service.ExecuteStmts(ctx, executor.NewViewDBSource(e.view), newSqlxIterator(e.dataUnit.Statements.Executable), dbOptions...) + err := service.ExecuteStmts(ctx, executor.NewViewDBSource(e.view), newSqlxIterator(e.dataUnit.Statements.Snapshot()), dbOptions...) if err != nil { - return err + return e.completeOwnedTx(err) } + e.unitMu.Lock() + seen := map[*expand.DataUnit]bool{e.dataUnit: true} + units := make([]*expand.DataUnit, 0, len(e.dataUnits)+len(e.unitsByDB)) for _, unit := range e.dataUnits { + if !seen[unit] { + seen[unit] = true + units = append(units, unit) + } + } + for _, unit := range e.unitsByDB { + if !seen[unit] { + seen[unit] = true + units = append(units, unit) + } + } + unitTx := make(map[*expand.DataUnit]*sql.Tx, len(e.unitTx)) + for unit, tx := range e.unitTx { + unitTx[unit] = tx + } + e.unitMu.Unlock() + for _, unit := range units { dbSource := &DbSource{} dbSource.db, _ = unit.MetaSource.Db() - if err := service.ExecuteStmts(ctx, dbSource, newSqlxIterator(unit.Statements.Executable)); err != nil { - return err + unitOptions := []executor.DBOption(nil) + if tx := unitTx[unit]; tx != nil { + unitOptions = append(unitOptions, executor.WithTx(tx)) + } + if err := service.ExecuteStmts(ctx, dbSource, newSqlxIterator(unit.Statements.Snapshot()), unitOptions...); err != nil { + return e.completeOwnedTx(err) } } - return err + return e.completeOwnedTx(err) +} + +func (e *Executor) flushTemplate(ctx context.Context) error { + if e.scope == nil { + return e.Execute(ctx) + } + if err := e.getBufferErr(); err != nil { + return err + } + if buffer := e.bufferFor(e.dataUnit); buffer != nil { + return buffer.Flush(ctx, "") + } + return nil } func (e *Executor) ExpandAndExecute(ctx context.Context) (*executor.Session, error) { @@ -270,6 +365,18 @@ func (e *Executor) ExpandAndExecute(ctx context.Context) (*executor.Session, err return nil, err } service := executor.New() + if e.scope != nil { + ordered, buildErr := service.BuildBuffered(ctx, sess) + if buildErr != nil { + return nil, buildErr + } + if buffer := e.bufferFor(e.dataUnit); buffer != nil { + if buildErr = buffer.Reconcile(ordered); buildErr != nil { + return nil, buildErr + } + } + return sess, e.getBufferErr() + } var dbOptions []executor.DBOption if e.tx != nil { @@ -278,11 +385,128 @@ func (e *Executor) ExpandAndExecute(ctx context.Context) (*executor.Session, err return sess, service.Exec(ctx, sess, dbOptions...) } +func (e *Executor) bufferFor(unit *expand.DataUnit) *uow.Buffer { + e.unitMu.Lock() + defer e.unitMu.Unlock() + return e.buffers[unit] +} + +func (e *Executor) ensureUnitOfWork(ctx context.Context) error { + e.ctx = ctx + scope, frame, ok := uow.FromContext(ctx) + if !ok { + return nil + } + if e.scope != nil { + if e.scope != scope || e.frame != frame { + return fmt.Errorf("executor mutation scope mismatch") + } + return e.getBufferErr() + } + e.scope, e.frame = scope, frame + if e.tx == nil && e.session != nil { + e.tx = e.session.Options.SqlTx() + } + e.unitMu.Lock() + defer e.unitMu.Unlock() + if e.buffers == nil { + e.buffers = map[*expand.DataUnit]*uow.Buffer{} + } + if e.unitsByDB == nil { + e.unitsByDB = map[*sql.DB]*expand.DataUnit{} + } + return e.attachBuffer(e.dataUnit, func(context.Context) (*sql.DB, error) { + if e.view == nil || e.view.Connector == nil { + return nil, fmt.Errorf("view connector is required") + } + return e.view.Connector.DB() + }, e.tx) +} + +func (e *Executor) attachBuffer(unit *expand.DataUnit, resolve func(context.Context) (*sql.DB, error), tx *sql.Tx) error { + if e.scope == nil || e.frame == nil || unit == nil { + return nil + } + if e.buffers[unit] != nil { + return nil + } + if tx != nil { + db, err := resolve(e.ctx) + if err != nil { + return err + } + if err = e.scope.AdoptTransaction(db, tx); err != nil { + return err + } + } + buffer := e.frame.NewBuffer(resolve, tx, func(ctx context.Context, transaction *sql.Tx, value any) error { + db, err := resolve(ctx) + if err != nil { + return err + } + source := &DbSource{db: db} + return executor.New().ExecuteStmts(ctx, source, &sqlxIterator{toExecute: []interface{}{value}}, executor.WithTx(transaction)) + }) + buffer.SetBatchExecutor(func(ctx context.Context, transaction *sql.Tx, values []any) error { + db, err := resolve(ctx) + if err != nil { + return err + } + return executor.New().ExecuteStmts(ctx, &DbSource{db: db}, newSqlxIterator(values), executor.WithTx(transaction)) + }) + e.buffers[unit] = buffer + unit.SetTransactionRunner(e.ctx, func(fn func(*sql.Tx) error) error { + return buffer.UseTransaction(e.ctx, fn) + }) + unit.Statements.SetAppendObserver(func(value interface{}) { + if err := buffer.Append(value); err != nil { + e.setBufferErr(err) + } + }) + for _, value := range unit.Statements.Snapshot() { + if err := buffer.Append(value); err != nil { + return err + } + } + return nil +} + +func (e *Executor) setBufferErr(err error) { + if err == nil { + return + } + e.bufferMu.Lock() + if e.bufferErr == nil { + e.bufferErr = err + } + e.bufferMu.Unlock() +} + +func (e *Executor) getBufferErr() error { + e.bufferMu.Lock() + defer e.bufferMu.Unlock() + return e.bufferErr +} + func (e *Executor) txStarted(tx *sql.Tx) { e.tx = tx + e.txOwned = tx != nil +} + +func (e *Executor) completeOwnedTx(cause error) error { + if !e.txOwned || e.tx == nil { + return cause + } + tx := e.tx + e.tx, e.txOwned = nil, false + if cause != nil { + return errors.Join(cause, tx.Rollback()) + } + return tx.Commit() } func (e *Executor) redirect(ctx context.Context, route *http2.Route, opts ...hstate.Option) (handler.Session, error) { + ctx = uow.Propagate(e.ctx, ctx) registry := e.session.Registry() if registry == nil { return nil, fmt.Errorf("registry was empty") @@ -291,6 +515,13 @@ func (e *Executor) redirect(ctx context.Context, route *http2.Route, opts ...hst if err != nil { return nil, err } + ctx = uow.PrepareChild(ctx, uow.RelationImperative, "") + if _, _, scoped := uow.FromContext(ctx); scoped { + ctx, _, _, _, err = uow.Enter(ctx, route.Method+" "+route.URL) + if err != nil { + return nil, err + } + } originalRequest, _ := e.session.HttpRequest(ctx, e.session.Clone()) request, _ := http.NewRequest(route.Method, route.URL, nil) @@ -299,7 +530,7 @@ func (e *Executor) redirect(ctx context.Context, route *http2.Route, opts ...hst } stateOptions := hstate.NewOptions(opts...) unmarshal := aComponent.UnmarshalFunc(request) - locatorOptions := append(aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal)) + locatorOptions := aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal) if stateOptions.Query() != nil { locatorOptions = append(locatorOptions, locator.WithQuery(stateOptions.Query())) } diff --git a/service/executor/handler/locator/handler.go b/service/executor/handler/locator/handler.go index 6f0c31aeb..d4f3fad2e 100644 --- a/service/executor/handler/locator/handler.go +++ b/service/executor/handler/locator/handler.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/viant/datly/repository/handler" ehandler "github.com/viant/datly/service/executor/handler" + "github.com/viant/datly/service/executor/uow" "github.com/viant/datly/service/session" "github.com/viant/datly/view" "github.com/viant/datly/view/extension" @@ -47,6 +48,16 @@ func (v *Handler) Value(ctx context.Context, _ reflect.Type, name string) (inter aView.Connector = resource.Connectors[0] } aSession := session.Context(ctx) + var frame *uow.Frame + _, _, scoped := uow.FromContext(ctx) + if scoped { + ctx = uow.PrepareChild(ctx, uow.RelationBinding, uow.BindingOrder(ctx)) + ctx, _, frame, _, err = uow.Enter(ctx, "handler "+name) + if err != nil { + return nil, false, err + } + defer frame.Seal() + } anExecutor := ehandler.NewExecutor(aView, aSession) handlerSession, err := anExecutor.NewHandlerSession(ctx, ehandler.WithTypes(v.types...), ehandler.WithAuth(aSession.Auth())) @@ -54,6 +65,9 @@ func (v *Handler) Value(ctx context.Context, _ reflect.Type, name string) (inter return nil, false, fmt.Errorf("failed to create handler session: %w", err) } result, err := anHandler.Call(ctx, handlerSession) + if err == nil && !scoped { + err = anExecutor.Execute(ctx) + } return result, err == nil, err } diff --git a/service/executor/handler/sqlx.go b/service/executor/handler/sqlx.go index 970482513..ac4a76815 100644 --- a/service/executor/handler/sqlx.go +++ b/service/executor/handler/sqlx.go @@ -7,6 +7,7 @@ import ( "github.com/viant/datly/service/executor" expand "github.com/viant/datly/service/executor/expand" "github.com/viant/datly/service/executor/sequencer" + "github.com/viant/datly/service/executor/uow" "github.com/viant/datly/view" "github.com/viant/sqlx/io/config" "github.com/viant/sqlx/io/read" @@ -34,6 +35,7 @@ type ( request *http.Request txNotifier func(tx *sql.Tx) tx *sql.Tx + buffer *uow.Buffer } sqlxIterator struct { @@ -61,24 +63,39 @@ func (s *sqlxIterator) HasAny() bool { } func (s *Service) Flush(ctx context.Context, tableName string) error { + if s.buffer != nil { + return s.buffer.Flush(ctx, tableName) + } var options []executor.DBOption tx := s.options.WithTx + owned := false + if tx == nil { + tx = s.tx + } if tx == nil { - var err error - tx, err = s.Tx(ctx) + db, err := s.Db(ctx) + if err != nil { + return err + } + tx, err = db.BeginTx(ctx, nil) if err != nil { return err } + owned = true } - options = append(options, executor.WithTx(tx)) - exec := executor.New() if err := exec.ExecuteStmts(ctx, s, &sqlxIterator{ - toExecute: s.dataUnit.Statements.FilterByTableName(tableName), + toExecute: s.dataUnit.Statements.CausalPrefixByTableName(tableName), }, options...); err != nil { + if owned { + _ = tx.Rollback() + } return err } + if owned { + return tx.Commit() + } return nil } @@ -179,6 +196,9 @@ func (s *Service) openDBConnection() (*sql.DB, error) { } func (s *Service) Tx(ctx context.Context) (*sql.Tx, error) { + if s.buffer != nil { + return nil, uow.ErrTransactionAccess + } if s.tx != nil { return s.tx, nil } @@ -230,8 +250,16 @@ func (s *Service) Allocate(ctx context.Context, tableName string, dest interface if err != nil { return err } - service := sequencer.New(context.Background(), db) - return service.Next(tableName, dest, selector) + if s.buffer != nil { + return s.buffer.UseTransaction(ctx, func(tx *sql.Tx) error { + return sequencer.New(ctx, db, tx).Next(tableName, dest, selector) + }) + } + tx := s.options.WithTx + if tx == nil { + tx = s.tx + } + return sequencer.New(ctx, db, tx).Next(tableName, dest, selector) } func (s *Service) CanBatchGlobally() bool { diff --git a/service/executor/handler/transaction_scope_test.go b/service/executor/handler/transaction_scope_test.go new file mode 100644 index 000000000..46c55209f --- /dev/null +++ b/service/executor/handler/transaction_scope_test.go @@ -0,0 +1,108 @@ +package handler + +import ( + "context" + "errors" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/viant/datly/service/executor/uow" + "github.com/viant/datly/service/session" + "github.com/viant/datly/view" + xsqlx "github.com/viant/xdatly/handler/sqlx" +) + +func TestScopedSQLServiceRejectsConflictingTransaction(t *testing.T) { + connector := view.NewConnector("main", "sqlite3", t.TempDir()+"/conflict.db") + db, err := connector.DB() + if err != nil { + t.Fatal(err) + } + defer db.Close() + txA, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer txA.Rollback() + txB, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer txB.Rollback() + aView := &view.View{Connector: connector} + aSession := session.New(aView, session.WithSQLTx(txA)) + ctx, scope, root := uow.NewRoot(context.Background(), "root") + if err = scope.AdoptTransaction(db, txA); err != nil { + t.Fatal(err) + } + executor := NewExecutor(aView, aSession) + if err = executor.ensureUnitOfWork(ctx); err != nil { + t.Fatal(err) + } + if _, err = executor.newSqlService(&xsqlx.Options{WithTx: txB}); !errors.Is(err, uow.ErrTransactionConflict) { + t.Fatalf("newSqlService() error=%v", err) + } + root.Seal() + if err = scope.Finish(ctx, errors.New("abort")); err == nil { + t.Fatal("expected root abort") + } +} + +func TestScopedSQLServiceDoesNotExposeRootTransaction(t *testing.T) { + connector := view.NewConnector("main", "sqlite3", t.TempDir()+"/access.db") + db, err := connector.DB() + if err != nil { + t.Fatal(err) + } + defer db.Close() + aView := &view.View{Connector: connector} + aSession := session.New(aView) + ctx, scope, root := uow.NewRoot(context.Background(), "root") + executor := NewExecutor(aView, aSession) + if err = executor.ensureUnitOfWork(ctx); err != nil { + t.Fatal(err) + } + service, err := executor.newSqlService(&xsqlx.Options{}) + if err != nil { + t.Fatal(err) + } + if _, err = service.Tx(ctx); !errors.Is(err, uow.ErrTransactionAccess) { + t.Fatalf("Tx() error=%v", err) + } + root.Seal() + if err = scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } +} + +func TestScopedSQLServiceDoesNotExposeSuppliedTransaction(t *testing.T) { + connector := view.NewConnector("main", "sqlite3", t.TempDir()+"/external-access.db") + db, err := connector.DB() + if err != nil { + t.Fatal(err) + } + defer db.Close() + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + aView := &view.View{Connector: connector} + aSession := session.New(aView, session.WithSQLTx(tx)) + ctx, scope, root := uow.NewRoot(context.Background(), "root") + executor := NewExecutor(aView, aSession) + if err = executor.ensureUnitOfWork(ctx); err != nil { + t.Fatal(err) + } + service, err := executor.newSqlService(&xsqlx.Options{}) + if err != nil { + t.Fatal(err) + } + if _, err = service.Tx(ctx); !errors.Is(err, uow.ErrTransactionAccess) { + t.Fatalf("Tx() error=%v", err) + } + root.Seal() + if err = scope.Finish(ctx, errors.New("abort")); err == nil { + t.Fatal("expected root abort") + } +} diff --git a/service/executor/sequencer/service.go b/service/executor/sequencer/service.go index 92ffc66b1..04b90708b 100644 --- a/service/executor/sequencer/service.go +++ b/service/executor/sequencer/service.go @@ -6,11 +6,13 @@ import ( "fmt" "github.com/viant/sqlx/io/insert" "github.com/viant/sqlx/metadata/info/dialect" + "github.com/viant/sqlx/option" "strings" ) type Service struct { db *sql.DB + tx *sql.Tx ctx context.Context } @@ -43,7 +45,18 @@ func (s *Service) next(table string, any interface{}, selector string) error { if err != nil { return err } - nextSeq, err := inserter.NextSequence(s.ctx, record, emptyRecordCount, dialect.PresetIDWithTransientTransaction) + strategy := dialect.PresetIDWithTransientTransaction + if s.tx != nil { + // The transient strategy opens and completes its own transaction on + // some products (notably MySQL). Invocation mode must remain inside + // the root transaction, so use the transaction-aware MAX strategy. + strategy = dialect.PresetIDWithMax + } + options := []option.Option{strategy} + if s.tx != nil { + options = append(options, s.tx) + } + nextSeq, err := inserter.NextSequence(s.ctx, record, emptyRecordCount, options...) if err != nil { return err } @@ -53,6 +66,10 @@ func (s *Service) next(table string, any interface{}, selector string) error { return err } -func New(ctx context.Context, db *sql.DB) *Service { - return &Service{db: db, ctx: ctx} +func New(ctx context.Context, db *sql.DB, tx ...*sql.Tx) *Service { + ret := &Service{db: db, ctx: ctx} + if len(tx) > 0 { + ret.tx = tx[0] + } + return ret } diff --git a/service/executor/sequencer/service_test.go b/service/executor/sequencer/service_test.go index d9970653f..3c540fb88 100644 --- a/service/executor/sequencer/service_test.go +++ b/service/executor/sequencer/service_test.go @@ -12,6 +12,43 @@ import ( "testing" ) +func TestServiceNextUsesSuppliedTransaction(t *testing.T) { + db, err := sql.Open("sqlite3", t.TempDir()+"/sequence_tx.db") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err = db.Exec("CREATE TABLE EMP (ID INTEGER PRIMARY KEY AUTOINCREMENT, NAME TEXT)"); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + type Emp struct { + ID int64 `sqlx:"ID,primaryKey=true"` + Name string `sqlx:"NAME"` + } + values := []*Emp{{Name: "reserved"}} + if err = New(context.Background(), db, tx).Next("EMP", values, "ID"); err != nil { + t.Fatal(err) + } + if err = tx.Rollback(); err != nil { + t.Fatal(err) + } + result, err := db.Exec("INSERT INTO EMP(NAME) VALUES ('actual')") + if err != nil { + t.Fatal(err) + } + id, err := result.LastInsertId() + if err != nil { + t.Fatal(err) + } + if id != 1 { + t.Fatalf("sequence allocation escaped supplied transaction: next id=%d", id) + } +} + func TestService_Next(t *testing.T) { _ = os.Remove("/tmp/datly_sequnece_test.db") diff --git a/service/executor/service.go b/service/executor/service.go index 086a40695..de72f1909 100644 --- a/service/executor/service.go +++ b/service/executor/service.go @@ -99,6 +99,30 @@ func (e *Executor) Exec(ctx context.Context, sess *Session, options ...DBOption) return state.Flush(expand2.StatusSuccess) } +// BuildBuffered materializes a template execution sequence without executing +// its DML. Root-owned units of work use the returned order at final completion. +func (e *Executor) BuildBuffered(ctx context.Context, sess *Session) ([]any, error) { + state, data, err := e.sqlBuilder.Build(ctx, sess.View, sess.Lookup(sess.View), sess.SessionHandler, sess.DataUnit) + if state != nil { + sess.TemplateState = state + } + if err != nil { + if state != nil { + _ = state.Flush(expand2.StatusFailure) + } + return nil, err + } + iterator := NewTemplateStmtIterator(state.DataUnit, data) + var result []any + for iterator.HasNext() { + result = append(result, iterator.Next()) + } + if err = state.Flush(expand2.StatusSuccess); err != nil { + return nil, err + } + return result, nil +} + func (e *Executor) ExecuteStmts(ctx context.Context, dbSource DBSource, it StmtIterator, options ...DBOption) error { if !it.HasAny() { return nil @@ -138,20 +162,24 @@ func (e *Executor) execData(ctx context.Context, sess *dbSession, data interface if actual.Executed() { return nil } - actual.MarkAsExecuted() + var err error switch actual.ExecType { case expand2.ExecTypeInsert: - return e.handleInsert(ctx, sess, actual, db) + err = e.handleInsert(ctx, sess, actual, db) case expand2.ExecTypeUpdate: - return e.handleUpdate(ctx, sess, db, actual) + err = e.handleUpdate(ctx, sess, db, actual) case expand2.ExecTypeDelete: - return e.handleDelete(ctx, sess, db, actual) + err = e.handleDelete(ctx, sess, db, actual) default: return fmt.Errorf("unsupported '%v' db operation\n", actual.ExecType.String()) } + if err == nil { + actual.MarkAsExecuted() + } + return err case *expand2.SQLStatment: - if len(actual.SQL) == 0 { + if actual.Executed() || len(actual.SQL) == 0 { return nil } @@ -160,7 +188,11 @@ func (e *Executor) execData(ctx context.Context, sess *dbSession, data interface return err } - return e.executeStatement(ctx, tx, actual, sess) + err = e.executeStatement(ctx, tx, actual, sess) + if err == nil { + actual.MarkAsExecuted() + } + return err } return fmt.Errorf("unsupported query type %T", data) } diff --git a/service/executor/service_execution_test.go b/service/executor/service_execution_test.go new file mode 100644 index 000000000..c48a9fe8b --- /dev/null +++ b/service/executor/service_execution_test.go @@ -0,0 +1,61 @@ +package executor + +import ( + "context" + "database/sql" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/viant/datly/service/executor/expand" + "github.com/viant/sqlx/io/config" + "github.com/viant/sqlx/metadata/info" +) + +type executionTestSource struct{ db *sql.DB } + +func (s executionTestSource) Db(context.Context) (*sql.DB, error) { return s.db, nil } +func (s executionTestSource) Dialect(ctx context.Context) (*info.Dialect, error) { + return config.Dialect(ctx, s.db) +} + +type executionTestIterator struct { + items []interface{} + index int +} + +func (i *executionTestIterator) HasAny() bool { return len(i.items) > 0 } +func (i *executionTestIterator) HasNext() bool { return i.index < len(i.items) } +func (i *executionTestIterator) Next() interface{} { + value := i.items[i.index] + i.index++ + return value +} + +func TestExecuteStmtsMarksRawSQLOnlyAfterSuccess(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + failed := &expand.SQLStatment{SQL: "INSERT INTO missing(id) VALUES (1)"} + err = New().ExecuteStmts(context.Background(), executionTestSource{db: db}, &executionTestIterator{items: []interface{}{failed}}) + if err == nil { + t.Fatal("expected execution failure") + } + if failed.Executed() { + t.Fatal("failed raw SQL was marked executed") + } + if _, err = db.Exec("CREATE TABLE audit (id INTEGER PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + success := &expand.SQLStatment{SQL: "INSERT INTO audit(id) VALUES (1)"} + if err = New().ExecuteStmts(context.Background(), executionTestSource{db: db}, &executionTestIterator{items: []interface{}{success}}); err != nil { + t.Fatal(err) + } + if !success.Executed() { + t.Fatal("successful raw SQL was not marked executed") + } + if err = New().ExecuteStmts(context.Background(), executionTestSource{db: db}, &executionTestIterator{items: []interface{}{success}}); err != nil { + t.Fatalf("executed raw SQL ran twice: %v", err) + } +} diff --git a/service/executor/uow/scope.go b/service/executor/uow/scope.go new file mode 100644 index 000000000..9612c6263 --- /dev/null +++ b/service/executor/uow/scope.go @@ -0,0 +1,748 @@ +// Package uow owns one invocation-scoped mutation unit of work. +package uow + +import ( + "context" + "database/sql" + "errors" + "fmt" + "reflect" + "sort" + "strings" + "sync" +) + +type Relation uint8 + +const ( + RelationRoot Relation = iota + RelationBinding + RelationImperative +) + +var ( + ErrCompleted = errors.New("mutation unit of work is completed") + ErrFailed = errors.New("mutation unit of work has failed") + ErrFrameSealed = errors.New("component mutation frame is sealed") + ErrBindingFlush = errors.New("cannot flush a binding child while its ancestor is open") + ErrTransactionConflict = errors.New("conflicting transaction for database") + ErrTransactionAccess = errors.New("direct transaction access is unavailable in invocation mode") +) + +type contextKey struct{} +type bindingOrderKey struct{} + +type carrier struct { + scope *Scope + frame *Frame + relation Relation + order string +} + +// Scope coordinates component frames and database transactions for one root operation. +type Scope struct { + mu sync.Mutex + flushMu sync.Mutex + root *Frame + databases map[*sql.DB]*databaseUnit + completed bool + failed error + nextFrame uint64 + nextOp uint64 + nextDB uint64 +} + +// Frame owns the mutation timeline of one component invocation. +type Frame struct { + scope *Scope + id uint64 + name string + relation Relation + order string + parent *Frame + open bool + timeline []timelineEntry + bindings []*Frame + nextBinding uint64 +} + +type timelineEntry struct { + operation *Operation + child *Frame +} + +// Buffer associates statements with one component frame and database source. +type Buffer struct { + scope *Scope + frame *Frame + resolveDB func(context.Context) (*sql.DB, error) + externalTx *sql.Tx + execute func(context.Context, *sql.Tx, any) error + executeBatch func(context.Context, *sql.Tx, []any) error +} + +// Operation is one ordered buffered mutation. +type Operation struct { + id uint64 + buffer *Buffer + value any + table string + executed bool + reserved bool +} + +type databaseUnit struct { + mu sync.Mutex + db *sql.DB + tx *sql.Tx + external bool + failed error + order uint64 +} + +// NewRoot creates a new root scope and installs it in ctx. +func NewRoot(ctx context.Context, name string) (context.Context, *Scope, *Frame) { + scope := &Scope{databases: map[*sql.DB]*databaseUnit{}, nextFrame: 1} + frame := &Frame{scope: scope, id: 1, name: name, relation: RelationRoot, open: true} + scope.root = frame + return context.WithValue(ctx, contextKey{}, &carrier{scope: scope, frame: frame}), scope, frame +} + +// FromContext returns the active scope and frame. +func FromContext(ctx context.Context) (*Scope, *Frame, bool) { + if ctx == nil { + return nil, nil, false + } + value, _ := ctx.Value(contextKey{}).(*carrier) + if value == nil || value.scope == nil || value.frame == nil { + return nil, nil, false + } + return value.scope, value.frame, true +} + +// Propagate copies only Datly's private invocation carrier from source to +// destination. Dispatcher closures use it so caller cancellation is retained +// while a nested session cannot accidentally lose the active unit of work. +func Propagate(source, destination context.Context) context.Context { + if destination == nil { + destination = context.Background() + } + if source == nil { + return destination + } + value, _ := source.Value(contextKey{}).(*carrier) + if value == nil { + return destination + } + return context.WithValue(destination, contextKey{}, value) +} + +// PrepareChild marks the next component dispatch as a child of the active frame. +func PrepareChild(ctx context.Context, relation Relation, order string) context.Context { + scope, frame, ok := FromContext(ctx) + if !ok { + return ctx + } + return context.WithValue(ctx, contextKey{}, &carrier{scope: scope, frame: frame, relation: relation, order: order}) +} + +// WithBindingOrder records the declaration order reserved by the parameter +// resolver before concurrently evaluating component-valued bindings. +func WithBindingOrder(ctx context.Context, order string) context.Context { + return context.WithValue(ctx, bindingOrderKey{}, order) +} + +// WithBindingOrderIndex extends a reserved parameter slot with the authored +// index of a concurrently resolved repeated item. +func WithBindingOrderIndex(ctx context.Context, index int) context.Context { + parent := BindingOrder(ctx) + return WithBindingOrder(ctx, fmt.Sprintf("%s/%020d", parent, index+1)) +} + +// BindingOrder returns a previously reserved binding declaration order. +func BindingOrder(ctx context.Context) string { + if ctx == nil { + return "" + } + order, _ := ctx.Value(bindingOrderKey{}).(string) + return order +} + +// ReserveBindingOrder allocates a stable frame-wide declaration slot before +// a binding resolver goroutine is started. +func ReserveBindingOrder(ctx context.Context) (string, error) { + scope, frame, ok := FromContext(ctx) + if !ok { + return "", nil + } + scope.mu.Lock() + defer scope.mu.Unlock() + if scope.completed { + return "", ErrCompleted + } + if scope.failed != nil { + return "", errors.Join(ErrFailed, scope.failed) + } + if !frame.open { + return "", ErrFrameSealed + } + frame.nextBinding++ + return fmt.Sprintf("%020d", frame.nextBinding), nil +} + +// Enter creates a child frame when ctx was prepared for dispatch. Otherwise it +// returns the current root frame, or creates a root for an external entry. +func Enter(ctx context.Context, name string) (context.Context, *Scope, *Frame, bool, error) { + value, _ := ctx.Value(contextKey{}).(*carrier) + if value == nil || value.scope == nil || value.frame == nil { + ctx, scope, frame := NewRoot(ctx, name) + return ctx, scope, frame, true, nil + } + scope := value.scope + scope.mu.Lock() + if scope.completed { + if value.relation == RelationRoot && value.frame.parent == nil { + scope.mu.Unlock() + ctx, freshScope, freshFrame := NewRoot(ctx, name) + return ctx, freshScope, freshFrame, true, nil + } + scope.mu.Unlock() + return nil, nil, nil, false, ErrCompleted + } + if scope.failed != nil { + err := scope.failed + scope.mu.Unlock() + return nil, nil, nil, false, errors.Join(ErrFailed, err) + } + if value.relation == RelationRoot { + if !value.frame.open { + scope.mu.Unlock() + return nil, nil, nil, false, ErrFrameSealed + } + scope.mu.Unlock() + return ctx, scope, value.frame, false, nil + } + if !value.frame.open { + scope.mu.Unlock() + return nil, nil, nil, false, ErrFrameSealed + } + scope.nextFrame++ + child := &Frame{ + scope: scope, id: scope.nextFrame, name: name, relation: value.relation, + order: value.order, parent: value.frame, open: true, + } + if value.relation == RelationBinding { + value.frame.bindings = append(value.frame.bindings, child) + sort.SliceStable(value.frame.bindings, func(i, j int) bool { + return value.frame.bindings[i].order < value.frame.bindings[j].order + }) + } else { + value.frame.timeline = append(value.frame.timeline, timelineEntry{child: child}) + } + ctx = context.WithValue(ctx, contextKey{}, &carrier{scope: scope, frame: child}) + scope.mu.Unlock() + return ctx, scope, child, false, nil +} + +// IsCompleted reports whether root completion has run. +func (s *Scope) IsCompleted() bool { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + return s.completed +} + +// AdoptTransaction registers a caller-owned transaction for db before child +// binding evaluation can lazily create a local transaction for that database. +func (s *Scope) AdoptTransaction(db *sql.DB, tx *sql.Tx) error { + if s == nil || db == nil || tx == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + if s.completed { + return ErrCompleted + } + if existing := s.databases[db]; existing != nil { + if existing.tx != tx { + return ErrTransactionConflict + } + return nil + } + s.nextDB++ + s.databases[db] = &databaseUnit{db: db, tx: tx, external: true, order: s.nextDB} + return nil +} + +// Seal closes a component frame to new semantic children. +func (f *Frame) Seal() { + if f == nil || f.scope == nil { + return + } + f.scope.mu.Lock() + f.open = false + f.scope.mu.Unlock() +} + +// NewBuffer creates a frame-scoped statement buffer. +func (f *Frame) NewBuffer(resolveDB func(context.Context) (*sql.DB, error), externalTx *sql.Tx, execute func(context.Context, *sql.Tx, any) error) *Buffer { + if f == nil { + return nil + } + return &Buffer{scope: f.scope, frame: f, resolveDB: resolveDB, externalTx: externalTx, execute: execute} +} + +// SetBatchExecutor enables batching for contiguous operations that expose the +// same non-empty BatchKey. Ordering barriers are never crossed. +func (b *Buffer) SetBatchExecutor(execute func(context.Context, *sql.Tx, []any) error) { + if b != nil { + b.executeBatch = execute + } +} + +// Append adds one operation at the exact current position in the frame timeline. +func (b *Buffer) Append(value any) error { + if b == nil || b.scope == nil || b.frame == nil { + return fmt.Errorf("mutation buffer is not configured") + } + s := b.scope + s.mu.Lock() + defer s.mu.Unlock() + if s.completed { + return ErrCompleted + } + if s.failed != nil { + return errors.Join(ErrFailed, s.failed) + } + if !b.frame.open { + return ErrFrameSealed + } + s.nextOp++ + op := &Operation{id: s.nextOp, buffer: b, value: value, table: operationTable(value)} + b.frame.timeline = append(b.frame.timeline, timelineEntry{operation: op}) + return nil +} + +// Reconcile replaces this buffer's observed entries with the exact execution +// sequence produced by template materialization while retaining child markers. +func (b *Buffer) Reconcile(values []any) error { + if b == nil || b.scope == nil || b.frame == nil { + return fmt.Errorf("mutation buffer is not configured") + } + s := b.scope + s.flushMu.Lock() + defer s.flushMu.Unlock() + s.mu.Lock() + defer s.mu.Unlock() + if s.completed { + return ErrCompleted + } + if s.failed != nil { + return errors.Join(ErrFailed, s.failed) + } + if !b.frame.open { + return ErrFrameSealed + } + existing := make([]*Operation, 0) + for _, entry := range b.frame.timeline { + if entry.operation != nil && entry.operation.buffer == b && !entry.operation.executed { + existing = append(existing, entry.operation) + } + } + desired := make([]*Operation, 0, len(values)) + used := make([]bool, len(existing)) + for _, value := range values { + var op *Operation + for i, candidate := range existing { + if !used[i] && sameValue(candidate.value, value) { + op = candidate + used[i] = true + break + } + } + if op == nil { + s.nextOp++ + op = &Operation{id: s.nextOp, buffer: b, value: value, table: operationTable(value)} + } + desired = append(desired, op) + } + result := make([]timelineEntry, 0, len(b.frame.timeline)+len(desired)) + next := 0 + insertAt := -1 + for _, entry := range b.frame.timeline { + if entry.operation != nil && entry.operation.buffer == b && !entry.operation.executed { + if next < len(desired) { + result = append(result, timelineEntry{operation: desired[next]}) + next++ + insertAt = len(result) + } + continue + } + result = append(result, entry) + } + if insertAt < 0 { + insertAt = len(result) + } + if next < len(desired) { + tail := append([]timelineEntry(nil), result[insertAt:]...) + result = result[:insertAt] + for ; next < len(desired); next++ { + result = append(result, timelineEntry{operation: desired[next]}) + } + result = append(result, tail...) + } + b.frame.timeline = result + return nil +} + +// Flush executes the causal prefix ending at the requested table for b's database. +func (b *Buffer) Flush(ctx context.Context, table string) error { + if b == nil || b.scope == nil { + return fmt.Errorf("mutation buffer is not configured") + } + return b.scope.flush(ctx, b.frame, b, table) +} + +// UseTransaction serializes work which must share this buffer's database +// transaction, including identity sequencing and buffered DML execution. +func (b *Buffer) UseTransaction(ctx context.Context, fn func(*sql.Tx) error) error { + if b == nil || b.scope == nil { + return fmt.Errorf("mutation buffer is not configured") + } + b.scope.flushMu.Lock() + defer b.scope.flushMu.Unlock() + b.scope.mu.Lock() + if b.scope.completed { + b.scope.mu.Unlock() + return ErrCompleted + } + if b.scope.failed != nil { + err := b.scope.failed + b.scope.mu.Unlock() + return errors.Join(ErrFailed, err) + } + b.scope.mu.Unlock() + db, err := b.resolveDB(ctx) + if err != nil { + return err + } + unit, err := b.scope.database(ctx, db, b.externalTx) + if err != nil { + return err + } + unit.mu.Lock() + defer unit.mu.Unlock() + if unit.failed != nil { + return errors.Join(ErrFailed, unit.failed) + } + if err = fn(unit.tx); err != nil { + unit.failed = err + b.scope.mu.Lock() + b.scope.failed = err + b.scope.mu.Unlock() + } + return err +} + +// Finish drains and completes locally owned transactions at the root boundary. +func (s *Scope) Finish(ctx context.Context, cause error) error { + if s == nil { + return cause + } + s.flushMu.Lock() + defer s.flushMu.Unlock() + s.mu.Lock() + if s.completed { + s.mu.Unlock() + return errors.Join(cause, ErrCompleted) + } + s.completed = true + sealFrameTree(s.root) + s.mu.Unlock() + + if cause == nil { + cause = s.flushLocked(ctx, s.root, nil, "") + } + if cause != nil { + return errors.Join(cause, s.rollbackLocal()) + } + return s.commitLocal() +} + +func sealFrameTree(frame *Frame) { + if frame == nil { + return + } + frame.open = false + for _, entry := range frame.timeline { + if entry.child != nil { + sealFrameTree(entry.child) + } + } + for _, child := range frame.bindings { + sealFrameTree(child) + } +} + +func (s *Scope) flush(ctx context.Context, caller *Frame, target *Buffer, table string) error { + s.flushMu.Lock() + defer s.flushMu.Unlock() + s.mu.Lock() + if s.completed { + s.mu.Unlock() + return ErrCompleted + } + s.mu.Unlock() + return s.flushLocked(ctx, caller, target, table) +} + +func (s *Scope) flushLocked(ctx context.Context, caller *Frame, target *Buffer, table string) error { + s.mu.Lock() + if s.failed != nil { + err := s.failed + s.mu.Unlock() + return errors.Join(ErrFailed, err) + } + if hasOpenAncestorOfBinding(caller) { + s.mu.Unlock() + return ErrBindingFlush + } + ordered := flatten(s.root) + s.mu.Unlock() + + var targetDB *sql.DB + var err error + if target != nil { + targetDB, err = target.resolveDB(ctx) + if err != nil { + return err + } + } + selected := make([]*Operation, 0, len(ordered)) + lastMatch := -1 + for _, op := range ordered { + if op.executed { + continue + } + db, resolveErr := op.buffer.resolveDB(ctx) + if resolveErr != nil { + return resolveErr + } + if targetDB != nil && db != targetDB { + continue + } + selected = append(selected, op) + if target == nil || (op.buffer == target && (table == "" || strings.EqualFold(op.table, table))) { + lastMatch = len(selected) - 1 + } + } + if lastMatch < 0 { + return nil + } + selected = selected[:lastMatch+1] + + s.mu.Lock() + for _, op := range selected { + if op.executed || op.reserved { + s.mu.Unlock() + return fmt.Errorf("operation %d is already reserved or executed", op.id) + } + op.reserved = true + } + s.mu.Unlock() + + for index := 0; index < len(selected); { + group := selected[index : index+1] + key := operationBatchKey(selected[index].value) + if key != "" && selected[index].buffer.executeBatch != nil { + end := index + 1 + for end < len(selected) && selected[end].buffer == selected[index].buffer && operationBatchKey(selected[end].value) == key { + end++ + } + group = selected[index:end] + } + if err = s.executeGroup(ctx, group); err != nil { + s.mu.Lock() + s.failed = err + for _, pending := range selected { + pending.reserved = false + } + s.mu.Unlock() + return err + } + s.mu.Lock() + for _, op := range group { + op.executed = true + op.reserved = false + } + s.mu.Unlock() + index += len(group) + } + return nil +} + +func hasOpenAncestorOfBinding(caller *Frame) bool { + for frame := caller; frame != nil; frame = frame.parent { + if frame.relation != RelationBinding { + continue + } + for ancestor := frame.parent; ancestor != nil; ancestor = ancestor.parent { + if ancestor.open { + return true + } + } + } + return false +} + +func (s *Scope) executeGroup(ctx context.Context, operations []*Operation) error { + if len(operations) == 0 { + return nil + } + buffer := operations[0].buffer + db, err := buffer.resolveDB(ctx) + if err != nil { + return err + } + unit, err := s.database(ctx, db, buffer.externalTx) + if err != nil { + return err + } + unit.mu.Lock() + defer unit.mu.Unlock() + if unit.failed != nil { + return unit.failed + } + if len(operations) > 1 && buffer.executeBatch != nil { + values := make([]any, len(operations)) + for index, operation := range operations { + values[index] = operation.value + } + err = buffer.executeBatch(ctx, unit.tx, values) + } else { + err = buffer.execute(ctx, unit.tx, operations[0].value) + } + if err != nil { + unit.failed = err + return err + } + return nil +} + +func (s *Scope) database(ctx context.Context, db *sql.DB, external *sql.Tx) (*databaseUnit, error) { + if db == nil { + return nil, fmt.Errorf("mutation database is required") + } + s.mu.Lock() + defer s.mu.Unlock() + if existing := s.databases[db]; existing != nil { + if external != nil && existing.tx != external { + return nil, ErrTransactionConflict + } + return existing, nil + } + s.nextDB++ + unit := &databaseUnit{db: db, tx: external, external: external != nil, order: s.nextDB} + if unit.tx == nil { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + unit.tx = tx + } + s.databases[db] = unit + return unit, nil +} + +func (s *Scope) rollbackLocal() error { + s.mu.Lock() + units := make([]*databaseUnit, 0, len(s.databases)) + for _, unit := range s.databases { + units = append(units, unit) + } + s.mu.Unlock() + sort.Slice(units, func(i, j int) bool { return units[i].order > units[j].order }) + var result error + for _, unit := range units { + if !unit.external && unit.tx != nil { + result = errors.Join(result, unit.tx.Rollback()) + } + } + return result +} + +func (s *Scope) commitLocal() error { + s.mu.Lock() + units := make([]*databaseUnit, 0, len(s.databases)) + for _, unit := range s.databases { + units = append(units, unit) + } + s.mu.Unlock() + sort.Slice(units, func(i, j int) bool { return units[i].order < units[j].order }) + for index, unit := range units { + if !unit.external && unit.tx != nil { + if err := unit.tx.Commit(); err != nil { + result := err + for _, pending := range units[index+1:] { + if !pending.external && pending.tx != nil { + result = errors.Join(result, pending.tx.Rollback()) + } + } + return result + } + } + } + return nil +} + +func flatten(frame *Frame) []*Operation { + if frame == nil { + return nil + } + var result []*Operation + for _, entry := range frame.timeline { + if entry.operation != nil { + result = append(result, entry.operation) + } + if entry.child != nil { + result = append(result, flatten(entry.child)...) + } + } + for _, child := range frame.bindings { + result = append(result, flatten(child)...) + } + return result +} + +type tableNamer interface{ TableName() string } +type batchKeyer interface{ BatchKey() string } + +func operationTable(value any) string { + if named, ok := value.(tableNamer); ok { + return named.TableName() + } + return "" +} + +func operationBatchKey(value any) string { + if keyed, ok := value.(batchKeyer); ok { + return keyed.BatchKey() + } + return "" +} + +func sameValue(a, b any) bool { + if a == nil || b == nil { + return a == b + } + av, bv := reflect.ValueOf(a), reflect.ValueOf(b) + if av.Type() != bv.Type() { + return false + } + if av.Kind() == reflect.Ptr || av.Kind() == reflect.Map || av.Kind() == reflect.Slice || av.Kind() == reflect.Func || av.Kind() == reflect.Chan { + return av.Pointer() == bv.Pointer() + } + return av.Type().Comparable() && av.Interface() == bv.Interface() +} diff --git a/service/executor/uow/scope_test.go b/service/executor/uow/scope_test.go new file mode 100644 index 000000000..5e61bc033 --- /dev/null +++ b/service/executor/uow/scope_test.go @@ -0,0 +1,664 @@ +package uow + +import ( + "context" + "database/sql" + "errors" + "reflect" + "sync" + "testing" + + _ "github.com/mattn/go-sqlite3" +) + +type namedOperation string + +func (o namedOperation) TableName() string { return string(o) } + +type sqlOperation struct { + table string + query string +} + +func (o sqlOperation) TableName() string { return o.table } + +type batchOperation struct{ key, name string } + +func (o batchOperation) TableName() string { return o.key } +func (o batchOperation) BatchKey() string { return o.key } + +func TestScopeOrdersParentBeforeBindingAndImperativeAtMarker(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + var order []string + newBuffer := func(frame *Frame) *Buffer { + return frame.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(_ context.Context, _ *sql.Tx, value any) error { + order = append(order, string(value.(namedOperation))) + return nil + }) + } + rootBuffer := newBuffer(root) + if err = rootBuffer.Append(namedOperation("parent-before")); err != nil { + t.Fatal(err) + } + childCtx := PrepareChild(ctx, RelationImperative, "") + _, _, imperative, _, err := Enter(childCtx, "imperative") + if err != nil { + t.Fatal(err) + } + if err = newBuffer(imperative).Append(namedOperation("imperative")); err != nil { + t.Fatal(err) + } + imperative.Seal() + if err = rootBuffer.Append(namedOperation("parent-after")); err != nil { + t.Fatal(err) + } + bindingCtx := PrepareChild(ctx, RelationBinding, "0001") + _, _, binding, _, err := Enter(bindingCtx, "binding") + if err != nil { + t.Fatal(err) + } + if err = newBuffer(binding).Append(namedOperation("binding")); err != nil { + t.Fatal(err) + } + binding.Seal() + root.Seal() + if err = scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + want := []string{"parent-before", "imperative", "parent-after", "binding"} + if !reflect.DeepEqual(order, want) { + t.Fatalf("order=%v want %v", order, want) + } +} + +func TestScopeRejectsBindingFlushWithOpenParent(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, _, _ := NewRoot(context.Background(), "root") + childCtx := PrepareChild(ctx, RelationBinding, "0001") + _, _, child, _, err := Enter(childCtx, "child") + if err != nil { + t.Fatal(err) + } + buffer := child.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, func(context.Context, *sql.Tx, any) error { return nil }) + if err = buffer.Append(namedOperation("child")); err != nil { + t.Fatal(err) + } + if err = buffer.Flush(ctx, "child"); !errors.Is(err, ErrBindingFlush) { + t.Fatalf("err=%v", err) + } +} + +func TestScopeRejectsImperativeDescendantFlushInsideOpenBinding(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, _, _ := NewRoot(context.Background(), "root") + bindingCtx := PrepareChild(ctx, RelationBinding, "0001") + bindingCtx, _, _, _, err := Enter(bindingCtx, "binding") + if err != nil { + t.Fatal(err) + } + imperativeCtx := PrepareChild(bindingCtx, RelationImperative, "") + _, _, imperative, _, err := Enter(imperativeCtx, "imperative") + if err != nil { + t.Fatal(err) + } + buffer := imperative.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(context.Context, *sql.Tx, any) error { return nil }) + if err = buffer.Append(namedOperation("child")); err != nil { + t.Fatal(err) + } + if err = buffer.Flush(ctx, "child"); !errors.Is(err, ErrBindingFlush) { + t.Fatalf("Flush() error=%v", err) + } +} + +func TestScopeCausalFlushIncludesPredecessorsOnce(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + var order []string + buffer := root.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(_ context.Context, _ *sql.Tx, value any) error { + order = append(order, string(value.(namedOperation))) + return nil + }) + for _, operation := range []namedOperation{"parent", "raw", "child"} { + if err := buffer.Append(operation); err != nil { + t.Fatal(err) + } + } + if err := buffer.Flush(ctx, "child"); err != nil { + t.Fatal(err) + } + root.Seal() + if err := scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + if want := []string{"parent", "raw", "child"}; !reflect.DeepEqual(order, want) { + t.Fatalf("order=%v want %v", order, want) + } +} + +func TestScopeBindingOrderUsesReservedDeclarationOrder(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + var order []string + for _, declaration := range []struct { + order string + name namedOperation + }{{"00000002", "third"}, {"00000000", "first"}, {"00000001", "second"}} { + childCtx := PrepareChild(ctx, RelationBinding, declaration.order) + _, _, child, _, err := Enter(childCtx, string(declaration.name)) + if err != nil { + t.Fatal(err) + } + buffer := child.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(_ context.Context, _ *sql.Tx, value any) error { + order = append(order, string(value.(namedOperation))) + return nil + }) + if err = buffer.Append(declaration.name); err != nil { + t.Fatal(err) + } + child.Seal() + } + root.Seal() + if err := scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + if want := []string{"first", "second", "third"}; !reflect.DeepEqual(order, want) { + t.Fatalf("order=%v want %v", order, want) + } +} + +func TestImperativeFlushStopsBeforeBindingChildren(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + var order []string + newBuffer := func(frame *Frame) *Buffer { + return frame.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(_ context.Context, _ *sql.Tx, value any) error { + order = append(order, string(value.(namedOperation))) + return nil + }) + } + bindingCtx := PrepareChild(ctx, RelationBinding, "00000000") + _, _, binding, _, err := Enter(bindingCtx, "binding") + if err != nil { + t.Fatal(err) + } + if err = newBuffer(binding).Append(namedOperation("binding")); err != nil { + t.Fatal(err) + } + binding.Seal() + rootBuffer := newBuffer(root) + if err = rootBuffer.Append(namedOperation("parent")); err != nil { + t.Fatal(err) + } + imperativeCtx := PrepareChild(ctx, RelationImperative, "") + _, _, imperative, _, err := Enter(imperativeCtx, "imperative") + if err != nil { + t.Fatal(err) + } + imperativeBuffer := newBuffer(imperative) + if err = imperativeBuffer.Append(namedOperation("imperative")); err != nil { + t.Fatal(err) + } + if err = imperativeBuffer.Flush(ctx, ""); err != nil { + t.Fatal(err) + } + if want := []string{"parent", "imperative"}; !reflect.DeepEqual(order, want) { + t.Fatalf("flush order=%v want %v", order, want) + } + imperative.Seal() + root.Seal() + if err = scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + if want := []string{"parent", "imperative", "binding"}; !reflect.DeepEqual(order, want) { + t.Fatalf("final order=%v want %v", order, want) + } +} + +func TestScopeCommitsLocalAndRollsBackOnFailure(t *testing.T) { + for _, testCase := range []struct { + name string + cause error + wantCount int + }{{"commit", nil, 1}, {"rollback", errors.New("root failed"), 0}} { + t.Run(testCase.name, func(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + if _, err := db.Exec(`CREATE TABLE mutations (id INTEGER PRIMARY KEY)`); err != nil { + t.Fatal(err) + } + ctx, scope, root := NewRoot(context.Background(), "root") + buffer := root.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(ctx context.Context, tx *sql.Tx, _ any) error { + _, err := tx.ExecContext(ctx, `INSERT INTO mutations(id) VALUES (1)`) + return err + }) + if err := buffer.Append(namedOperation("mutations")); err != nil { + t.Fatal(err) + } + root.Seal() + err := scope.Finish(ctx, testCase.cause) + if testCase.cause == nil && err != nil { + t.Fatal(err) + } + if testCase.cause != nil && !errors.Is(err, testCase.cause) { + t.Fatalf("err=%v", err) + } + var count int + if err = db.QueryRow(`SELECT COUNT(*) FROM mutations`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != testCase.wantCount { + t.Fatalf("count=%d want %d", count, testCase.wantCount) + } + }) + } +} + +func TestScopeLeavesAdoptedTransactionOpen(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + if _, err := db.Exec(`CREATE TABLE mutations (id INTEGER PRIMARY KEY)`); err != nil { + t.Fatal(err) + } + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + ctx, scope, root := NewRoot(context.Background(), "root") + if err = scope.AdoptTransaction(db, tx); err != nil { + t.Fatal(err) + } + buffer := root.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(ctx context.Context, actual *sql.Tx, _ any) error { + if actual != tx { + t.Fatal("scope did not use adopted transaction") + } + _, execErr := actual.ExecContext(ctx, `INSERT INTO mutations(id) VALUES (1)`) + return execErr + }) + if err = buffer.Append(namedOperation("mutations")); err != nil { + t.Fatal(err) + } + root.Seal() + if err = scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(`INSERT INTO mutations(id) VALUES (2)`); err != nil { + t.Fatalf("adopted transaction was completed: %v", err) + } + if err = tx.Rollback(); err != nil { + t.Fatal(err) + } +} + +func TestScopeCampaignFlightForeignKeyOrderAndRollback(t *testing.T) { + for _, testCase := range []struct { + name string + cause error + wantCount int + }{{name: "commit", wantCount: 1}, {name: "rollback", cause: errors.New("parent failed")}} { + t.Run(testCase.name, func(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + if _, err := db.Exec("PRAGMA foreign_keys = ON"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("CREATE TABLE campaign (id INTEGER PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("CREATE TABLE campaign_flight (id INTEGER PRIMARY KEY, campaign_id INTEGER REFERENCES campaign(id))"); err != nil { + t.Fatal(err) + } + ctx, scope, root := NewRoot(context.Background(), "campaign") + newBuffer := func(frame *Frame) *Buffer { + return frame.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(ctx context.Context, tx *sql.Tx, value any) error { + _, err := tx.ExecContext(ctx, value.(sqlOperation).query) + return err + }) + } + bindingCtx := PrepareChild(ctx, RelationBinding, "00000000") + _, _, flight, _, err := Enter(bindingCtx, "flight") + if err != nil { + t.Fatal(err) + } + if err = newBuffer(flight).Append(sqlOperation{table: "campaign_flight", query: "INSERT INTO campaign_flight(id, campaign_id) VALUES (20, 10)"}); err != nil { + t.Fatal(err) + } + flight.Seal() + if err = newBuffer(root).Append(sqlOperation{table: "campaign", query: "INSERT INTO campaign(id) VALUES (10)"}); err != nil { + t.Fatal(err) + } + root.Seal() + err = scope.Finish(ctx, testCase.cause) + if testCase.cause == nil && err != nil { + t.Fatal(err) + } + if testCase.cause != nil && !errors.Is(err, testCase.cause) { + t.Fatalf("Finish() error=%v", err) + } + for _, table := range []string{"campaign", "campaign_flight"} { + var count int + if err = db.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count); err != nil || count != testCase.wantCount { + t.Fatalf("%s count=%d want=%d err=%v", table, count, testCase.wantCount, err) + } + } + }) + } +} + +func TestScopeConcurrentAppendAndRepeatedCompletion(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + var mu sync.Mutex + executed := 0 + buffer := root.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(context.Context, *sql.Tx, any) error { + mu.Lock() + executed++ + mu.Unlock() + return nil + }) + var wait sync.WaitGroup + for i := 0; i < 100; i++ { + wait.Add(1) + go func() { + defer wait.Done() + if err := buffer.Append(namedOperation("audit")); err != nil { + t.Errorf("Append() error=%v", err) + } + }() + } + wait.Wait() + root.Seal() + if err := scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + if executed != 100 { + t.Fatalf("executed=%d", executed) + } + if err := scope.Finish(ctx, nil); !errors.Is(err, ErrCompleted) { + t.Fatalf("second Finish() error=%v", err) + } +} + +func TestScopeAppendDuringReservedFlushRemainsForCompletion(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + started := make(chan struct{}) + release := make(chan struct{}) + var mu sync.Mutex + var order []string + buffer := root.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(_ context.Context, _ *sql.Tx, value any) error { + name := string(value.(namedOperation)) + if name == "first" { + close(started) + <-release + } + mu.Lock() + order = append(order, name) + mu.Unlock() + return nil + }) + if err := buffer.Append(namedOperation("first")); err != nil { + t.Fatal(err) + } + flushErr := make(chan error, 1) + go func() { flushErr <- buffer.Flush(ctx, "first") }() + <-started + if err := buffer.Append(namedOperation("later")); err != nil { + t.Fatal(err) + } + close(release) + if err := <-flushErr; err != nil { + t.Fatal(err) + } + root.Seal() + if err := scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + if want := []string{"first", "later"}; !reflect.DeepEqual(order, want) { + t.Fatalf("order=%v want=%v", order, want) + } +} + +func TestScopeBatchesOnlyContiguousCompatibleOperations(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + var trace []string + buffer := root.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(_ context.Context, _ *sql.Tx, value any) error { + trace = append(trace, "single:"+value.(batchOperation).name) + return nil + }) + buffer.SetBatchExecutor(func(_ context.Context, _ *sql.Tx, values []any) error { + names := "batch" + for _, value := range values { + names += ":" + value.(batchOperation).name + } + trace = append(trace, names) + return nil + }) + for _, operation := range []batchOperation{{"insert:a", "1"}, {"insert:a", "2"}, {"update:a", "3"}, {"insert:a", "4"}} { + if err := buffer.Append(operation); err != nil { + t.Fatal(err) + } + } + root.Seal() + if err := scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + want := []string{"batch:1:2", "single:3", "single:4"} + if !reflect.DeepEqual(trace, want) { + t.Fatalf("trace=%v want=%v", trace, want) + } +} + +func TestBufferReconcilePreservesImperativeMarkers(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + var order []string + newBuffer := func(frame *Frame) *Buffer { + return frame.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(_ context.Context, _ *sql.Tx, value any) error { + order = append(order, string(value.(namedOperation))) + return nil + }) + } + rootBuffer := newBuffer(root) + _ = rootBuffer.Append(namedOperation("old-1")) + imperativeCtx := PrepareChild(ctx, RelationImperative, "") + _, _, imperative, _, err := Enter(imperativeCtx, "imperative") + if err != nil { + t.Fatal(err) + } + _ = newBuffer(imperative).Append(namedOperation("child")) + imperative.Seal() + _ = rootBuffer.Append(namedOperation("old-2")) + if err = rootBuffer.Reconcile([]any{namedOperation("new-1"), namedOperation("new-2")}); err != nil { + t.Fatal(err) + } + root.Seal() + if err = scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + want := []string{"new-1", "child", "new-2"} + if !reflect.DeepEqual(order, want) { + t.Fatalf("order=%v want=%v", order, want) + } +} + +func TestEnterStartsFreshRootButRejectsCapturedCompletedChild(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + childCtx := PrepareChild(ctx, RelationImperative, "") + childCtx, _, child, _, err := Enter(childCtx, "child") + if err != nil { + t.Fatal(err) + } + child.Seal() + root.Seal() + if err = scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } + if _, _, _, _, err = Enter(childCtx, "stale-child"); !errors.Is(err, ErrCompleted) { + t.Fatalf("captured child Enter() error=%v", err) + } + _, fresh, _, created, err := Enter(ctx, "fresh-root") + if err != nil { + t.Fatal(err) + } + if !created || fresh == scope { + t.Fatalf("created=%v fresh scope reused=%v", created, fresh == scope) + } +} + +func TestSealedFrameRejectsReuseAndAppendBeforeRootCompletion(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + childCtx := PrepareChild(ctx, RelationImperative, "") + childCtx, _, child, _, err := Enter(childCtx, "child") + if err != nil { + t.Fatal(err) + } + buffer := child.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(context.Context, *sql.Tx, any) error { return nil }) + child.Seal() + if err = buffer.Append(namedOperation("late")); !errors.Is(err, ErrFrameSealed) { + t.Fatalf("Append() error=%v", err) + } + if _, _, _, _, err = Enter(childCtx, "reused"); !errors.Is(err, ErrFrameSealed) { + t.Fatalf("Enter() error=%v", err) + } + root.Seal() + if err = scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } +} + +func TestFailureIsTerminalForAppendFlushAndFinish(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + wantErr := errors.New("write failed") + buffer := root.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(context.Context, *sql.Tx, any) error { return wantErr }) + if err := buffer.Append(namedOperation("first")); err != nil { + t.Fatal(err) + } + if err := buffer.Flush(ctx, "first"); !errors.Is(err, wantErr) { + t.Fatalf("Flush() error=%v", err) + } + if err := buffer.Append(namedOperation("second")); !errors.Is(err, ErrFailed) || !errors.Is(err, wantErr) { + t.Fatalf("Append() error=%v", err) + } + if err := buffer.Flush(ctx, "first"); !errors.Is(err, ErrFailed) || !errors.Is(err, wantErr) { + t.Fatalf("second Flush() error=%v", err) + } + if err := scope.Finish(ctx, nil); !errors.Is(err, ErrFailed) || !errors.Is(err, wantErr) { + t.Fatalf("Finish() error=%v", err) + } +} + +func TestFlushAndFinishAreSerialized(t *testing.T) { + db, _ := sql.Open("sqlite3", ":memory:") + defer db.Close() + ctx, scope, root := NewRoot(context.Background(), "root") + started := make(chan struct{}) + release := make(chan struct{}) + buffer := root.NewBuffer(func(context.Context) (*sql.DB, error) { return db, nil }, nil, + func(context.Context, *sql.Tx, any) error { + close(started) + <-release + return nil + }) + if err := buffer.Append(namedOperation("first")); err != nil { + t.Fatal(err) + } + flushErr := make(chan error, 1) + finishErr := make(chan error, 1) + go func() { flushErr <- buffer.Flush(ctx, "first") }() + <-started + go func() { finishErr <- scope.Finish(ctx, nil) }() + close(release) + if err := <-flushErr; err != nil { + t.Fatal(err) + } + if err := <-finishErr; err != nil { + t.Fatal(err) + } + if err := buffer.Flush(ctx, "first"); !errors.Is(err, ErrCompleted) { + t.Fatalf("post-completion Flush() error=%v", err) + } +} + +func TestReserveBindingOrderIsUniqueAcrossConcurrentResolvers(t *testing.T) { + ctx, _, _ := NewRoot(context.Background(), "root") + const count = 100 + orders := make(chan string, count) + errs := make(chan error, count) + var wait sync.WaitGroup + for i := 0; i < count; i++ { + wait.Add(1) + go func() { + defer wait.Done() + order, err := ReserveBindingOrder(ctx) + orders <- order + errs <- err + }() + } + wait.Wait() + close(orders) + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + unique := map[string]bool{} + for order := range orders { + if unique[order] { + t.Fatalf("duplicate binding order %q", order) + } + unique[order] = true + } + if len(unique) != count { + t.Fatalf("orders=%d want=%d", len(unique), count) + } +} + +func TestPropagateRetainsDestinationCancellationAndInvocationFrame(t *testing.T) { + source, scope, frame := NewRoot(context.Background(), "root") + destination, cancel := context.WithCancel(context.Background()) + cancel() + propagated := Propagate(source, destination) + gotScope, gotFrame, ok := FromContext(propagated) + if !ok || gotScope != scope || gotFrame != frame { + t.Fatalf("scope propagated=%v scopeMatch=%v frameMatch=%v", ok, gotScope == scope, gotFrame == frame) + } + if !errors.Is(propagated.Err(), context.Canceled) { + t.Fatalf("Err()=%v", propagated.Err()) + } +} diff --git a/service/operator/executor.go b/service/operator/executor.go index 7fbba3ef1..093c52114 100644 --- a/service/operator/executor.go +++ b/service/operator/executor.go @@ -8,6 +8,7 @@ import ( "github.com/viant/datly/repository" "github.com/viant/datly/repository/contract" "github.com/viant/datly/service/executor/handler" + "github.com/viant/datly/service/executor/uow" "github.com/viant/gmetric/counter" xhandler "github.com/viant/xdatly/handler" @@ -17,6 +18,13 @@ import ( // HandlerSession returns a handler session func (s *Service) HandlerSession(ctx context.Context, aComponent *repository.Component, aSession *session.Session) (xhandler.Session, error) { + if _, _, scoped := uow.FromContext(ctx); scoped { + var err error + ctx, _, _, _, err = uow.Enter(ctx, aComponent.Method+" "+aComponent.URI) + if err != nil { + return nil, err + } + } anExecutor := handler.NewExecutor(aComponent.View, aSession) return anExecutor.NewHandlerSession(ctx, handler.WithTypes(aComponent.Types()...), handler.WithAuth(aSession.Auth())) } diff --git a/service/operator/invocation_injector.go b/service/operator/invocation_injector.go new file mode 100644 index 000000000..311ed0813 --- /dev/null +++ b/service/operator/invocation_injector.go @@ -0,0 +1,28 @@ +package operator + +import ( + "context" + + xstate "github.com/viant/xdatly/handler/state" +) + +type invocationInjector struct { + ctx context.Context + delegate xstate.Injector +} + +func (i *invocationInjector) Into(_ context.Context, value interface{}, options ...xstate.Option) error { + return i.delegate.Into(i.ctx, value, options...) +} + +func (i *invocationInjector) Bind(_ context.Context, value interface{}, options ...xstate.Option) error { + return i.delegate.Bind(i.ctx, value, options...) +} + +func (i *invocationInjector) Value(_ context.Context, key string) (interface{}, bool, error) { + return i.delegate.Value(i.ctx, key) +} + +func (i *invocationInjector) ValuesOf(_ context.Context, value interface{}) (map[string]interface{}, error) { + return i.delegate.ValuesOf(i.ctx, value) +} diff --git a/service/operator/service.go b/service/operator/service.go index dc5865ec5..410549d33 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "reflect" + "sync" "time" "github.com/viant/afs" @@ -18,6 +19,7 @@ import ( "github.com/viant/datly/repository/content" "github.com/viant/datly/repository/contract" "github.com/viant/datly/service" + "github.com/viant/datly/service/executor/uow" "github.com/viant/datly/service/reader" "github.com/viant/datly/service/session" "github.com/viant/datly/utils/types" @@ -44,8 +46,28 @@ type Service struct { } // Operate processes data component with data session -func (s *Service) Operate(ctx context.Context, aSession *session.Session, aComponent *repository.Component) (interface{}, error) { - if err := s.updateBackgroundJob(ctx, aComponent); err != nil { +func (s *Service) Operate(ctx context.Context, aSession *session.Session, aComponent *repository.Component) (result interface{}, err error) { + identity := aComponent.Method + " " + aComponent.URI + ctx, scope, frame, owner, err := uow.Enter(ctx, identity) + if err != nil { + return nil, err + } + defer func() { + frame.Seal() + if owner { + err = scope.Finish(ctx, err) + } + }() + if tx := aSession.Options.SqlTx(); tx != nil && aComponent.View != nil && aComponent.View.Connector != nil { + db, dbErr := aComponent.View.Connector.DB() + if dbErr != nil { + return nil, dbErr + } + if err = scope.AdoptTransaction(db, tx); err != nil { + return nil, err + } + } + if err = s.updateBackgroundJob(ctx, aComponent); err != nil { return nil, err } return s.operate(ctx, aComponent, aSession) @@ -132,19 +154,23 @@ func (s *Service) operate(ctx context.Context, aComponent *repository.Component, func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSession *session.Session) (interface{}, error) { if injectorFinalizer, ok := ret.(state.InjectorFinalizer); ok { + var childFramesMu sync.Mutex + var childFrames []*uow.Frame - lookup := func(ctx context.Context, route xhttp.Route) (xstate.Injector, error) { - aComponent, err := aSession.Registry().Lookup(ctx, contract.NewPath(route.Method, route.URL)) + lookup := func(lookupCtx context.Context, route xhttp.Route) (xstate.Injector, error) { + lookupCtx = uow.Propagate(ctx, lookupCtx) + lookupCtx = uow.PrepareChild(lookupCtx, uow.RelationImperative, "") + aComponent, err := aSession.Registry().Lookup(lookupCtx, contract.NewPath(route.Method, route.URL)) if err != nil { return nil, err } - originalRequest, _ := aSession.HttpRequest(ctx, aSession.Clone()) + originalRequest, _ := aSession.HttpRequest(lookupCtx, aSession.Clone()) request, _ := http.NewRequest(route.Method, route.URL, nil) if originalRequest != nil { request.Header = originalRequest.Header } unmarshal := aComponent.UnmarshalFunc(request) - locatorOptions := append(aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal)) + locatorOptions := aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal) childSession := session.New(aComponent.View, session.WithAuth(aSession.Auth()), session.WithLocatorOptions(locatorOptions...), @@ -161,10 +187,23 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes if err := childSession.InitKinds(state.KindComponent, state.KindHeader, state.KindRequestBody, state.KindForm, state.KindQuery); err != nil { return nil, err } - return childSession, nil + childCtx, _, childFrame, _, enterErr := uow.Enter(lookupCtx, route.Method+" "+route.URL) + if enterErr != nil { + return nil, enterErr + } + childFramesMu.Lock() + childFrames = append(childFrames, childFrame) + childFramesMu.Unlock() + childCtx = childSession.Context(childCtx, true) + return &invocationInjector{ctx: childCtx, delegate: childSession}, nil } err = injectorFinalizer.Finalize(ctx, lookup) + childFramesMu.Lock() + for _, childFrame := range childFrames { + childFrame.Seal() + } + childFramesMu.Unlock() if err != nil { return ret, err } @@ -213,21 +252,23 @@ func (s *Service) finalizeMCPOutput(ctx context.Context, ret interface{}, aSessi if !ok { return nil } - getBinder := func(ctx context.Context, route xhttp.Route) (xhandler.Session, error) { + getBinder := func(binderCtx context.Context, route xhttp.Route) (xhandler.Session, error) { + binderCtx = uow.Propagate(ctx, binderCtx) + binderCtx = uow.PrepareChild(binderCtx, uow.RelationImperative, "") if aSession == nil || aSession.Registry() == nil { return nil, fmt.Errorf("session registry unavailable") } - aComponent, err := aSession.Registry().Lookup(ctx, contract.NewPath(route.Method, route.URL)) + aComponent, err := aSession.Registry().Lookup(binderCtx, contract.NewPath(route.Method, route.URL)) if err != nil { return nil, err } - originalRequest, _ := aSession.HttpRequest(ctx, aSession.Clone()) + originalRequest, _ := aSession.HttpRequest(binderCtx, aSession.Clone()) request, _ := http.NewRequest(route.Method, route.URL, nil) if originalRequest != nil { request.Header = originalRequest.Header } unmarshal := aComponent.UnmarshalFunc(request) - locatorOptions := append(aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal)) + locatorOptions := aComponent.LocatorOptions(request, hstate.NewForm(), unmarshal) childSession := session.New(aComponent.View, session.WithAuth(aSession.Auth()), session.WithLocatorOptions(locatorOptions...), @@ -243,7 +284,7 @@ func (s *Service) finalizeMCPOutput(ctx context.Context, ret interface{}, aSessi if err := childSession.InitKinds(state.KindComponent, state.KindHeader, state.KindRequestBody, state.KindForm, state.KindQuery); err != nil { return nil, err } - return s.HandlerSession(ctx, aComponent, childSession) + return s.HandlerSession(binderCtx, aComponent, childSession) } return finalizer.FinalizeMCP(ctx, mcp, getBinder) } diff --git a/service/session/jwt_codec_test.go b/service/session/jwt_codec_test.go new file mode 100644 index 000000000..d39400326 --- /dev/null +++ b/service/session/jwt_codec_test.go @@ -0,0 +1,57 @@ +package session + +import ( + "context" + "testing" + "time" + + jwtv5 "github.com/golang-jwt/jwt/v5" + "github.com/viant/datly/service/auth" + "github.com/viant/datly/service/auth/config" + "github.com/viant/datly/service/auth/mock" + dcodec "github.com/viant/datly/view/extension/codec" + jwtclaims "github.com/viant/scy/auth/jwt" + "github.com/viant/scy/auth/jwt/signer" + "github.com/viant/scy/auth/jwt/verifier" + _ "github.com/viant/scy/kms/blowfish" +) + +func TestSessionCodecOptionsUseConfiguredVerifierRules(t *testing.T) { + ctx := context.Background() + hmacSigner := mock.HmacJwtSigner() + signerService := signer.New(&signer.Config{Rules: []*signer.Rule{{ + Resource: []string{"mcp"}, + Algorithm: "HS256", + HMAC: hmacSigner.HMAC, + }}}) + if err := signerService.Init(ctx); err != nil { + t.Fatal(err) + } + verifierConfig := &verifier.Config{Rules: []*verifier.Rule{{ + Resource: []string{"mcp"}, + Algorithm: "HS256", + HMAC: mock.HmacJwtVerifier().HMAC, + }}} + authService := auth.New(&config.Config{JWTValidator: verifierConfig}) + if err := authService.Init(ctx); err != nil { + t.Fatal(err) + } + token, err := signerService.Create(time.Hour, &jwtclaims.Claims{ + UserID: 73, + RegisteredClaims: jwtv5.RegisteredClaims{ + Audience: jwtv5.ClaimStrings{"mcp"}, + }, + }) + if err != nil { + t.Fatal(err) + } + options := NewOptions(WithAuth(authService)) + actual, err := (&dcodec.JwtClaim{}).Value(ctx, token, options.codecOptionsWithAuth()...) + if err != nil { + t.Fatalf("JwtClaim.Value() failed to use session verifier: %v", err) + } + claims, ok := actual.(*jwtclaims.Claims) + if !ok || claims.UserID != 73 { + t.Fatalf("claims=%T %+v", actual, actual) + } +} diff --git a/service/session/state.go b/service/session/state.go index 6911045d0..97d28c21e 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -16,8 +16,10 @@ import ( "github.com/viant/datly/internal/converter" "github.com/viant/datly/repository" "github.com/viant/datly/service/auth" + "github.com/viant/datly/service/executor/uow" "github.com/viant/datly/utils/types" "github.com/viant/datly/view" + dcodec "github.com/viant/datly/view/extension/codec" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind/locator" "github.com/viant/datly/view/tags" @@ -278,8 +280,12 @@ func (s *Session) SetState(ctx context.Context, parameters state.Parameters, aSt if parameter.Scope != opts.scope { continue } + order, reserveErr := uow.ReserveBindingOrder(ctx) + if reserveErr != nil { + return reserveErr + } wg.Add(1) - go s.populateParameterInBackground(ctx, parameter, aState, opts, err, &wg) + go s.populateParameterInBackground(ctx, parameter, aState, opts, err, &wg, order) } wg.Wait() if err.HasError() { @@ -289,8 +295,9 @@ func (s *Session) SetState(ctx context.Context, parameters state.Parameters, aSt return nil } -func (s *Session) populateParameterInBackground(ctx context.Context, parameter *state.Parameter, aState *structology.State, options *Options, errors *response.Errors, wg *sync.WaitGroup) { +func (s *Session) populateParameterInBackground(ctx context.Context, parameter *state.Parameter, aState *structology.State, options *Options, errors *response.Errors, wg *sync.WaitGroup, order string) { defer wg.Done() + ctx = uow.WithBindingOrder(ctx, order) if err := s.populateParameter(ctx, parameter, aState, options); err != nil { s.handleParameterError(parameter, err, errors) } @@ -802,7 +809,7 @@ func (s *Session) adjustAndCache(ctx context.Context, parameter *state.Parameter return nil, false, initErr } } - transformed, err := parameter.Output.Transform(ctx, value, opts.codecOptions...) + transformed, err := parameter.Output.Transform(ctx, value, opts.codecOptionsWithAuth()...) if err != nil { return nil, false, fmt.Errorf("failed to transform %s with %s: %v, %w", parameter.Name, parameter.Output.Name, value, err) } @@ -817,6 +824,14 @@ func (s *Session) adjustAndCache(ctx context.Context, parameter *state.Parameter return value, has, err } +func (o *Options) codecOptionsWithAuth() []codec.Option { + result := append([]codec.Option(nil), o.codecOptions...) + if o.auth != nil && o.auth.Verifier() != nil { + result = append(result, dcodec.WithJWTVerifier(o.auth.Verifier())) + } + return result +} + // SetValue sets value to session cache func (s *Session) setValue(parameter *state.Parameter, value interface{}) { s.cache.put(parameter, value) diff --git a/view/extension/codec/jwt.go b/view/extension/codec/jwt.go index 0352d22ef..20e647062 100644 --- a/view/extension/codec/jwt.go +++ b/view/extension/codec/jwt.go @@ -4,13 +4,15 @@ import ( "context" "encoding/base64" "fmt" - "github.com/viant/scy/auth/gcp" - "github.com/viant/scy/auth/jwt" - "github.com/viant/xdatly/codec" "reflect" "strings" "sync" "time" + + "github.com/viant/scy/auth/gcp" + "github.com/viant/scy/auth/jwt" + "github.com/viant/scy/auth/jwt/verifier" + "github.com/viant/xdatly/codec" ) const ( @@ -26,28 +28,60 @@ type ( } JwtCache struct { - entries map[string]*JwtEntry + entries map[jwtCacheKey]*JwtEntry mux sync.RWMutex } JwtClaim struct{} + + jwtCacheKey struct { + token string + verifier *verifier.Service + } + + jwtVerifierOption struct { + service *verifier.Service + } ) +// WithJWTVerifier makes the session-configured verifier available to JwtClaim. +// It appends to the codec's untyped extension options without replacing other +// options installed by the view/session pipeline. +func WithJWTVerifier(service *verifier.Service) codec.Option { + return func(options *codec.Options) { + if service != nil { + options.Options = append(options.Options, &jwtVerifierOption{service: service}) + } + } +} + +// Put retains the legacy GCP-cache API. Configured verifier paths use the +// verifier-scoped private variant below. func (j *JwtCache) Put(token string, claims *jwt.Claims) { - if !claims.VerifyExpiresAt(time.Now(), true) { + j.put(token, nil, claims) +} + +func (j *JwtCache) put(token string, service *verifier.Service, claims *jwt.Claims) { + if claims == nil || !claims.VerifyExpiresAt(time.Now(), true) { return } j.mux.Lock() defer j.mux.Unlock() if len(j.entries) > 100 { - j.entries = map[string]*JwtEntry{} + j.entries = map[jwtCacheKey]*JwtEntry{} } - j.entries[token] = &JwtEntry{Token: token, Claims: claims} + j.entries[jwtCacheKey{token: token, verifier: service}] = &JwtEntry{Token: token, Claims: claims} } +// Lookup retains the legacy GCP-cache API. func (j *JwtCache) Lookup(token string) *jwt.Claims { + return j.lookup(token, nil) +} + +func (j *JwtCache) lookup(token string, service *verifier.Service) *jwt.Claims { + key := jwtCacheKey{token: token, verifier: service} j.mux.RLock() - entry, ok := j.entries[token] + entry, ok := j.entries[key] j.mux.RUnlock() if !ok { return nil @@ -56,12 +90,12 @@ func (j *JwtCache) Lookup(token string) *jwt.Claims { return entry.Claims } j.mux.Lock() - delete(j.entries, token) + delete(j.entries, key) defer j.mux.Unlock() return nil } -var jwtCache = &JwtCache{entries: map[string]*JwtEntry{}} +var jwtCache = &JwtCache{entries: map[jwtCacheKey]*JwtEntry{}} func (j *JwtClaim) ResultType(paramType reflect.Type) (reflect.Type, error) { return reflect.TypeOf(&jwt.Claims{}), nil @@ -79,13 +113,30 @@ func (j *JwtClaim) Value(ctx context.Context, raw interface{}, options ...codec. if decoded, err := base64.StdEncoding.DecodeString(rawString); err == nil { data = string(decoded) } - if claim := jwtCache.Lookup(data); claim != nil { + service := jwtVerifier(options) + if claim := jwtCache.lookup(data, service); claim != nil { return claim, nil } - info, err := gcp.JwtClaims(ctx, data) + var info *jwt.Claims + var err error + if service != nil { + info, err = service.VerifyClaims(ctx, data) + } else { + info, err = gcp.JwtClaims(ctx, data) + } if err != nil { return nil, err } - jwtCache.Put(data, info) + jwtCache.put(data, service, info) return info, nil } + +func jwtVerifier(options []codec.Option) *verifier.Service { + codecOptions := codec.NewOptions(options) + for index := len(codecOptions.Options) - 1; index >= 0; index-- { + if candidate, ok := codecOptions.Options[index].(*jwtVerifierOption); ok { + return candidate.service + } + } + return nil +} diff --git a/view/extension/codec/jwt_test.go b/view/extension/codec/jwt_test.go new file mode 100644 index 000000000..b55692ba8 --- /dev/null +++ b/view/extension/codec/jwt_test.go @@ -0,0 +1,29 @@ +package codec + +import ( + "testing" + "time" + + jwtv5 "github.com/golang-jwt/jwt/v5" + "github.com/viant/scy/auth/jwt" + "github.com/viant/scy/auth/jwt/verifier" +) + +func TestJwtCacheIsScopedByVerifier(t *testing.T) { + first := verifier.New(nil) + second := verifier.New(nil) + claims := &jwt.Claims{RegisteredClaims: jwtv5.RegisteredClaims{ + ExpiresAt: jwtv5.NewNumericDate(time.Now().Add(time.Hour)), + }} + cache := &JwtCache{entries: map[jwtCacheKey]*JwtEntry{}} + cache.put("token", first, claims) + if actual := cache.lookup("token", first); actual != claims { + t.Fatalf("same-verifier lookup=%p want=%p", actual, claims) + } + if actual := cache.lookup("token", second); actual != nil { + t.Fatalf("cross-verifier lookup=%p", actual) + } + if actual := cache.lookup("token", nil); actual != nil { + t.Fatalf("GCP fallback lookup reused configured-verifier entry: %p", actual) + } +} diff --git a/view/state/kind/locator/repeated.go b/view/state/kind/locator/repeated.go index 2f19e91ae..17dd05421 100644 --- a/view/state/kind/locator/repeated.go +++ b/view/state/kind/locator/repeated.go @@ -7,6 +7,7 @@ import ( "sync" "sync/atomic" + "github.com/viant/datly/service/executor/uow" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind" "github.com/viant/xunsafe" @@ -67,7 +68,8 @@ func (p *Repeated) getRepeatedItems(ctx context.Context, parameter *state.Parame go func(index int, item *state.Parameter) { defer wg.Done() anEntry := &temp[index] - if anEntry.value, anEntry.has, anEntry.err = p.ParameterLookup(ctx, item); anEntry.has || anEntry.err != nil { + itemCtx := uow.WithBindingOrderIndex(ctx, index) + if anEntry.value, anEntry.has, anEntry.err = p.ParameterLookup(itemCtx, item); anEntry.has || anEntry.err != nil { atomic.AddInt32(&hasCount, 1) } }(i, parameter.Repeated[i]) diff --git a/view/state/kind/locator/repeated_uow_test.go b/view/state/kind/locator/repeated_uow_test.go new file mode 100644 index 000000000..4032211e1 --- /dev/null +++ b/view/state/kind/locator/repeated_uow_test.go @@ -0,0 +1,37 @@ +package locator + +import ( + "context" + "reflect" + "sync" + "testing" + + "github.com/viant/datly/service/executor/uow" + "github.com/viant/datly/view/state" +) + +func TestRepeatedExtendsBindingOrderByAuthoredIndex(t *testing.T) { + items := state.Parameters{{Name: "first"}, {Name: "second"}, {Name: "third"}} + parameter := &state.Parameter{Repeated: items} + orders := map[string]string{} + var mu sync.Mutex + locator := &Repeated{ParameterLookup: func(ctx context.Context, item *state.Parameter) (interface{}, bool, error) { + mu.Lock() + orders[item.Name] = uow.BindingOrder(ctx) + mu.Unlock() + return item.Name, true, nil + }} + ctx := uow.WithBindingOrder(context.Background(), "parent") + entries, count := locator.getRepeatedItems(ctx, parameter) + if count != 3 || len(entries) != 3 { + t.Fatalf("count=%d entries=%d", count, len(entries)) + } + want := map[string]string{ + "first": "parent/00000000000000000001", + "second": "parent/00000000000000000002", + "third": "parent/00000000000000000003", + } + if !reflect.DeepEqual(orders, want) { + t.Fatalf("orders=%v want=%v", orders, want) + } +} From 18de0463411a2a6ebcc7b631af573509b2288f46 Mon Sep 17 00:00:00 2001 From: adranwit Date: Wed, 22 Jul 2026 09:48:39 +0200 Subject: [PATCH 266/279] - refactored global transaction handling --- service/executor/extension/options.go | 6 ++ service/executor/extension/session.go | 12 +++ service/executor/handler/executor.go | 76 +++++++++++++++- service/executor/handler/http.go | 10 +++ .../handler/transaction_scope_test.go | 88 +++++++++++++++++++ service/executor/sequencer/service.go | 9 +- service/executor/uow/scope.go | 53 +++++++++++ service/executor/uow/scope_test.go | 32 +++++++ service/operator/invocation_injector.go | 37 +++++++- service/operator/service.go | 39 ++++---- 10 files changed, 327 insertions(+), 35 deletions(-) diff --git a/service/executor/extension/options.go b/service/executor/extension/options.go index 2e2511083..c98f11976 100644 --- a/service/executor/extension/options.go +++ b/service/executor/extension/options.go @@ -1,5 +1,11 @@ package extension +func WithTransaction(fn TransactionFn) Option { + return func(s *Session) { + s.transaction = fn + } +} + // Option represen session option type Option func(s *Session) diff --git a/service/executor/extension/session.go b/service/executor/extension/session.go index 9fbfe51fc..7e94a7213 100644 --- a/service/executor/extension/session.go +++ b/service/executor/extension/session.go @@ -2,6 +2,7 @@ package extension import ( "context" + "database/sql" "sync" "github.com/viant/cloudless/async/mbus" @@ -30,6 +31,7 @@ type ( http HttpFn auth AuthFn logger logger.Logger + transaction TransactionFn } SqlServiceFn func(options *sqlx.Options) (sqlx.Sqlx, error) @@ -38,8 +40,18 @@ type ( RouterFn func(ctx context.Context, route *http.Route) (handler.Session, error) HttpFn func() http.Http AuthFn func() hauth.Auth + TransactionFn func(context.Context) (*sql.Tx, error) ) +// Transaction returns the transaction owned by the current Datly invocation. +// Commit and rollback remain the responsibility of the root invocation. +func (s *Session) Transaction(ctx context.Context) (*sql.Tx, error) { + if s.transaction == nil { + return nil, nil + } + return s.transaction(ctx) +} + func (s *Session) Session(ctx context.Context, route *http.Route, opts ...state.Option) (handler.Session, error) { return s.redirect(ctx, route, opts...) } diff --git a/service/executor/handler/executor.go b/service/executor/handler/executor.go index b6e34b990..5ed1d455f 100644 --- a/service/executor/handler/executor.go +++ b/service/executor/handler/executor.go @@ -160,9 +160,10 @@ func (e *Executor) newSession(aSession *session.Session, opts ...Option) *extens extension.WithTemplateFlush(func(ctx context.Context) error { return e.flushTemplate(ctx) }), - extension.WithStater(aSession), + extension.WithStater(&sessionInjector{executor: e, session: aSession}), extension.WithRedirect(e.redirect), extension.WithSql(e.newSqlService), + extension.WithTransaction(e.transaction), extension.WithHttp(e.newHttp), extension.WithLogger(e.logger), extension.WithAuth(e.newAuth), @@ -171,6 +172,62 @@ func (e *Executor) newSession(aSession *session.Session, opts ...Option) *extens return sess } +// sessionInjector restores the invocation context captured by the handler +// session before delegating state operations. Public handler APIs accept a +// context supplied by the caller, which can still identify the parent frame; +// component binding must run in the child frame owned by this session. +type sessionInjector struct { + executor *Executor + session *session.Session +} + +func (i *sessionInjector) context(ctx context.Context) context.Context { + if i.executor != nil { + ctx = i.executor.invocationContext(ctx) + } + if i.session != nil { + ctx = i.session.Context(ctx, true) + } + return ctx +} + +// invocationContext keeps public handler facades on the component frame that +// created them. Legacy handlers commonly pass their outer request context to a +// child session's Stater or Http facade; that context must not move the child +// executor back to the parent frame. +func (e *Executor) invocationContext(ctx context.Context) context.Context { + if e == nil { + return ctx + } + return uow.Propagate(e.ctx, ctx) +} + +func (i *sessionInjector) Into(ctx context.Context, value interface{}, opts ...hstate.Option) error { + return i.session.Into(i.context(ctx), value, opts...) +} + +func (i *sessionInjector) Bind(ctx context.Context, value interface{}, opts ...hstate.Option) error { + return i.session.Bind(i.context(ctx), value, opts...) +} + +func (i *sessionInjector) Value(ctx context.Context, key string) (interface{}, bool, error) { + return i.session.Value(i.context(ctx), key) +} + +func (i *sessionInjector) ValuesOf(ctx context.Context, value interface{}) (map[string]interface{}, error) { + return i.session.ValuesOf(i.context(ctx), value) +} + +func (e *Executor) transaction(ctx context.Context) (*sql.Tx, error) { + e.unitMu.Lock() + buffer := e.buffers[e.dataUnit] + e.unitMu.Unlock() + if buffer == nil { + return nil, fmt.Errorf("invocation transaction is unavailable") + } + return buffer.Transaction(ctx) +} + func (e *Executor) newValidator() *validator.Service { return validator.New(&Validator{ validator: expand.CommonValidator(), @@ -392,17 +449,30 @@ func (e *Executor) bufferFor(unit *expand.DataUnit) *uow.Buffer { } func (e *Executor) ensureUnitOfWork(ctx context.Context) error { - e.ctx = ctx scope, frame, ok := uow.FromContext(ctx) if !ok { + // Backward-compatible callers may invoke a returned handler session with + // an unscoped context. Once this executor belongs to a unit of work, keep + // its captured invocation context so subsequent nested dispatches remain + // children of this component rather than falling back to an ancestor. + if e.scope != nil { + return e.getBufferErr() + } + e.ctx = ctx return nil } if e.scope != nil { if e.scope != scope || e.frame != frame { - return fmt.Errorf("executor mutation scope mismatch") + viewName := "" + if e.view != nil { + viewName = e.view.Name + } + return fmt.Errorf("executor mutation scope mismatch: view=%s executor scope=%p frame=%s, context scope=%p frame=%s", viewName, e.scope, e.frame.DebugLabel(), scope, frame.DebugLabel()) } + e.ctx = ctx return e.getBufferErr() } + e.ctx = ctx e.scope, e.frame = scope, frame if e.tx == nil && e.session != nil { e.tx = e.session.Options.SqlTx() diff --git a/service/executor/handler/http.go b/service/executor/handler/http.go index 0a8e6849b..1ded672c4 100644 --- a/service/executor/handler/http.go +++ b/service/executor/handler/http.go @@ -26,6 +26,7 @@ type ( ) func (h *Httper) rawRequest(ctx context.Context, opts ...state.Option) (*http.Request, error) { + ctx = h.invocationContext(ctx) aSession, err := h.executor.Session(ctx) if err != nil { return nil, err @@ -70,6 +71,7 @@ func (h *Httper) NewRequest(ctx context.Context, opts ...hstate.Option) (*http.R } func (h *Httper) Redirect(ctx context.Context, route *dhttp.Route, request *http.Request) error { + ctx = h.invocationContext(ctx) aSession, err := h.executor.Session(ctx) if err != nil { return err @@ -86,6 +88,7 @@ func (h *Httper) FailWithCode(statusCode int, err error) error { } func (h *Httper) buildRequestOptions(ctx context.Context, params []*state.Parameter) ([]hstate.Option, error) { + ctx = h.invocationContext(ctx) aSession, err := h.executor.Session(ctx) if err != nil { return nil, err @@ -179,6 +182,13 @@ func (h *Httper) buildRequestOptions(ctx context.Context, params []*state.Parame return opts, nil } +func (h *Httper) invocationContext(ctx context.Context) context.Context { + if h == nil || h.executor == nil { + return ctx + } + return h.executor.invocationContext(ctx) +} + func mergeOptionsIntoRequest(req *http.Request, opts *hstate.Options) { // 1. Replace path parameters in the URL req.URL.Path = replacePathParams(req.URL.Path, opts.PathParameters()) diff --git a/service/executor/handler/transaction_scope_test.go b/service/executor/handler/transaction_scope_test.go index 46c55209f..764e12d7f 100644 --- a/service/executor/handler/transaction_scope_test.go +++ b/service/executor/handler/transaction_scope_test.go @@ -2,6 +2,7 @@ package handler import ( "context" + "database/sql" "errors" "testing" @@ -12,6 +13,52 @@ import ( xsqlx "github.com/viant/xdatly/handler/sqlx" ) +func TestHandlerSessionProvidesRootOwnedTransaction(t *testing.T) { + connector := view.NewConnector("main", "sqlite3", t.TempDir()+"/provider.db") + db, err := connector.DB() + if err != nil { + t.Fatal(err) + } + defer db.Close() + aView := &view.View{Connector: connector} + aView.SetResource(&view.Resource{}) + aSession := session.New(aView) + ctx, scope, root := uow.NewRoot(context.Background(), "root") + executor := NewExecutor(aView, aSession) + handlerSession, err := executor.NewHandlerSession(ctx) + if err != nil { + t.Fatal(err) + } + provider, ok := handlerSession.(interface { + Transaction(context.Context) (*sql.Tx, error) + }) + if !ok { + t.Fatal("handler session does not expose transaction capability") + } + tx, err := provider.Transaction(ctx) + if err != nil { + t.Fatal(err) + } + if tx == nil { + t.Fatal("expected invocation transaction") + } + service, err := executor.newSqlService(&xsqlx.Options{}) + if err != nil { + t.Fatal(err) + } + bufferTx, err := service.(*Service).buffer.Transaction(ctx) + if err != nil { + t.Fatal(err) + } + if bufferTx != tx { + t.Fatal("session capability and mutation buffer use different transactions") + } + root.Seal() + if err = scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } +} + func TestScopedSQLServiceRejectsConflictingTransaction(t *testing.T) { connector := view.NewConnector("main", "sqlite3", t.TempDir()+"/conflict.db") db, err := connector.DB() @@ -48,6 +95,47 @@ func TestScopedSQLServiceRejectsConflictingTransaction(t *testing.T) { } } +func TestScopedExecutorRetainsInvocationContextForLegacyUnscopedCall(t *testing.T) { + aView := &view.View{Name: "patch"} + aSession := session.New(aView) + ctx, _, frame := uow.NewRoot(context.Background(), "PATCH /component") + executor := NewExecutor(aView, aSession) + if err := executor.ensureUnitOfWork(ctx); err != nil { + t.Fatal(err) + } + + if err := executor.ensureUnitOfWork(context.Background()); err != nil { + t.Fatal(err) + } + _, captured, ok := uow.FromContext(executor.ctx) + if !ok { + t.Fatal("scoped executor lost its invocation context") + } + if captured != frame { + t.Fatalf("captured frame=%s, want %s", captured.DebugLabel(), frame.DebugLabel()) + } +} + +func TestHandlerHttpFacadeRestoresChildInvocationContext(t *testing.T) { + parentCtx, _, _ := uow.NewRoot(context.Background(), "PATCH /parent") + childCtx := uow.PrepareChild(parentCtx, uow.RelationImperative, "") + childCtx, _, childFrame, _, err := uow.Enter(childCtx, "GET /child") + if err != nil { + t.Fatal(err) + } + executor := &Executor{ctx: childCtx} + httpFacade := &Httper{executor: executor} + + restored := httpFacade.invocationContext(parentCtx) + _, restoredFrame, ok := uow.FromContext(restored) + if !ok { + t.Fatal("HTTP facade lost the child invocation context") + } + if restoredFrame != childFrame { + t.Fatalf("restored frame=%s, want %s", restoredFrame.DebugLabel(), childFrame.DebugLabel()) + } +} + func TestScopedSQLServiceDoesNotExposeRootTransaction(t *testing.T) { connector := view.NewConnector("main", "sqlite3", t.TempDir()+"/access.db") db, err := connector.DB() diff --git a/service/executor/sequencer/service.go b/service/executor/sequencer/service.go index 04b90708b..6c40c6342 100644 --- a/service/executor/sequencer/service.go +++ b/service/executor/sequencer/service.go @@ -45,14 +45,7 @@ func (s *Service) next(table string, any interface{}, selector string) error { if err != nil { return err } - strategy := dialect.PresetIDWithTransientTransaction - if s.tx != nil { - // The transient strategy opens and completes its own transaction on - // some products (notably MySQL). Invocation mode must remain inside - // the root transaction, so use the transaction-aware MAX strategy. - strategy = dialect.PresetIDWithMax - } - options := []option.Option{strategy} + options := []option.Option{dialect.PresetIDWithTransientTransaction} if s.tx != nil { options = append(options, s.tx) } diff --git a/service/executor/uow/scope.go b/service/executor/uow/scope.go index 9612c6263..fbd4bb0a5 100644 --- a/service/executor/uow/scope.go +++ b/service/executor/uow/scope.go @@ -66,6 +66,15 @@ type Frame struct { nextBinding uint64 } +// DebugLabel identifies a frame in diagnostics without exposing its mutable +// timeline or ownership internals. +func (f *Frame) DebugLabel() string { + if f == nil { + return "" + } + return fmt.Sprintf("%d:%s", f.id, f.name) +} + type timelineEntry struct { operation *Operation child *Frame @@ -143,6 +152,16 @@ func PrepareChild(ctx context.Context, relation Relation, order string) context. if !ok { return ctx } + // A dispatcher/session may outlive the component frame in which it was + // created. A subsequent explicit child dispatch is a sibling of that sealed + // invocation, not an attempt to reopen it. Attach it to the nearest open + // ancestor so repeated imperative dispatch remains ordered in the shared + // scope. Enter called directly with the stale context still rejects reuse. + scope.mu.Lock() + for frame != nil && !frame.open && frame.parent != nil { + frame = frame.parent + } + scope.mu.Unlock() return context.WithValue(ctx, contextKey{}, &carrier{scope: scope, frame: frame, relation: relation, order: order}) } @@ -446,6 +465,40 @@ func (b *Buffer) UseTransaction(ctx context.Context, fn func(*sql.Tx) error) err return err } +// Transaction returns the database transaction owned by the invocation scope. +// The caller may issue work through it, but must not commit or roll it back; +// completion remains the responsibility of the root scope. +func (b *Buffer) Transaction(ctx context.Context) (*sql.Tx, error) { + if b == nil || b.scope == nil { + return nil, fmt.Errorf("mutation buffer is not configured") + } + b.scope.mu.Lock() + if b.scope.completed { + b.scope.mu.Unlock() + return nil, ErrCompleted + } + if b.scope.failed != nil { + err := b.scope.failed + b.scope.mu.Unlock() + return nil, errors.Join(ErrFailed, err) + } + b.scope.mu.Unlock() + db, err := b.resolveDB(ctx) + if err != nil { + return nil, err + } + unit, err := b.scope.database(ctx, db, b.externalTx) + if err != nil { + return nil, err + } + unit.mu.Lock() + defer unit.mu.Unlock() + if unit.failed != nil { + return nil, errors.Join(ErrFailed, unit.failed) + } + return unit.tx, nil +} + // Finish drains and completes locally owned transactions at the root boundary. func (s *Scope) Finish(ctx context.Context, cause error) error { if s == nil { diff --git a/service/executor/uow/scope_test.go b/service/executor/uow/scope_test.go index 5e61bc033..3bea921a7 100644 --- a/service/executor/uow/scope_test.go +++ b/service/executor/uow/scope_test.go @@ -558,6 +558,38 @@ func TestSealedFrameRejectsReuseAndAppendBeforeRootCompletion(t *testing.T) { } } +func TestPrepareChildFromSealedFrameUsesOpenAncestor(t *testing.T) { + ctx, scope, root := NewRoot(context.Background(), "root") + firstCtx := PrepareChild(ctx, RelationImperative, "") + firstCtx, _, first, _, err := Enter(firstCtx, "first") + if err != nil { + t.Fatal(err) + } + first.Seal() + + // The captured child context is stale, but PrepareChild explicitly starts a + // new invocation. It must create a sibling under the still-open root. + secondCtx := PrepareChild(firstCtx, RelationImperative, "") + _, gotScope, second, _, err := Enter(secondCtx, "second") + if err != nil { + t.Fatal(err) + } + if gotScope != scope { + t.Fatal("new child did not retain the invocation scope") + } + if second.parent != root { + t.Fatalf("second parent=%v, want root", second.parent) + } + if second == first { + t.Fatal("sealed frame was reused") + } + second.Seal() + root.Seal() + if err = scope.Finish(ctx, nil); err != nil { + t.Fatal(err) + } +} + func TestFailureIsTerminalForAppendFlushAndFinish(t *testing.T) { db, _ := sql.Open("sqlite3", ":memory:") defer db.Close() diff --git a/service/operator/invocation_injector.go b/service/operator/invocation_injector.go index 311ed0813..046ec60c3 100644 --- a/service/operator/invocation_injector.go +++ b/service/operator/invocation_injector.go @@ -3,26 +3,55 @@ package operator import ( "context" + "github.com/viant/datly/service/executor/uow" xstate "github.com/viant/xdatly/handler/state" ) type invocationInjector struct { ctx context.Context + name string delegate xstate.Injector } func (i *invocationInjector) Into(_ context.Context, value interface{}, options ...xstate.Option) error { - return i.delegate.Into(i.ctx, value, options...) + return i.invoke(func(ctx context.Context) error { + return i.delegate.Into(ctx, value, options...) + }) } func (i *invocationInjector) Bind(_ context.Context, value interface{}, options ...xstate.Option) error { - return i.delegate.Bind(i.ctx, value, options...) + return i.invoke(func(ctx context.Context) error { + return i.delegate.Bind(ctx, value, options...) + }) } func (i *invocationInjector) Value(_ context.Context, key string) (interface{}, bool, error) { - return i.delegate.Value(i.ctx, key) + var value interface{} + var ok bool + err := i.invoke(func(ctx context.Context) error { + var err error + value, ok, err = i.delegate.Value(ctx, key) + return err + }) + return value, ok, err } func (i *invocationInjector) ValuesOf(_ context.Context, value interface{}) (map[string]interface{}, error) { - return i.delegate.ValuesOf(i.ctx, value) + var result map[string]interface{} + err := i.invoke(func(ctx context.Context) error { + var err error + result, err = i.delegate.ValuesOf(ctx, value) + return err + }) + return result, err +} + +func (i *invocationInjector) invoke(fn func(context.Context) error) error { + ctx := uow.PrepareChild(i.ctx, uow.RelationImperative, "") + ctx, _, frame, _, err := uow.Enter(ctx, i.name) + if err != nil { + return err + } + defer frame.Seal() + return fn(ctx) } diff --git a/service/operator/service.go b/service/operator/service.go index 410549d33..f6d87a66e 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -8,7 +8,6 @@ import ( "fmt" "net/http" "reflect" - "sync" "time" "github.com/viant/afs" @@ -48,12 +47,18 @@ type Service struct { // Operate processes data component with data session func (s *Service) Operate(ctx context.Context, aSession *session.Session, aComponent *repository.Component) (result interface{}, err error) { identity := aComponent.Method + " " + aComponent.URI + _, previousFrame, hadFrame := uow.FromContext(ctx) ctx, scope, frame, owner, err := uow.Enter(ctx, identity) if err != nil { return nil, err } + createdFrame := !hadFrame || previousFrame != frame defer func() { - frame.Seal() + // A backward-compatible nested Operate call can arrive without an + // explicit child marker and therefore borrow the current frame. Only the + // operation that created a frame may seal it; otherwise a nested read can + // close the root while its handler still has mutations to dispatch. + sealCreatedFrame(createdFrame, frame) if owner { err = scope.Finish(ctx, err) } @@ -73,6 +78,12 @@ func (s *Service) Operate(ctx context.Context, aSession *session.Session, aCompo return s.operate(ctx, aComponent, aSession) } +func sealCreatedFrame(created bool, frame *uow.Frame) { + if created { + frame.Seal() + } +} + // HandleError processes output with error func (s *Service) HandleError(ctx context.Context, aSession *session.Session, aComponent *repository.Component, err error) (interface{}, error) { ctx = vcontext.WithValue(ctx, exec.ErrorKey, err) @@ -154,12 +165,8 @@ func (s *Service) operate(ctx context.Context, aComponent *repository.Component, func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSession *session.Session) (interface{}, error) { if injectorFinalizer, ok := ret.(state.InjectorFinalizer); ok { - var childFramesMu sync.Mutex - var childFrames []*uow.Frame - lookup := func(lookupCtx context.Context, route xhttp.Route) (xstate.Injector, error) { lookupCtx = uow.Propagate(ctx, lookupCtx) - lookupCtx = uow.PrepareChild(lookupCtx, uow.RelationImperative, "") aComponent, err := aSession.Registry().Lookup(lookupCtx, contract.NewPath(route.Method, route.URL)) if err != nil { return nil, err @@ -187,23 +194,15 @@ func (s *Service) finalize(ctx context.Context, ret interface{}, err error, aSes if err := childSession.InitKinds(state.KindComponent, state.KindHeader, state.KindRequestBody, state.KindForm, state.KindQuery); err != nil { return nil, err } - childCtx, _, childFrame, _, enterErr := uow.Enter(lookupCtx, route.Method+" "+route.URL) - if enterErr != nil { - return nil, enterErr - } - childFramesMu.Lock() - childFrames = append(childFrames, childFrame) - childFramesMu.Unlock() - childCtx = childSession.Context(childCtx, true) - return &invocationInjector{ctx: childCtx, delegate: childSession}, nil + childCtx := childSession.Context(lookupCtx, true) + return &invocationInjector{ + ctx: childCtx, + name: route.Method + " " + route.URL, + delegate: childSession, + }, nil } err = injectorFinalizer.Finalize(ctx, lookup) - childFramesMu.Lock() - for _, childFrame := range childFrames { - childFrame.Seal() - } - childFramesMu.Unlock() if err != nil { return ret, err } From b8408a99c403b95a95e78de23541b697d227f25e Mon Sep 17 00:00:00 2001 From: arao Date: Wed, 22 Jul 2026 16:43:08 -0700 Subject: [PATCH 267/279] TX-00001: error handling changes --- service/operator/service.go | 24 ++++++++- service/operator/service_operation_test.go | 56 ++++++++++++++++++++ service/session/state.go | 51 ++++++++++++++++--- service/session/state_error_test.go | 59 ++++++++++++++++++++++ 4 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 service/operator/service_operation_test.go create mode 100644 service/session/state_error_test.go diff --git a/service/operator/service.go b/service/operator/service.go index f6d87a66e..50388aca5 100644 --- a/service/operator/service.go +++ b/service/operator/service.go @@ -60,7 +60,7 @@ func (s *Service) Operate(ctx context.Context, aSession *session.Session, aCompo // close the root while its handler still has mutations to dispatch. sealCreatedFrame(createdFrame, frame) if owner { - err = scope.Finish(ctx, err) + result, err = finishOperation(ctx, scope, result, err) } }() if tx := aSession.Options.SqlTx(); tx != nil && aComponent.View != nil && aComponent.View.Connector != nil { @@ -84,6 +84,28 @@ func sealCreatedFrame(created bool, frame *uow.Frame) { } } +// finishOperation preserves structured output for an operation that already +// failed, while preventing output produced before a flush or commit failure +// from being returned as a successful response. +func finishOperation(ctx context.Context, scope *uow.Scope, result interface{}, operationErr error) (interface{}, error) { + finishErr := scope.Finish(ctx, operationErr) + if operationErr != nil { + // scope.Finish reports the original cause combined (via errors.Join) with any + // rollback error. That join erases the concrete error type the gateway relies + // on for status-code and nested-error propagation (e.g. a 401 Jwt error nested + // under an Auth component parameter). Prefer the original typed error and only + // fall back to Finish's error when it surfaced an additional rollback failure. + if finishErr != nil && !errors.Is(finishErr, operationErr) { + return result, finishErr + } + return result, operationErr + } + if finishErr != nil { + return nil, finishErr + } + return result, nil +} + // HandleError processes output with error func (s *Service) HandleError(ctx context.Context, aSession *session.Session, aComponent *repository.Component, err error) (interface{}, error) { ctx = vcontext.WithValue(ctx, exec.ErrorKey, err) diff --git a/service/operator/service_operation_test.go b/service/operator/service_operation_test.go new file mode 100644 index 000000000..e0752b087 --- /dev/null +++ b/service/operator/service_operation_test.go @@ -0,0 +1,56 @@ +package operator + +import ( + "context" + "database/sql" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/service/executor/uow" +) + +func TestFinishOperation(t *testing.T) { + t.Run("preserves structured output for operation error", func(t *testing.T) { + ctx, scope, _ := uow.NewRoot(context.Background(), "test") + expectedOutput := &struct{}{} + expectedErr := errors.New("handler error") + + actualOutput, actualErr := finishOperation(ctx, scope, expectedOutput, expectedErr) + + require.ErrorIs(t, actualErr, expectedErr) + assert.Same(t, expectedOutput, actualOutput) + }) + + t.Run("discards output when finish introduces an error", func(t *testing.T) { + ctx, scope, frame := uow.NewRoot(context.Background(), "test") + expectedOutput := &struct{}{} + expectedErr := errors.New("generation error") + buffer := frame.NewBuffer( + func(context.Context) (*sql.DB, error) { + return nil, expectedErr + }, + nil, + func(context.Context, *sql.Tx, any) error { + return nil + }, + ) + require.NoError(t, buffer.Append(struct{}{})) + + actualOutput, actualErr := finishOperation(ctx, scope, expectedOutput, nil) + + require.ErrorIs(t, actualErr, expectedErr) + assert.Nil(t, actualOutput) + }) + + t.Run("preserves output when finish succeeds", func(t *testing.T) { + ctx, scope, _ := uow.NewRoot(context.Background(), "test") + expectedOutput := &struct{}{} + + actualOutput, actualErr := finishOperation(ctx, scope, expectedOutput, nil) + + require.NoError(t, actualErr) + assert.Same(t, expectedOutput, actualOutput) + }) +} diff --git a/service/session/state.go b/service/session/state.go index 97d28c21e..e57542428 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -1023,23 +1023,60 @@ func isZeroValue(value interface{}) bool { return v.IsZero() } -func (s *Session) handleParameterError(parameter *state.Parameter, err error, errors *response.Errors) { +func (s *Session) handleParameterError(parameter *state.Parameter, err error, result *response.Errors) { if parameter.ErrorMessage != "" && err != nil { msg := strings.ReplaceAll(parameter.ErrorMessage, "${error}", err.Error()) err = fmt.Errorf("%s", msg) } + // Directly-typed *response.Error (the common parameter-binding path): reset its + // status to the parameter's configured code. When the parameter has no explicit + // code (e.g. an object parameter such as Auth), this leaves Code=0 so the router's + // NormalizeErr can recurse into the wrapped cause (e.g. a nested Jwt error) to + // derive the real status code (401) and build the nested object payload. if pErr, ok := err.(*response.Error); ok { pErr.Code = parameter.ErrorStatusCode - errors.Append(pErr) - } else { - errors.AddError("", parameter.Name, err, response.WithErrorStatusCode(parameter.ErrorStatusCode)) + result.Append(pErr) + if parameter.ErrorStatusCode != 0 { + result.SetStatusCode(parameter.ErrorStatusCode) + } + return } + + result.AddError("", parameter.Name, err, response.WithErrorStatusCode(parameter.ErrorStatusCode)) + if parameter.ErrorStatusCode != 0 { - errors.SetStatusCode(parameter.ErrorStatusCode) - } else if asErrors, ok := err.(*response.Errors); ok && asErrors.StatusCode() != http.StatusBadRequest { - errors.SetStatusCode(asErrors.StatusCode()) + result.SetStatusCode(parameter.ErrorStatusCode) + return } + // Directly-typed *response.Errors: preserve an aggregated status that is more + // specific than a generic 400. + if asErrors, ok := err.(*response.Errors); ok { + if code := asErrors.StatusCode(); code != 0 && code != http.StatusBadRequest { + result.SetStatusCode(code) + } + return + } + + // Fallback: the error may be wrapped (fmt.Errorf %w / errors.Join). Resolve a + // meaningful status via errors.As so wrapped response errors keep their status. + var statusCoder response.StatusCoder + if errors.As(err, &statusCoder) { + if code := statusCoder.StatusCode(); code != 0 { + result.SetStatusCode(code) + if last := lastError(result); last != nil && last.Code == 0 { + last.Code = code + } + } + } +} + +// lastError returns the most recently appended error, or nil when empty. +func lastError(result *response.Errors) *response.Error { + if result == nil || len(result.Errors) == 0 { + return nil + } + return result.Errors[len(result.Errors)-1] } func (s *Session) InitKinds(kinds ...state.Kind) error { diff --git a/service/session/state_error_test.go b/service/session/state_error_test.go new file mode 100644 index 000000000..3596fcfda --- /dev/null +++ b/service/session/state_error_test.go @@ -0,0 +1,59 @@ +package session + +import ( + stderrors "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/view/state" + "github.com/viant/xdatly/handler/response" +) + +func TestSessionHandleParameterErrorPreservesWrappedResponseStatus(t *testing.T) { + testCases := []struct { + name string + err error + parameterStatus int + expectedStatus int + expectedErrorCode int + }{ + { + name: "wrapped response error", + err: stderrors.Join(response.NewError(http.StatusUnauthorized, "unauthorized"), nil), + expectedStatus: http.StatusUnauthorized, + expectedErrorCode: http.StatusUnauthorized, + }, + { + name: "wrapped response errors", + err: func() error { + previous := response.NewErrors() + previous.Append(response.NewError(http.StatusUnauthorized, "unauthorized")) + return stderrors.Join(previous, nil) + }(), + expectedStatus: http.StatusUnauthorized, + expectedErrorCode: http.StatusUnauthorized, + }, + { + name: "parameter status overrides wrapped status", + err: stderrors.Join(response.NewError(http.StatusUnauthorized, "unauthorized"), nil), + parameterStatus: http.StatusForbidden, + expectedStatus: http.StatusForbidden, + expectedErrorCode: http.StatusForbidden, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + actual := response.NewErrors() + parameter := &state.Parameter{Name: "Auth", ErrorStatusCode: testCase.parameterStatus} + + (&Session{}).handleParameterError(parameter, testCase.err, actual) + + assert.Equal(t, testCase.expectedStatus, actual.StatusCode()) + require.Len(t, actual.Errors, 1) + assert.Equal(t, testCase.expectedErrorCode, actual.Errors[0].StatusCode()) + }) + } +} From 74098bfe5b1126991b4955d303fe4dc318b17a21 Mon Sep 17 00:00:00 2001 From: vcarey Date: Thu, 23 Jul 2026 18:28:17 -0400 Subject: [PATCH 268/279] Update sqlx dependency --- go.mod | 4 ++-- go.sum | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index fe9188b85..ae00e07cc 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.33.1 - github.com/viant/sqlx v0.23.1-0.20260712191511-2534f58bdccc + github.com/viant/sqlx v0.23.1-0.20260721202550-583cf232e734 github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 @@ -46,6 +46,7 @@ require ( ) require ( + github.com/golang-jwt/jwt/v5 v5.2.2 github.com/viant/aerospike v0.2.11-0.20241108195857-ed524b97800d github.com/viant/firebase v0.1.1 github.com/viant/jsonrpc v0.17.0 @@ -126,7 +127,6 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/s2a-go v0.1.8 // indirect diff --git a/go.sum b/go.sum index 7525af70e..109d15f67 100644 --- a/go.sum +++ b/go.sum @@ -1192,14 +1192,12 @@ github.com/viant/parsly v0.3.3 h1:7ytgfLOG4Ils+wviGacWxRD0gAUvVEH/iGsSE+UI8YM= github.com/viant/parsly v0.3.3/go.mod h1:85fneXJbErKMGhSQto3A5ElTQCwl3t74U9cSV0waBHw= github.com/viant/pgo v0.11.0 h1:PNuYVhwTfyrAHGBO6lxaMFuHP4NkjKV8ULecz3OWk8c= github.com/viant/pgo v0.11.0/go.mod h1:MFzHmkRFZlciugEgUvpl/3grK789PBSH4dUVSLOSo+Q= -github.com/viant/scy v0.24.0 h1:KAC3IUARkQxTNSuwBK2YhVBJMOOLN30YaLKHbbuSkMU= -github.com/viant/scy v0.24.0/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/scy v0.33.1 h1:jlSgOxwsLvY1/YvAd6y5shvig/6hfhX+/sHVKmh4B5s= github.com/viant/scy v0.33.1/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQnwf69oc86xW1e4wEEzKUg5OC6tOYegE= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.23.1-0.20260712191511-2534f58bdccc h1:uxPdh1l7dBvMUqJT2aMdPU9ubz3ErgQSmFoaDDe8row= -github.com/viant/sqlx v0.23.1-0.20260712191511-2534f58bdccc/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= +github.com/viant/sqlx v0.23.1-0.20260721202550-583cf232e734 h1:vZF9F8r3lUSfdRBMZyWje0eabeI0Q5sMwbd0QF3pq8c= +github.com/viant/sqlx v0.23.1-0.20260721202550-583cf232e734/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= github.com/viant/structology v0.9.0/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= From ff9ed9563e1d567b20dde398288f7cf2dabe6a09 Mon Sep 17 00:00:00 2001 From: vcarey Date: Mon, 27 Jul 2026 17:49:36 -0400 Subject: [PATCH 269/279] Add internal output projection support --- service.go | 8 + service/session/option.go | 24 +++ service/session/projection.go | 41 +++++ service/session/reader.go | 3 + service/session/selector.go | 3 + service/session/state.go | 64 +++++++ service_projection_test.go | 313 ++++++++++++++++++++++++++++++++++ view/projection.go | 132 ++++++++++++++ view/state.go | 5 + 9 files changed, 593 insertions(+) create mode 100644 service/session/projection.go create mode 100644 service_projection_test.go create mode 100644 view/projection.go diff --git a/service.go b/service.go index 98d5ac027..f6f893f58 100644 --- a/service.go +++ b/service.go @@ -186,6 +186,14 @@ func WithOutput(output interface{}) OperateOption { } } +func ContextWithOutputProjection(ctx context.Context, output interface{}) context.Context { + return session.ContextWithOutputProjection(ctx, output) +} + +func ContextWithViewOutputProjection(ctx context.Context, viewName string, output interface{}) context.Context { + return session.ContextWithViewOutputProjection(ctx, viewName, output) +} + func WithSession(session *session.Session) OperateOption { return func(o *operateOptions) { o.session = session diff --git a/service/session/option.go b/service/session/option.go index 40ee8c6c5..5464f9a63 100644 --- a/service/session/option.go +++ b/service/session/option.go @@ -37,6 +37,8 @@ type ( preseedCache bool cacheDisabled bool sqlTx *sql.Tx + viewProjections map[string][]string + outputProjection *OutputProjection } Option func(o *Options) @@ -138,6 +140,28 @@ func WithLocatorOptions(options ...locator.Option) Option { } } +func WithViewProjectionColumns(viewName string, columns []string) Option { + return func(s *Options) { + if len(columns) == 0 { + return + } + if s.viewProjections == nil { + s.viewProjections = map[string][]string{} + } + s.viewProjections[viewName] = append([]string(nil), columns...) + } +} + +func WithOutputProjection(output interface{}) Option { + return WithViewOutputProjection("", output) +} + +func WithViewOutputProjection(viewName string, output interface{}) Option { + return func(s *Options) { + s.outputProjection = &OutputProjection{View: viewName, Output: output} + } +} + func WithStateResource(resource state.Resource) Option { return func(s *Options) { s.resource = resource diff --git a/service/session/projection.go b/service/session/projection.go new file mode 100644 index 000000000..53b88debf --- /dev/null +++ b/service/session/projection.go @@ -0,0 +1,41 @@ +package session + +import "context" + +type outputProjectionKey struct{} + +type OutputProjection struct { + View string + Output interface{} +} + +func ContextWithOutputProjection(ctx context.Context, output interface{}) context.Context { + return ContextWithViewOutputProjection(ctx, "", output) +} + +func ContextWithViewOutputProjection(ctx context.Context, viewName string, output interface{}) context.Context { + return context.WithValue(ctx, outputProjectionKey{}, &OutputProjection{View: viewName, Output: output}) +} + +func OutputProjectionFromContext(ctx context.Context, viewName string) interface{} { + if ctx == nil { + return nil + } + value := ctx.Value(outputProjectionKey{}) + switch actual := value.(type) { + case *OutputProjection: + if actual == nil { + return nil + } + if actual.View != "" && normalizeViewProjectionName(actual.View) != normalizeViewProjectionName(viewName) { + return nil + } + return actual.Output + case OutputProjection: + if actual.View != "" && normalizeViewProjectionName(actual.View) != normalizeViewProjectionName(viewName) { + return nil + } + return actual.Output + } + return nil +} diff --git a/service/session/reader.go b/service/session/reader.go index 94a91d528..4574d0b81 100644 --- a/service/session/reader.go +++ b/service/session/reader.go @@ -18,6 +18,9 @@ func (s *Session) ReadInto(ctx context.Context, dest interface{}, aView *view.Vi } }() } + if err := s.ApplyOutputProjection(ctx, aView); err != nil { + return err + } if err := s.SetViewState(ctx, aView); err != nil { return err } diff --git a/service/session/selector.go b/service/session/selector.go index 78818d033..7183b4117 100644 --- a/service/session/selector.go +++ b/service/session/selector.go @@ -100,6 +100,9 @@ func (s *Session) setQuerySelector(ctx context.Context, ns *view.NamespaceView, // but still validate against view selector constraints. if injected != nil { selector.QuerySelector = injected.QuerySelector + if len(injected.Columns) > 0 { + selector.SetColumns(injected.Columns) + } if err := s.applyInjectedQuerySelector(ns, selector, injected); err != nil { return err } diff --git a/service/session/state.go b/service/session/state.go index e57542428..de32d93fa 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log" "net/http" "os" "reflect" @@ -177,9 +178,72 @@ func (s *Session) setViewState(ctx context.Context, aView *view.View) (err error return err } } + s.applyViewProjection(aView) return err } +func (s *Session) applyViewProjection(aView *view.View) { + if s == nil || aView == nil || len(s.viewProjections) == 0 { + return + } + columns, ok := s.viewProjections[aView.Name] + if !ok { + normalizedViewName := normalizeViewProjectionName(aView.Name) + for name, candidate := range s.viewProjections { + if normalizeViewProjectionName(name) == normalizedViewName { + columns = candidate + ok = true + break + } + } + } + if !ok || len(columns) == 0 { + return + } + statelet := s.state.Lookup(aView) + statelet.SetColumns(columns) +} + +func (s *Session) ApplyOutputProjection(ctx context.Context, aView *view.View) error { + var output interface{} + if s.outputProjection != nil { + output = projectionOutputForView(*s.outputProjection, aView.Name) + } + if output == nil { + output = OutputProjectionFromContext(ctx, aView.Name) + } + if output == nil { + return nil + } + columns, err := view.ProjectionColumnsForOutput(aView, output) + if err != nil { + return err + } + if columns == nil { + return nil + } + log.Printf("[PROJECTION] view=%s columns=%v", aView.Name, columns) + s.Apply(WithViewProjectionColumns(aView.Name, columns)) + statelet := s.state.Lookup(aView) + statelet.SetColumns(columns) + return nil +} + +func projectionOutputForView(projection OutputProjection, viewName string) interface{} { + if projection.View != "" && normalizeViewProjectionName(projection.View) != normalizeViewProjectionName(viewName) { + return nil + } + return projection.Output +} + +func normalizeViewProjectionName(name string) string { + name = strings.ToLower(strings.TrimSpace(name)) + name = strings.ReplaceAll(name, "_", "") + name = strings.ReplaceAll(name, "-", "") + name = strings.ReplaceAll(name, ".", "") + return name +} + func (s *Session) viewNamespace(aView *view.View) *view.NamespaceView { ns := s.namespacedView.ByName(aView.Name) if ns == nil { diff --git a/service_projection_test.go b/service_projection_test.go new file mode 100644 index 000000000..fed405c16 --- /dev/null +++ b/service_projection_test.go @@ -0,0 +1,313 @@ +package datly + +import ( + "context" + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/service/reader" + "github.com/viant/datly/service/session" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" +) + +type fullProjectionOutput struct { + AccountID int `json:"accountId" sqlx:"account_id"` + CampaignID int `json:"campaignId" sqlx:"campaign_id"` + Impressions int `json:"impressions"` + Spend float64 `json:"spend"` +} + +type alternateProjectionOutput struct { + Campaign int `json:"campaignId"` + Spend int `sqlx:"spend"` +} + +type sourceProjectionOutput struct { + AliasValue int `source:"src.alias_value"` +} + +type ignoredEmbeddedProjection struct { + AccountID int `json:"accountId"` +} + +func TestProjectionColumnsForOutput_BuildsProjectionFromDestinationType(t *testing.T) { + aComponent := projectionTestComponent(t, reflect.TypeOf(fullProjectionOutput{})) + var output []alternateProjectionOutput + + columns, err := view.ProjectionColumnsForOutput(aComponent.View, &output) + + require.NoError(t, err) + require.Equal(t, []string{"campaign_id", "spend"}, columns) +} + +func TestProjectionColumnsForOutput_IgnoresNonSliceOutput(t *testing.T) { + aComponent := projectionTestComponent(t, reflect.TypeOf(fullProjectionOutput{})) + var output struct { + Status string `json:"status"` + } + + columns, err := view.ProjectionColumnsForOutput(aComponent.View, &output) + + require.NoError(t, err) + require.Nil(t, columns) +} + +func TestProjectionColumnsForOutput_FailsForEmptyProjectionDTO(t *testing.T) { + aComponent := projectionTestComponent(t, reflect.TypeOf(fullProjectionOutput{})) + var output []struct{} + + columns, err := view.ProjectionColumnsForOutput(aComponent.View, &output) + + require.Error(t, err) + require.Nil(t, columns) +} + +func TestProjectionColumnsForOutput_UsesSourceAliases(t *testing.T) { + aComponent := projectionTestComponent(t, reflect.TypeOf(fullProjectionOutput{})) + var output []sourceProjectionOutput + + columns, err := view.ProjectionColumnsForOutput(aComponent.View, &output) + + require.NoError(t, err) + require.Equal(t, []string{"alias_value"}, columns) +} + +func TestProjectionColumnsForOutput_IgnoresSkippedAnonymousEmbeds(t *testing.T) { + aComponent := projectionTestComponent(t, reflect.TypeOf(fullProjectionOutput{})) + var output []struct { + ignoredEmbeddedProjection `json:"-"` + Spend int `json:"spend"` + } + + columns, err := view.ProjectionColumnsForOutput(aComponent.View, &output) + + require.NoError(t, err) + require.Equal(t, []string{"spend"}, columns) +} + +func TestProjectionColumnsForOutput_FailsForUnknownField(t *testing.T) { + aComponent := projectionTestComponent(t, reflect.TypeOf(fullProjectionOutput{})) + var output []struct { + Unknown int `json:"unknown"` + } + + columns, err := view.ProjectionColumnsForOutput(aComponent.View, &output) + + require.Error(t, err) + require.Nil(t, columns) +} + +func TestSessionViewProjectionColumns_NarrowsAfterStatePopulation(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + columns := []string{"account_id", "bids"} + aSession := session.New(aComponent.View, session.WithViewProjectionColumns(aComponent.View.Name, columns)) + + err := aSession.SetViewState(context.Background(), aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.True(t, statelet.Has("account_id")) + require.True(t, statelet.Has("bids")) + require.False(t, statelet.Has("campaign_id")) + + query, err := reader.NewBuilder().CacheSQL(context.Background(), aComponent.View, statelet) + + require.NoError(t, err) + require.Contains(t, query.SQL, "SELECT account_id, bids FROM") + require.Contains(t, query.SQL, "GROUP BY 1") + require.NotContains(t, query.SQL, "campaign_id") +} + +func TestSessionApplyOutputProjectionFromContext_NarrowsChildView(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + var output []struct { + AccountID int `json:"accountId"` + Bids int `json:"bids"` + } + aSession := session.New(aComponent.View) + ctx := ContextWithOutputProjection(context.Background(), &output) + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) + require.True(t, statelet.Has("account_id")) + require.True(t, statelet.Has("bids")) + require.False(t, statelet.Has("campaign_id")) + + query, err := reader.NewBuilder().Build(context.Background(), reader.WithBuilderView(aComponent.View), reader.WithBuilderStatelet(statelet)) + + require.NoError(t, err) + require.Contains(t, query.SQL, "SELECT account_id, bids FROM") + require.Contains(t, query.SQL, "GROUP BY 1") + require.NotContains(t, query.SQL, "campaign_id") +} + +func TestSessionApplyOutputProjectionFromScopedContext_OnlyAppliesToMatchingView(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + var output []struct { + AccountID int `json:"accountId"` + Bids int `json:"bids"` + } + aSession := session.New(aComponent.View) + ctx := ContextWithViewOutputProjection(context.Background(), "other_view", &output) + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Empty(t, statelet.Columns) + + ctx = ContextWithViewOutputProjection(context.Background(), aComponent.View.Name, &output) + err = aSession.ApplyOutputProjection(ctx, aComponent.View) + require.NoError(t, err) + require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) +} + +func TestSessionApplyOutputProjectionFromOption_NarrowsChildView(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + var output []struct { + AccountID int `json:"accountId"` + Bids int `json:"bids"` + } + aSession := session.New(aComponent.View, session.WithOutputProjection(&output)) + + err := aSession.ApplyOutputProjection(context.Background(), aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) +} + +func TestSessionApplyOutputProjectionFromScopedOption_OnlyAppliesToMatchingView(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + var output []struct { + AccountID int `json:"accountId"` + Bids int `json:"bids"` + } + aSession := session.New(aComponent.View, session.WithViewOutputProjection("other_view", &output)) + + err := aSession.ApplyOutputProjection(context.Background(), aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Empty(t, statelet.Columns) + + aSession.Apply(session.WithViewOutputProjection(aComponent.View.Name, &output)) + err = aSession.ApplyOutputProjection(context.Background(), aComponent.View) + require.NoError(t, err) + require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) +} + +func TestSessionApplyOutputProjectionWithoutHint_LeavesChildViewFullWidth(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View) + + err := aSession.ApplyOutputProjection(context.Background(), aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Empty(t, statelet.Columns) + + query, err := reader.NewBuilder().Build(context.Background(), reader.WithBuilderView(aComponent.View), reader.WithBuilderStatelet(statelet)) + + require.NoError(t, err) + require.Contains(t, query.SQL, "SELECT t.account_id, t.campaign_id, t.bids FROM") + require.Contains(t, query.SQL, "GROUP BY 1, 2") +} + +func TestSessionApplyOutputProjectionNonSliceHint_DoesNotClearExistingProjection(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View) + statelet := aSession.State().Lookup(aComponent.View) + statelet.SetColumns([]string{"account_id", "bids"}) + var output struct { + Status string `json:"status"` + } + ctx := ContextWithOutputProjection(context.Background(), &output) + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + + require.NoError(t, err) + require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) +} + +func TestSessionApplyOutputProjectionFromContext_FailsForEmptyProjectionDTO(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + var output []struct{} + aSession := session.New(aComponent.View) + ctx := ContextWithOutputProjection(context.Background(), &output) + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + + require.Error(t, err) + require.Contains(t, err.Error(), "did not map any columns") +} + +func TestSessionApplyOutputProjectionFromContext_FailsForUnknownField(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + var output []struct { + Unknown int `json:"unknown"` + } + aSession := session.New(aComponent.View) + ctx := ContextWithOutputProjection(context.Background(), &output) + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + + require.Error(t, err) + require.Contains(t, err.Error(), "failed to map output field Unknown") +} + +func TestWithOutput_DoesNotEnableProjection(t *testing.T) { + options := newOperateOptions([]OperateOption{WithOutput(&[]alternateProjectionOutput{})}) + + require.NotNil(t, options.output) +} + +func projectionTestComponent(t *testing.T, outputType reflect.Type) *repository.Component { + t.Helper() + aView := view.NewView("projection", "projection", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + &view.Column{Name: "account_id", DataType: "int"}, + &view.Column{Name: "campaign_id", DataType: "int"}, + &view.Column{Name: "impressions", DataType: "int"}, + &view.Column{Name: "spend", DataType: "float"}, + &view.Column{Name: "alias_value", DataType: "int", Tag: `source:"src.alias_value"`}, + }), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + output, err := state.NewType(state.WithSchema(state.NewSchema(outputType))) + require.NoError(t, err) + return &repository.Component{ + View: aView, + Contract: contract.Contract{ + Output: contract.Output{ + Type: *output, + }, + }, + } +} + +func groupableProjectionTestComponent(t *testing.T) *repository.Component { + t.Helper() + aView := view.NewView("groupable_projection", "(SELECT account_id, campaign_id, SUM(bids) AS bids FROM bids GROUP BY 1, 2)", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + &view.Column{Name: "account_id", DataType: "int", Tag: `groupable:"true"`}, + &view.Column{Name: "campaign_id", DataType: "int", Tag: `groupable:"true"`}, + &view.Column{Name: "bids", DataType: "int", Aggregate: true}, + }), + view.WithGroupable(true), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + output, err := state.NewType(state.WithSchema(state.NewSchema(reflect.TypeOf(fullProjectionOutput{})))) + require.NoError(t, err) + return &repository.Component{ + View: aView, + Contract: contract.Contract{ + Output: contract.Output{ + Type: *output, + }, + }, + } +} diff --git a/view/projection.go b/view/projection.go new file mode 100644 index 000000000..c63c60584 --- /dev/null +++ b/view/projection.go @@ -0,0 +1,132 @@ +package view + +import ( + "fmt" + "reflect" + "strings" +) + +func ProjectionColumnsForOutput(aView *View, output interface{}) ([]string, error) { + if output == nil { + return nil, nil + } + return ProjectionColumnsForType(aView, ProjectionOutputStructType(reflect.TypeOf(output))) +} + +func ProjectionColumnsForType(aView *View, rType reflect.Type) ([]string, error) { + if aView == nil || rType == nil { + return nil, nil + } + var result []string + seen := map[string]bool{} + for i := 0; i < rType.NumField(); i++ { + field := rType.Field(i) + if field.PkgPath != "" { + continue + } + if skipProjectionField(field) { + continue + } + if field.Anonymous { + if nested := ProjectionStructType(field.Type); nested != nil { + columns, err := ProjectionColumnsForType(aView, nested) + if err != nil { + return nil, err + } + for _, column := range columns { + if !seen[column] { + result = append(result, column) + seen[column] = true + } + } + continue + } + } + column, err := projectionColumnForField(aView, field) + if err != nil { + return nil, err + } + if !seen[column.Name] { + result = append(result, column.Name) + seen[column.Name] = true + } + } + if len(result) == 0 { + return nil, fmt.Errorf("output projection for view %s did not map any columns", aView.Name) + } + return result, nil +} + +func ProjectionStructType(rType reflect.Type) reflect.Type { + if rType == nil { + return nil + } + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + if rType.Kind() == reflect.Slice || rType.Kind() == reflect.Array { + rType = rType.Elem() + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + } + if rType.Kind() != reflect.Struct { + return nil + } + return rType +} + +func ProjectionOutputStructType(rType reflect.Type) reflect.Type { + if rType == nil { + return nil + } + for rType.Kind() == reflect.Ptr { + rType = rType.Elem() + } + if rType.Kind() != reflect.Slice && rType.Kind() != reflect.Array { + return nil + } + return ProjectionStructType(rType) +} + +func projectionColumnForField(aView *View, field reflect.StructField) (*Column, error) { + for _, candidate := range projectionFieldCandidates(field) { + if column, ok := aView.ColumnByName(candidate); ok { + return column, nil + } + } + return nil, fmt.Errorf("failed to map output field %s to a column in view %s", field.Name, aView.Name) +} + +func projectionFieldCandidates(field reflect.StructField) []string { + var result []string + add := func(value string) { + value = strings.TrimSpace(value) + if value == "" || value == "-" { + return + } + for _, item := range strings.Split(value, "|") { + item = strings.TrimSpace(item) + if item == "" || item == "-" { + continue + } + result = append(result, item) + } + } + add(tagName(field.Tag.Get("sqlx"))) + add(tagName(field.Tag.Get("source"))) + add(tagName(field.Tag.Get("json"))) + add(field.Name) + return result +} + +func skipProjectionField(field reflect.StructField) bool { + return tagName(field.Tag.Get("json")) == "-" || tagName(field.Tag.Get("sqlx")) == "-" +} + +func tagName(tag string) string { + if index := strings.Index(tag, ","); index != -1 { + tag = tag[:index] + } + return strings.TrimSpace(tag) +} diff --git a/view/state.go b/view/state.go index 48761aa7e..7aaee7169 100644 --- a/view/state.go +++ b/view/state.go @@ -73,6 +73,11 @@ func (s *Statelet) Add(fieldName string, isHolder bool) { } } +func (s *Statelet) SetColumns(columns []string) { + s.Columns = append([]string(nil), columns...) + s._columnNames = Names(s.Columns).Index() +} + // AppendFilters safely appends filters to the selector's Filters to avoid data races. func (s *Statelet) AppendFilters(filters predicate.Filters) { if len(filters) == 0 { From 1257d1bfda50bc3c538e48f109865fab5788de23 Mon Sep 17 00:00:00 2001 From: vcarey Date: Mon, 27 Jul 2026 17:55:16 -0400 Subject: [PATCH 270/279] bypass internal true for warmup --- gateway/route_warmup_test.go | 59 ++++++++++++++++++++++++++++++++++++ gateway/router.go | 18 +++++++++++ 2 files changed, 77 insertions(+) diff --git a/gateway/route_warmup_test.go b/gateway/route_warmup_test.go index 4fad3f078..75069da50 100644 --- a/gateway/route_warmup_test.go +++ b/gateway/route_warmup_test.go @@ -52,6 +52,35 @@ func TestRouterAppendCacheWarmupRoute_NonGET(t *testing.T) { require.Empty(t, routes) } +func TestRouterInternalGETAllowsWarmupRouteOnly(t *testing.T) { + aPath := &path.Path{ + Path: *contract.NewPath(http.MethodGet, "/v1/api/internal/order"), + Internal: true, + } + router := newWarmupTestRouter(t, aPath) + + _, err := router.Match(http.MethodGet, "/v1/api/internal/order", nil) + require.Error(t, err) + + route, err := router.Match(http.MethodPost, "/v1/api/cache/warmup/internal/order", nil) + require.NoError(t, err) + require.Equal(t, RouteWarmupKind, route.Kind) +} + +func TestRouterInternalPOSTDoesNotAllowWarmupRoute(t *testing.T) { + aPath := &path.Path{ + Path: *contract.NewPath(http.MethodPost, "/v1/api/internal/order"), + Internal: true, + } + router := newWarmupTestRouter(t, aPath) + + _, err := router.Match(http.MethodPost, "/v1/api/internal/order", nil) + require.Error(t, err) + + _, err = router.Match(http.MethodPost, "/v1/api/cache/warmup/internal/order", nil) + require.Error(t, err) +} + func TestRouterHandleCacheWarmupWithErr_NoCacheViews(t *testing.T) { router := &Router{} provider := repository.NewProvider( @@ -96,3 +125,33 @@ func TestRouterHandleCacheWarmupWithErr_DetachesRequestContext(t *testing.T) { require.Equal(t, http.StatusOK, statusCode, string(body)) } + +func newWarmupTestRouter(t *testing.T, routes ...*path.Path) *Router { + t.Helper() + ctx := context.Background() + repo, err := repository.New(ctx, repository.WithComponentURL(""), repository.WithNoPlugin()) + require.NoError(t, err) + + item := &path.Item{Paths: routes} + repo.Container().Items = []*path.Item{item} + + providers := make([]*repository.Provider, 0, len(routes)) + for _, routePath := range routes { + routePath := routePath + component, err := repository.NewComponent(&routePath.Path, repository.WithView(&view.View{Name: "order"})) + require.NoError(t, err) + providers = append(providers, repository.NewProvider(routePath.Path, &version.Control{}, func(ctx context.Context, opts ...repository.Option) (*repository.Component, error) { + return component, nil + })) + } + repo.Registry().SetProviders(providers) + + router, err := NewRouter(ctx, repo, &Config{ + ExposableConfig: ExposableConfig{ + APIPrefix: "/v1/api", + Meta: meta.Config{CacheWarmURI: "/v1/api/cache/warmup"}, + }, + }, nil, nil, nil) + require.NoError(t, err) + return router +} diff --git a/gateway/router.go b/gateway/router.go index c520fe902..143d82e5d 100644 --- a/gateway/router.go +++ b/gateway/router.go @@ -342,6 +342,24 @@ func (r *Router) newMatcher(ctx context.Context) (*matcher.Matcher, []*contract. for _, anItem := range container.Items { for _, aPath := range anItem.Paths { if aPath.Internal { + if aPath.ContentURL == "" { + var apiKeys []*path.APIKey + if matched := r.config.APIKeys.Match(aPath.URI); matched != nil { + aPath.APIKey = matched + apiKeys = append(apiKeys, matched) + } + offset := len(routes) + provider, err := r.repository.Registry().LookupProvider(ctx, &aPath.Path) + if err != nil { + return nil, nil, fmt.Errorf("failed to locate component provider: %w", err) + } + routes = r.appendCacheWarmupRoute(routes, aPath, provider) + if len(apiKeys) > 0 { + for i := offset; i < len(routes); i++ { + routes[i].ApiKeys = apiKeys + } + } + } continue } var apiKeys []*path.APIKey From c907d7227af934cc9cf8b204d8d1fc5806872def Mon Sep 17 00:00:00 2001 From: vcarey Date: Wed, 29 Jul 2026 10:31:16 -0400 Subject: [PATCH 271/279] Improve warmup projection reuse --- go.mod | 2 +- go.sum | 2 + repository/locator/component/component.go | 60 +++- .../locator/component/component_uow_test.go | 127 +++++++ service.go | 8 + service/reader/handler/handler.go | 3 + service/reader/service.go | 77 ++++- service/reader/service_warmup_test.go | 269 +++++++++++++++ service/reader/sql_groupable_test.go | 61 +++- service/session/option.go | 10 + service/session/projection.go | 69 +++- service/session/state.go | 51 ++- service_projection_test.go | 99 ++++++ view/cache.go | 205 ++++++++++- view/collector.go | 18 + view/projection.go | 182 ++++++++++ view/projection_test.go | 78 +++++ warmup/cache.go | 1 + warmup/cache_test.go | 320 ++++++++++++++++++ 19 files changed, 1615 insertions(+), 27 deletions(-) create mode 100644 view/projection_test.go diff --git a/go.mod b/go.mod index ae00e07cc..74736b694 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.33.1 - github.com/viant/sqlx v0.23.1-0.20260721202550-583cf232e734 + github.com/viant/sqlx v0.23.1-0.20260729142839-24da78934871 github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 diff --git a/go.sum b/go.sum index 109d15f67..029eec27b 100644 --- a/go.sum +++ b/go.sum @@ -1198,6 +1198,8 @@ github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQ github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.23.1-0.20260721202550-583cf232e734 h1:vZF9F8r3lUSfdRBMZyWje0eabeI0Q5sMwbd0QF3pq8c= github.com/viant/sqlx v0.23.1-0.20260721202550-583cf232e734/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= +github.com/viant/sqlx v0.23.1-0.20260729142839-24da78934871 h1:9RqxSYtQfUGiMoT9YNpBfPZVjtorUrJ3uQISvS43EKI= +github.com/viant/sqlx v0.23.1-0.20260729142839-24da78934871/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= github.com/viant/structology v0.9.0/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= diff --git a/repository/locator/component/component.go b/repository/locator/component/component.go index 86c9f38f0..0e37a12ef 100644 --- a/repository/locator/component/component.go +++ b/repository/locator/component/component.go @@ -7,10 +7,12 @@ import ( "net/http" "net/url" "reflect" + "strings" "github.com/viant/datly/repository/contract" "github.com/viant/datly/service/executor/uow" "github.com/viant/datly/shared" + "github.com/viant/datly/view" "github.com/viant/datly/view/state" "github.com/viant/datly/view/state/kind" "github.com/viant/datly/view/state/kind/locator" @@ -48,11 +50,12 @@ func (l *componentLocator) Value(ctx context.Context, _ reflect.Type, name strin if err != nil { return nil, false, err } + request = sanitizeSelectorRequest(request) form := l.form value, err := l.dispatch.Dispatch(ctx, &contract.Path{Method: method, URI: URI}, contract.WithRequest(request), contract.WithConstants(l.constants), contract.WithPath(l.path), - contract.WithQuery(l.query), + contract.WithQuery(sanitizeSelectorQuery(l.query)), contract.WithForm(form), contract.WithLogger(l.logger), contract.WithHeader(l.header), @@ -61,6 +64,61 @@ func (l *componentLocator) Value(ctx context.Context, _ reflect.Type, name strin return value, err == nil, err } +func sanitizeSelectorQuery(query url.Values) url.Values { + sanitized, _ := sanitizeSelectorQueryWithRemoval(query) + return sanitized +} + +func sanitizeSelectorQueryWithRemoval(query url.Values) (url.Values, bool) { + if len(query) == 0 { + return query, false + } + removed := false + result := make(url.Values, len(query)) + for key, values := range query { + if isSelectorQueryKey(key) { + removed = true + continue + } + result[key] = append([]string(nil), values...) + } + if !removed { + return query, false + } + return result, true +} + +func sanitizeSelectorRequest(request *http.Request) *http.Request { + if request == nil || request.URL == nil || request.URL.RawQuery == "" { + return request + } + sanitized, removed := sanitizeSelectorQueryWithRemoval(request.URL.Query()) + if !removed { + return request + } + cloned := request.Clone(request.Context()) + cloned.URL = cloneURL(request.URL) + cloned.URL.RawQuery = sanitized.Encode() + return cloned +} + +func cloneURL(src *url.URL) *url.URL { + if src == nil { + return nil + } + cloned := *src + return &cloned +} + +func isSelectorQueryKey(name string) bool { + switch strings.ToLower(strings.TrimSpace(name)) { + case view.FieldsQuery, view.OrderByQuery, view.LimitQuery, view.OffsetQuery, view.PageQuery, view.CriteriaQuery: + return true + default: + return false + } +} + func updateErrWithResponseStatus(err error, response interface{}) error { var statusErr error responseStatus, ok := tryExtractResponseStatus(response) diff --git a/repository/locator/component/component_uow_test.go b/repository/locator/component/component_uow_test.go index f92e17cd6..3a7c419da 100644 --- a/repository/locator/component/component_uow_test.go +++ b/repository/locator/component/component_uow_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "net/http" + "net/url" "reflect" "testing" @@ -39,6 +40,18 @@ func (d *componentScopeDispatcher) Dispatch(ctx context.Context, path *contract. return struct{}{}, nil } +type componentQueryDispatcher struct { + query url.Values + request *http.Request +} + +func (d *componentQueryDispatcher) Dispatch(_ context.Context, _ *contract.Path, opts ...contract.Option) (interface{}, error) { + options := contract.NewOptions(opts...) + d.query = options.Query + d.request = options.Request + return struct{}{}, nil +} + func TestComponentLocatorCreatesOrderedBindingFrames(t *testing.T) { db, _ := sql.Open("sqlite3", ":memory:") defer db.Close() @@ -79,6 +92,120 @@ func TestComponentLocatorCreatesOrderedBindingFrames(t *testing.T) { } } +func TestComponentLocatorDropsSelectorQueryParamsForChildDispatch(t *testing.T) { + dispatcher := &componentQueryDispatcher{} + request, _ := http.NewRequest(http.MethodGet, "/?_fields=AudienceId&_orderby=AudienceId&_limit=10&_offset=5&_page=2&_criteria=AudienceId+%3D+1&criteria=business+criteria&audience_id=123&order_id=456&from=2026-07-01&to=2026-07-02", nil) + query := url.Values{ + "_fields": {"AudienceId"}, + "_orderby": {"AudienceId"}, + "_limit": {"10"}, + "_offset": {"5"}, + "_page": {"2"}, + "_criteria": {"AudienceId = 1"}, + "criteria": {"business criteria"}, + "audience_id": {"123"}, + "order_id": {"456"}, + "from": {"2026-07-01"}, + "to": {"2026-07-02"}, + } + componentLocator := &componentLocator{ + dispatch: dispatcher, + query: query, + getRequest: func() (*http.Request, error) { + return request, nil + }, + } + + _, found, err := componentLocator.Value(context.Background(), reflect.TypeOf(""), "GET:/child") + if err != nil || !found { + t.Fatalf("Value() found=%v err=%v", found, err) + } + + for _, key := range []string{"_fields", "_orderby", "_limit", "_offset", "_page", "_criteria"} { + if _, ok := dispatcher.query[key]; ok { + t.Fatalf("selector query key %q was forwarded: %v", key, dispatcher.query) + } + } + for key, want := range map[string]string{ + "criteria": "business criteria", + "audience_id": "123", + "order_id": "456", + "from": "2026-07-01", + "to": "2026-07-02", + } { + if got := dispatcher.query.Get(key); got != want { + t.Fatalf("query[%s]=%q want %q; query=%v", key, got, want, dispatcher.query) + } + } + if dispatcher.request == nil || dispatcher.request.URL == nil { + t.Fatal("expected forwarded request") + } + requestQuery := dispatcher.request.URL.Query() + for _, key := range []string{"_fields", "_orderby", "_limit", "_offset", "_page", "_criteria"} { + if _, ok := requestQuery[key]; ok { + t.Fatalf("selector query key %q was forwarded on request URL: %s", key, dispatcher.request.URL.RawQuery) + } + } + for key, want := range map[string]string{ + "criteria": "business criteria", + "audience_id": "123", + "order_id": "456", + "from": "2026-07-01", + "to": "2026-07-02", + } { + if got := requestQuery.Get(key); got != want { + t.Fatalf("request query[%s]=%q want %q; raw=%s", key, got, want, dispatcher.request.URL.RawQuery) + } + } + if request.URL.Query().Get("_fields") != "AudienceId" { + t.Fatal("original parent request was mutated") + } +} + +func TestSanitizeSelectorQueryClonesForwardedValues(t *testing.T) { + query := url.Values{"_fields": {"AudienceId"}, "order_id": {"456"}} + sanitized := sanitizeSelectorQuery(query) + + query.Set("order_id", "mutated") + + if got, want := sanitized.Get("order_id"), "456"; got != want { + t.Fatalf("sanitized query was not cloned, got %q want %q", got, want) + } +} + +func TestSanitizeSelectorRequestClonesForwardedRequest(t *testing.T) { + request, _ := http.NewRequest(http.MethodGet, "/?_fields=AudienceId&order_id=456", nil) + + sanitized := sanitizeSelectorRequest(request) + + if sanitized == request { + t.Fatal("expected sanitized request clone") + } + if got := sanitized.URL.Query().Get("_fields"); got != "" { + t.Fatalf("sanitized request still has _fields=%q", got) + } + if got, want := sanitized.URL.Query().Get("order_id"), "456"; got != want { + t.Fatalf("sanitized request order_id=%q want %q", got, want) + } + if got, want := request.URL.Query().Get("_fields"), "AudienceId"; got != want { + t.Fatalf("original request was mutated, _fields=%q want %q", got, want) + } +} + +func TestSanitizeSelectorRequestDoesNotReencodeWhenNoSelectorParams(t *testing.T) { + request, _ := http.NewRequest(http.MethodGet, "/?b=two%20words&a=1", nil) + originalRawQuery := request.URL.RawQuery + + sanitized := sanitizeSelectorRequest(request) + + if sanitized != request { + t.Fatal("expected original request when no selector params are present") + } + if sanitized.URL.RawQuery != originalRawQuery { + t.Fatalf("raw query changed, got %q want %q", sanitized.URL.RawQuery, originalRawQuery) + } +} + func TestComponentLocatorRequiresInvocationDispatcher(t *testing.T) { if _, err := newComponentLocator(locator.WithConstants(nil)); err == nil { t.Fatal("expected missing dispatcher error") diff --git a/service.go b/service.go index f6f893f58..27cf7a25f 100644 --- a/service.go +++ b/service.go @@ -194,6 +194,14 @@ func ContextWithViewOutputProjection(ctx context.Context, viewName string, outpu return session.ContextWithViewOutputProjection(ctx, viewName, output) } +func ContextWithOutputFields(ctx context.Context, fields ...string) context.Context { + return session.ContextWithOutputFields(ctx, fields...) +} + +func ContextWithViewOutputFields(ctx context.Context, viewName string, fields ...string) context.Context { + return session.ContextWithViewOutputFields(ctx, viewName, fields...) +} + func WithSession(session *session.Session) OperateOption { return func(o *operateOptions) { o.session = session diff --git a/service/reader/handler/handler.go b/service/reader/handler/handler.go index 98a3e47d3..3f568231d 100644 --- a/service/reader/handler/handler.go +++ b/service/reader/handler/handler.go @@ -118,6 +118,9 @@ func (h *Handler) readData(ctx context.Context, aView *view.View, aState *sessio return err } } + if err = aState.ApplyOutputProjection(ctx, aView); err != nil { + return err + } if err = aState.Populate(ctx); err != nil { return err } diff --git a/service/reader/service.go b/service/reader/service.go index f635064c8..70e7fb95b 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -630,8 +630,83 @@ func (s *Service) warmupMatcher(ctx context.Context, aView *view.View, statelet } cloned := *statelet cloned.Template = clonedTemplate + ok, err := applyWarmupIdentityProjection(aView, &cloned) + if err != nil { + return nil, err + } + if !ok { + return nil, nil + } + + matcher, err := s.sqlBuilder.CacheSQLWithOptions(ctx, aView, &cloned, nil, nil, parent) + if err != nil || matcher == nil { + return matcher, err + } + if err = applyRequestedFields(aView, statelet, matcher); err != nil { + fmt.Printf("[INFO] datly warmup projection metadata error view=%s fields=%v error=%v\n", aView.Name, requestedFieldNames(statelet), err) + return nil, nil + } + return matcher, nil +} - return s.sqlBuilder.CacheSQLWithOptions(ctx, aView, &cloned, nil, nil, parent) +func applyWarmupIdentityProjection(aView *view.View, statelet *view.Statelet) (bool, error) { + if aView == nil || aView.Cache == nil || aView.Cache.Warmup == nil || statelet == nil { + return true, nil + } + fieldNames, ok := aView.Cache.WarmupFieldNamesForSelector(statelet) + if !ok { + return false, nil + } + if len(fieldNames) == 0 { + statelet.SetColumns(nil) + statelet.Fields = nil + return true, nil + } + columns, err := view.ProjectionColumnsForNames(aView, fieldNames) + if err != nil { + return false, err + } + fields := make([]string, 0, len(columns)) + for _, columnName := range columns { + column, ok := aView.ColumnByName(columnName) + if !ok { + return false, fmt.Errorf("failed to map warmup identity column %s to view %s column", columnName, aView.Name) + } + fieldName := column.FieldName() + if fieldName == "" { + fieldName = column.Name + } + fields = append(fields, fieldName) + } + statelet.SetColumns(columns) + statelet.Fields = fields + return true, nil +} + +func applyRequestedFields(aView *view.View, statelet *view.Statelet, matcher *cache.ParmetrizedQuery) error { + if aView == nil || statelet == nil || matcher == nil { + return nil + } + names := statelet.Columns + if len(names) == 0 { + names = statelet.Fields + } + fields, err := view.ProjectionFieldsForNames(aView, names) + if err != nil { + return err + } + matcher.RequestedFields = view.SQLXProjectionFields(fields) + return nil +} + +func requestedFieldNames(statelet *view.Statelet) []string { + if statelet == nil { + return nil + } + if len(statelet.Columns) != 0 { + return statelet.Columns + } + return statelet.Fields } func warmupIndexParameter(aView *view.View) *state.Parameter { diff --git a/service/reader/service_warmup_test.go b/service/reader/service_warmup_test.go index 53ca54d05..ea4aae56b 100644 --- a/service/reader/service_warmup_test.go +++ b/service/reader/service_warmup_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/viant/datly/view" "github.com/viant/datly/view/state" + "github.com/viant/sqlx/io/read/cache" "github.com/viant/structology" ) @@ -205,3 +206,271 @@ func TestWarmupIndexParameterDoesNotMatchUnrelatedParameter(t *testing.T) { require.Nil(t, parameter) } + +func TestApplyRequestedFieldsPopulatesMatcherProjection(t *testing.T) { + aView := view.NewView("events", "events", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + {Name: "order_id", DataType: "int", Tag: `json:"orderId"`, Groupable: true}, + {Name: "bids", DataType: "int", Aggregate: true}, + {Name: "impressions", DataType: "int", Aggregate: true}, + }), + view.WithGroupable(true), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + statelet := view.NewStatelet() + statelet.SetColumns([]string{"impressions", "order_id"}) + matcher := &cache.ParmetrizedQuery{} + + require.NoError(t, applyRequestedFields(aView, statelet, matcher)) + + require.Len(t, matcher.RequestedFields, 2) + require.Equal(t, "impressions", matcher.RequestedFields[0].Name) + require.Equal(t, "impressions", matcher.RequestedFields[0].MeasureKey) + require.Equal(t, "order_id", matcher.RequestedFields[1].Name) + require.Equal(t, "order_id", matcher.RequestedFields[1].DimensionKey) +} + +func TestApplyRequestedFieldsIgnoresUnmappedProjection(t *testing.T) { + aView := view.NewView("events", "events", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + {Name: "order_id", DataType: "int"}, + }), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + statelet := view.NewStatelet() + statelet.SetColumns([]string{"missing"}) + matcher := &cache.ParmetrizedQuery{} + + require.Error(t, applyRequestedFields(aView, statelet, matcher)) + + require.Empty(t, matcher.RequestedFields) +} + +func TestApplyRequestedFieldsUsesFullProjectionWhenRequestHasNoProjection(t *testing.T) { + aView := view.NewView("events", "events", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + {Name: "audience_id", DataType: "int", Groupable: true}, + {Name: "spend", DataType: "float", Aggregate: true}, + }), + view.WithGroupable(true), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + matcher := &cache.ParmetrizedQuery{} + + require.NoError(t, applyRequestedFields(aView, view.NewStatelet(), matcher)) + + require.Len(t, matcher.RequestedFields, 2) + require.Equal(t, "audience_id", matcher.RequestedFields[0].DimensionKey) + require.Equal(t, "spend", matcher.RequestedFields[1].MeasureKey) +} + +func TestApplyWarmupIdentityProjectionUsesWarmupFieldNames(t *testing.T) { + aView := view.NewView("events", "events", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + {Name: "audience_id", DataType: "int", Tag: `json:"audienceId"`, Groupable: true}, + {Name: "bids", DataType: "int", Aggregate: true}, + {Name: "spend", DataType: "float", Aggregate: true}, + {Name: "period_ecpm", DataType: "float", Aggregate: true}, + }), + view.WithGroupable(true), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + aView.Cache = &view.Cache{ + Warmup: &view.Warmup{ + FieldNames: []string{"audience_id", "bids", "spend", "period_ecpm"}, + }, + } + statelet := view.NewStatelet() + statelet.SetColumns([]string{"audience_id", "spend", "period_ecpm"}) + statelet.Fields = []string{"AudienceId", "Spend", "PeriodEcpm"} + + ok, err := applyWarmupIdentityProjection(aView, statelet) + require.NoError(t, err) + require.True(t, ok) + + require.Equal(t, []string{"audience_id", "bids", "spend", "period_ecpm"}, statelet.Columns) + require.Equal(t, []string{"audience_id", "bids", "spend", "period_ecpm"}, statelet.Fields) +} + +func TestApplyWarmupIdentityProjectionUsesMatchingCaseFieldNames(t *testing.T) { + periodParam := state.NewParameter("Period", state.NewQueryLocation("period"), state.WithParameterType(reflect.TypeOf(""))) + aView := view.NewView("events", "events", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + {Name: "audience_id", DataType: "int", Groupable: true}, + {Name: "bids", DataType: "int", Aggregate: true}, + {Name: "spend", DataType: "float", Aggregate: true}, + {Name: "period_ecpm", DataType: "float", Aggregate: true}, + }), + view.WithTemplate(view.NewTemplate("", view.WithTemplateParameters(periodParam))), + view.WithGroupable(true), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + aView.Cache = &view.Cache{ + Warmup: &view.Warmup{ + FieldNames: []string{"audience_id", "bids"}, + Cases: []*view.CacheParameters{ + { + Set: []*view.ParamValue{ + {Name: "Period", Values: []interface{}{"today"}}, + }, + FieldNames: []string{"audience_id", "spend", "period_ecpm"}, + }, + }, + }, + } + statelet := view.NewStatelet() + statelet.Init(aView) + require.NoError(t, periodParam.Set(statelet.Template, "today")) + + ok, err := applyWarmupIdentityProjection(aView, statelet) + require.NoError(t, err) + require.True(t, ok) + + require.Equal(t, []string{"audience_id", "spend", "period_ecpm"}, statelet.Columns) +} + +func TestApplyWarmupIdentityProjectionMatchesDefaultOptionalCase(t *testing.T) { + periodParam := state.NewParameter("Period", state.NewQueryLocation("period"), state.WithParameterType(reflect.TypeOf(""))) + aView := view.NewView("events", "events", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + {Name: "audience_id", DataType: "int", Groupable: true}, + {Name: "bids", DataType: "int", Aggregate: true}, + {Name: "spend", DataType: "float", Aggregate: true}, + }), + view.WithTemplate(view.NewTemplate("", view.WithTemplateParameters(periodParam))), + view.WithGroupable(true), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + aView.Cache = &view.Cache{ + Warmup: &view.Warmup{ + FieldNames: []string{"audience_id", "bids"}, + Cases: []*view.CacheParameters{ + { + Set: []*view.ParamValue{ + {Name: "Period"}, + }, + FieldNames: []string{"audience_id", "spend"}, + }, + }, + }, + } + statelet := view.NewStatelet() + statelet.Init(aView) + + ok, err := applyWarmupIdentityProjection(aView, statelet) + require.NoError(t, err) + require.True(t, ok) + + require.Equal(t, []string{"audience_id", "spend"}, statelet.Columns) +} + +func TestApplyWarmupIdentityProjectionClearsProjectionForAmbiguousCaseFieldNames(t *testing.T) { + periodParam := state.NewParameter("Period", state.NewQueryLocation("period"), state.WithParameterType(reflect.TypeOf(""))) + aView := view.NewView("events", "events", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + {Name: "audience_id", DataType: "int", Groupable: true}, + {Name: "bids", DataType: "int", Aggregate: true}, + {Name: "spend", DataType: "float", Aggregate: true}, + }), + view.WithTemplate(view.NewTemplate("", view.WithTemplateParameters(periodParam))), + view.WithGroupable(true), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + aView.Cache = &view.Cache{ + Warmup: &view.Warmup{ + Cases: []*view.CacheParameters{ + { + Set: []*view.ParamValue{ + {Name: "Period"}, + }, + FieldNames: []string{"audience_id", "bids"}, + }, + { + Set: []*view.ParamValue{ + {Name: "Period"}, + }, + FieldNames: []string{"audience_id", "spend"}, + }, + }, + }, + } + statelet := view.NewStatelet() + statelet.Init(aView) + statelet.SetColumns([]string{"audience_id", "spend"}) + + ok, err := applyWarmupIdentityProjection(aView, statelet) + require.NoError(t, err) + require.False(t, ok) + + require.Equal(t, []string{"audience_id", "spend"}, statelet.Columns) + require.Empty(t, statelet.Fields) +} + +func TestApplyWarmupIdentityProjectionAllowsEquivalentCaseFieldAliases(t *testing.T) { + periodParam := state.NewParameter("Period", state.NewQueryLocation("period"), state.WithParameterType(reflect.TypeOf(""))) + aView := view.NewView("events", "events", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + {Name: "audience_id", DataType: "int", Tag: `json:"audienceId"`, Groupable: true}, + {Name: "spend", DataType: "float", Aggregate: true}, + }), + view.WithTemplate(view.NewTemplate("", view.WithTemplateParameters(periodParam))), + view.WithGroupable(true), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + aView.Cache = &view.Cache{ + Warmup: &view.Warmup{ + Cases: []*view.CacheParameters{ + { + Set: []*view.ParamValue{ + {Name: "Period"}, + }, + FieldNames: []string{"AudienceId", "Spend"}, + }, + { + Set: []*view.ParamValue{ + {Name: "Period"}, + }, + FieldNames: []string{"audience_id", "spend"}, + }, + }, + }, + } + statelet := view.NewStatelet() + statelet.Init(aView) + + ok, err := applyWarmupIdentityProjection(aView, statelet) + require.NoError(t, err) + require.True(t, ok) + + require.Equal(t, []string{"audience_id", "spend"}, statelet.Columns) +} + +func TestApplyWarmupIdentityProjectionClearsProjectionForFullWarmup(t *testing.T) { + aView := view.NewView("events", "events", + view.WithConnector(view.NewConnector("test", "sqlite3", ":memory:")), + view.WithColumns(view.Columns{ + {Name: "audience_id", DataType: "int"}, + {Name: "spend", DataType: "float"}, + }), + ) + require.NoError(t, aView.Init(context.Background(), view.EmptyResource())) + aView.Cache = &view.Cache{Warmup: &view.Warmup{}} + statelet := view.NewStatelet() + statelet.SetColumns([]string{"audience_id", "spend"}) + statelet.Fields = []string{"AudienceId", "Spend"} + + ok, err := applyWarmupIdentityProjection(aView, statelet) + require.NoError(t, err) + require.True(t, ok) + + require.Empty(t, statelet.Columns) + require.Empty(t, statelet.Fields) +} diff --git a/service/reader/sql_groupable_test.go b/service/reader/sql_groupable_test.go index 16e652c3d..1dcb85076 100644 --- a/service/reader/sql_groupable_test.go +++ b/service/reader/sql_groupable_test.go @@ -193,6 +193,59 @@ func TestBuilder_rewriteGroupBy(t *testing.T) { }, expected: "(SELECT (99 * SUM(avails)) + MOD(SUM(avails), 99) AS avails FROM audience_event_v1 v)", }, + { + description: "rewrite site cube shared diagnostic projection", + sql: "(SELECT ao.event_date, ao.advertiser_date, ao.agency_id, ao.advertiser_id, ao.campaign_id, " + + "ao.ad_order_id, ao.audience_id, ao.creative_id, ao.deal_id, ao.publisher_id, ao.channel_id, " + + "ao.country, ao.site_type, ao.site_id, SUM(ao.bids) AS bids, SUM(ao.impressions) AS impressions, " + + "SUM(ao.clicks) AS clicks, SUM(ao.conversions) AS conversions, SUM(ao.view_conversions) AS view_conversions, " + + "SUM(ao.total_spend) AS total_spend, SUM(ao.v_start) AS v_start, SUM(ao.v_100) AS v_100 " + + "FROM fact_perf_site_daily_v ao GROUP BY ao.event_date, ao.advertiser_date, ao.agency_id, ao.advertiser_id, " + + "ao.campaign_id, ao.ad_order_id, ao.audience_id, ao.creative_id, ao.deal_id, ao.external_deal_id, " + + "ao.publisher_id, ao.channel_id, ao.country, ao.site_type, ao.is_pg, ao.media_execution_id, ao.site_id)", + allColumns: []*view.Column{ + {Name: "event_date", Groupable: true}, + {Name: "advertiser_date", Groupable: true}, + {Name: "agency_id", Groupable: true}, + {Name: "advertiser_id", Groupable: true}, + {Name: "campaign_id", Groupable: true}, + {Name: "ad_order_id", Groupable: true}, + {Name: "audience_id", Groupable: true}, + {Name: "creative_id", Groupable: true}, + {Name: "deal_id", Groupable: true}, + {Name: "publisher_id", Groupable: true}, + {Name: "channel_id", Groupable: true}, + {Name: "country", Groupable: true}, + {Name: "site_type", Groupable: true}, + {Name: "site_id", Groupable: true}, + {Name: "bids"}, + {Name: "impressions"}, + {Name: "clicks"}, + {Name: "conversions"}, + {Name: "view_conversions"}, + {Name: "total_spend"}, + {Name: "v_start"}, + {Name: "v_100"}, + }, + projected: []*view.Column{ + {Name: "event_date", Groupable: true}, + {Name: "advertiser_date", Groupable: true}, + {Name: "site_id", Groupable: true}, + {Name: "site_type", Groupable: true}, + {Name: "bids"}, + {Name: "impressions"}, + {Name: "clicks"}, + {Name: "conversions"}, + {Name: "view_conversions"}, + {Name: "total_spend"}, + {Name: "v_start"}, + {Name: "v_100"}, + }, + expected: "(SELECT ao.event_date, ao.advertiser_date, ao.site_id, ao.site_type, SUM(ao.bids) AS bids, SUM(ao.impressions) AS impressions, " + + "SUM(ao.clicks) AS clicks, SUM(ao.conversions) AS conversions, SUM(ao.view_conversions) AS view_conversions, " + + "SUM(ao.total_spend) AS total_spend, SUM(ao.v_start) AS v_start, SUM(ao.v_100) AS v_100 " + + "FROM fact_perf_site_daily_v ao GROUP BY 1, 2, 3, 4)", + }, { description: "rewrite grouped aggregates matches reordered forecasting measures by alias not metadata order", sql: "(SELECT IFNULL(STRING_AGG(DISTINCT IAB[SAFE_OFFSET(0)], ', ' LIMIT 20), '') AS iab_cats, " + @@ -221,11 +274,11 @@ func TestBuilder_rewriteGroupBy(t *testing.T) { {Name: "hh_uniqs"}, {Name: "device_uniqs"}, }, - expected: "(SELECT (99 * SUM(avails)) + MOD(SUM(avails), 99) AS avails, " + + expected: "(SELECT AVG(v.clearing_price) AS min_clearing_price, " + + "MAX(v.clearing_price) AS max_clearing_price, " + + "(99 * SUM(avails)) + MOD(SUM(avails), 99) AS avails, " + "(9100 * APPROX_COUNT_DISTINCT(IF(avails = 100, COALESCE(alias_ip, IF(hhip_flag = 1, ip, NULL)), NULL))) AS hh_uniqs, " + - "(9100 * APPROX_COUNT_DISTINCT(uid)) AS device_uniqs, " + - "AVG(v.clearing_price) AS min_clearing_price, " + - "MAX(v.clearing_price) AS max_clearing_price " + + "(9100 * APPROX_COUNT_DISTINCT(uid)) AS device_uniqs " + "FROM audience_event_v1 v)", }, { diff --git a/service/session/option.go b/service/session/option.go index 5464f9a63..48e85b6d9 100644 --- a/service/session/option.go +++ b/service/session/option.go @@ -162,6 +162,16 @@ func WithViewOutputProjection(viewName string, output interface{}) Option { } } +func WithOutputFields(fields ...string) Option { + return WithViewOutputFields("", fields...) +} + +func WithViewOutputFields(viewName string, fields ...string) Option { + return func(s *Options) { + s.outputProjection = &OutputProjection{View: viewName, Fields: append([]string(nil), fields...), FieldsHint: true} + } +} + func WithStateResource(resource state.Resource) Option { return func(s *Options) { s.resource = resource diff --git a/service/session/projection.go b/service/session/projection.go index 53b88debf..a43e01cac 100644 --- a/service/session/projection.go +++ b/service/session/projection.go @@ -1,12 +1,17 @@ package session -import "context" +import ( + "context" + "fmt" +) type outputProjectionKey struct{} type OutputProjection struct { - View string - Output interface{} + View string + Output interface{} + Fields []string + FieldsHint bool } func ContextWithOutputProjection(ctx context.Context, output interface{}) context.Context { @@ -17,6 +22,14 @@ func ContextWithViewOutputProjection(ctx context.Context, viewName string, outpu return context.WithValue(ctx, outputProjectionKey{}, &OutputProjection{View: viewName, Output: output}) } +func ContextWithOutputFields(ctx context.Context, fields ...string) context.Context { + return ContextWithViewOutputFields(ctx, "", fields...) +} + +func ContextWithViewOutputFields(ctx context.Context, viewName string, fields ...string) context.Context { + return context.WithValue(ctx, outputProjectionKey{}, &OutputProjection{View: viewName, Fields: append([]string(nil), fields...), FieldsHint: true}) +} + func OutputProjectionFromContext(ctx context.Context, viewName string) interface{} { if ctx == nil { return nil @@ -39,3 +52,53 @@ func OutputProjectionFromContext(ctx context.Context, viewName string) interface } return nil } + +func OutputFieldsFromContext(ctx context.Context, viewName string) []string { + if ctx == nil { + return nil + } + value := ctx.Value(outputProjectionKey{}) + switch actual := value.(type) { + case *OutputProjection: + if actual == nil { + return nil + } + if actual.View != "" && normalizeViewProjectionName(actual.View) != normalizeViewProjectionName(viewName) { + return nil + } + return append([]string(nil), actual.Fields...) + case OutputProjection: + if actual.View != "" && normalizeViewProjectionName(actual.View) != normalizeViewProjectionName(viewName) { + return nil + } + return append([]string(nil), actual.Fields...) + } + return nil +} + +func OutputFieldsHintFromContext(ctx context.Context, viewName string) bool { + if ctx == nil { + return false + } + value := ctx.Value(outputProjectionKey{}) + switch actual := value.(type) { + case *OutputProjection: + if actual == nil { + return false + } + if actual.View != "" && normalizeViewProjectionName(actual.View) != normalizeViewProjectionName(viewName) { + return false + } + return actual.FieldsHint + case OutputProjection: + if actual.View != "" && normalizeViewProjectionName(actual.View) != normalizeViewProjectionName(viewName) { + return false + } + return actual.FieldsHint + } + return false +} + +func EmptyOutputFieldsError(viewName string) error { + return fmt.Errorf("output projection for view %s did not specify any field names", viewName) +} diff --git a/service/session/state.go b/service/session/state.go index de32d93fa..c00b7428e 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "log" "net/http" "os" "reflect" @@ -205,6 +204,18 @@ func (s *Session) applyViewProjection(aView *view.View) { } func (s *Session) ApplyOutputProjection(ctx context.Context, aView *view.View) error { + if hasFieldsHint := s.outputFieldsHintForView(ctx, aView.Name); hasFieldsHint { + fields := s.outputFieldsForView(ctx, aView.Name) + if len(fields) == 0 { + return EmptyOutputFieldsError(aView.Name) + } + columns, err := view.ProjectionColumnsForNames(aView, fields) + if err != nil { + return err + } + s.applyProjectionColumns(aView, columns) + return nil + } var output interface{} if s.outputProjection != nil { output = projectionOutputForView(*s.outputProjection, aView.Name) @@ -222,11 +233,31 @@ func (s *Session) ApplyOutputProjection(ctx context.Context, aView *view.View) e if columns == nil { return nil } - log.Printf("[PROJECTION] view=%s columns=%v", aView.Name, columns) + s.applyProjectionColumns(aView, columns) + return nil +} + +func (s *Session) outputFieldsForView(ctx context.Context, viewName string) []string { + if s.outputProjection != nil && len(s.outputProjection.Fields) > 0 { + return projectionFieldsForView(*s.outputProjection, viewName) + } + return OutputFieldsFromContext(ctx, viewName) +} + +func (s *Session) outputFieldsHintForView(ctx context.Context, viewName string) bool { + if s.outputProjection != nil && s.outputProjection.FieldsHint { + return projectionFieldsHintForView(*s.outputProjection, viewName) + } + return OutputFieldsHintFromContext(ctx, viewName) +} + +func (s *Session) applyProjectionColumns(aView *view.View, columns []string) { + if len(columns) == 0 { + return + } s.Apply(WithViewProjectionColumns(aView.Name, columns)) statelet := s.state.Lookup(aView) statelet.SetColumns(columns) - return nil } func projectionOutputForView(projection OutputProjection, viewName string) interface{} { @@ -236,6 +267,20 @@ func projectionOutputForView(projection OutputProjection, viewName string) inter return projection.Output } +func projectionFieldsForView(projection OutputProjection, viewName string) []string { + if projection.View != "" && normalizeViewProjectionName(projection.View) != normalizeViewProjectionName(viewName) { + return nil + } + return append([]string(nil), projection.Fields...) +} + +func projectionFieldsHintForView(projection OutputProjection, viewName string) bool { + if projection.View != "" && normalizeViewProjectionName(projection.View) != normalizeViewProjectionName(viewName) { + return false + } + return projection.FieldsHint +} + func normalizeViewProjectionName(name string) string { name = strings.ToLower(strings.TrimSpace(name)) name = strings.ReplaceAll(name, "_", "") diff --git a/service_projection_test.go b/service_projection_test.go index fed405c16..83ce0afc1 100644 --- a/service_projection_test.go +++ b/service_projection_test.go @@ -166,6 +166,36 @@ func TestSessionApplyOutputProjectionFromScopedContext_OnlyAppliesToMatchingView require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) } +func TestSessionApplyOutputFieldsFromContext_NarrowsChildView(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View) + ctx := ContextWithOutputFields(context.Background(), "account_id", "bids") + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) + require.True(t, statelet.Has("account_id")) + require.True(t, statelet.Has("bids")) + require.False(t, statelet.Has("campaign_id")) +} + +func TestSessionApplyOutputFieldsFromScopedContext_OnlyAppliesToMatchingView(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View) + ctx := ContextWithViewOutputFields(context.Background(), "other_view", "account_id", "bids") + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Empty(t, statelet.Columns) + + ctx = ContextWithViewOutputFields(context.Background(), aComponent.View.Name, "account_id", "bids") + err = aSession.ApplyOutputProjection(ctx, aComponent.View) + require.NoError(t, err) + require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) +} + func TestSessionApplyOutputProjectionFromOption_NarrowsChildView(t *testing.T) { aComponent := groupableProjectionTestComponent(t) var output []struct { @@ -180,6 +210,16 @@ func TestSessionApplyOutputProjectionFromOption_NarrowsChildView(t *testing.T) { require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) } +func TestSessionApplyOutputFieldsFromOption_NarrowsChildView(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View, session.WithOutputFields("account_id", "bids")) + + err := aSession.ApplyOutputProjection(context.Background(), aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) +} + func TestSessionApplyOutputProjectionFromScopedOption_OnlyAppliesToMatchingView(t *testing.T) { aComponent := groupableProjectionTestComponent(t) var output []struct { @@ -199,6 +239,21 @@ func TestSessionApplyOutputProjectionFromScopedOption_OnlyAppliesToMatchingView( require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) } +func TestSessionApplyOutputFieldsFromScopedOption_OnlyAppliesToMatchingView(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View, session.WithViewOutputFields("other_view", "account_id", "bids")) + + err := aSession.ApplyOutputProjection(context.Background(), aComponent.View) + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Empty(t, statelet.Columns) + + aSession.Apply(session.WithViewOutputFields(aComponent.View.Name, "account_id", "bids")) + err = aSession.ApplyOutputProjection(context.Background(), aComponent.View) + require.NoError(t, err) + require.Equal(t, []string{"account_id", "bids"}, statelet.Columns) +} + func TestSessionApplyOutputProjectionWithoutHint_LeavesChildViewFullWidth(t *testing.T) { aComponent := groupableProjectionTestComponent(t) aSession := session.New(aComponent.View) @@ -257,6 +312,50 @@ func TestSessionApplyOutputProjectionFromContext_FailsForUnknownField(t *testing require.Contains(t, err.Error(), "failed to map output field Unknown") } +func TestSessionApplyOutputFieldsFromContext_FailsForUnknownField(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View) + ctx := ContextWithOutputFields(context.Background(), "unknown") + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + + require.Error(t, err) + require.Contains(t, err.Error(), "failed to map output field unknown") +} + +func TestSessionApplyOutputFieldsFromContext_FailsForEmptyFields(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View) + ctx := ContextWithOutputFields(context.Background()) + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + + require.Error(t, err) + require.Contains(t, err.Error(), "did not specify any field names") +} + +func TestSessionApplyOutputFieldsFromOption_FailsForEmptyFields(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View, session.WithOutputFields()) + + err := aSession.ApplyOutputProjection(context.Background(), aComponent.View) + + require.Error(t, err) + require.Contains(t, err.Error(), "did not specify any field names") +} + +func TestSessionApplyOutputFieldsFromScopedContext_IgnoresEmptyFieldsForNonMatchingView(t *testing.T) { + aComponent := groupableProjectionTestComponent(t) + aSession := session.New(aComponent.View) + ctx := ContextWithViewOutputFields(context.Background(), "other_view") + + err := aSession.ApplyOutputProjection(ctx, aComponent.View) + + require.NoError(t, err) + statelet := aSession.State().Lookup(aComponent.View) + require.Empty(t, statelet.Columns) +} + func TestWithOutput_DoesNotEnableProjection(t *testing.T) { options := newOperateOptions([]OperateOption{WithOutput(&[]alternateProjectionOutput{})}) diff --git a/view/cache.go b/view/cache.go index 3c59bbab7..51a4e1683 100644 --- a/view/cache.go +++ b/view/cache.go @@ -78,12 +78,13 @@ type ( } CacheInput struct { - Selector *Statelet - Column string - MetaColumn string - IndexMeta bool - Label string - FieldNames []string + Selector *Statelet + Column string + MetaColumn string + IndexMeta bool + Label string + FieldNames []string + StoredFields []ProjectionField } CacheInputFn func() ([]*CacheInput, error) @@ -430,7 +431,10 @@ func (p *ParamValue) clone() *ParamValue { func (c *Cache) GenerateCacheInput(ctx context.Context) ([]*CacheInput, error) { if len(c.Warmup.Cases) == 0 { - input := c.NewInput(NewStatelet()) + input, err := c.newInputWithError(NewStatelet(), nil) + if err != nil { + return nil, err + } if c.maxCasesExceeded(0, 0, input) { if maxCases := c.maxCases(); maxCases > 0 { fmt.Printf("[INFO] cache warmup selector cap view=%s max_cases=%d selected_entries=0 selected_selectors=0\n", c.owner.Name, maxCases) @@ -673,7 +677,10 @@ func (c *Cache) appendSelectors(set *CacheParameters, paramValues [][]interface{ indexes := make([]int, len(paramValues)) generatedEntries := 0 if len(indexes) == 0 { - input := c.newInput(NewStatelet(), set) + input, err := c.newInputWithError(NewStatelet(), set) + if err != nil { + return err + } if c.maxCasesExceeded(selectedEntries, generatedEntries, input) { return nil } @@ -702,7 +709,10 @@ outer: } label := strings.Join(debugParams, ",") - input := c.newInput(selector, set) + input, err := c.newInputWithError(selector, set) + if err != nil { + return err + } input.Label = label if c.maxCasesExceeded(selectedEntries, generatedEntries, input) { return nil @@ -733,18 +743,41 @@ func (c *Cache) NewInput(selector *Statelet) *CacheInput { } func (c *Cache) newInput(selector *Statelet, set *CacheParameters) *CacheInput { + input, err := c.newInputWithError(selector, set) + if err == nil { + return input + } + if c != nil && c.owner != nil { + fmt.Printf("[INFO] cache warmup projection metadata error view=%s field_names=%v error=%v\n", c.owner.Name, c.fieldNamesFor(set), err) + } + return c.newInputWithoutStoredFields(selector, set) +} + +func (c *Cache) newInputWithError(selector *Statelet, set *CacheParameters) (*CacheInput, error) { fieldNames := c.fieldNamesFor(set) if selector != nil && c.Warmup != nil && c.Warmup.Limit != nil { selector.Limit = *c.Warmup.Limit selector.WarmupNoLimit = *c.Warmup.Limit == 0 } c.applyWarmupFieldNames(selector, fieldNames) + storedFields, err := ProjectionFieldsForNames(c.owner, fieldNames) + if err != nil { + return nil, err + } + input := c.newInputWithoutStoredFields(selector, set) + input.StoredFields = append([]ProjectionField(nil), storedFields...) + return input, nil +} + +func (c *Cache) newInputWithoutStoredFields(selector *Statelet, set *CacheParameters) *CacheInput { + fieldNames := c.fieldNamesFor(set) return &CacheInput{ - Selector: selector, - Column: c.Warmup.IndexColumn, - MetaColumn: c.Warmup.IndexColumn, - IndexMeta: (c.Warmup.IndexMeta || c.Warmup.IndexColumn != "") && c.owner.Template.Summary != nil, - FieldNames: append([]string(nil), fieldNames...), + Selector: selector, + Column: c.Warmup.IndexColumn, + MetaColumn: c.Warmup.IndexColumn, + IndexMeta: (c.Warmup.IndexMeta || c.Warmup.IndexColumn != "") && c.owner.Template.Summary != nil, + FieldNames: append([]string(nil), fieldNames...), + StoredFields: nil, } } @@ -758,6 +791,150 @@ func (c *Cache) fieldNamesFor(set *CacheParameters) []string { return c.Warmup.FieldNames } +func (c *Cache) WarmupFieldNamesForSelector(selector *Statelet) ([]string, bool) { + if c == nil || c.Warmup == nil { + return nil, true + } + matchedAny := false + var matchedColumns []string + for _, candidate := range c.Warmup.Cases { + if candidate == nil || len(candidate.FieldNames) == 0 || !c.warmupCaseMatchesSelector(candidate, selector) { + continue + } + columns, ok := c.warmupProjectionColumns(candidate.FieldNames) + if !ok { + return nil, false + } + if !matchedAny { + matchedAny = true + matchedColumns = columns + continue + } + if !stringSlicesEqual(matchedColumns, columns) { + return nil, false + } + } + if matchedAny { + return matchedColumns, true + } + return c.Warmup.FieldNames, true +} + +func (c *Cache) warmupProjectionColumns(fieldNames []string) ([]string, bool) { + if len(fieldNames) == 0 { + return nil, true + } + if c == nil || c.owner == nil { + return fieldNames, true + } + columns, err := ProjectionColumnsForNames(c.owner, fieldNames) + if err == nil { + return columns, true + } + columns = make([]string, 0, len(fieldNames)) + for _, fieldName := range fieldNames { + column, ok := c.owner.ColumnByName(fieldName) + if !ok { + column, ok = c.warmupColumnByNormalizedName(fieldName) + } + if !ok { + return nil, false + } + columns = append(columns, column.Name) + } + return columns, true +} + +func (c *Cache) warmupColumnByNormalizedName(name string) (*Column, bool) { + if c == nil || c.owner == nil { + return nil, false + } + normalized := normalizeProjectionFieldName(name) + for _, column := range c.owner.Columns { + if column == nil { + continue + } + if normalizeProjectionFieldName(column.Name) == normalized || + normalizeProjectionFieldName(column.FieldName()) == normalized || + normalizeProjectionFieldName(column.DatabaseColumn) == normalized { + return column, true + } + } + return nil, false +} + +func stringSlicesEqual(left, right []string) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if normalizeProjectionFieldName(left[i]) != normalizeProjectionFieldName(right[i]) { + return false + } + } + return true +} + +func (c *Cache) warmupCaseMatchesSelector(candidate *CacheParameters, selector *Statelet) bool { + if candidate == nil || selector == nil || selector.Template == nil { + return false + } + for _, paramValue := range candidate.Set { + if paramValue == nil { + return false + } + actual, ok := warmupSelectorValue(selector, paramValue) + if !ok { + return false + } + candidates := paramValue.Values + if paramValue._param != nil { + var err error + candidates, err = c.getParamValues(context.Background(), paramValue) + if err != nil { + return false + } + } + if !warmupValueMatches(actual, candidates) { + return false + } + } + return true +} + +func warmupSelectorValue(selector *Statelet, paramValue *ParamValue) (interface{}, bool) { + if selector == nil || selector.Template == nil || paramValue == nil { + return nil, false + } + if paramValue._param != nil && paramValue._param.Selector() != nil { + stateSelector := paramValue._param.Selector() + if !stateSelector.Has(selector.Template.Pointer()) { + return nil, true + } + return stateSelector.Value(selector.Template.Pointer()), true + } + stateSelector, err := selector.Template.Selector(paramValue.Name) + if err != nil || stateSelector == nil { + return nil, false + } + if !stateSelector.Has(selector.Template.Pointer()) { + return nil, true + } + return stateSelector.Value(selector.Template.Pointer()), true +} + +func warmupValueMatches(actual interface{}, candidates []interface{}) bool { + if len(candidates) == 0 { + return actual == nil || reflect.ValueOf(actual).IsZero() + } + for _, candidate := range candidates { + if reflect.DeepEqual(actual, candidate) || fmt.Sprint(actual) == fmt.Sprint(candidate) { + return true + } + } + return false +} + func (c *Cache) maxCases() int { if c == nil || c.Warmup == nil || c.Warmup.MaxCases == nil || *c.Warmup.MaxCases <= 0 { return 0 diff --git a/view/collector.go b/view/collector.go index b6c5fab5f..9e3bc24fe 100644 --- a/view/collector.go +++ b/view/collector.go @@ -98,14 +98,26 @@ func normalizeValues(value interface{}) []interface{} { case []string: result := make([]interface{}, 0, len(actual)) for _, item := range actual { + if isBlankStringKey(item) { + continue + } result = append(result, io.NormalizeKey(item)) } return result + case string: + if isBlankStringKey(actual) { + return nil + } + return []interface{}{io.NormalizeKey(value)} default: return []interface{}{io.NormalizeKey(value)} } } +func isBlankStringKey(value string) bool { + return strings.TrimSpace(value) == "" +} + func compositeRows(parts [][]interface{}) [][]interface{} { if len(parts) == 0 { return nil @@ -1143,6 +1155,9 @@ outer: case []string: for j := range actual { + if isBlankStringKey(actual[j]) { + continue + } if _, ok := unique[actual[j]]; ok { continue } @@ -1150,6 +1165,9 @@ outer: result = append(result, actual[j]) } default: + if actual, ok := fieldValue.(string); ok && isBlankStringKey(actual) { + continue + } if count := len(result); count > 0 { if result[count-1] == fieldValue { //value already added continue diff --git a/view/projection.go b/view/projection.go index c63c60584..eae4b7de1 100644 --- a/view/projection.go +++ b/view/projection.go @@ -4,8 +4,20 @@ import ( "fmt" "reflect" "strings" + + "github.com/viant/sqlx/io/read/cache" ) +type ProjectionField struct { + Name string `json:",omitempty" yaml:",omitempty"` + FieldName string `json:",omitempty" yaml:",omitempty"` + ColumnName string `json:",omitempty" yaml:",omitempty"` + Source string `json:",omitempty" yaml:",omitempty"` + DimensionKey string `json:",omitempty" yaml:",omitempty"` + MeasureKey string `json:",omitempty" yaml:",omitempty"` + Lookup []string `json:",omitempty" yaml:",omitempty"` +} + func ProjectionColumnsForOutput(aView *View, output interface{}) ([]string, error) { if output == nil { return nil, nil @@ -13,6 +25,122 @@ func ProjectionColumnsForOutput(aView *View, output interface{}) ([]string, erro return ProjectionColumnsForType(aView, ProjectionOutputStructType(reflect.TypeOf(output))) } +func ProjectionColumnsForNames(aView *View, names []string) ([]string, error) { + if len(names) == 0 { + return nil, nil + } + columns := make([]string, 0, len(names)) + seen := map[string]bool{} + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" { + return nil, fmt.Errorf("output projection for view %s contains empty field name", aView.Name) + } + column, ok := aView.ColumnByName(name) + if !ok { + return nil, fmt.Errorf("failed to map output field %s to view %s column", name, aView.Name) + } + if seen[column.Name] { + continue + } + seen[column.Name] = true + columns = append(columns, column.Name) + } + return columns, nil +} + +func ProjectionFieldsForNames(aView *View, names []string) ([]ProjectionField, error) { + if len(names) == 0 { + return ProjectionFieldsForColumns(aView, aView.Columns), nil + } + fields := make([]ProjectionField, 0, len(names)) + seen := map[string]bool{} + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" { + return nil, fmt.Errorf("output projection for view %s contains empty field name", aView.Name) + } + column, ok := aView.ColumnByName(name) + if !ok { + return nil, fmt.Errorf("failed to map output field %s to view %s column", name, aView.Name) + } + if seen[column.Name] { + continue + } + seen[column.Name] = true + fields = append(fields, ProjectionFieldForViewColumn(aView, column)) + } + return fields, nil +} + +func ProjectionFieldsForColumns(aView *View, columns Columns) []ProjectionField { + if len(columns) == 0 { + return nil + } + fields := make([]ProjectionField, 0, len(columns)) + seen := map[string]bool{} + for _, column := range columns { + if column == nil || column.Name == "" || seen[column.Name] { + continue + } + seen[column.Name] = true + fields = append(fields, ProjectionFieldForViewColumn(aView, column)) + } + return fields +} + +func ProjectionFieldForColumn(column *Column) ProjectionField { + if column == nil { + return ProjectionField{} + } + fieldName := column.FieldName() + if fieldName == "" { + fieldName = column.Name + } + source := projectionFieldSource(column) + return ProjectionField{ + Name: column.Name, + FieldName: fieldName, + ColumnName: column.Name, + Source: source, + Lookup: projectionFieldLookup(column.Name, fieldName, column.DatabaseColumn), + } +} + +func ProjectionFieldForViewColumn(aView *View, column *Column) ProjectionField { + field := ProjectionFieldForColumn(column) + if aView == nil || column == nil || !aView.Groupable { + return field + } + if column.Groupable { + field.MeasureKey = "" + field.DimensionKey = strings.TrimSpace(column.Name) + return field + } + field.DimensionKey = "" + field.MeasureKey = strings.TrimSpace(column.Name) + return field +} + +func SQLXProjectionFields(fields []ProjectionField) []cache.ProjectionField { + if fields == nil { + return nil + } + result := make([]cache.ProjectionField, 0, len(fields)) + for _, field := range fields { + result = append(result, cache.ProjectionField{ + Name: field.Name, + FieldName: field.FieldName, + ColumnName: field.ColumnName, + Source: field.Source, + DimensionKey: field.DimensionKey, + MeasureKey: field.MeasureKey, + Lookup: append([]string(nil), field.Lookup...), + }) + } + return result +} + func ProjectionColumnsForType(aView *View, rType reflect.Type) ([]string, error) { if aView == nil || rType == nil { return nil, nil @@ -130,3 +258,57 @@ func tagName(tag string) string { } return strings.TrimSpace(tag) } + +func projectionFieldSource(column *Column) string { + if column == nil || column.Tag == "" { + return "" + } + return reflect.StructTag(column.Tag).Get("source") +} + +func projectionFieldLookup(values ...string) []string { + seen := map[string]bool{} + var result []string + for _, value := range values { + for _, candidate := range projectionFieldLookupCandidates(value) { + if seen[candidate] { + continue + } + seen[candidate] = true + result = append(result, candidate) + } + } + return result +} + +func projectionFieldLookupCandidates(value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + values := []string{value} + if index := strings.LastIndex(value, "."); index != -1 && index+1 < len(value) { + values = append(values, value[index+1:]) + } + result := make([]string, 0, len(values)*3) + for _, candidate := range values { + result = append(result, candidate) + normalized := normalizeProjectionFieldName(candidate) + if normalized != "" && normalized != candidate { + result = append(result, normalized) + } + lower := strings.ToLower(candidate) + if lower != candidate && lower != normalized { + result = append(result, lower) + } + } + return result +} + +func normalizeProjectionFieldName(value string) string { + value = strings.TrimSpace(strings.ToLower(value)) + value = strings.ReplaceAll(value, "_", "") + value = strings.ReplaceAll(value, "-", "") + value = strings.ReplaceAll(value, ".", "") + return value +} diff --git a/view/projection_test.go b/view/projection_test.go new file mode 100644 index 000000000..9216041c9 --- /dev/null +++ b/view/projection_test.go @@ -0,0 +1,78 @@ +package view + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProjectionFieldForColumn_DoesNotSetGroupedSemanticKeys(t *testing.T) { + dimension := ProjectionFieldForColumn(&Column{Name: "audience_id", Groupable: true}) + + assert.Empty(t, dimension.DimensionKey) + assert.Empty(t, dimension.MeasureKey) + + measure := ProjectionFieldForColumn(&Column{Name: "spend", Aggregate: true}) + + assert.Empty(t, measure.DimensionKey) + assert.Empty(t, measure.MeasureKey) +} + +func TestProjectionFieldForColumn_KeepsSourceOutOfLookup(t *testing.T) { + field := ProjectionFieldForColumn(&Column{ + Name: "campaign_id", + DatabaseColumn: "CAMPAIGN_ID", + Tag: `source:"ID"`, + }) + + assert.Equal(t, "ID", field.Source) + assert.NotContains(t, field.Lookup, "ID") + assert.NotContains(t, field.Lookup, "id") + assert.Contains(t, field.Lookup, "campaign_id") + assert.Contains(t, field.Lookup, "campaignid") +} + +func TestProjectionFieldsForNames_NonGroupedViewDoesNotSetGroupedSemanticKeys(t *testing.T) { + aView := NewView("events", "events", + WithConnector(NewConnector("test", "sqlite3", ":memory:")), + WithColumns(Columns{ + {Name: "order_id", DataType: "int", Groupable: true}, + {Name: "spend", DataType: "float", Aggregate: true}, + }), + ) + require.NoError(t, aView.Init(context.Background(), EmptyResource())) + fields, err := ProjectionFieldsForNames(aView, []string{"order_id", "spend"}) + require.NoError(t, err) + require.Len(t, fields, 2) + + assert.Empty(t, fields[0].DimensionKey) + assert.Empty(t, fields[0].MeasureKey) + assert.Empty(t, fields[1].DimensionKey) + assert.Empty(t, fields[1].MeasureKey) +} + +func TestProjectionFieldsForNames_GroupedViewTreatsNonGroupableColumnsAsMeasures(t *testing.T) { + aView := NewView("linePeriodSummary", "line_period_summary", + WithConnector(NewConnector("test", "sqlite3", ":memory:")), + WithGroupable(true), + WithColumns(Columns{ + {Name: "audience_id", DataType: "int", Groupable: true}, + {Name: "SPEND", DataType: "float"}, + {Name: "PERIOD_ECPM", DataType: "float"}, + }), + ) + require.NoError(t, aView.Init(context.Background(), EmptyResource())) + fields, err := ProjectionFieldsForNames(aView, []string{"audience_id", "spend", "period_ecpm"}) + require.NoError(t, err) + require.Len(t, fields, 3) + + assert.Equal(t, "audience_id", fields[0].DimensionKey) + + assert.Empty(t, fields[1].DimensionKey) + assert.Equal(t, "SPEND", fields[1].MeasureKey) + + assert.Empty(t, fields[2].DimensionKey) + assert.Equal(t, "PERIOD_ECPM", fields[2].MeasureKey) +} diff --git a/warmup/cache.go b/warmup/cache.go index a3a3ec38c..46f39d8f5 100644 --- a/warmup/cache.go +++ b/warmup/cache.go @@ -171,6 +171,7 @@ func (c *matchersCollector) createIndexWarmupEntry(ctx context.Context, aView *v } return } + build.StoredFields = view.SQLXProjectionFields(cacheInput.StoredFields) aChan <- func() (*warmupEntry, error) { return &warmupEntry{ diff --git a/warmup/cache_test.go b/warmup/cache_test.go index 9b1f7c797..b73345213 100644 --- a/warmup/cache_test.go +++ b/warmup/cache_test.go @@ -244,6 +244,7 @@ Connectors: Views: - Name: events + Groupable: true Connector: Ref: db Table: events @@ -283,6 +284,10 @@ Views: require.NoError(t, err) require.NotEmpty(t, fieldInput) assert.Equal(t, []string{"Quantity"}, fieldInput[0].FieldNames) + require.Len(t, fieldInput[0].StoredFields, 1) + assert.Equal(t, "quantity", fieldInput[0].StoredFields[0].Name) + assert.Equal(t, "quantity", fieldInput[0].StoredFields[0].FieldName) + assert.Contains(t, fieldInput[0].StoredFields[0].Lookup, "quantity") fieldQuery, err := builder.CacheSQL(context.Background(), aView, fieldInput[0].Selector) require.NoError(t, err) @@ -294,6 +299,321 @@ Views: assert.Contains(t, fieldQuery.SQL, "quantity") } +func TestGenerateCacheInput_StoresDefaultProjectionFieldMetadata(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Groupable: true + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + Tag: 'source:"e.event_type_id"' + Groupable: true + - Name: quantity + DataType: int + Aggregate: true + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + Selector: + Constraints: + Projection: true + Template: + Source: SELECT * FROM EVENTS +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + aView := resource.Views[0] + + input, err := aView.Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.NotEmpty(t, input) + require.Len(t, input[0].StoredFields, 2) + + assert.Equal(t, "event_type_id", input[0].StoredFields[0].Name) + assert.Equal(t, "event_type_id", input[0].StoredFields[0].DimensionKey) + assert.Empty(t, input[0].StoredFields[0].MeasureKey) + assert.NotContains(t, input[0].StoredFields[0].Lookup, "e.event_type_id") + assert.Contains(t, input[0].StoredFields[0].Lookup, "event_type_id") + + assert.Equal(t, "quantity", input[0].StoredFields[1].Name) + assert.Empty(t, input[0].StoredFields[1].DimensionKey) + assert.Equal(t, "quantity", input[0].StoredFields[1].MeasureKey) +} + +func TestGenerateCacheInput_ReturnsStoredFieldMetadataError(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Groupable: true + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + - Name: quantity + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + Selector: + Constraints: + Projection: true + Template: + Source: SELECT * FROM EVENTS +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + aView := resource.Views[0] + + aView.Cache.Warmup.FieldNames = []string{"missing"} + _, err = aView.Cache.GenerateCacheInput(context.Background()) + + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to map output field missing") +} + +func TestCacheNewInput_ReturnsInputWhenStoredFieldMetadataFails(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Groupable: true + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + Selector: + Constraints: + Projection: true + Template: + Source: SELECT * FROM EVENTS +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + aView := resource.Views[0] + aView.Cache.Warmup.FieldNames = []string{"missing"} + + input := aView.Cache.NewInput(view.NewStatelet()) + + require.NotNil(t, input) + assert.Equal(t, []string{"missing"}, input.FieldNames) + assert.Empty(t, input.StoredFields) +} + +func TestSQLXProjectionFieldsCopiesStoredFieldMetadata(t *testing.T) { + actual := view.SQLXProjectionFields([]view.ProjectionField{ + { + Name: "order_id", + FieldName: "OrderId", + ColumnName: "order_id", + Source: "o.order_id", + DimensionKey: "order_id", + Lookup: []string{"order_id", "OrderId", "orderid"}, + }, + { + Name: "bids", + FieldName: "Bids", + MeasureKey: "bids", + Lookup: []string{"bids", "Bids"}, + }, + }) + + require.Len(t, actual, 2) + assert.Equal(t, "order_id", actual[0].Name) + assert.Equal(t, "OrderId", actual[0].FieldName) + assert.Equal(t, "order_id", actual[0].ColumnName) + assert.Equal(t, "o.order_id", actual[0].Source) + assert.Equal(t, "order_id", actual[0].DimensionKey) + assert.Empty(t, actual[0].MeasureKey) + assert.Equal(t, []string{"order_id", "OrderId", "orderid"}, actual[0].Lookup) + + assert.Equal(t, "bids", actual[1].Name) + assert.Empty(t, actual[1].DimensionKey) + assert.Equal(t, "bids", actual[1].MeasureKey) +} + +func TestCreateIndexWarmupEntrySetsMatcherStoredFields(t *testing.T) { + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: ":memory:" + +Views: + - Name: events + Groupable: true + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + Groupable: true + - Name: quantity + DataType: int + Aggregate: true + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + FieldNames: + - quantity + Selector: + Constraints: + Projection: true + Template: + Source: SELECT * FROM EVENTS +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + aView := resource.Views[0] + inputs, err := aView.Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.NotEmpty(t, inputs) + + collector := make(chan warmupEntryFn, 1) + (&matchersCollector{builder: reader.NewBuilder(), view: aView}).createIndexWarmupEntry(context.Background(), aView, collector, inputs[0]) + entry, err := (<-collector)() + + require.NoError(t, err) + require.NotNil(t, entry.matcher) + require.Len(t, entry.matcher.StoredFields, 1) + assert.Equal(t, "quantity", entry.matcher.StoredFields[0].Name) + assert.Equal(t, "quantity", entry.matcher.StoredFields[0].MeasureKey) +} + +func TestCreateMetaWarmupEntryDoesNotSetDataStoredFields(t *testing.T) { + dbPath := path.Join(t.TempDir(), "events.db") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + _, err = db.Exec(`CREATE TABLE EVENTS (event_type_id INTEGER, quantity INTEGER)`) + require.NoError(t, err) + + resourcePath := path.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourcePath, []byte(` +CacheProviders: + - Name: aerospike + Location: ${view.Name} + Provider: 'aerospike://127.0.0.1:3000/test' + TimeToLiveMs: 3600000 + +Connectors: + - Name: db + Driver: sqlite3 + DSN: "`+dbPath+`" + +Views: + - Name: events + Connector: + Ref: db + Table: events + Columns: + - Name: event_type_id + DataType: int + Groupable: true + - Name: quantity + DataType: int + Aggregate: true + Cache: + Ref: aerospike + Warmup: + IndexColumn: event_type_id + FieldNames: + - quantity + Selector: + Constraints: + Projection: true + Template: + Summary: + Name: EventsMeta + Source: 'SELECT COUNT(*) AS TOTAL_RECORDS, event_type_id FROM ($View.Expand($criteria)) GROUP BY event_type_id' + Source: SELECT * FROM EVENTS +`), 0644)) + + resource, err := view.NewResourceFromURL(context.Background(), resourcePath, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, resource.Views) + aView := resource.Views[0] + inputs, err := aView.Cache.GenerateCacheInput(context.Background()) + require.NoError(t, err) + require.NotEmpty(t, inputs) + require.NotEmpty(t, inputs[0].StoredFields) + + collector := make(chan warmupEntryFn, 1) + (&matchersCollector{builder: reader.NewBuilder(), view: aView}).createMetaWarmupEntry(context.Background(), aView, collector, inputs[0]) + entry, err := (<-collector)() + + require.NoError(t, err) + require.NotNil(t, entry.matcher) + assert.Empty(t, entry.matcher.StoredFields) +} + func TestGenerateCacheInput_AppliesWarmupLimitOverride(t *testing.T) { resourcePath := path.Join(t.TempDir(), "resource.yaml") require.NoError(t, os.WriteFile(resourcePath, []byte(` From 1ebc636450e671593e4387c0b18e49c521765d21 Mon Sep 17 00:00:00 2001 From: vcarey Date: Sun, 2 Aug 2026 18:42:04 -0400 Subject: [PATCH 272/279] Improve SQL error logging context --- logger/adapter.go | 38 ++++++++++++++++++++++-- logger/adapter_test.go | 58 +++++++++++++++++++++++++++++++++++++ service/executor/service.go | 10 ++++++- service/reader/service.go | 8 ++--- view/sql.go | 2 +- 5 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 logger/adapter_test.go diff --git a/logger/adapter.go b/logger/adapter.go index e060cdd62..6617fdeaf 100644 --- a/logger/adapter.go +++ b/logger/adapter.go @@ -1,7 +1,9 @@ package logger import ( + "context" "fmt" + "github.com/viant/datly/internal/requesttrace" "github.com/viant/datly/shared" "github.com/viant/datly/utils/debug" "strings" @@ -86,9 +88,41 @@ func (l *Adapter) Inherit(adapter *Adapter) { l.log = adapter.log } -func (l *Adapter) LogDatabaseErr(SQL string, err error, args ...interface{}) { +func (l *Adapter) LogDatabaseErr(ctx context.Context, view string, SQL string, err error, args ...interface{}) { SQL = shared.ExpandSQL(SQL, args) - fmt.Printf("error occured while executing SQL: %v, SQL: %v, params: %v\n", err, strings.ReplaceAll(SQL, "\n", "\\n"), args) + fmt.Printf("[ERROR] datly sql reqTraceId=%s view=%s error=%q sql=%q params=%v\n", + reqTraceID(ctx), + view, + normalizeDatabaseError(err), + strings.ReplaceAll(SQL, "\n", "\\n"), + args) +} + +func reqTraceID(ctx context.Context) string { + if traceID := requesttrace.Current(ctx); traceID != "" { + return traceID + } + return "unknown" +} + +func normalizeDatabaseError(err error) string { + if err == nil { + return "" + } + message := err.Error() + if idx := strings.LastIndex(message, ", due to "); idx >= 0 { + return strings.TrimSpace(message[idx+len(", due to "):]) + } + if idx := strings.LastIndex(message, " due to "); idx >= 0 { + return strings.TrimSpace(message[idx+len(" due to "):]) + } + if idx := strings.LastIndex(message, " failed to run query: "); idx >= 0 { + return strings.TrimSpace(message[:idx]) + } + if strings.HasPrefix(message, "failed to run query: ") { + return "failed to run query" + } + return message } func NewLogger(name string, logger Logger) *Adapter { diff --git a/logger/adapter_test.go b/logger/adapter_test.go new file mode 100644 index 000000000..ae931e314 --- /dev/null +++ b/logger/adapter_test.go @@ -0,0 +1,58 @@ +package logger + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/datly/internal/requesttrace" +) + +func TestReqTraceID(t *testing.T) { + require.Equal(t, "unknown", reqTraceID(nil)) + require.Equal(t, "unknown", reqTraceID(context.Background())) + + ctx := requesttrace.Ensure(context.Background(), "trace-123") + require.Equal(t, "trace-123", reqTraceID(ctx)) +} + +func TestNormalizeDatabaseError(t *testing.T) { + testCases := []struct { + name string + err error + expected string + }{ + { + name: "nil", + err: nil, + expected: "", + }, + { + name: "bigquery due to", + err: errors.New("failed to run query: SELECT * FROM table, due to googleapi: Error 400: invalidQuery"), + expected: "googleapi: Error 400: invalidQuery", + }, + { + name: "sqlx wrapped query", + err: errors.New("database error occured while fetching Data for view v failed to run query: SELECT * FROM table WHERE id = ?"), + expected: "database error occured while fetching Data for view v", + }, + { + name: "raw failed query", + err: errors.New("failed to run query: SELECT * FROM table WHERE id = ?"), + expected: "failed to run query", + }, + { + name: "plain error", + err: errors.New("connection refused"), + expected: "connection refused", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + require.Equal(t, testCase.expected, normalizeDatabaseError(testCase.err)) + }) + } +} diff --git a/service/executor/service.go b/service/executor/service.go index de72f1909..c52550a9f 100644 --- a/service/executor/service.go +++ b/service/executor/service.go @@ -333,7 +333,7 @@ func (e *Executor) executeStatement(ctx context.Context, tx *sql.Tx, stmt *expan _, err := tx.ExecContext(ctx, stmt.SQL, stmt.Args...) if err != nil { if sess.logger != nil { - sess.logger.LogDatabaseErr(stmt.SQL, err, stmt.Args...) + sess.logger.LogDatabaseErr(ctx, databaseLogView(ctx), stmt.SQL, err, stmt.Args...) } err = fmt.Errorf("error occured while connecting to database") @@ -342,6 +342,14 @@ func (e *Executor) executeStatement(ctx context.Context, tx *sql.Tx, stmt *expan return err } +func databaseLogView(ctx context.Context) string { + aView := view.Context(ctx) + if aView == nil { + return "" + } + return aView.Name +} + func (s *dbSession) collection(executable *expand2.Executable) *batcher.Collection { if collection, ok := s.collections[executable.Table]; ok { return collection diff --git a/service/reader/service.go b/service/reader/service.go index 70e7fb95b..28c49b3e2 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -911,7 +911,7 @@ BEGIN: } if err != nil { stats.SetError(err) - anExec, err := s.HandleSQLError(err, session, aView, parametrizedSQL, stats) + anExec, err := s.HandleSQLError(ctx, err, aView, parametrizedSQL, stats) return []*response.SQLExecution{anExec}, err } @@ -938,7 +938,7 @@ BEGIN: logCacheRead(ctx, aView, cacheStats, end.Sub(begin), *readData, parametrizedSQL.Args) if err != nil { stats.SetError(err) - anExec, err := s.HandleSQLError(err, session, aView, parametrizedSQL, stats) + anExec, err := s.HandleSQLError(ctx, err, aView, parametrizedSQL, stats) return []*response.SQLExecution{anExec}, err } return []*response.SQLExecution{stats}, nil @@ -1043,8 +1043,8 @@ func (s *Service) queryWithPartitions(ctx context.Context, session *Session, aVi return executions, err } -func (s *Service) HandleSQLError(err error, session *Session, aView *view.View, matcher *cache.ParmetrizedQuery, stats *response.SQLExecution) (*response.SQLExecution, error) { - aView.Logger.LogDatabaseErr(matcher.SQL, err, matcher.Args...) +func (s *Service) HandleSQLError(ctx context.Context, err error, aView *view.View, matcher *cache.ParmetrizedQuery, stats *response.SQLExecution) (*response.SQLExecution, error) { + aView.Logger.LogDatabaseErr(ctx, aView.Name, matcher.SQL, err, matcher.Args...) stats.Error = err.Error() return stats, fmt.Errorf("database error occured while fetching Data for view %v %w", aView.Name, err) } diff --git a/view/sql.go b/view/sql.go index ad5276829..b9d1de67c 100644 --- a/view/sql.go +++ b/view/sql.go @@ -53,7 +53,7 @@ func detectColumns(ctx context.Context, evaluation *TemplateEvaluation, v *View) } query, err := aDb.QueryContext(ctx, SQL, args...) if err != nil { - v.Logger.LogDatabaseErr(SQL, err, args...) + v.Logger.LogDatabaseErr(ctx, v.Name, SQL, err, args...) return nil, SQL, err } defer query.Close() From b3788d365e7388dfcd5a494eb00989f9890d0679 Mon Sep 17 00:00:00 2001 From: vcarey Date: Mon, 3 Aug 2026 12:47:45 -0400 Subject: [PATCH 273/279] Upgrade viant bigquery dependency --- go.mod | 92 +++++++++++++------------ go.sum | 209 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 149 insertions(+), 152 deletions(-) diff --git a/go.mod b/go.mod index 74736b694..69bdca461 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/viant/afs v1.29.0 github.com/viant/afsc v1.16.0 github.com/viant/assertly v0.9.1-0.20220620174148-bab013f93a60 - github.com/viant/bigquery v0.4.1 + github.com/viant/bigquery v0.5.2-0.20260803163621-2873b21b58e5 github.com/viant/cloudless v1.12.0 github.com/viant/dsc v0.16.2 // indirect github.com/viant/dsunit v0.10.8 @@ -34,9 +34,9 @@ require ( github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e github.com/viant/xunsafe v0.11.0 - golang.org/x/mod v0.28.0 - golang.org/x/oauth2 v0.32.0 - google.golang.org/api v0.201.0 + golang.org/x/mod v0.37.0 + golang.org/x/oauth2 v0.36.0 + google.golang.org/api v0.287.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -62,28 +62,28 @@ require ( github.com/viant/xdatly/types/custom v0.0.0-20240801144911-4c2bfca4c23a github.com/viant/xlsy v0.3.1 github.com/viant/xmlify v0.1.1 - golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 - golang.org/x/tools v0.37.0 + golang.org/x/net v0.57.0 + golang.org/x/tools v0.47.0 modernc.org/sqlite v1.18.1 ) require ( - cel.dev/expr v0.24.0 // indirect - cloud.google.com/go v0.116.0 // indirect - cloud.google.com/go/auth v0.9.8 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cel.dev/expr v0.25.1 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.22.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/firestore v1.17.0 // indirect - cloud.google.com/go/iam v1.2.1 // indirect - cloud.google.com/go/longrunning v0.6.1 // indirect - cloud.google.com/go/monitoring v1.21.1 // indirect - cloud.google.com/go/secretmanager v1.14.1 // indirect - cloud.google.com/go/storage v1.45.0 // indirect + cloud.google.com/go/firestore v1.21.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/longrunning v0.8.0 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/secretmanager v1.16.0 // indirect + cloud.google.com/go/storage v1.56.0 // indirect firebase.google.com/go v3.13.0+incompatible // indirect firebase.google.com/go/v4 v4.14.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect github.com/aerospike/aerospike-client-go/v6 v6.15.1 // indirect github.com/aws/aws-sdk-go v1.51.23 // indirect @@ -115,23 +115,22 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect github.com/aws/smithy-go v1.22.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect - github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-errors/errors v1.5.1 // indirect - github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/s2a-go v0.1.8 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect @@ -156,29 +155,28 @@ require ( github.com/xuri/excelize/v2 v2.8.0 // indirect github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a // indirect github.com/yuin/gopher-lua v1.1.1 // indirect - go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel v1.38.0 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect - go.opentelemetry.io/otel/sdk v1.38.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect - go.opentelemetry.io/otel/trace v1.38.0 // indirect - golang.org/x/crypto v0.43.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/term v0.36.0 // indirect - golang.org/x/text v0.30.0 // indirect - golang.org/x/time v0.7.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/time v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/appengine/v2 v2.0.2 // indirect - google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect - google.golang.org/grpc v1.77.0 // indirect - google.golang.org/protobuf v1.36.10 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260724162435-b2f20204f0df // indirect + google.golang.org/grpc v1.82.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect lukechampine.com/uint128 v1.2.0 // indirect modernc.org/cc/v3 v3.36.3 // indirect diff --git a/go.sum b/go.sum index 029eec27b..1595850df 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= -cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -41,8 +41,8 @@ cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFO cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= -cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= -cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -104,10 +104,10 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= -cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8= -cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= -cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= -cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= +cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ= +cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= @@ -285,8 +285,8 @@ cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLY cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= -cloud.google.com/go/firestore v1.17.0 h1:iEd1LBbkDZTFsLw3sTH50eyg4qe8eoG6CjocmEXO9aQ= -cloud.google.com/go/firestore v1.17.0/go.mod h1:69uPx1papBsY8ZETooc71fOhoKkD70Q1DwMrtKuOT/Y= +cloud.google.com/go/firestore v1.21.0 h1:BhopUsx7kh6NFx77ccRsHhrtkbJUmDAxNY3uapWdjcM= +cloud.google.com/go/firestore v1.21.0/go.mod h1:1xH6HNcnkf/gGyR8udd6pFO4Z7GWJSwLKQMx/u6UrP4= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= @@ -325,8 +325,8 @@ cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGE cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= -cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= @@ -356,13 +356,13 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6 cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= -cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= -cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= +cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= +cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= -cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= -cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= @@ -386,8 +386,8 @@ cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhI cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= -cloud.google.com/go/monitoring v1.21.1 h1:zWtbIoBMnU5LP9A/fz8LmWMGHpk4skdfeiaa66QdFGc= -cloud.google.com/go/monitoring v1.21.1/go.mod h1:Rj++LKrlht9uBi8+Eb530dIrzG/cU/lB8mt+lbeFK1c= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= @@ -496,8 +496,8 @@ cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISI cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= -cloud.google.com/go/secretmanager v1.14.1 h1:xlWSIg8rtBn5qCr2f3XtQP19+5COyf/ll49SEvi/0vM= -cloud.google.com/go/secretmanager v1.14.1/go.mod h1:L+gO+u2JA9CCyXpSR8gDH0o8EV7i/f0jdBOrUXcIV0U= +cloud.google.com/go/secretmanager v1.16.0 h1:19QT7ZsLJ8FSP1k+4esQvuCD7npMJml6hYzilxVyT+k= +cloud.google.com/go/secretmanager v1.16.0/go.mod h1://C/e4I8D26SDTz1f3TQcddhcmiC3rMEl0S1Cakvs3Q= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= @@ -553,8 +553,8 @@ cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeL cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= -cloud.google.com/go/storage v1.45.0 h1:5av0QcIVj77t+44mV4gffFC/LscFRUhto6UBMB5SimM= -cloud.google.com/go/storage v1.45.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE= +cloud.google.com/go/storage v1.56.0 h1:iixmq2Fse2tqxMbWhLWC9HfBj1qdxqAmiK8/eqtsLxI= +cloud.google.com/go/storage v1.56.0/go.mod h1:Tpuj6t4NweCLzlNbw9Z9iwxEkrSem20AetIeH/shgVU= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= @@ -574,8 +574,8 @@ cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= -cloud.google.com/go/trace v1.11.1 h1:UNqdP+HYYtnm6lb91aNA5JQ0X14GnxkABGlfz2PzPew= -cloud.google.com/go/trace v1.11.1/go.mod h1:IQKNQuBzH72EGaXEodKlNJrWykGZxet2zgjtS60OtjA= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= @@ -636,14 +636,14 @@ git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGy git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1 h1:oTX4vsorBZo/Zdum6OKPA4o7544hm6smoRv1QjpTwGo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= @@ -756,8 +756,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= -github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -781,18 +781,18 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= -github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= -github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs= -github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= -github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= -github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= -github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -816,8 +816,8 @@ github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmn github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -852,8 +852,6 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -939,8 +937,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= -github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= -github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -950,8 +948,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= -github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrmAy9B9bH+yC4563vmnJ34QR8i4= +github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -968,8 +966,8 @@ github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38 github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= -github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= -github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -1160,8 +1158,8 @@ github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49u github.com/viant/assertly v0.9.0/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/assertly v0.9.1-0.20220620174148-bab013f93a60 h1:VFJvCOHKXv4IqX8rJwn1otpHWQGgMDv2bXtAPgEzndM= github.com/viant/assertly v0.9.1-0.20220620174148-bab013f93a60/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= -github.com/viant/bigquery v0.4.1 h1:O3/7G+F6ZH3wAeYhIB1OGYxVGP/VXF0xwRyNCHnpa9w= -github.com/viant/bigquery v0.4.1/go.mod h1:9xYllhrjuHujXhTKfm8uIfAW719GSFTMjZGHwovnXW8= +github.com/viant/bigquery v0.5.2-0.20260803163621-2873b21b58e5 h1:n0b8XMi4UZO532AP17Scc2vCDJTVSMzLHOUXFg5RyvQ= +github.com/viant/bigquery v0.5.2-0.20260803163621-2873b21b58e5/go.mod h1:IIlW5q1E6xqg1p/91b3AGRCeLVZpRi+9C1kUL9PFNqw= github.com/viant/cloudless v1.12.0 h1:EVLki/Ontsj1viXGUh+BCB1y0XMkDd9VQypjZKzIgfc= github.com/viant/cloudless v1.12.0/go.mod h1:jxHtSl2HrPfqU8M0pZV5kIWmHJo6sA1G0KwUDj6+rBo= github.com/viant/dsc v0.16.2 h1:Kw8zNct6dTISVZpartYK4MlKiwSSqIdRSq5CYjtZcc4= @@ -1196,8 +1194,6 @@ github.com/viant/scy v0.33.1 h1:jlSgOxwsLvY1/YvAd6y5shvig/6hfhX+/sHVKmh4B5s= github.com/viant/scy v0.33.1/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQnwf69oc86xW1e4wEEzKUg5OC6tOYegE= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.23.1-0.20260721202550-583cf232e734 h1:vZF9F8r3lUSfdRBMZyWje0eabeI0Q5sMwbd0QF3pq8c= -github.com/viant/sqlx v0.23.1-0.20260721202550-583cf232e734/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/sqlx v0.23.1-0.20260729142839-24da78934871 h1:9RqxSYtQfUGiMoT9YNpBfPZVjtorUrJ3uQISvS43EKI= github.com/viant/sqlx v0.23.1-0.20260729142839-24da78934871/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= @@ -1258,26 +1254,29 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs= -go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -1303,8 +1302,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= -golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1367,8 +1366,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= -golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1438,8 +1437,8 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 h1:6/3JGEh1C88g7m+qzzTbl3A0FtsLguXieqofVLU/JAo= -golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1473,8 +1472,8 @@ golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= -golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= -golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1494,8 +1493,8 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1587,8 +1586,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1604,8 +1603,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= -golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1625,8 +1624,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1634,8 +1633,8 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1700,8 +1699,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= -golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1714,8 +1713,8 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= @@ -1784,8 +1783,8 @@ google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjY google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= -google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0= -google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4= +google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI= +google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1938,21 +1937,21 @@ google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE= -google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4= -google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260724162435-b2f20204f0df h1:O3ig1i5WDDzsVzRp+cCdgelT9vXnlnOFdlEeFtL4HCc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260724162435-b2f20204f0df/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1997,8 +1996,8 @@ google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= -google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= -google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2018,8 +2017,8 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 43fbe3c4e4fcf39613979a72a765c9650153aaae Mon Sep 17 00:00:00 2001 From: vcarey Date: Mon, 3 Aug 2026 12:52:08 -0400 Subject: [PATCH 274/279] Upgrade sqlx dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 69bdca461..f8882f624 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.33.1 - github.com/viant/sqlx v0.23.1-0.20260729142839-24da78934871 + github.com/viant/sqlx v0.23.1-0.20260803165008-da07533d2e8f github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 diff --git a/go.sum b/go.sum index 1595850df..c456cb73f 100644 --- a/go.sum +++ b/go.sum @@ -1194,8 +1194,8 @@ github.com/viant/scy v0.33.1 h1:jlSgOxwsLvY1/YvAd6y5shvig/6hfhX+/sHVKmh4B5s= github.com/viant/scy v0.33.1/go.mod h1:7uNRS67X45YN+JqTLCcMEhehffVjqrejULEDln9p0Ao= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQnwf69oc86xW1e4wEEzKUg5OC6tOYegE= github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= -github.com/viant/sqlx v0.23.1-0.20260729142839-24da78934871 h1:9RqxSYtQfUGiMoT9YNpBfPZVjtorUrJ3uQISvS43EKI= -github.com/viant/sqlx v0.23.1-0.20260729142839-24da78934871/go.mod h1:dizufL+nTNqDCpivUnE2HqtddTp2TdA6WFghGfZo11c= +github.com/viant/sqlx v0.23.1-0.20260803165008-da07533d2e8f h1:yte+MMDo1mWS6+YpM1OYmfZe3+ieYZXsCWfjAf4c+PY= +github.com/viant/sqlx v0.23.1-0.20260803165008-da07533d2e8f/go.mod h1:yZOQRVCMZAkexsTaoqCPGJvsNO2qajQRU1VuYu23fX8= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= github.com/viant/structology v0.9.0/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= From 3bbe151923366583580861b6ee850590169a6f74 Mon Sep 17 00:00:00 2001 From: Badr Ezzir Date: Tue, 4 Aug 2026 00:11:06 +0100 Subject: [PATCH 275/279] ENG-00000: Make generated code and paths.yaml deterministic Generated artifacts differed between machines and between runs, so regenerating produced large spurious diffs. Fix each source of non-determinism where the unordered data enters the pipeline. - codegen: run emitted code through go/format and correct the EmbedFS snippet indentation. On a format error the unformatted result is kept. - column discovery: sort information_schema columns by ordinal position. Those queries carry no ORDER BY, so driver ordering leaked into generated struct field order. Columns inferred from a result set carry no position and are left untouched, preserving the projection order the query already established. - route listing: sort paths.yaml entries by URL. The recursive listing reflected directory enumeration order, which varies by filesystem and shifts whenever route files are rewritten, so partial regeneration reshuffled thousands of lines. Route lookup is order independent: the matcher builds a trie and prefers exact matches. Adds unit tests covering each sort and the gofmt stability of generated output. Co-Authored-By: Claude Opus 5 (1M context) --- repository/codegen.go | 11 +++- repository/codegen_embedfs_test.go | 96 ++++++++++++++++++++++++++++++ repository/path/service.go | 14 +++++ repository/path/sort_test.go | 94 +++++++++++++++++++++++++++++ view/column/discover.go | 19 ++++++ view/column/discover_sort_test.go | 83 ++++++++++++++++++++++++++ 6 files changed, 314 insertions(+), 3 deletions(-) create mode 100644 repository/codegen_embedfs_test.go create mode 100644 repository/path/sort_test.go create mode 100644 view/column/discover_sort_test.go diff --git a/repository/codegen.go b/repository/codegen.go index 137ac8c34..11b9135b6 100644 --- a/repository/codegen.go +++ b/repository/codegen.go @@ -13,6 +13,7 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/toolbox/data" "github.com/viant/xreflect" + "go/format" "path" "reflect" "strconv" @@ -190,14 +191,18 @@ func (c *Component) GenerateOutputCode(ctx context.Context, withDefineComponent, if withEmbed { embedderCode := fmt.Sprintf(` - func (i *%vInput) EmbedFS() *embed.FS { - return &%vFS - }`, componentName, componentName) +func (i *%vInput) EmbedFS() *embed.FS { + return &%vFS +} +`, componentName, componentName) builder.WriteString(embedderCode) } result := builder.String() result = c.View.Resource().ReverseSubstitutes(result) + if formatted, err := format.Source([]byte(result)); err == nil { + result = string(formatted) + } return result } diff --git a/repository/codegen_embedfs_test.go b/repository/codegen_embedfs_test.go new file mode 100644 index 000000000..ccd5efc70 --- /dev/null +++ b/repository/codegen_embedfs_test.go @@ -0,0 +1,96 @@ +package repository + +import ( + "context" + "go/format" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/viant/datly/repository/contract" + "github.com/viant/datly/view" + "github.com/viant/datly/view/state" + "github.com/viant/xreflect" +) + +func newEmbedFSTestComponent(t *testing.T) *Component { + t.Helper() + + resource := view.EmptyResource() + rootView := view.NewView("active_advertiser", "ACTIVE_ADVERTISER") + rootView.Connector = &view.Connector{Connection: view.Connection{DBConfig: view.DBConfig{Name: "ci_ads"}}} + rootView.Template = &view.Template{Source: "SELECT ID FROM ACTIVE_ADVERTISER"} + rootView.Schema = state.NewSchema(reflect.TypeOf([]*struct { + Id *int `sqlx:"ID"` + }{})) + resource.Types = []*view.TypeDefinition{ + {Name: "ActiveAdvertiserView", Package: "universalpixel", DataType: `struct{Id *int ` + "`sqlx:\"ID\"`" + `;}`}, + } + require.NoError(t, resource.TypeRegistry().Register("ActiveAdvertiserView", + xreflect.WithPackage("universalpixel"), + xreflect.WithReflectType(reflect.TypeOf(struct { + Id *int `sqlx:"ID"` + }{})))) + rootView.SetResource(resource) + resource.AddViews(rootView) + + inputType, err := state.NewType(state.WithParameters(state.Parameters{ + &state.Parameter{Name: "AdvertiserId", In: state.NewQueryLocation("advertiserId"), Schema: state.NewSchema(reflect.TypeOf(0))}, + }), state.WithResource(&reportTestResource{})) + require.NoError(t, err) + inputType.Name = "ActiveAdvertiserInput" + inputType.Package = "universalpixel" + + outputType, err := state.NewType(state.WithParameters(state.Parameters{ + &state.Parameter{Name: "Data", In: state.NewOutputLocation("view"), Schema: &state.Schema{Name: "ActiveAdvertiserView", Package: "universalpixel", Cardinality: state.Many}}, + }), state.WithResource(rootView.Resource())) + require.NoError(t, err) + outputType.Name = "ActiveAdvertiserOutput" + outputType.Package = "universalpixel" + + return &Component{ + Path: contract.Path{Method: "GET", URI: "/v1/api/platform/universalpixel/activeadvertiser"}, + Meta: contract.Meta{Name: "ActiveAdvertiser"}, + View: rootView, + Contract: contract.Contract{ + Input: contract.Input{Type: *inputType}, + Output: contract.Output{Type: *outputType}, + }, + } +} + +// The generated EmbedFS accessor used to be emitted from a raw string literal +// indented by one tab and without a trailing newline, so every generated reader +// was unformatted Go. Consumers that ran gofmt saw the file churn back on the +// next `datly gen`. +func TestGenerateOutputCode_EmbedFSAccessorIsGoFormatted(t *testing.T) { + component := newEmbedFSTestComponent(t) + + code := component.GenerateOutputCode(context.Background(), false, true, map[string]string{}) + + assert.Contains(t, code, "\nfunc (i *ActiveAdvertiserInput) EmbedFS() *embed.FS {\n\treturn &ActiveAdvertiserFS\n}\n", + "EmbedFS accessor must be emitted at column 0 with a tab-indented body") + assert.NotContains(t, code, "\n\tfunc (", "no top-level func may be indented") + assert.True(t, strings.HasSuffix(code, "\n"), "generated file must end with a newline") +} + +func TestGenerateOutputCode_IsGofmtStable(t *testing.T) { + component := newEmbedFSTestComponent(t) + + code := component.GenerateOutputCode(context.Background(), false, true, map[string]string{}) + + formatted, err := format.Source([]byte(code)) + require.NoError(t, err, "generated code must parse") + assert.Equal(t, string(formatted), code, "generated code must already be gofmt-clean") +} + +// Regenerating an unchanged component must not produce a different file, or +// consumers get spurious diffs on every codegen run. +func TestGenerateOutputCode_IsDeterministic(t *testing.T) { + first := newEmbedFSTestComponent(t).GenerateOutputCode(context.Background(), false, true, map[string]string{}) + second := newEmbedFSTestComponent(t).GenerateOutputCode(context.Background(), false, true, map[string]string{}) + + assert.Equal(t, first, second) +} diff --git a/repository/path/service.go b/repository/path/service.go index 44118dbbd..83e735f31 100644 --- a/repository/path/service.go +++ b/repository/path/service.go @@ -13,6 +13,7 @@ import ( "github.com/viant/datly/repository/version" "gopkg.in/yaml.v3" "path" + "sort" "strings" "sync" "time" @@ -151,6 +152,7 @@ func (s *Service) createPathFiles(ctx context.Context) error { if err != nil { return err } + sortByURL(candidates) rootPath := url.Path(s.URL) for _, candidate := range candidates { if candidate.IsDir() { @@ -187,6 +189,18 @@ func (s *Service) createPathFiles(ctx context.Context) error { return nil } +// sortByURL gives paths.yaml a stable entry order. The recursive listing that +// feeds it reflects directory enumeration order, which differs between +// filesystems and shifts whenever route files are rewritten - so without this +// the same repository yields a different paths.yaml on every machine, and any +// partial regeneration reshuffles thousands of lines. Route lookup itself is +// order independent: the matcher builds a trie and prefers exact matches. +func sortByURL(candidates []storage.Object) { + sort.SliceStable(candidates, func(i, j int) bool { + return candidates[i].URL() < candidates[j].URL() + }) +} + func (s *Service) buildPaths(ctx context.Context, candidate storage.Object, rootPath string) (*Item, error) { data, err := s.fs.Download(ctx, candidate) if err != nil { diff --git a/repository/path/sort_test.go b/repository/path/sort_test.go new file mode 100644 index 000000000..03ee61cf6 --- /dev/null +++ b/repository/path/sort_test.go @@ -0,0 +1,94 @@ +package path + +import ( + spath "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/viant/afs/file" + "github.com/viant/afs/object" + "github.com/viant/afs/storage" +) + +func objectURLs(candidates []storage.Object) []string { + var result []string + for _, candidate := range candidates { + result = append(result, candidate.URL()) + } + return result +} + +func newObject(URL string) storage.Object { + name := spath.Base(URL) + return object.New(URL, file.NewInfo(name, 0, file.DefaultFileOsMode, time.Now(), false), nil) +} + +func newObjects(URLs ...string) []storage.Object { + var result []storage.Object + for _, URL := range URLs { + result = append(result, newObject(URL)) + } + return result +} + +// The recursive listing behind paths.yaml reflects directory enumeration order, +// so without an explicit sort the same repository produces a different +// paths.yaml on every machine and any partial regeneration reshuffles it. +func TestSortByURL(t *testing.T) { + testCases := []struct { + description string + urls []string + expect []string + }{ + { + description: "enumeration order is normalised to lexical order", + urls: []string{ + "file:///repo/routes/system/session/session.yaml", + "file:///repo/routes/mdp/adorder/forecast.yaml", + "file:///repo/routes/platform/agency/agency.yaml", + }, + expect: []string{ + "file:///repo/routes/mdp/adorder/forecast.yaml", + "file:///repo/routes/platform/agency/agency.yaml", + "file:///repo/routes/system/session/session.yaml", + }, + }, + { + description: "already sorted stays sorted", + urls: []string{"file:///repo/routes/a.yaml", "file:///repo/routes/b.yaml"}, + expect: []string{"file:///repo/routes/a.yaml", "file:///repo/routes/b.yaml"}, + }, + { + description: "single entry", + urls: []string{"file:///repo/routes/only.yaml"}, + expect: []string{"file:///repo/routes/only.yaml"}, + }, + { + description: "empty listing is a no-op", + urls: nil, + expect: nil, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + candidates := newObjects(testCase.urls...) + sortByURL(candidates) + assert.Equal(t, testCase.expect, objectURLs(candidates)) + }) + } +} + +// Sorting must be idempotent, otherwise repeated regeneration still churns. +func TestSortByURL_Idempotent(t *testing.T) { + candidates := newObjects( + "file:///repo/routes/z.yaml", + "file:///repo/routes/a.yaml", + "file:///repo/routes/m.yaml", + ) + sortByURL(candidates) + first := objectURLs(candidates) + sortByURL(candidates) + assert.Equal(t, first, objectURLs(candidates)) +} diff --git a/view/column/discover.go b/view/column/discover.go index 5c779fd9c..7c03b4a79 100644 --- a/view/column/discover.go +++ b/view/column/discover.go @@ -15,6 +15,7 @@ import ( "github.com/viant/sqlx/io" "github.com/viant/sqlx/io/config" "reflect" + "sort" "github.com/viant/sqlx/metadata/sink" "github.com/viant/xreflect" @@ -250,9 +251,27 @@ func readSinkColumns(ctx context.Context, db *sql.DB, table string) ([]sink.Colu if len(columns) == 0 { return nil, vErr } + sortByPosition(columns) return columns, err } +// sortByPosition orders columns by their ordinal position in the table. +// The information_schema queries behind config.Columns carry no ORDER BY, so +// the driver may return columns in any order - which would otherwise leak into +// generated struct field order and produce spurious diffs between machines. +// Columns inferred from a result set carry no position; leaving them stable +// preserves the projection order the query already established. +func sortByPosition(columns []sink.Column) { + for _, column := range columns { + if column.Position == 0 { + return + } + } + sort.SliceStable(columns, func(i, j int) bool { + return columns[i].Position < columns[j].Position + }) +} + func parseQuery(SQL string) (string, string, sqlparser.Columns) { sqlQuery, _ := sqlparser.ParseQuery(SQL) var table string diff --git a/view/column/discover_sort_test.go b/view/column/discover_sort_test.go new file mode 100644 index 000000000..0e9d07980 --- /dev/null +++ b/view/column/discover_sort_test.go @@ -0,0 +1,83 @@ +package column + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/viant/sqlx/metadata/sink" +) + +func names(columns []sink.Column) []string { + var result []string + for _, column := range columns { + result = append(result, column.Name) + } + return result +} + +// config.Columns runs an information_schema query with no ORDER BY, so the +// driver may hand back columns in any order. Without a sort, that order leaks +// into generated struct field order and churns on every regeneration. +func TestSortByPosition(t *testing.T) { + testCases := []struct { + description string + columns []sink.Column + expect []string + }{ + { + description: "alphabetical metadata order is restored to table order", + columns: []sink.Column{ + {Name: "CREATED", Position: 5}, + {Name: "CREATED_USER", Position: 7}, + {Name: "FEE_DOMAIN", Position: 4}, + {Name: "FEE_TYPE_ID", Position: 3}, + {Name: "ID", Position: 1}, + {Name: "NAME", Position: 2}, + {Name: "UPDATED", Position: 6}, + {Name: "UPDATED_USER", Position: 8}, + }, + expect: []string{"ID", "NAME", "FEE_TYPE_ID", "FEE_DOMAIN", "CREATED", "UPDATED", "CREATED_USER", "UPDATED_USER"}, + }, + { + description: "already ordered stays ordered", + columns: []sink.Column{ + {Name: "ID", Position: 1}, + {Name: "NAME", Position: 2}, + }, + expect: []string{"ID", "NAME"}, + }, + { + description: "result-set inferred columns carry no position, so projection order is preserved", + columns: []sink.Column{ + {Name: "TOTAL_SPEND"}, + {Name: "AGENCY_ID"}, + }, + expect: []string{"TOTAL_SPEND", "AGENCY_ID"}, + }, + { + description: "empty input is a no-op", + columns: []sink.Column{}, + expect: nil, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + sortByPosition(testCase.columns) + assert.Equal(t, testCase.expect, names(testCase.columns)) + }) + } +} + +// Sorting must be idempotent, otherwise repeated regeneration would still churn. +func TestSortByPosition_Idempotent(t *testing.T) { + columns := []sink.Column{ + {Name: "UPDATED", Position: 6}, + {Name: "ID", Position: 1}, + {Name: "NAME", Position: 2}, + } + sortByPosition(columns) + first := names(columns) + sortByPosition(columns) + assert.Equal(t, first, names(columns)) +} From 2b496042a5bfab15ce2d5c9a959ae54c53855578 Mon Sep 17 00:00:00 2001 From: vagarwal-viant Date: Fri, 7 Aug 2026 11:14:52 -0700 Subject: [PATCH 276/279] ENG-00000 fix data race on Statelet.Filters --- service/reader/service.go | 6 +++--- service/session/state.go | 4 +--- view/state.go | 16 +++++++++++++--- view/state_test.go | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 view/state_test.go diff --git a/service/reader/service.go b/service/reader/service.go index 28c49b3e2..468e79677 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -628,9 +628,9 @@ func (s *Service) warmupMatcher(ctx context.Context, aView *view.View, statelet } } } - cloned := *statelet + cloned := statelet.CloneForSummary() cloned.Template = clonedTemplate - ok, err := applyWarmupIdentityProjection(aView, &cloned) + ok, err := applyWarmupIdentityProjection(aView, cloned) if err != nil { return nil, err } @@ -638,7 +638,7 @@ func (s *Service) warmupMatcher(ctx context.Context, aView *view.View, statelet return nil, nil } - matcher, err := s.sqlBuilder.CacheSQLWithOptions(ctx, aView, &cloned, nil, nil, parent) + matcher, err := s.sqlBuilder.CacheSQLWithOptions(ctx, aView, cloned, nil, nil, parent) if err != nil || matcher == nil { return matcher, err } diff --git a/service/session/state.go b/service/session/state.go index c00b7428e..919994604 100644 --- a/service/session/state.go +++ b/service/session/state.go @@ -73,9 +73,7 @@ func (s *Session) NewSession(component *repository.Component) *Session { if ret.Options.state != nil { ret.Options.state.RWMutex.Lock() for _, st := range ret.Options.state.Views { - if st != nil { - st.Filters = nil - } + st.ClearFilters() } ret.Options.state.RWMutex.Unlock() } diff --git a/view/state.go b/view/state.go index 7aaee7169..6bc6b6e79 100644 --- a/view/state.go +++ b/view/state.go @@ -88,6 +88,16 @@ func (s *Statelet) AppendFilters(filters predicate.Filters) { s.filtersMu.Unlock() } +// ClearFilters safely clears the selector's filters. +func (s *Statelet) ClearFilters() { + if s == nil { + return + } + s.filtersMu.Lock() + s.Filters = nil + s.filtersMu.Unlock() +} + // NewStatelet creates a selector func NewStatelet() *Statelet { return &Statelet{ @@ -186,9 +196,9 @@ func (s *Statelet) CloneForSummary() *Statelet { ret._columnNames = map[string]bool{} } - if len(s.Filters) > 0 { - ret.Filters = append(predicate.Filters(nil), s.Filters...) - } + s.filtersMu.Lock() + ret.Filters = append(predicate.Filters(nil), s.Filters...) + s.filtersMu.Unlock() if len(s.Fields) > 0 { ret.Fields = append([]string(nil), s.Fields...) diff --git a/view/state_test.go b/view/state_test.go new file mode 100644 index 000000000..51814ee09 --- /dev/null +++ b/view/state_test.go @@ -0,0 +1,37 @@ +package view + +import ( + "sync" + "testing" + + "github.com/viant/datly/view/state/predicate" +) + +func TestStateletCloneForSummaryConcurrentFilters(t *testing.T) { + statelet := NewStatelet() + filter := &predicate.Filter{Name: "active"} + + var waitGroup sync.WaitGroup + waitGroup.Add(2) + + go func() { + defer waitGroup.Done() + for i := 0; i < 1000; i++ { + statelet.AppendFilters(predicate.Filters{filter}) + statelet.ClearFilters() + } + }() + + go func() { + defer waitGroup.Done() + for i := 0; i < 1000; i++ { + clone := statelet.CloneForSummary() + if clone == statelet { + t.Errorf("CloneForSummary() returned the original statelet") + return + } + } + }() + + waitGroup.Wait() +} From b9c5d49e6d2924bc2b868d362a253eb3a24b5b20 Mon Sep 17 00:00:00 2001 From: vagarwal-viant Date: Fri, 7 Aug 2026 11:27:57 -0700 Subject: [PATCH 277/279] ENG-00000 fix npe on readRequestBody(nil) --- view/state/kind/locator/body.go | 9 ++++-- view/state/kind/locator/body_test.go | 44 ++++++++++++++++++++++++++++ view/state/kind/locator/http.go | 3 ++ 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 view/state/kind/locator/body_test.go diff --git a/view/state/kind/locator/body.go b/view/state/kind/locator/body.go index 6374362aa..d98fe3e52 100644 --- a/view/state/kind/locator/body.go +++ b/view/state/kind/locator/body.go @@ -58,12 +58,12 @@ func (r *Body) Value(ctx context.Context, rType reflect.Type, name string) (inte } } - if len(r.body) == 0 { - return nil, false, nil - } if r.err != nil { return nil, false, r.err } + if len(r.body) == 0 { + return nil, false, nil + } if r.bodyType.Kind() == reflect.Map { return r.decodeBodyMap(ctx) } @@ -100,6 +100,9 @@ func (r *Body) initOnce() { // Non-multipart: clone and read body safely var request *http.Request request, r.err = shared.CloneHTTPRequest(r.request) + if r.err != nil { + return + } r.body, r.err = readRequestBody(request) }) } diff --git a/view/state/kind/locator/body_test.go b/view/state/kind/locator/body_test.go new file mode 100644 index 000000000..0f9e21e76 --- /dev/null +++ b/view/state/kind/locator/body_test.go @@ -0,0 +1,44 @@ +package locator + +import ( + "context" + "errors" + "net/http/httptest" + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +type failingBody struct{} + +func (failingBody) Read([]byte) (int, error) { + return 0, errors.New("failed to read request body") +} + +func (failingBody) Close() error { + return nil +} + +func TestBodyValueReturnsRequestBodyReadError(t *testing.T) { + request := httptest.NewRequest("POST", "http://localhost/test", nil) + request.Body = failingBody{} + + aLocator, err := NewBody( + WithRequest(request), + WithBodyType(reflect.TypeOf(struct{}{})), + WithUnmarshal(func([]byte, interface{}) error { return nil }), + ) + require.NoError(t, err) + + value, ok, err := aLocator.Value(context.Background(), reflect.TypeOf(struct{}{}), "") + require.Nil(t, value) + require.False(t, ok) + require.EqualError(t, err, "failed to read request body") +} + +func TestReadRequestBodyRejectsNilRequest(t *testing.T) { + data, err := readRequestBody(nil) + require.Nil(t, data) + require.EqualError(t, err, "request was empty") +} diff --git a/view/state/kind/locator/http.go b/view/state/kind/locator/http.go index 62c9ed697..e76f95466 100644 --- a/view/state/kind/locator/http.go +++ b/view/state/kind/locator/http.go @@ -61,6 +61,9 @@ func NewHttpRequest(opts ...Option) (kind.Locator, error) { } func readRequestBody(request *http.Request) ([]byte, error) { + if request == nil { + return nil, fmt.Errorf("request was empty") + } if request.Body == nil { return nil, nil } From 2ef031b46c8ec14d2ca79183ffca49122e44fc33 Mon Sep 17 00:00:00 2001 From: vcarey Date: Fri, 7 Aug 2026 17:19:27 -0400 Subject: [PATCH 278/279] Expose authoritative warmup keys --- gateway/warmup/cache.go | 68 ++++++------ gateway/warmup/cache_test.go | 48 ++++----- go.mod | 2 +- go.sum | 2 + service/reader/service.go | 19 +++- service/reader/service_metrics_test.go | 39 +++++++ warmup/cache.go | 142 +++++++++++-------------- warmup/cache_test.go | 31 ++---- 8 files changed, 185 insertions(+), 166 deletions(-) diff --git a/gateway/warmup/cache.go b/gateway/warmup/cache.go index bb4215f43..02b0fd3b8 100644 --- a/gateway/warmup/cache.go +++ b/gateway/warmup/cache.go @@ -23,36 +23,37 @@ const ( warmupRunErrorKey = "run.error" warmupCasesCompletedKey = "cases.completed" warmupCasesFailedKey = "cases.failed" - warmupRowsKey = "rows" + warmupGroupsWrittenKey = "groupsWritten" warmupMetricFallbackPkg = "datly" warmupMetricRecentBuckets = 2 ) type PreCachables func(ctx context.Context, method, matchingURI string) ([]*view.View, error) type PreCached struct { - URI string - View string - Column string - Params string - CacheKey string - FieldNames string `json:",omitempty"` - Elapsed string - TimeTaken time.Duration - Rows int - Error string `json:"error,omitempty"` + URI string + View string + Column string + Params string + WarmupKey string + MarkerKey string `json:",omitempty"` + FieldNames string `json:",omitempty"` + Elapsed string + TimeTaken time.Duration + GroupsWritten int `json:"groupsWritten,omitempty"` + Error string `json:"error,omitempty"` } type Summary struct { CompletedCases int `json:"completedCases"` FailedCases int `json:"failedCases"` - WarmedRows int `json:"warmedRows"` + GroupsWritten int `json:"groupsWritten,omitempty"` } type viewSummary struct { View string CompletedCases int FailedCases int - WarmedRows int + GroupsWritten int Elapsed time.Duration } @@ -106,11 +107,11 @@ func PreCache(ctx context.Context, lookup PreCachables, warmupURIs ...string) *R setErr(e) } elapsed := time.Now().Sub(startTime) - rows := 0 + groupsWritten := 0 if result != nil { - rows = result.Rows + groupsWritten = result.GroupsWritten } - fmt.Printf("[INFO] cache warmup uri done uri=%s rows=%d elapsed=%s\n", URI, rows, elapsed) + fmt.Printf("[INFO] cache warmup uri done uri=%s groups_written=%d elapsed=%s\n", URI, groupsWritten, elapsed) if result == nil { return } @@ -139,16 +140,17 @@ func appendPreCached(response *Response, URI string, result *warmup.Result) { continue } response.PreCached = append(response.PreCached, &PreCached{ - URI: URI, - View: entry.View, - Column: entry.Column, - Params: entry.Params, - CacheKey: entry.CacheKey, - FieldNames: entry.FieldNames, - Elapsed: entry.Elapsed, - TimeTaken: entry.TimeTaken, - Rows: entry.Rows, - Error: entry.Error, + URI: URI, + View: entry.View, + Column: entry.Column, + Params: entry.Params, + WarmupKey: entry.WarmupKey, + MarkerKey: entry.MarkerKey, + FieldNames: entry.FieldNames, + Elapsed: entry.Elapsed, + TimeTaken: entry.TimeTaken, + GroupsWritten: entry.GroupsWritten, + Error: entry.Error, }) } } @@ -167,7 +169,7 @@ func summarize(entries []*PreCached) *Summary { continue } summary.CompletedCases++ - summary.WarmedRows += entry.Rows + summary.GroupsWritten += entry.GroupsWritten } return summary } @@ -192,7 +194,7 @@ func summarizeByView(entries []*warmup.EntryResult) []*viewSummary { continue } current.CompletedCases++ - current.WarmedRows += entry.Rows + current.GroupsWritten += entry.GroupsWritten } result := make([]*viewSummary, 0, len(index)) for _, item := range index { @@ -210,12 +212,12 @@ func logViewSummaries(uri string, views []*view.View, result *warmup.Result) { } viewsIndex := indexViewsByName(views) for _, summary := range summarizeByView(result.Entries) { - fmt.Printf("[INFO] cache warmup view summary uri=%s view=%s completed_cases=%d failed_cases=%d warmed_rows=%d elapsed=%s\n", + fmt.Printf("[INFO] cache warmup view summary uri=%s view=%s completed_cases=%d failed_cases=%d groups_written=%d elapsed=%s\n", uri, summary.View, summary.CompletedCases, summary.FailedCases, - summary.WarmedRows, + summary.GroupsWritten, summary.Elapsed) recordWarmupViewMetrics(viewsIndex[summary.View], summary) } @@ -265,8 +267,8 @@ func recordWarmupViewMetrics(aView *view.View, summary *viewSummary) { if summary.FailedCases > 0 { operation.IncrementValueBy(warmupCasesFailedKey, int64(summary.FailedCases)) } - if summary.WarmedRows > 0 { - operation.IncrementValueBy(warmupRowsKey, int64(summary.WarmedRows)) + if summary.GroupsWritten > 0 { + operation.IncrementValueBy(warmupGroupsWrittenKey, int64(summary.GroupsWritten)) } } @@ -287,7 +289,7 @@ func warmupMetricOperation(aView *view.View) *gmetricx.OperationRef { warmupRunErrorKey, warmupCasesCompletedKey, warmupCasesFailedKey, - warmupRowsKey, + warmupGroupsWrittenKey, )) }) } diff --git a/gateway/warmup/cache_test.go b/gateway/warmup/cache_test.go index 94574492d..208c3eb75 100644 --- a/gateway/warmup/cache_test.go +++ b/gateway/warmup/cache_test.go @@ -16,13 +16,13 @@ import ( "github.com/viant/gmetric/stat" ) -func TestAppendPreCachedUsesEntryRows(t *testing.T) { +func TestAppendPreCachedUsesGroupsWritten(t *testing.T) { response := &Response{} result := &datlywarmup.Result{ - Rows: 30, + GroupsWritten: 300, Entries: []*datlywarmup.EntryResult{ - {View: "periodSummary#", Column: "order_id", Params: "Period=today", CacheKey: "cache://today", Elapsed: "1s", TimeTaken: time.Second, Rows: 10}, - {View: "periodSummary#", Column: "order_id", Params: "Period=month", CacheKey: "cache://month", FieldNames: "OrderId,Spend", Elapsed: "2s", TimeTaken: 2 * time.Second, Rows: 20}, + {View: "periodSummary#", Column: "order_id", Params: "Period=today", WarmupKey: "cache://today", Elapsed: "1s", TimeTaken: time.Second, GroupsWritten: 100}, + {View: "periodSummary#", Column: "order_id", Params: "Period=month", WarmupKey: "cache://month", MarkerKey: "order_id#cache://month", FieldNames: "OrderId,Spend", Elapsed: "2s", TimeTaken: 2 * time.Second, GroupsWritten: 200}, }, } @@ -30,12 +30,13 @@ func TestAppendPreCachedUsesEntryRows(t *testing.T) { require.Len(t, response.PreCached, 2) require.Equal(t, "Period=today", response.PreCached[0].Params) - require.Equal(t, "cache://today", response.PreCached[0].CacheKey) - require.Equal(t, 10, response.PreCached[0].Rows) + require.Equal(t, "cache://today", response.PreCached[0].WarmupKey) + require.Equal(t, 100, response.PreCached[0].GroupsWritten) require.Equal(t, "Period=month", response.PreCached[1].Params) - require.Equal(t, "cache://month", response.PreCached[1].CacheKey) + require.Equal(t, "cache://month", response.PreCached[1].WarmupKey) + require.Equal(t, "order_id#cache://month", response.PreCached[1].MarkerKey) require.Equal(t, "OrderId,Spend", response.PreCached[1].FieldNames) - require.Equal(t, 20, response.PreCached[1].Rows) + require.Equal(t, 200, response.PreCached[1].GroupsWritten) require.Equal(t, "/v1/api/cache/warmup/order", response.PreCached[1].URI) } @@ -43,7 +44,7 @@ func TestAppendPreCachedPreservesEntryErrors(t *testing.T) { response := &Response{} result := &datlywarmup.Result{ Entries: []*datlywarmup.EntryResult{ - {View: "diagnostics", Column: "ad_order_id", Params: "From=2026-07-02", CacheKey: "cache://today", Elapsed: "250ms", TimeTaken: 250 * time.Millisecond, Rows: 7, Error: "failed to index"}, + {View: "diagnostics", Column: "ad_order_id", Params: "From=2026-07-02", WarmupKey: "cache://today", Elapsed: "250ms", TimeTaken: 250 * time.Millisecond, GroupsWritten: 7, Error: "failed to index"}, }, } @@ -55,16 +56,16 @@ func TestAppendPreCachedPreservesEntryErrors(t *testing.T) { func TestSummarize(t *testing.T) { summary := summarize([]*PreCached{ - {Rows: 10, TimeTaken: 100 * time.Millisecond}, - {Rows: 20, TimeTaken: 200 * time.Millisecond}, - {Rows: 99, TimeTaken: 300 * time.Millisecond, Error: "failed to index"}, - {Rows: 30, TimeTaken: 400 * time.Millisecond}, + {GroupsWritten: 100, TimeTaken: 100 * time.Millisecond}, + {GroupsWritten: 200, TimeTaken: 200 * time.Millisecond}, + {GroupsWritten: 999, TimeTaken: 300 * time.Millisecond, Error: "failed to index"}, + {GroupsWritten: 300, TimeTaken: 400 * time.Millisecond}, }) require.NotNil(t, summary) require.Equal(t, 3, summary.CompletedCases) require.Equal(t, 1, summary.FailedCases) - require.Equal(t, 60, summary.WarmedRows) + require.Equal(t, 600, summary.GroupsWritten) } func TestSummarizeEmpty(t *testing.T) { @@ -73,27 +74,26 @@ func TestSummarizeEmpty(t *testing.T) { require.NotNil(t, summary) require.Zero(t, summary.CompletedCases) require.Zero(t, summary.FailedCases) - require.Zero(t, summary.WarmedRows) } func TestSummarizeByView(t *testing.T) { summaries := summarizeByView([]*datlywarmup.EntryResult{ - {View: "periodSummary#", Rows: 10, TimeTaken: time.Second}, - {View: "periodSummary#", Rows: 99, TimeTaken: 2 * time.Second, Error: "failed"}, - {View: "timeline#", Rows: 20, TimeTaken: 3 * time.Second}, - {View: "periodSummary#", Rows: 30, TimeTaken: 4 * time.Second}, + {View: "periodSummary#", GroupsWritten: 100, TimeTaken: time.Second}, + {View: "periodSummary#", GroupsWritten: 999, TimeTaken: 2 * time.Second, Error: "failed"}, + {View: "timeline#", GroupsWritten: 200, TimeTaken: 3 * time.Second}, + {View: "periodSummary#", GroupsWritten: 300, TimeTaken: 4 * time.Second}, }) require.Len(t, summaries, 2) require.Equal(t, "periodSummary#", summaries[0].View) require.Equal(t, 2, summaries[0].CompletedCases) require.Equal(t, 1, summaries[0].FailedCases) - require.Equal(t, 40, summaries[0].WarmedRows) + require.Equal(t, 400, summaries[0].GroupsWritten) require.Equal(t, 7*time.Second, summaries[0].Elapsed) require.Equal(t, "timeline#", summaries[1].View) require.Equal(t, 1, summaries[1].CompletedCases) require.Equal(t, 0, summaries[1].FailedCases) - require.Equal(t, 20, summaries[1].WarmedRows) + require.Equal(t, 200, summaries[1].GroupsWritten) require.Equal(t, 3*time.Second, summaries[1].Elapsed) } @@ -149,7 +149,6 @@ Connectors: require.NotNil(t, response.Summary) require.Equal(t, 0, response.Summary.CompletedCases) require.Equal(t, 1, response.Summary.FailedCases) - require.Zero(t, response.Summary.WarmedRows) require.Len(t, response.PreCached, 1) } @@ -162,7 +161,6 @@ func TestPreCacheLookupFailureAccounting(t *testing.T) { require.NotNil(t, response.Summary) require.Equal(t, 0, response.Summary.CompletedCases) require.Equal(t, 1, response.Summary.FailedCases) - require.Zero(t, response.Summary.WarmedRows) require.Len(t, response.PreCached, 1) require.Equal(t, "lookup failed", response.PreCached[0].Error) } @@ -198,7 +196,7 @@ func TestRecordWarmupViewMetrics(t *testing.T) { View: aView.Name, CompletedCases: 2, FailedCases: 1, - WarmedRows: 40, + GroupsWritten: 400, Elapsed: 1500 * time.Millisecond, }) @@ -207,6 +205,6 @@ func TestRecordWarmupViewMetrics(t *testing.T) { require.Equal(t, int64(1), metrics.LookupOperationCumulativeMetric(metricName, warmupRunErrorKey)) require.Equal(t, int64(2), metrics.LookupOperationCumulativeMetric(metricName, warmupCasesCompletedKey)) require.Equal(t, int64(1), metrics.LookupOperationCumulativeMetric(metricName, warmupCasesFailedKey)) - require.Equal(t, int64(40), metrics.LookupOperationCumulativeMetric(metricName, warmupRowsKey)) + require.Equal(t, int64(400), metrics.LookupOperationCumulativeMetric(metricName, warmupGroupsWrittenKey)) require.GreaterOrEqual(t, metrics.LookupOperationCumulativeMetric(metricName, stat.CounterTimeTakenKey), int64(1500)) } diff --git a/go.mod b/go.mod index f8882f624..b978e3a25 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/viant/parsly v0.3.3 github.com/viant/pgo v0.11.0 github.com/viant/scy v0.33.1 - github.com/viant/sqlx v0.23.1-0.20260803165008-da07533d2e8f + github.com/viant/sqlx v0.23.1-0.20260807211629-027861517984 github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.1-0.20260408224432-5a1c31e1bd87 diff --git a/go.sum b/go.sum index c456cb73f..f8efa2602 100644 --- a/go.sum +++ b/go.sum @@ -1196,6 +1196,8 @@ github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7 h1:2FdVturjHBSQ github.com/viant/sqlparser v0.12.1-0.20260409013525-147f8fc299b7/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.23.1-0.20260803165008-da07533d2e8f h1:yte+MMDo1mWS6+YpM1OYmfZe3+ieYZXsCWfjAf4c+PY= github.com/viant/sqlx v0.23.1-0.20260803165008-da07533d2e8f/go.mod h1:yZOQRVCMZAkexsTaoqCPGJvsNO2qajQRU1VuYu23fX8= +github.com/viant/sqlx v0.23.1-0.20260807211629-027861517984 h1:/ayViIofvv1pA8wrN3ZhPmJZb0kFpfycqC6dMWXPOjg= +github.com/viant/sqlx v0.23.1-0.20260807211629-027861517984/go.mod h1:yZOQRVCMZAkexsTaoqCPGJvsNO2qajQRU1VuYu23fX8= github.com/viant/structology v0.9.0 h1:ibR/XmdQ3+/4XW3JK+pXRqugSnxJOm2bmIvbQ0hqztY= github.com/viant/structology v0.9.0/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= diff --git a/service/reader/service.go b/service/reader/service.go index 28c49b3e2..a6fc2cf90 100644 --- a/service/reader/service.go +++ b/service/reader/service.go @@ -1054,7 +1054,7 @@ func logCacheRead(ctx context.Context, aView *view.View, stats *cache.Stats, ela return } recordCacheReadMetrics(aView, stats) - fmt.Printf("[INFO] datly cache read reqTraceId=%s view=%s source=%s type=%s found_warmup=%t found_lazy=%t records=%d rows=%d namespace=%s set=%s elapsed=%s args=%v\n", + fmt.Printf("[INFO] datly cache read reqTraceId=%s view=%s source=%s type=%s found_warmup=%t found_lazy=%t records=%d rows=%d namespace=%s set=%s elapsed=%s args=%v%s\n", reqTraceID(ctx), aView.Name, cacheReadSource(stats), @@ -1066,7 +1066,22 @@ func logCacheRead(ctx context.Context, aView *view.View, stats *cache.Stats, ela stats.Namespace, stats.Dataset, elapsed, - args) + args, + warmupReadKeysSuffix(stats)) +} + +func warmupReadKeysSuffix(stats *cache.Stats) string { + if stats == nil { + return "" + } + var result string + if stats.WarmupKey != "" { + result += " warmup_key=" + stats.WarmupKey + } + if stats.MarkerKey != "" { + result += " marker_key=" + stats.MarkerKey + } + return result } func reqTraceID(ctx context.Context) string { diff --git a/service/reader/service_metrics_test.go b/service/reader/service_metrics_test.go index 7d65633a9..c8a32aa54 100644 --- a/service/reader/service_metrics_test.go +++ b/service/reader/service_metrics_test.go @@ -99,6 +99,45 @@ func TestRecordCacheReadMetrics(t *testing.T) { } } +func TestWarmupReadKeysSuffix(t *testing.T) { + testCases := []struct { + description string + stats *cache.Stats + expected string + }{ + { + description: "nil stats", + expected: "", + }, + { + description: "empty stats", + stats: &cache.Stats{}, + expected: "", + }, + { + description: "warmup key only", + stats: &cache.Stats{WarmupKey: "warmup-123"}, + expected: " warmup_key=warmup-123", + }, + { + description: "marker key only", + stats: &cache.Stats{MarkerKey: "order_id#warmup-123"}, + expected: " marker_key=order_id#warmup-123", + }, + { + description: "warmup and marker keys", + stats: &cache.Stats{WarmupKey: "warmup-123", MarkerKey: "order_id#warmup-123"}, + expected: " warmup_key=warmup-123 marker_key=order_id#warmup-123", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + require.Equal(t, testCase.expected, warmupReadKeysSuffix(testCase.stats)) + }) + } +} + type metricsTestRow struct { ID int } diff --git a/warmup/cache.go b/warmup/cache.go index 46f39d8f5..7f742d0ea 100644 --- a/warmup/cache.go +++ b/warmup/cache.go @@ -9,7 +9,6 @@ import ( errUtils "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/sqlx/io/read/cache" - cachehash "github.com/viant/sqlx/io/read/cache/hash" "strings" "sync" "time" @@ -34,27 +33,27 @@ type ( column string label string fields string - key string } warmupEntryFn func() (*warmupEntry, error) notifierFn func() (int, *EntryResult, error) EntryResult struct { - View string - Column string - Params string - CacheKey string - FieldNames string - Elapsed string - TimeTaken time.Duration - Rows int - Error string `json:",omitempty"` + View string + Column string + Params string + WarmupKey string + MarkerKey string + FieldNames string + Elapsed string + TimeTaken time.Duration + GroupsWritten int + Error string `json:",omitempty"` } Result struct { - Rows int - Entries []*EntryResult + GroupsWritten int + Entries []*EntryResult } ) @@ -66,7 +65,7 @@ func (c *matchersCollector) populate(ctx context.Context, collector chan warmupE if err == nil { return size, nil, nil } - return size, failedEntryResult(&warmupEntry{view: c.view}, 0, 0, err), err + return size, failedEntryResult(&warmupEntry{view: c.view}, 0, err), err } }() } @@ -118,20 +117,6 @@ func (c *matchersCollector) createMetaWarmupEntry(ctx context.Context, aView *vi } return } - cacheKey, err := warmupCacheKey(cacheIndex) - if err != nil { - fmt.Printf("[INFO] cache warmup entry build error view=%s type=meta column=%s field_names=%s error=%v\n", aView.Name, input.MetaColumn, strings.Join(input.FieldNames, ","), err) - aChan <- func() (*warmupEntry, error) { - return &warmupEntry{ - view: aView, - column: input.MetaColumn, - label: input.Label, - fields: strings.Join(input.FieldNames, ","), - }, err - } - return - } - aChan <- func() (*warmupEntry, error) { return &warmupEntry{ matcher: cacheIndex, @@ -139,7 +124,6 @@ func (c *matchersCollector) createMetaWarmupEntry(ctx context.Context, aView *vi column: input.MetaColumn, label: input.Label, fields: strings.Join(input.FieldNames, ","), - key: cacheKey, }, nil } } @@ -158,19 +142,6 @@ func (c *matchersCollector) createIndexWarmupEntry(ctx context.Context, aView *v } return } - cacheKey, err := warmupCacheKey(build) - if err != nil { - fmt.Printf("[INFO] cache warmup entry build error view=%s type=index column=%s field_names=%s error=%v\n", aView.Name, cacheInput.Column, strings.Join(cacheInput.FieldNames, ","), err) - aChan <- func() (*warmupEntry, error) { - return &warmupEntry{ - view: aView, - column: cacheInput.Column, - label: cacheInput.Label, - fields: strings.Join(cacheInput.FieldNames, ","), - }, err - } - return - } build.StoredFields = view.SQLXProjectionFields(cacheInput.StoredFields) aChan <- func() (*warmupEntry, error) { @@ -180,7 +151,6 @@ func (c *matchersCollector) createIndexWarmupEntry(ctx context.Context, aView *v column: cacheInput.Column, label: cacheInput.Label, fields: strings.Join(cacheInput.FieldNames, ","), - key: cacheKey, }, nil } } @@ -239,39 +209,68 @@ func readWithChan(ctx context.Context, entry *warmupEntry, notifier chan func() func readWithErr(ctx context.Context, entry *warmupEntry) (*EntryResult, error) { started := time.Now() - fmt.Printf("[INFO] cache warmup query start start_time=%s view=%s cache=%s cache_key=%s db_connector=%s column=%s params=%s field_names=%s args=%v sql=%q\n", started.Format(time.RFC3339), entry.view.Name, cacheLabel(entry.view), entry.key, warmupConnectorLabel(entry.view), entry.column, entry.label, entry.fields, entry.matcher.Args, truncateSQL(entry.matcher.SQL)) + fmt.Printf("[INFO] cache warmup query start start_time=%s view=%s cache=%s db_connector=%s column=%s params=%s field_names=%s args=%v sql=%q\n", started.Format(time.RFC3339), entry.view.Name, cacheLabel(entry.view), warmupConnectorLabel(entry.view), entry.column, entry.label, entry.fields, entry.matcher.Args, truncateSQL(entry.matcher.SQL)) db, err := DB(entry) if err != nil { elapsed := time.Since(started) - fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, elapsed, err) - return failedEntryResult(entry, elapsed, 0, err), err + fmt.Printf("[INFO] cache warmup query error view=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.column, entry.label, entry.fields, elapsed, err) + return failedEntryResult(entry, elapsed, err), err } service, err := entry.view.Cache.Service() if err != nil { elapsed := time.Since(started) - fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, elapsed, err) - return failedEntryResult(entry, elapsed, 0, err), err + fmt.Printf("[INFO] cache warmup query error view=%s column=%s params=%s field_names=%s elapsed=%s cache_write=skipped error=%v\n", entry.view.Name, entry.column, entry.label, entry.fields, elapsed, err) + return failedEntryResult(entry, elapsed, err), err } matcher := entry.matcher - indexed, err := service.IndexBy(indexProgressContext(ctx, entry), db, entry.column, matcher.SQL, matcher.Args, matcher) + indexResult, err := indexByWithResult(indexProgressContext(ctx, entry), service, db, entry.column, matcher.SQL, matcher.Args, matcher) elapsed := time.Since(started) + if indexResult == nil { + indexResult = &indexByResult{} + } if err != nil { - fmt.Printf("[INFO] cache warmup query error view=%s cache_key=%s column=%s params=%s field_names=%s rows=%d elapsed=%s cache_write=error error=%v\n", entry.view.Name, entry.key, entry.column, entry.label, entry.fields, indexed, elapsed, err) + fmt.Printf("[INFO] cache warmup query error view=%s warmup_key=%s marker_key=%s column=%s params=%s field_names=%s groups_written=%d elapsed=%s cache_write=error error=%v\n", entry.view.Name, indexResult.warmupKey, indexResult.markerKey, entry.column, entry.label, entry.fields, indexResult.groupsWritten, elapsed, err) indexErr := fmt.Errorf("failed to index: %w", err) - return failedEntryResult(entry, elapsed, indexed, indexErr), indexErr + result := failedEntryResult(entry, elapsed, indexErr) + result.WarmupKey = indexResult.warmupKey + result.MarkerKey = indexResult.markerKey + result.GroupsWritten = indexResult.groupsWritten + return result, indexErr + } + + fmt.Printf("[INFO] cache warmup query done view=%s cache=%s warmup_key=%s marker_key=%s db_connector=%s column=%s params=%s field_names=%s groups_written=%d elapsed=%s cache_write=success\n", entry.view.Name, cacheLabel(entry.view), indexResult.warmupKey, indexResult.markerKey, warmupConnectorLabel(entry.view), entry.column, entry.label, entry.fields, indexResult.groupsWritten, elapsed) + return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, WarmupKey: indexResult.warmupKey, MarkerKey: indexResult.markerKey, FieldNames: entry.fields, Elapsed: elapsed.String(), TimeTaken: elapsed, GroupsWritten: indexResult.groupsWritten}, nil +} + +type indexByResult struct { + groupsWritten int + warmupKey string + markerKey string +} + +func indexByWithResult(ctx context.Context, service cache.Cache, db *sql.DB, column, SQL string, args []interface{}, matcher *cache.ParmetrizedQuery) (*indexByResult, error) { + if indexer, ok := service.(cache.WarmupIndexer); ok { + result, err := indexer.IndexByWithResult(ctx, db, column, SQL, args, matcher) + if result == nil { + return nil, err + } + return &indexByResult{ + groupsWritten: result.GroupsWritten, + warmupKey: result.WarmupKey, + markerKey: result.MarkerKey, + }, err } - fmt.Printf("[INFO] cache warmup query done view=%s cache=%s cache_key=%s db_connector=%s column=%s params=%s field_names=%s rows=%d elapsed=%s cache_write=success\n", entry.view.Name, cacheLabel(entry.view), entry.key, warmupConnectorLabel(entry.view), entry.column, entry.label, entry.fields, indexed, elapsed) - return &EntryResult{View: entry.view.Name, Column: entry.column, Params: entry.label, CacheKey: entry.key, FieldNames: entry.fields, Elapsed: elapsed.String(), TimeTaken: elapsed, Rows: indexed}, nil + groupsWritten, err := service.IndexBy(ctx, db, column, SQL, args, matcher) + return &indexByResult{groupsWritten: groupsWritten}, err } -func failedEntryResult(entry *warmupEntry, elapsed time.Duration, rows int, err error) *EntryResult { +func failedEntryResult(entry *warmupEntry, elapsed time.Duration, err error) *EntryResult { result := &EntryResult{ Elapsed: elapsed.String(), TimeTaken: elapsed, - Rows: rows, } if entry != nil { if entry.view != nil { @@ -279,7 +278,6 @@ func failedEntryResult(entry *warmupEntry, elapsed time.Duration, rows int, err } result.Column = entry.column result.Params = entry.label - result.CacheKey = entry.key result.FieldNames = entry.fields } if err != nil { @@ -297,24 +295,6 @@ func firstError(errors []error) error { return nil } -func warmupCacheKey(query *cache.ParmetrizedQuery) (string, error) { - if query == nil { - return "", fmt.Errorf("warmup cache key query was nil") - } - return warmupIdentityURL(query) -} - -func warmupIdentityURL(query *cache.ParmetrizedQuery) (string, error) { - if query == nil { - return "", fmt.Errorf("warmup identity query was nil") - } - SQL, _, argsMarshal, err := query.WarmupIdentity() - if err != nil { - return "", err - } - return cachehash.GenerateWithMarshal(SQL, "", "", argsMarshal) -} - func DB(entry *warmupEntry) (*sql.DB, error) { if entry.view.Cache.Warmup.Connector != nil { return entry.view.Cache.Warmup.Connector.DB() @@ -328,7 +308,7 @@ func PopulateCache(views []*view.View) (int, error) { if result == nil { return 0, err } - return result.Rows, err + return result.GroupsWritten, err } func PopulateCacheWithDetails(views []*view.View) (*Result, error) { @@ -342,7 +322,7 @@ func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (* result := &Result{} if len(viewsWithCache) == 0 { - fmt.Printf("[INFO] cache warmup populate done rows=0 elapsed=%s\n", time.Since(started)) + fmt.Printf("[INFO] cache warmup populate done groups_written=0 elapsed=%s\n", time.Since(started)) return result, nil } @@ -376,7 +356,7 @@ func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (* } if collectorSize == 0 { - fmt.Printf("[INFO] cache warmup populate done rows=0 entries=0 elapsed=%s\n", time.Since(started)) + fmt.Printf("[INFO] cache warmup populate done groups_written=0 entries=0 elapsed=%s\n", time.Since(started)) err := errUtils.CombineErrors("errors while populating cache: ", errors) if err != nil { return result, err @@ -391,7 +371,7 @@ func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (* entry, err := fn() if err != nil { errors = append(errors, err) - result.Entries = append(result.Entries, failedEntryResult(entry, 0, 0, err)) + result.Entries = append(result.Entries, failedEntryResult(entry, 0, err)) } else { warmupEntries = append(warmupEntries, entry) } @@ -419,7 +399,7 @@ func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (* entryResult, err := actual() if entryResult != nil { result.Entries = append(result.Entries, entryResult) - result.Rows += entryResult.Rows + result.GroupsWritten += entryResult.GroupsWritten } if err != nil { errors = append(errors, err) @@ -429,10 +409,10 @@ func PopulateCacheWithDetailsContext(ctx context.Context, views []*view.View) (* close(notifier) err := errUtils.CombineErrors("errors while populating cache: ", errors) if err != nil { - fmt.Printf("[INFO] cache warmup populate error rows=%d entries=%d failures=%d elapsed=%s first_error=%v\n", result.Rows, len(warmupEntries), len(errors), time.Since(started), firstError(errors)) + fmt.Printf("[INFO] cache warmup populate error groups_written=%d entries=%d failures=%d elapsed=%s first_error=%v\n", result.GroupsWritten, len(warmupEntries), len(errors), time.Since(started), firstError(errors)) return result, err } - fmt.Printf("[INFO] cache warmup populate done rows=%d entries=%d elapsed=%s\n", result.Rows, len(warmupEntries), time.Since(started)) + fmt.Printf("[INFO] cache warmup populate done groups_written=%d entries=%d elapsed=%s\n", result.GroupsWritten, len(warmupEntries), time.Since(started)) return result, nil } diff --git a/warmup/cache_test.go b/warmup/cache_test.go index b73345213..0a574ad9a 100644 --- a/warmup/cache_test.go +++ b/warmup/cache_test.go @@ -17,7 +17,7 @@ import ( sqlcache "github.com/viant/sqlx/io/read/cache" ) -func TestPopulateCache(t *testing.T) { +func TestPopulateCacheWithDetails(t *testing.T) { if os.Getenv("DATLY_RUN_WARMUP_TESTS") == "" { t.Skip("set DATLY_RUN_WARMUP_TESTS=1 to run warmup integration test") } @@ -78,9 +78,10 @@ func TestPopulateCache(t *testing.T) { views = append(views, item) } - inserted, err := PopulateCache(views) + result, err := PopulateCacheWithDetails(views) assert.Nil(t, err, testCase.description) - assert.Equal(t, testCase.expectedInserted, inserted, testCase.description) + require.NotNil(t, result, testCase.description) + assert.Equal(t, testCase.expectedInserted, result.GroupsWritten, testCase.description) for _, aView := range views { cache := aView.Cache @@ -146,7 +147,7 @@ func TestWarmupWithLimitCapsConcurrency(t *testing.T) { } time.Sleep(5 * time.Millisecond) atomic.AddInt64(&active, -1) - return &EntryResult{Rows: 1}, nil + return &EntryResult{GroupsWritten: 1}, nil } notifier := make(chan func() (*EntryResult, error)) @@ -157,26 +158,13 @@ func TestWarmupWithLimitCapsConcurrency(t *testing.T) { actual := <-notifier result, err := actual() assert.Nil(t, err) - total += result.Rows + total += result.GroupsWritten } assert.Equal(t, len(entries), total) assert.LessOrEqual(t, atomic.LoadInt64(&maxActive), int64(maxWarmupConcurrency)) } -func TestWarmupCacheKeyNormalizesNilArgs(t *testing.T) { - nilArgsKey, err := warmupCacheKey(&sqlcache.ParmetrizedQuery{SQL: "SELECT * FROM events", Args: nil}) - assert.Nil(t, err) - - emptyArgsKey, err := warmupCacheKey(&sqlcache.ParmetrizedQuery{SQL: "SELECT * FROM events", Args: []interface{}{}}) - assert.Nil(t, err) - - assert.Equal(t, emptyArgsKey, nilArgsKey) - - _, err = warmupCacheKey(nil) - assert.ErrorContains(t, err, "query was nil") -} - func TestIndexProgressContext(t *testing.T) { aView := &view.View{ Name: "performanceTimeline", @@ -228,7 +216,7 @@ func TestIndexProgressContext(t *testing.T) { require.True(t, actual.Done) } -func TestWarmupFieldNamesAffectGeneratedCacheKey(t *testing.T) { +func TestWarmupFieldNamesAffectGeneratedProjection(t *testing.T) { resourcePath := path.Join(t.TempDir(), "resource.yaml") require.NoError(t, os.WriteFile(resourcePath, []byte(` CacheProviders: @@ -276,8 +264,6 @@ Views: builder := reader.NewBuilder() fullQuery, err := builder.CacheSQL(context.Background(), aView, input[0].Selector) require.NoError(t, err) - fullKey, err := warmupCacheKey(fullQuery) - require.NoError(t, err) aView.Cache.Warmup.FieldNames = []string{"Quantity"} fieldInput, err := aView.Cache.GenerateCacheInput(context.Background()) @@ -291,11 +277,8 @@ Views: fieldQuery, err := builder.CacheSQL(context.Background(), aView, fieldInput[0].Selector) require.NoError(t, err) - fieldKey, err := warmupCacheKey(fieldQuery) - require.NoError(t, err) assert.NotEqual(t, fullQuery.SQL, fieldQuery.SQL) - assert.NotEqual(t, fullKey, fieldKey) assert.Contains(t, fieldQuery.SQL, "quantity") } From 54ae48e3a4649d32aa3018051932fab2ad5f6b38 Mon Sep 17 00:00:00 2001 From: adranwit Date: Mon, 10 Aug 2026 23:26:54 +0200 Subject: [PATCH 279/279] - extended mcp integration --- cmd/command/mcp.go | 1 + cmd/command/run.go | 3 ++- cmd/option.go | 23 ++++++++++++----------- cmd/options/mcp.go | 1 + cmd/options/run.go | 25 +++++++++++++------------ gateway/config.go | 14 ++++++++++++++ gateway/mcp.go | 4 +--- mcp/server.go | 2 +- 8 files changed, 45 insertions(+), 28 deletions(-) diff --git a/cmd/command/mcp.go b/cmd/command/mcp.go index bbdb84897..df68c01da 100644 --- a/cmd/command/mcp.go +++ b/cmd/command/mcp.go @@ -46,6 +46,7 @@ func (s *Service) mcp(ctx context.Context, mcpOption *options.Mcp) error { Port: mcpOption.Port, OAuth2ConfigURL: mcpOption.OAuth2ConfigURL, IssuerURL: mcpOption.IssuerURL, + ResourceURL: mcpOption.ResourceURL, AuthorizerMode: mcpOption.AuthorizerMode, }) if err != nil { diff --git a/cmd/command/run.go b/cmd/command/run.go index db3ba7a1f..04b88b2e4 100644 --- a/cmd/command/run.go +++ b/cmd/command/run.go @@ -43,7 +43,7 @@ func (s *Service) run(ctx context.Context, run *options.Run) (*standalone.Server _ = s.fs.Copy(ctx, parent, s.config.Config.PluginsURL) } s.config.Version = run.Version - if run.MCPPort != nil || run.MCPAuthURL != "" || run.MCPIssuerURL != "" || run.MCPAuthMode != "" { + if run.MCPPort != nil || run.MCPAuthURL != "" || run.MCPIssuerURL != "" || run.MCPResourceURL != "" || run.MCPAuthMode != "" { if s.config.Config.MCP == nil { s.config.Config.MCP = &gateway.ModelContextProtocol{} } @@ -52,6 +52,7 @@ func (s *Service) run(ctx context.Context, run *options.Run) (*standalone.Server } setter.SetStringIfEmpty(&s.config.Config.MCP.OAuth2ConfigURL, run.MCPAuthURL) setter.SetStringIfEmpty(&s.config.Config.MCP.IssuerURL, run.MCPIssuerURL) + setter.SetStringIfEmpty(&s.config.Config.MCP.ResourceURL, run.MCPResourceURL) setter.SetStringIfEmpty(&s.config.Config.MCP.AuthorizerMode, run.MCPAuthMode) } return standalone.New(ctx, standalone.WithConfig(s.config)) diff --git a/cmd/option.go b/cmd/option.go index 85072d8d8..71157d8ce 100644 --- a/cmd/option.go +++ b/cmd/option.go @@ -35,16 +35,17 @@ type ( Plugins Package Module - AssetsURL string `short:"a" long:"assetsURL" description:"assets destination"` - ConstURL string `long:"constURL" description:"path where const files are stored"` - Legacy bool `short:"l"` - cache *view.Cache - SubstituesURL []string `long:"substituesURL" description:"substitues URL, expands template before processing"` - JobURL string `short:"z" long:"joburl" description:"job url"` - MCPPort int `long:"mcpPort" description:"enable MCP HTTP server on the specified port"` - MCPAuthURL string `long:"mcpAuthClient" description:"auth client url for MCP server"` - MCPIssuerURL string `long:"mcpIssuerURL" description:"issuer url for MCP server"` - MCPAuthMode string `long:"mcpAuth" description:"authorizer S - server authorizer, F fallback authorizer"` + AssetsURL string `short:"a" long:"assetsURL" description:"assets destination"` + ConstURL string `long:"constURL" description:"path where const files are stored"` + Legacy bool `short:"l"` + cache *view.Cache + SubstituesURL []string `long:"substituesURL" description:"substitues URL, expands template before processing"` + JobURL string `short:"z" long:"joburl" description:"job url"` + MCPPort int `long:"mcpPort" description:"enable MCP HTTP server on the specified port"` + MCPAuthURL string `long:"mcpAuthClient" description:"auth client url for MCP server"` + MCPIssuerURL string `long:"mcpIssuerURL" description:"issuer url for MCP server"` + MCPResourceURL string `long:"mcpResourceURL" description:"protected resource identifier for MCP server"` + MCPAuthMode string `long:"mcpAuth" description:"authorizer S - server authorizer, F fallback authorizer"` } Package struct { @@ -170,7 +171,7 @@ func (o *Options) BuildOption() *options.Options { } if o.ConfigURL != "" && repo == nil { - result.Run = &options.Run{ConfigURL: o.ConfigURL, JobURL: o.JobURL, MCPAuthURL: o.MCPAuthURL, MCPIssuerURL: o.MCPIssuerURL, MCPAuthMode: o.MCPAuthMode} + result.Run = &options.Run{ConfigURL: o.ConfigURL, JobURL: o.JobURL, MCPAuthURL: o.MCPAuthURL, MCPIssuerURL: o.MCPIssuerURL, MCPResourceURL: o.MCPResourceURL, MCPAuthMode: o.MCPAuthMode} if o.MCPPort > 0 { result.Run.MCPPort = &o.MCPPort } diff --git a/cmd/options/mcp.go b/cmd/options/mcp.go index 5ca5db3b1..c53034493 100644 --- a/cmd/options/mcp.go +++ b/cmd/options/mcp.go @@ -9,6 +9,7 @@ type Mcp struct { Port *int `short:"p" long:"port" description:"http port"` OAuth2ConfigURL string `short:"C" long:"authclient" description:"auth client url"` IssuerURL string `short:"I" long:"issuerurl" description:"issuer url"` + ResourceURL string `long:"resourceurl" description:"protected resource identifier"` AuthorizerMode string `short:"A" long:"auth" description:"authorizer S - server authorizer, F fallback authorizer (server size)" choice:"F" choice:"S"` } diff --git a/cmd/options/run.go b/cmd/options/run.go index 19c1f991f..5b294e34d 100644 --- a/cmd/options/run.go +++ b/cmd/options/run.go @@ -5,18 +5,19 @@ import ( ) type Run struct { - ConfigURL string `short:"c" long:"conf" description:"datly config"` - WarmupURIs []string `short:"w" long:"warmup" description:"warmup uris"` - JobURL string `short:"z" long:"joburl" description:"job url"` - MaxJobs int `short:"W" long:"mjobs" description:"max jobs" default:"40" ` - FailedJobURL string `short:"F" long:"fjobs" description:"failed jobs" ` - LoadPlugin bool `short:"L" long:"lplugin" description:"load plugin"` - MCPPort *int `long:"mcpPort" description:"enable MCP HTTP server on the specified port"` - MCPAuthURL string `long:"mcpAuthClient" description:"auth client url for MCP server"` - MCPIssuerURL string `long:"mcpIssuerURL" description:"issuer url for MCP server"` - MCPAuthMode string `long:"mcpAuth" description:"authorizer S - server authorizer, F fallback authorizer" choice:"F" choice:"S"` - PluginInfo string - Version string + ConfigURL string `short:"c" long:"conf" description:"datly config"` + WarmupURIs []string `short:"w" long:"warmup" description:"warmup uris"` + JobURL string `short:"z" long:"joburl" description:"job url"` + MaxJobs int `short:"W" long:"mjobs" description:"max jobs" default:"40" ` + FailedJobURL string `short:"F" long:"fjobs" description:"failed jobs" ` + LoadPlugin bool `short:"L" long:"lplugin" description:"load plugin"` + MCPPort *int `long:"mcpPort" description:"enable MCP HTTP server on the specified port"` + MCPAuthURL string `long:"mcpAuthClient" description:"auth client url for MCP server"` + MCPIssuerURL string `long:"mcpIssuerURL" description:"issuer url for MCP server"` + MCPResourceURL string `long:"mcpResourceURL" description:"protected resource identifier for MCP server"` + MCPAuthMode string `long:"mcpAuth" description:"authorizer S - server authorizer, F fallback authorizer" choice:"F" choice:"S"` + PluginInfo string + Version string } func (r *Run) Init() error { diff --git a/gateway/config.go b/gateway/config.go index dfc84072c..aff266826 100644 --- a/gateway/config.go +++ b/gateway/config.go @@ -56,6 +56,7 @@ type ( Port *int OAuth2ConfigURL string IssuerURL string + ResourceURL string AuthorizerMode string BFFExchangeHeader string BFFRedirectURI string @@ -86,6 +87,19 @@ type ( } ) +const defaultMCPProtectedResource = "https://datly.viantinc.com" + +// ProtectedResourceURL returns the RFC 9728 resource-server identifier used +// in MCP authorization challenges and protected-resource metadata. +func (m *ModelContextProtocol) ProtectedResourceURL() string { + if m != nil { + if resourceURL := strings.TrimSpace(m.ResourceURL); resourceURL != "" { + return resourceURL + } + } + return defaultMCPProtectedResource +} + const ( DQLBootstrapPrecedenceRoutesWins = "routes_wins" DQLBootstrapPrecedenceDQLWins = "dql_wins" diff --git a/gateway/mcp.go b/gateway/mcp.go index ccafb6922..05a11657f 100644 --- a/gateway/mcp.go +++ b/gateway/mcp.go @@ -399,8 +399,6 @@ func (r *Router) addAuthTokenIfPresent(ctx context.Context, httpRequest *http.Re } } -const defaultMCPProtectedResource = "https://datly.viantinc.com" - func (r *Router) mcpUnauthorizedError() *jsonrpc.Error { if r == nil || r.config == nil || r.config.MCP == nil { return jsonrpc.NewError(schema.Unauthorized, "Unauthorized", nil) @@ -413,7 +411,7 @@ func (r *Router) mcpUnauthorizedError() *jsonrpc.Error { RequiredScopes: []string{}, UseIdToken: true, ProtectedResourceMetadata: &oauthmeta.ProtectedResourceMetadata{ - Resource: defaultMCPProtectedResource, + Resource: r.config.MCP.ProtectedResourceURL(), AuthorizationServers: []string{issuerURL}, }, }) diff --git a/mcp/server.go b/mcp/server.go index b3e1c3e45..a0b6c6e88 100644 --- a/mcp/server.go +++ b/mcp/server.go @@ -64,7 +64,7 @@ func (s *Server) init() error { ExcludeURI: "/sse", Global: &authorization.Authorization{ ProtectedResourceMetadata: &meta.ProtectedResourceMetadata{ - Resource: "https://datly.viantinc.com", + Resource: s.config.ProtectedResourceURL(), AuthorizationServers: []string{issuerURL}, }, UseIdToken: true,