diff --git a/backend/internal/handler/openai_chat_completions.go b/backend/internal/handler/openai_chat_completions.go index 6b509121138e..e78a66a1e956 100644 --- a/backend/internal/handler/openai_chat_completions.go +++ b/backend/internal/handler/openai_chat_completions.go @@ -130,6 +130,20 @@ func (h *OpenAIGatewayHandler) ChatCompletions(c *gin.Context) { subscription, _ := middleware2.GetSubscriptionFromContext(c) requestPlatform := openAICompatibleRequestPlatform(c.Request.Context(), apiKey) + effectiveModel := reqModel + if channelMapping.Mapped && strings.TrimSpace(channelMapping.MappedModel) != "" { + effectiveModel = channelMapping.MappedModel + } + if sanitizedBody, changed, sanitizeErr := service.SanitizeUnsupportedCNImageInput(body, requestPlatform, effectiveModel); sanitizeErr != nil { + h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to normalize image input") + return + } else if changed { + body = sanitizedBody + reqLog.Info("openai_chat_completions.unsupported_image_input_sanitized", + zap.String("platform", requestPlatform), + zap.String("effective_model", effectiveModel), + ) + } service.SetOpsLatencyMs(c, service.OpsAuthLatencyMsKey, time.Since(requestStart).Milliseconds()) diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index a44dbfb47fb1..610128e51eb7 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -513,6 +513,17 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { routingModel, groupModelMapped := resolveGroupRequestModel(apiKey, reqModel) channelMapping, _ := h.gatewayService.ResolveChannelMappingAndRestrict(c.Request.Context(), apiKey.GroupID, routingModel) forwardModel, requestModelMapped := effectiveOpenAIForwardModel(routingModel, groupModelMapped, channelMapping) + requestPlatform := openAICompatibleRequestPlatform(c.Request.Context(), apiKey) + if sanitizedBody, changed, sanitizeErr := service.SanitizeUnsupportedCNImageInput(body, requestPlatform, forwardModel); sanitizeErr != nil { + h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to normalize image input") + return + } else if changed { + body = sanitizedBody + reqLog.Info("openai.responses.unsupported_image_input_sanitized", + zap.String("platform", requestPlatform), + zap.String("effective_model", forwardModel), + ) + } forwardBody := openAIModelMappedBody(body, requestModelMapped, forwardModel, h.gatewayService.ReplaceModelInBody) seedOpenAIForwardImageIntentHint(c, requestModelMapped, imageIntent) c.Request = c.Request.WithContext(service.WithOpenAIForwardModel( @@ -533,7 +544,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { // Get subscription info (may be nil) subscription, _ := middleware2.GetSubscriptionFromContext(c) - requestPlatform := openAICompatibleRequestPlatform(c.Request.Context(), apiKey) + requestPlatform = openAICompatibleRequestPlatform(c.Request.Context(), apiKey) if reqStream && requestPlatform == service.PlatformNvidia && isBareOpenAIResponsesPath(c) { service.MarkOpenAINvidiaResponsesStream(c) } diff --git a/backend/internal/service/openai_image_input_policy.go b/backend/internal/service/openai_image_input_policy.go new file mode 100644 index 000000000000..a0d5768043d7 --- /dev/null +++ b/backend/internal/service/openai_image_input_policy.go @@ -0,0 +1,109 @@ +package service + +import ( + "encoding/json" + "strings" +) + +const unsupportedCNImageInputPlaceholder = "[Image input omitted: selected model does not support vision.]" + +// SanitizeUnsupportedCNImageInput prevents text-only Chinese-provider models +// from receiving image parts that their upstream rejects with a 400. The +// original request is returned untouched when the platform/model is allowed +// to receive images or when no image part is present. +func SanitizeUnsupportedCNImageInput(body []byte, platform, model string) ([]byte, bool, error) { + if !unsupportedCNImageInputPolicyApplies(platform, model) { + return body, false, nil + } + + var request map[string]any + if err := decodeOpenAIJSONUseNumber(body, &request); err != nil { + return body, false, err + } + if !hasOpenAIInputImage(request) { + return body, false, nil + } + + changed := false + for _, key := range []string{"input", "messages"} { + value, ok := request[key] + if !ok { + continue + } + sanitized, valueChanged := sanitizeUnsupportedCNImageInputValue(value) + if valueChanged { + request[key] = sanitized + changed = true + } + } + if !changed { + return body, false, nil + } + + rebuilt, err := json.Marshal(request) + if err != nil { + return body, false, err + } + return rebuilt, true, nil +} + +func unsupportedCNImageInputPolicyApplies(platform, model string) bool { + platform = strings.ToLower(strings.TrimSpace(platform)) + if platform != PlatformDeepseek && platform != PlatformZhipu { + return false + } + return !cnModelSupportsImageInput(model) +} + +func cnModelSupportsImageInput(model string) bool { + model = strings.ToLower(strings.TrimSpace(model)) + if model == "" { + return false + } + if model == "deepseek-v4-flash-vision-exp" { + return true + } + for _, marker := range []string{"vision", "-vl", "glm-4v", "glm-4.5v", "glm-4.6v", "glm-5v"} { + if strings.Contains(model, marker) { + return true + } + } + return strings.HasSuffix(model, "-v") +} + +func sanitizeUnsupportedCNImageInputValue(value any) (any, bool) { + switch value := value.(type) { + case []any: + changed := false + for i, item := range value { + sanitized, itemChanged := sanitizeUnsupportedCNImageInputValue(item) + if itemChanged { + value[i] = sanitized + changed = true + } + } + return value, changed + case map[string]any: + typ := strings.ToLower(strings.TrimSpace(firstNonEmptyString(value["type"]))) + switch typ { + case "image_url": + return map[string]any{"type": "text", "text": unsupportedCNImageInputPlaceholder}, true + case "input_image": + return map[string]any{"type": "input_text", "text": unsupportedCNImageInputPlaceholder}, true + } + if _, ok := value["image_url"]; ok { + return map[string]any{"type": "text", "text": unsupportedCNImageInputPlaceholder}, true + } + changed := false + for key, item := range value { + sanitized, itemChanged := sanitizeUnsupportedCNImageInputValue(item) + if itemChanged { + value[key] = sanitized + changed = true + } + } + return value, changed + default: + return value, false + } +} diff --git a/backend/internal/service/openai_image_input_policy_test.go b/backend/internal/service/openai_image_input_policy_test.go new file mode 100644 index 000000000000..e77bf1b489ef --- /dev/null +++ b/backend/internal/service/openai_image_input_policy_test.go @@ -0,0 +1,120 @@ +package service + +import ( + "encoding/json" + "testing" +) + +func TestSanitizeUnsupportedCNImageInput(t *testing.T) { + tests := []struct { + name string + platform string + model string + body string + wantChange bool + wantType string + }{ + { + name: "chat image url", + platform: PlatformDeepseek, + model: "deepseek-v4-flash-0731", + body: `{"model":"deepseek-v4-flash-0731","messages":[{"role":"user","content":[{"type":"text","text":"describe"},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]}]}`, + wantChange: true, + wantType: "text", + }, + { + name: "responses input image", + platform: PlatformDeepseek, + model: "deepseek-v4-pro-0813", + body: `{"model":"deepseek-v4-pro-0813","input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AAAA"}]}]}`, + wantChange: true, + wantType: "input_text", + }, + { + name: "zhipu historical message", + platform: PlatformZhipu, + model: "glm-5.3", + body: `{"model":"glm-5.3","messages":[{"role":"user","content":"first"},{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.invalid/image.png"}}]}]}`, + wantChange: true, + wantType: "text", + }, + { + name: "deepseek vision model is allowed", + platform: PlatformDeepseek, + model: "deepseek-v4-flash-vision-exp", + body: `{"model":"deepseek-v4-flash-vision-exp","input":[{"type":"input_image","image_url":"https://example.invalid/image.png"}]}`, + wantChange: false, + }, + { + name: "non cn platform is unchanged", + platform: PlatformOpenAI, + model: "gpt-5", + body: `{"model":"gpt-5","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.invalid/image.png"}}]}]}`, + wantChange: false, + }, + { + name: "text only request is unchanged", + platform: PlatformZhipu, + model: "glm-5.3", + body: `{"model":"glm-5.3","messages":[{"role":"user","content":"hello"}]}`, + wantChange: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, changed, err := SanitizeUnsupportedCNImageInput([]byte(tt.body), tt.platform, tt.model) + if err != nil { + t.Fatalf("SanitizeUnsupportedCNImageInput() error = %v", err) + } + if changed != tt.wantChange { + t.Fatalf("changed = %v, want %v", changed, tt.wantChange) + } + if !tt.wantChange { + if string(got) != tt.body { + t.Fatalf("unchanged body = %s, want %s", got, tt.body) + } + return + } + + var request map[string]any + if err := json.Unmarshal(got, &request); err != nil { + t.Fatalf("sanitized body is invalid JSON: %v", err) + } + if hasOpenAIInputImage(request) { + t.Fatal("sanitized request still contains image input") + } + if tt.wantType == "input_text" { + input, ok := request["input"].([]any) + if !ok || len(input) == 0 { + t.Fatal("sanitized input is missing") + } + item, ok := input[0].(map[string]any) + if !ok { + t.Fatal("sanitized input item has unexpected type") + } + contentItems, ok := item["content"].([]any) + if !ok || len(contentItems) == 0 { + t.Fatal("sanitized content is missing") + } + content, ok := contentItems[0].(map[string]any) + if !ok { + t.Fatal("sanitized content item has unexpected type") + } + if content["type"] != tt.wantType { + t.Fatalf("replacement type = %v, want %s", content["type"], tt.wantType) + } + } + }) + } +} + +func TestSanitizeUnsupportedCNImageInputRejectsMalformedJSON(t *testing.T) { + _, changed, err := SanitizeUnsupportedCNImageInput([]byte(`{"input":[`), PlatformDeepseek, "deepseek-v4-flash") + if err == nil { + t.Fatal("expected malformed JSON error") + } + if changed { + t.Fatal("malformed request must not be marked changed") + } +} diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index b12d9b089211..ee37134001fd 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -421,6 +421,14 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( } normalized = next } + if group := apiKeyGroup(getAPIKeyFromContext(c)); group != nil { + if sanitizedBody, changed, sanitizeErr := SanitizeUnsupportedCNImageInput(normalized, group.Platform, upstreamModel); sanitizeErr != nil { + return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket request payload", sanitizeErr) + } else if changed { + normalized = sanitizedBody + logOpenAIWSModeInfo("ingress_ws_unsupported_image_input_sanitized account_id=%d platform=%s model=%s", account.ID, group.Platform, upstreamModel) + } + } SetOpsUpstreamModel(c, upstreamModel) if isCodexCLI && codexImageGenerationExplicitToolPolicy == codexImageGenerationExplicitToolPolicyStrip { if stripped, changed, stripErr := stripOpenAIImageGenerationToolsFromRawPayload(normalized); stripErr != nil {