Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions backend/internal/handler/openai_chat_completions.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,20 @@ func (h *OpenAIGatewayHandler) ChatCompletions(c *gin.Context) {

subscription, _ := middleware2.GetSubscriptionFromContext(c)
requestPlatform := openAICompatibleRequestPlatform(c.Request.Context(), apiKey)
effectiveModel := reqModel
if channelMapping.Mapped && strings.TrimSpace(channelMapping.MappedModel) != "" {
effectiveModel = channelMapping.MappedModel
}
if sanitizedBody, changed, sanitizeErr := service.SanitizeUnsupportedCNImageInput(body, requestPlatform, effectiveModel); sanitizeErr != nil {
h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to normalize image input")
return
} else if changed {
body = sanitizedBody
reqLog.Info("openai_chat_completions.unsupported_image_input_sanitized",
zap.String("platform", requestPlatform),
zap.String("effective_model", effectiveModel),
)
}

service.SetOpsLatencyMs(c, service.OpsAuthLatencyMsKey, time.Since(requestStart).Milliseconds())

Expand Down
13 changes: 12 additions & 1 deletion backend/internal/handler/openai_gateway_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -513,6 +513,17 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) {
routingModel, groupModelMapped := resolveGroupRequestModel(apiKey, reqModel)
channelMapping, _ := h.gatewayService.ResolveChannelMappingAndRestrict(c.Request.Context(), apiKey.GroupID, routingModel)
forwardModel, requestModelMapped := effectiveOpenAIForwardModel(routingModel, groupModelMapped, channelMapping)
requestPlatform := openAICompatibleRequestPlatform(c.Request.Context(), apiKey)
if sanitizedBody, changed, sanitizeErr := service.SanitizeUnsupportedCNImageInput(body, requestPlatform, forwardModel); sanitizeErr != nil {
h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to normalize image input")
return
} else if changed {
body = sanitizedBody
reqLog.Info("openai.responses.unsupported_image_input_sanitized",
zap.String("platform", requestPlatform),
zap.String("effective_model", forwardModel),
)
}
forwardBody := openAIModelMappedBody(body, requestModelMapped, forwardModel, h.gatewayService.ReplaceModelInBody)
seedOpenAIForwardImageIntentHint(c, requestModelMapped, imageIntent)
c.Request = c.Request.WithContext(service.WithOpenAIForwardModel(
Expand All @@ -533,7 +544,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) {

// Get subscription info (may be nil)
subscription, _ := middleware2.GetSubscriptionFromContext(c)
requestPlatform := openAICompatibleRequestPlatform(c.Request.Context(), apiKey)
requestPlatform = openAICompatibleRequestPlatform(c.Request.Context(), apiKey)
if reqStream && requestPlatform == service.PlatformNvidia && isBareOpenAIResponsesPath(c) {
service.MarkOpenAINvidiaResponsesStream(c)
}
Expand Down
109 changes: 109 additions & 0 deletions backend/internal/service/openai_image_input_policy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package service

import (
"encoding/json"
"strings"
)

const unsupportedCNImageInputPlaceholder = "[Image input omitted: selected model does not support vision.]"

// SanitizeUnsupportedCNImageInput prevents text-only Chinese-provider models
// from receiving image parts that their upstream rejects with a 400. The
// original request is returned untouched when the platform/model is allowed
// to receive images or when no image part is present.
func SanitizeUnsupportedCNImageInput(body []byte, platform, model string) ([]byte, bool, error) {
if !unsupportedCNImageInputPolicyApplies(platform, model) {
return body, false, nil
}

var request map[string]any
if err := decodeOpenAIJSONUseNumber(body, &request); err != nil {
return body, false, err
}
if !hasOpenAIInputImage(request) {
return body, false, nil
}

changed := false
for _, key := range []string{"input", "messages"} {
value, ok := request[key]
if !ok {
continue
}
sanitized, valueChanged := sanitizeUnsupportedCNImageInputValue(value)
if valueChanged {
request[key] = sanitized
changed = true
}
}
if !changed {
return body, false, nil
}

rebuilt, err := json.Marshal(request)
if err != nil {
return body, false, err
}
return rebuilt, true, nil
}

func unsupportedCNImageInputPolicyApplies(platform, model string) bool {
platform = strings.ToLower(strings.TrimSpace(platform))
if platform != PlatformDeepseek && platform != PlatformZhipu {
return false
}
return !cnModelSupportsImageInput(model)
}

func cnModelSupportsImageInput(model string) bool {
model = strings.ToLower(strings.TrimSpace(model))
if model == "" {
return false
}
if model == "deepseek-v4-flash-vision-exp" {
return true
}
for _, marker := range []string{"vision", "-vl", "glm-4v", "glm-4.5v", "glm-4.6v", "glm-5v"} {
if strings.Contains(model, marker) {
return true
}
}
return strings.HasSuffix(model, "-v")
}

func sanitizeUnsupportedCNImageInputValue(value any) (any, bool) {
switch value := value.(type) {
case []any:
changed := false
for i, item := range value {
sanitized, itemChanged := sanitizeUnsupportedCNImageInputValue(item)
if itemChanged {
value[i] = sanitized
changed = true
}
}
return value, changed
case map[string]any:
typ := strings.ToLower(strings.TrimSpace(firstNonEmptyString(value["type"])))
switch typ {
case "image_url":
return map[string]any{"type": "text", "text": unsupportedCNImageInputPlaceholder}, true
case "input_image":
return map[string]any{"type": "input_text", "text": unsupportedCNImageInputPlaceholder}, true
}
if _, ok := value["image_url"]; ok {
return map[string]any{"type": "text", "text": unsupportedCNImageInputPlaceholder}, true
}
changed := false
for key, item := range value {
sanitized, itemChanged := sanitizeUnsupportedCNImageInputValue(item)
if itemChanged {
value[key] = sanitized
changed = true
}
}
return value, changed
default:
return value, false
}
}
120 changes: 120 additions & 0 deletions backend/internal/service/openai_image_input_policy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package service

import (
"encoding/json"
"testing"
)

func TestSanitizeUnsupportedCNImageInput(t *testing.T) {
tests := []struct {
name string
platform string
model string
body string
wantChange bool
wantType string
}{
{
name: "chat image url",
platform: PlatformDeepseek,
model: "deepseek-v4-flash-0731",
body: `{"model":"deepseek-v4-flash-0731","messages":[{"role":"user","content":[{"type":"text","text":"describe"},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]}]}`,
wantChange: true,
wantType: "text",
},
{
name: "responses input image",
platform: PlatformDeepseek,
model: "deepseek-v4-pro-0813",
body: `{"model":"deepseek-v4-pro-0813","input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AAAA"}]}]}`,
wantChange: true,
wantType: "input_text",
},
{
name: "zhipu historical message",
platform: PlatformZhipu,
model: "glm-5.3",
body: `{"model":"glm-5.3","messages":[{"role":"user","content":"first"},{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.invalid/image.png"}}]}]}`,
wantChange: true,
wantType: "text",
},
{
name: "deepseek vision model is allowed",
platform: PlatformDeepseek,
model: "deepseek-v4-flash-vision-exp",
body: `{"model":"deepseek-v4-flash-vision-exp","input":[{"type":"input_image","image_url":"https://example.invalid/image.png"}]}`,
wantChange: false,
},
{
name: "non cn platform is unchanged",
platform: PlatformOpenAI,
model: "gpt-5",
body: `{"model":"gpt-5","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.invalid/image.png"}}]}]}`,
wantChange: false,
},
{
name: "text only request is unchanged",
platform: PlatformZhipu,
model: "glm-5.3",
body: `{"model":"glm-5.3","messages":[{"role":"user","content":"hello"}]}`,
wantChange: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, changed, err := SanitizeUnsupportedCNImageInput([]byte(tt.body), tt.platform, tt.model)
if err != nil {
t.Fatalf("SanitizeUnsupportedCNImageInput() error = %v", err)
}
if changed != tt.wantChange {
t.Fatalf("changed = %v, want %v", changed, tt.wantChange)
}
if !tt.wantChange {
if string(got) != tt.body {
t.Fatalf("unchanged body = %s, want %s", got, tt.body)
}
return
}

var request map[string]any
if err := json.Unmarshal(got, &request); err != nil {
t.Fatalf("sanitized body is invalid JSON: %v", err)
}
if hasOpenAIInputImage(request) {
t.Fatal("sanitized request still contains image input")
}
if tt.wantType == "input_text" {
input, ok := request["input"].([]any)
if !ok || len(input) == 0 {
t.Fatal("sanitized input is missing")
}
item, ok := input[0].(map[string]any)
if !ok {
t.Fatal("sanitized input item has unexpected type")
}
contentItems, ok := item["content"].([]any)
if !ok || len(contentItems) == 0 {
t.Fatal("sanitized content is missing")
}
content, ok := contentItems[0].(map[string]any)
if !ok {
t.Fatal("sanitized content item has unexpected type")
}
if content["type"] != tt.wantType {
t.Fatalf("replacement type = %v, want %s", content["type"], tt.wantType)
}
}
})
}
}

func TestSanitizeUnsupportedCNImageInputRejectsMalformedJSON(t *testing.T) {
_, changed, err := SanitizeUnsupportedCNImageInput([]byte(`{"input":[`), PlatformDeepseek, "deepseek-v4-flash")
if err == nil {
t.Fatal("expected malformed JSON error")
}
if changed {
t.Fatal("malformed request must not be marked changed")
}
}
8 changes: 8 additions & 0 deletions backend/internal/service/openai_ws_forwarder_ingress.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,14 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient(
}
normalized = next
}
if group := apiKeyGroup(getAPIKeyFromContext(c)); group != nil {
if sanitizedBody, changed, sanitizeErr := SanitizeUnsupportedCNImageInput(normalized, group.Platform, upstreamModel); sanitizeErr != nil {
return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket request payload", sanitizeErr)
} else if changed {
normalized = sanitizedBody
logOpenAIWSModeInfo("ingress_ws_unsupported_image_input_sanitized account_id=%d platform=%s model=%s", account.ID, group.Platform, upstreamModel)
}
}
SetOpsUpstreamModel(c, upstreamModel)
if isCodexCLI && codexImageGenerationExplicitToolPolicy == codexImageGenerationExplicitToolPolicyStrip {
if stripped, changed, stripErr := stripOpenAIImageGenerationToolsFromRawPayload(normalized); stripErr != nil {
Expand Down
Loading