Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/librecode/cli_helpers_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func TestPrintSessionSummaryAndEntry(t *testing.T) {
Role: database.RoleUser,
Content: "message text",
Provider: "",
Model: "",
Model: "", Parts: nil,
},
Summary: "",
ToolStatus: "",
Expand Down
1 change: 1 addition & 0 deletions cmd/librecode/prompt.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ func buildPromptRequest(cwd, message string, options promptRunOptions) *assistan
ParentEntryID: nil,
SessionID: options.SessionID,
CWD: cwd,
Images: nil,
Text: message,
Name: options.SessionName,
ResumeLatest: options.Resume,
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ require (
github.com/yuin/goldmark v1.8.5
github.com/yuin/gopher-lua v1.1.2
golang.design/x/clipboard v0.8.0
golang.org/x/image v0.44.0
golang.org/x/net v0.57.0
golang.org/x/text v0.40.0
gopkg.in/yaml.v3 v3.0.1
Expand Down Expand Up @@ -107,7 +108,6 @@ require (
golang.design/x/x11 v0.2.0 // indirect
golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirect
golang.org/x/exp/shiny v0.0.0-20250606033433-dcc06ee1d476 // indirect
golang.org/x/image v0.41.0 // indirect
golang.org/x/mobile v0.0.0-20250606033058-a2a15c67f36f // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,8 @@ golang.org/x/exp v0.0.0-20260718201538-764159d718ef h1:LkZ48HFgy/TvhTI0bcWkjgFkg
golang.org/x/exp v0.0.0-20260718201538-764159d718ef/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
golang.org/x/exp/shiny v0.0.0-20250606033433-dcc06ee1d476 h1:Wdx0vgH5Wgsw+lF//LJKmWOJBLWX6nprsMqnf99rYDE=
golang.org/x/exp/shiny v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:ygj7T6vSGhhm/9yTpOQQNvuAUFziTH7RUiH74EoE2C8=
golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo=
golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA=
golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I=
golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY=
golang.org/x/mobile v0.0.0-20250606033058-a2a15c67f36f h1:/n+PL2HlfqeSiDCuhdBbRNlGS/g2fM4OHufalHaTVG8=
golang.org/x/mobile v0.0.0-20250606033058-a2a15c67f36f/go.mod h1:ESkJ836Z6LpG6mTVAhA48LpfW/8fNR0ifStlH2axyfg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
Expand Down
2 changes: 1 addition & 1 deletion internal/agenttask/runtime_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func (runner *RuntimeRunner) Run(
},
OnRetry: nil, OnUserEntry: nil, ParentEntryID: nil,
SessionID: task.ChildSessionID, CWD: session.CWD, Text: task.Prompt,
Name: "", ResumeLatest: false, HideUserPrompt: false,
Images: nil, Name: "", ResumeLatest: false, HideUserPrompt: false,
})
usageJSON, usageErr := agentUsageJSON(response, metrics.Snapshot())

Expand Down
2 changes: 1 addition & 1 deletion internal/assistant/context_compaction_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,7 @@ func appendRuntimeTestMessage(
Role: role,
Content: content,
Provider: "",
Model: "",
Model: "", Parts: nil,
})
require.NoError(t, err)

Expand Down
7 changes: 7 additions & 0 deletions internal/assistant/lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ func lifecyclePromptRequest(request *PromptRequest) *lifecyclepayload.PromptRequ
if request == nil {
return &lifecyclepayload.PromptRequest{
ParentEntryID: nil,
Attachments: nil,
CWD: "",
Name: "",
SessionID: "",
Expand All @@ -281,8 +282,14 @@ func lifecyclePromptRequest(request *PromptRequest) *lifecyclepayload.PromptRequ
}
}

attachments := make([]map[string]any, len(request.Images))
for index := range request.Images {
attachments[index] = imageMetadata(request.Images[index])
}

return &lifecyclepayload.PromptRequest{
ParentEntryID: request.ParentEntryID,
Attachments: attachments,
CWD: request.CWD,
Name: request.Name,
SessionID: request.SessionID,
Expand Down
2 changes: 1 addition & 1 deletion internal/assistant/lifecyclepayload/behavior_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ func messageEntity(role database.Role, content string) database.MessageEntity {
Role: role,
Content: content,
Provider: "",
Model: "",
Model: "", Parts: nil,
}
}

Expand Down
15 changes: 9 additions & 6 deletions internal/assistant/lifecyclepayload/lifecyclepayload.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ type PromptRequest struct {
Name string
SessionID string
Text string
Attachments []map[string]any
ResumeLatest bool
}

Expand Down Expand Up @@ -145,12 +146,14 @@ func Prompt(request *PromptRequest) map[string]any {
}

return map[string]any{
CWDKey: request.CWD,
ToolNameKey: request.Name,
ParentEntryIDKey: stringPtrValue(request.ParentEntryID),
PromptKey: request.Text,
"resume_latest": request.ResumeLatest,
SessionIDKey: request.SessionID,
"attachments": request.Attachments,
"attachment_count": len(request.Attachments),
CWDKey: request.CWD,
ToolNameKey: request.Name,
ParentEntryIDKey: stringPtrValue(request.ParentEntryID),
PromptKey: request.Text,
"resume_latest": request.ResumeLatest,
SessionIDKey: request.SessionID,
}
}

Expand Down
45 changes: 29 additions & 16 deletions internal/assistant/lifecyclepayload/lifecyclepayload_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,37 @@ const (
func TestPromptAndTurnPayloads(t *testing.T) {
t.Parallel()

t.Run("prompt", func(t *testing.T) {
t.Run("prompt attachments", func(t *testing.T) {
t.Parallel()

parentID := lifecycleTestParentID
prompt := lifecyclepayload.Prompt(&lifecyclepayload.PromptRequest{
ParentEntryID: &parentID,
CWD: "/work",
Name: "agent",
SessionID: lifecycleTestSessionID,
Text: "hello",
ResumeLatest: true,
})
assert.Equal(t, "/work", prompt[lifecyclepayload.CWDKey])
assert.Equal(t, "agent", prompt[lifecyclepayload.ToolNameKey])
assert.Equal(t, parentID, prompt[lifecyclepayload.ParentEntryIDKey])
resumeLatest, ok := prompt["resume_latest"].(bool)
require.True(t, ok)
assert.True(t, resumeLatest)

tests := []struct {
name string
attachments []map[string]any
wantCount int
}{
{name: "none", attachments: nil, wantCount: 0},
{name: "one", attachments: []map[string]any{{"name": "screen.webp", "size": 442}}, wantCount: 1},
}
for _, testCase := range tests {
t.Run(testCase.name, func(t *testing.T) {
t.Parallel()

prompt := lifecyclepayload.Prompt(&lifecyclepayload.PromptRequest{
ParentEntryID: &parentID, Attachments: testCase.attachments, CWD: "/work",
Name: "agent", SessionID: lifecycleTestSessionID, Text: "hello", ResumeLatest: true,
})
assert.Equal(t, "/work", prompt[lifecyclepayload.CWDKey])
assert.Equal(t, "agent", prompt[lifecyclepayload.ToolNameKey])
assert.Equal(t, parentID, prompt[lifecyclepayload.ParentEntryIDKey])
assert.Equal(t, testCase.wantCount, prompt["attachment_count"])
assert.Equal(t, testCase.attachments, prompt["attachments"])
resumeLatest, ok := prompt["resume_latest"].(bool)
require.True(t, ok)
assert.True(t, resumeLatest)
})
}
})

t.Run("nil prompt", func(t *testing.T) {
Expand Down Expand Up @@ -109,7 +122,7 @@ func TestSessionEntryAndContextPayloads(t *testing.T) {
Role: database.RoleAssistant,
Content: "answer",
Provider: "provider-1",
Model: lifecycleTestModel,
Model: lifecycleTestModel, Parts: nil,
},
Summary: "summary",
ToolStatus: "",
Expand Down
35 changes: 33 additions & 2 deletions internal/assistant/llm_conversion.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package assistant

import (
"encoding/base64"
"strings"

"github.com/samber/lo"
Expand Down Expand Up @@ -47,7 +48,7 @@ func llmMessagesFromDatabase(messages []database.MessageEntity) []llm.Message {
}

func llmMessageFromDatabase(message *database.MessageEntity) (llm.Message, bool) {
if message == nil || strings.TrimSpace(message.Content) == "" {
if message == nil {
return emptyLLMMessage(), false
}

Expand All @@ -56,7 +57,37 @@ func llmMessageFromDatabase(message *database.MessageEntity) (llm.Message, bool)
return emptyLLMMessage(), false
}

return llm.TextMessage(role, message.Content), true
parts := llmPartsFromDatabase(message.Parts)

if len(parts) == 0 && strings.TrimSpace(message.Content) != "" {
parts = append(parts, llm.TextPart(message.Content))
}

if len(parts) == 0 {
return emptyLLMMessage(), false
}

return llm.Message{Metadata: nil, Role: role, Content: parts}, true
}

func llmPartsFromDatabase(databaseParts []database.MessagePartEntity) []llm.Part {
parts := make([]llm.Part, 0, len(databaseParts))
for index := range databaseParts {
part := &databaseParts[index]
if part.Type == database.MessagePartText && strings.TrimSpace(part.Text) != "" {
parts = append(parts, llm.TextPart(part.Text))
}

if part.Type == database.MessagePartImage && len(part.Data) != 0 {
parts = append(parts, llm.Part{
Metadata: imagePartMetadata(part.Name, part.Width, part.Height),
ToolCall: nil, ToolResult: nil, Type: llm.PartImage, Text: "",
Data: base64.StdEncoding.EncodeToString(part.Data), MIMEType: part.MIMEType,
})
}
}

return parts
}

func emptyLLMMessage() llm.Message {
Expand Down
81 changes: 79 additions & 2 deletions internal/assistant/llm_conversion_internal_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package assistant

import (
"encoding/base64"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -25,7 +27,7 @@ func TestLLMRequestFromCompletionRequestConvertsAssistantState(t *testing.T) {
ToolRegistry: registry,
ExecuteTools: nil,
SessionID: "session-1",
SystemPrompt: jsonSystemRole,
SystemPrompt: "system instructions",
ThinkingLevel: "high",
CWD: t.TempDir(),
Auth: model.RequestAuth{
Expand Down Expand Up @@ -65,10 +67,12 @@ func TestLLMRequestFromCompletionRequestConvertsAssistantState(t *testing.T) {
converted := llmRequestFromCompletionRequest(request)

assert.Equal(t, "session-1", converted.SessionID)
assert.Equal(t, expectedSystemRole, converted.SystemPrompt)
assert.Equal(t, "system instructions", converted.SystemPrompt)
assert.Equal(t, "high", converted.ThinkingLevel)
assert.Equal(t, "openai", converted.Model.Provider)
assert.Equal(t, "gpt-test", converted.Model.ID)
assert.Equal(t, apiOpenAIResponses, converted.Model.API)
assert.Equal(t, "https://example.test", converted.Model.BaseURL)
assert.Equal(t, "secret", converted.Auth.APIKey)
assert.Equal(t, "value", converted.Auth.Headers["x-test"])
assert.Len(t, converted.Messages, 4)
Expand All @@ -85,6 +89,79 @@ func TestLLMRequestFromCompletionRequestConvertsAssistantState(t *testing.T) {
assert.Equal(t, "yes", request.Model.Compat["compat"])
}

func TestLLMMessageFromDatabaseConvertsContentParts(t *testing.T) {
t.Parallel()

data := []byte{0, 1, 2, 3}
tests := []struct {
name string
wantText string
message database.MessageEntity
wantTypes []llm.PartType
}{
{
name: "ordered multipart",
message: database.MessageEntity{
Timestamp: time.Time{}, Role: database.RoleUser, Content: "", Provider: "", Model: "",
Parts: []database.MessagePartEntity{
{Text: "inspect image", MIMEType: "", Name: "", Type: database.MessagePartText,
Data: nil, Width: 0, Height: 0},
{Text: "", MIMEType: imageMIMEPNG, Name: "screen.png", Type: database.MessagePartImage,
Data: data, Width: 10, Height: 20},
},
},
wantTypes: []llm.PartType{llm.PartText, llm.PartImage},
wantText: "inspect image",
},
{
name: "image only",
message: database.MessageEntity{
Timestamp: time.Time{}, Role: database.RoleUser, Content: "", Provider: "", Model: "",
Parts: []database.MessagePartEntity{{
Text: "", MIMEType: imageMIMEPNG, Name: "", Type: database.MessagePartImage,
Data: data, Width: 1, Height: 1,
}},
},
wantTypes: []llm.PartType{llm.PartImage},
wantText: "",
},
{
name: "legacy content fallback",
message: database.MessageEntity{
Timestamp: time.Time{}, Role: database.RoleAssistant, Content: "answer",
Provider: "", Model: "", Parts: nil,
},
wantTypes: []llm.PartType{llm.PartText},
wantText: "answer",
},
}

for _, testCase := range tests {
t.Run(testCase.name, func(t *testing.T) {
t.Parallel()

message, converted := llmMessageFromDatabase(&testCase.message)
require.True(t, converted)
require.Len(t, message.Content, len(testCase.wantTypes))

for index := range testCase.wantTypes {
assert.Equal(t, testCase.wantTypes[index], message.Content[index].Type)
}

if testCase.wantText != "" {
assert.Equal(t, testCase.wantText, message.Content[0].Text)
}

if message.Content[len(message.Content)-1].Type == llm.PartImage {
imagePart := message.Content[len(message.Content)-1]
assert.Equal(t, base64.StdEncoding.EncodeToString(data), imagePart.Data)
assert.Equal(t, imageMIMEPNG, imagePart.MIMEType)
assert.NotEmpty(t, imagePart.Metadata)
}
})
}
}

func TestLLMRequestFromCompletionRequestNilAndDisabledTools(t *testing.T) {
t.Parallel()

Expand Down
Loading
Loading