/* * Copyright 2025 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package adk import ( "context" "errors" "io" "strings" "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "github.com/cloudwego/eino/components/model" "github.com/cloudwego/eino/components/tool" "github.com/cloudwego/eino/compose" mockModel "github.com/cloudwego/eino/internal/mock/components/model" "github.com/cloudwego/eino/schema" ) var errRetryAble = errors.New("retry-able error") var errNonRetryAble = errors.New("non-retry-able error") var instantBackoff = func(_ context.Context, _ int) time.Duration { return time.Millisecond } type agentEvent struct { Err error Output *AgentOutput StreamContent string } func drainAgentEvents(t *testing.T, iterator *AsyncIterator[*AgentEvent]) []agentEvent { t.Helper() var events []agentEvent for { event, ok := iterator.Next() if !ok { break } events = append(events, agentEvent{Err: event.Err, Output: event.Output}) } return events } func drainStreamingAgentEvents(t *testing.T, iterator *AsyncIterator[*AgentEvent]) (events []agentEvent, streamTermErrs []error) { t.Helper() for { event, ok := iterator.Next() if !ok { break } ae := agentEvent{Err: event.Err, Output: event.Output} if event.Output != nil && event.Output.MessageOutput != nil { mo := event.Output.MessageOutput if mo.IsStreaming && mo.MessageStream != nil { var chunks []string for { msg, recvErr := mo.MessageStream.Recv() if recvErr != nil { streamTermErrs = append(streamTermErrs, recvErr) break } if msg != nil { chunks = append(chunks, msg.Content) } } ae.StreamContent = strings.Join(chunks, "") } } events = append(events, ae) } return events, streamTermErrs } func TestChatModelAgentRetry_NoTools_DirectError_Generate(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) if count < 3 { return nil, errRetryAble } return schema.AssistantMessage("Success after retry", nil), nil }).Times(3) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.Nil(t, event.Err) assert.NotNil(t, event.Output) assert.Equal(t, "Success after retry", event.Output.MessageOutput.Message.Content) _, ok = iterator.Next() assert.False(t, ok) assert.Equal(t, int32(3), atomic.LoadInt32(&callCount)) } func TestChatModelAgentRetry_NoTools_DirectError_Stream(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { count := atomic.AddInt32(&callCount, 1) if count < 2 { return nil, errRetryAble } return schema.StreamReaderFromArray([]*schema.Message{ schema.AssistantMessage("Success", nil), }), nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.Nil(t, event.Err) assert.NotNil(t, event.Output) assert.True(t, event.Output.MessageOutput.IsStreaming) _, ok = iterator.Next() assert.False(t, ok) assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) } type streamErrorModel struct { callCount int32 failAtChunk int maxFailures int tools []*schema.ToolInfo returnTool bool } func (m *streamErrorModel) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) { return schema.AssistantMessage("Generated", nil), nil } func (m *streamErrorModel) Stream(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) { count := atomic.AddInt32(&m.callCount, 1) sr, sw := schema.Pipe[*schema.Message](10) go func() { defer sw.Close() for i := 0; i < 5; i++ { if i == m.failAtChunk && int(count) <= m.maxFailures { sw.Send(nil, errRetryAble) return } if m.returnTool && i == 0 { sw.Send(schema.AssistantMessage("", []schema.ToolCall{{ ID: "call-1", Function: schema.FunctionCall{Name: "test_tool", Arguments: "{}"}, }}), nil) } else { sw.Send(schema.AssistantMessage("chunk", nil), nil) } } }() return sr, nil } func (m *streamErrorModel) WithTools(tools []*schema.ToolInfo) (model.ToolCallingChatModel, error) { m.tools = tools return m, nil } func TestChatModelAgentRetry_StreamError(t *testing.T) { t.Run("WithTools", func(t *testing.T) { ctx := context.Background() m := &streamErrorModel{ failAtChunk: 2, maxFailures: 2, returnTool: false, } config := &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: m, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, ToolsConfig: ToolsConfig{ ToolsNodeConfig: compose.ToolsNodeConfig{ Tools: []tool.BaseTool{&fakeToolForTest{tarCount: 0}}, }, }, } agent, err := NewChatModelAgent(ctx, config) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) var events []*AgentEvent for { event, ok := iterator.Next() if !ok { break } events = append(events, event) } assert.Equal(t, 3, len(events)) var streamErrEventCount int var errs []error for i, event := range events { if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming { sr := event.Output.MessageOutput.MessageStream for { msg, err := sr.Recv() if err == io.EOF { break } if err != nil { streamErrEventCount++ errs = append(errs, err) t.Logf("event %d: err: %v", i, err) break } t.Logf("event %d: %v", i, msg.Content) } } } assert.Equal(t, 2, streamErrEventCount) assert.Equal(t, 2, len(errs)) var willRetryErr *WillRetryError assert.True(t, errors.As(errs[0], &willRetryErr)) assert.True(t, errors.As(errs[1], &willRetryErr)) assert.Equal(t, int32(3), atomic.LoadInt32(&m.callCount)) }) t.Run("NoTools", func(t *testing.T) { ctx := context.Background() m := &streamErrorModel{ failAtChunk: 2, maxFailures: 2, returnTool: false, } config := &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: m, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, } agent, err := NewChatModelAgent(ctx, config) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) var events []*AgentEvent for { event, ok := iterator.Next() if !ok { break } events = append(events, event) } assert.Equal(t, 3, len(events)) var streamErrEventCount int var errs []error for i, event := range events { if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming { sr := event.Output.MessageOutput.MessageStream for { msg, err := sr.Recv() if err == io.EOF { break } if err != nil { streamErrEventCount++ errs = append(errs, err) t.Logf("event %d: err: %v", i, err) break } t.Logf("event %d: %v", i, msg.Content) } } } assert.Equal(t, 2, streamErrEventCount) assert.Equal(t, 2, len(errs)) var willRetryErr *WillRetryError assert.True(t, errors.As(errs[0], &willRetryErr)) assert.True(t, errors.As(errs[1], &willRetryErr)) assert.Equal(t, int32(3), atomic.LoadInt32(&m.callCount)) }) } func TestChatModelAgentRetry_WithTools_DirectError_Generate(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) if count < 2 { return nil, errRetryAble } return schema.AssistantMessage("Success after retry", nil), nil }).Times(2) cm.EXPECT().WithTools(gomock.Any()).Return(cm, nil).AnyTimes() fakeTool := &fakeToolForTest{tarCount: 0} agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: cm, ToolsConfig: ToolsConfig{ ToolsNodeConfig: compose.ToolsNodeConfig{ Tools: []tool.BaseTool{fakeTool}, }, }, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.Nil(t, event.Err) assert.NotNil(t, event.Output) assert.Equal(t, "Success after retry", event.Output.MessageOutput.Message.Content) _, ok = iterator.Next() assert.False(t, ok) assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) } func TestChatModelAgentRetry_NonRetryableError(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, errNonRetryAble).Times(1) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.NotNil(t, event.Err) assert.True(t, errors.Is(event.Err, errNonRetryAble)) _, ok = iterator.Next() assert.False(t, ok) } type inputCapturingModel struct { capturedInputs [][]Message } func (m *inputCapturingModel) Generate(_ context.Context, input []*schema.Message, _ ...model.Option) (*schema.Message, error) { m.capturedInputs = append(m.capturedInputs, input) return schema.AssistantMessage("Response from capturing model", nil), nil } func (m *inputCapturingModel) Stream(_ context.Context, input []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) { m.capturedInputs = append(m.capturedInputs, input) return schema.StreamReaderFromArray([]*schema.Message{ schema.AssistantMessage("Response from capturing model", nil), }), nil } func (m *inputCapturingModel) WithTools(_ []*schema.ToolInfo) (model.ToolCallingChatModel, error) { return m, nil } func TestChatModelAgentRetry_MaxRetriesExhausted(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, errRetryAble).Times(4) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.NotNil(t, event.Err) assert.True(t, errors.Is(event.Err, ErrExceedMaxRetries)) var retryErr *RetryExhaustedError assert.True(t, errors.As(event.Err, &retryErr)) assert.True(t, errors.Is(retryErr.LastErr, errRetryAble)) _, ok = iterator.Next() assert.False(t, ok) } func TestChatModelAgentRetry_BackoffFunction(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var backoffCalls []int var callCount int32 cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) if count < 3 { return nil, errRetryAble } return schema.AssistantMessage("Success", nil), nil }).Times(3) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, BackoffFunc: func(ctx context.Context, attempt int) time.Duration { backoffCalls = append(backoffCalls, attempt) return time.Millisecond }, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.Nil(t, event.Err) _, ok = iterator.Next() assert.False(t, ok) assert.Equal(t, []int{1, 2}, backoffCalls) } func TestChatModelAgentRetry_NoRetryConfig(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, errRetryAble).Times(1) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "TestAgent", Description: "Test agent without retry config", Instruction: "You are a helpful assistant.", Model: cm, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.NotNil(t, event.Err) assert.True(t, errors.Is(event.Err, errRetryAble)) _, ok = iterator.Next() assert.False(t, ok) } func TestChatModelAgentRetry_WithTools_NonRetryAbleStreamError(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, errNonRetryAble).Times(1) cm.EXPECT().WithTools(gomock.Any()).Return(cm, nil).AnyTimes() fakeTool := &fakeToolForTest{tarCount: 0} agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: cm, ToolsConfig: ToolsConfig{ ToolsNodeConfig: compose.ToolsNodeConfig{ Tools: []tool.BaseTool{fakeTool}, }, }, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.NotNil(t, event.Err) assert.True(t, errors.Is(event.Err, errNonRetryAble)) _, ok = iterator.Next() assert.False(t, ok) } type nonRetryAbleStreamErrorModel struct { tools []*schema.ToolInfo } func (m *nonRetryAbleStreamErrorModel) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) { return schema.AssistantMessage("Generated", nil), nil } func (m *nonRetryAbleStreamErrorModel) Stream(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) { sr, sw := schema.Pipe[*schema.Message](10) go func() { defer sw.Close() sw.Send(schema.AssistantMessage("chunk1", nil), nil) sw.Send(nil, errNonRetryAble) }() return sr, nil } func (m *nonRetryAbleStreamErrorModel) WithTools(tools []*schema.ToolInfo) (model.ToolCallingChatModel, error) { m.tools = tools return m, nil } func TestChatModelAgentRetry_NoTools_NonRetryAbleStreamError(t *testing.T) { ctx := context.Background() m := &nonRetryAbleStreamErrorModel{} agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent for retry functionality", Instruction: "You are a helpful assistant.", Model: m, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) var events []*AgentEvent for { event, ok := iterator.Next() if !ok { break } events = append(events, event) } assert.Equal(t, 2, len(events)) event0 := events[0] assert.NotNil(t, event0.Output) assert.NotNil(t, event0.Output.MessageOutput) assert.True(t, event0.Output.MessageOutput.IsStreaming) sr := event0.Output.MessageOutput.MessageStream var streamErr error for { _, err := sr.Recv() if err == io.EOF { break } if err != nil { streamErr = err break } } assert.NotNil(t, streamErr) assert.True(t, errors.Is(streamErr, errNonRetryAble), "Stream error should be the original error") event1 := events[1] assert.NotNil(t, event1.Err) assert.True(t, errors.Is(event1.Err, errNonRetryAble)) } func TestDefaultBackoff(t *testing.T) { ctx := context.Background() d1 := defaultBackoff(ctx, 1) d2 := defaultBackoff(ctx, 2) d3 := defaultBackoff(ctx, 3) t.Logf("Backoff delays: d1=%v, d2=%v, d3=%v", d1, d2, d3) assert.True(t, d1 >= 100*time.Millisecond && d1 < 150*time.Millisecond, "First retry should be ~100ms + jitter (0-50ms), got %v", d1) assert.True(t, d2 >= 200*time.Millisecond && d2 < 300*time.Millisecond, "Second retry should be ~200ms + jitter (0-100ms), got %v", d2) assert.True(t, d3 >= 400*time.Millisecond && d3 < 600*time.Millisecond, "Third retry should be ~400ms + jitter (0-200ms), got %v", d3) d10 := defaultBackoff(ctx, 10) t.Logf("Backoff delay for attempt 10: %v", d10) assert.True(t, d10 >= 10*time.Second && d10 <= 15*time.Second, "Delay should be capped at 10s + jitter (0-5s), got %v", d10) d100 := defaultBackoff(ctx, 100) t.Logf("Backoff delay for attempt 100: %v", d100) assert.True(t, d100 >= 10*time.Second && d100 <= 15*time.Second, "Delay should still be capped at 10s + jitter for very high attempts, got %v", d100) } type customError struct { code int msg string } func (e *customError) Error() string { return e.msg } func TestWillRetryError_Unwrap(t *testing.T) { originalErr := &customError{code: 500, msg: "internal error"} willRetry := &WillRetryError{ErrStr: originalErr.Error(), RetryAttempt: 1, err: originalErr} assert.True(t, errors.Is(willRetry, originalErr)) var targetErr *customError assert.True(t, errors.As(willRetry, &targetErr)) assert.Equal(t, 500, targetErr.code) assert.Equal(t, "internal error", targetErr.msg) } func TestChatModelAgentRetry_DefaultIsRetryAble(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) if count < 2 { return nil, errors.New("any error should be retried") } return schema.AssistantMessage("Success", nil), nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTestAgent", Description: "Test agent with default IsRetryAble", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.Nil(t, event.Err) assert.Equal(t, "Success", event.Output.MessageOutput.Message.Content) _, ok = iterator.Next() assert.False(t, ok) assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) } func TestSequentialWorkflow_RetryAbleStreamError_SuccessfulRetry(t *testing.T) { ctx := context.Background() retryModel := &streamErrorModel{ failAtChunk: 2, maxFailures: 2, } agentA, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "AgentA", Description: "Agent A with retry that emits stream errors then succeeds", Instruction: "You are agent A.", Model: retryModel, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, }) assert.NoError(t, err) capturingModel := &inputCapturingModel{} agentB, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "AgentB", Description: "Agent B that captures input", Instruction: "You are agent B.", Model: capturingModel, }) assert.NoError(t, err) sequentialAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{ Name: "SequentialAgent", Description: "Sequential agent A->B", SubAgents: []Agent{agentA, agentB}, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } ctx, _ = initRunCtx(ctx, sequentialAgent.Name(ctx), input) iterator := sequentialAgent.Run(ctx, input) var willRetryErrCount int for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming { sr := event.Output.MessageOutput.MessageStream for { _, err := sr.Recv() if err == io.EOF { break } if err != nil { var retryErr *WillRetryError if errors.As(err, &retryErr) { willRetryErrCount++ } break } } } } assert.Equal(t, 2, willRetryErrCount, "End-user should receive 2 WillRetryError events") assert.Equal(t, 1, len(capturingModel.capturedInputs), "Agent B should be called exactly once") successorInput := capturingModel.capturedInputs[0] var hasSuccessfulMessage bool for _, msg := range successorInput { if strings.Contains(msg.Content, "chunkchunkchunkchunkchunk") { hasSuccessfulMessage = true break } } assert.True(t, hasSuccessfulMessage, "Agent B should receive the successful message from Agent A") for _, msg := range successorInput { assert.NotContains(t, msg.Content, "retry-able error", "Agent B should not receive failed stream messages") } } type streamErrorModelNoRetry struct { callCount int32 } func (m *streamErrorModelNoRetry) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) { return schema.AssistantMessage("Generated", nil), nil } func (m *streamErrorModelNoRetry) Stream(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) { atomic.AddInt32(&m.callCount, 1) sr, sw := schema.Pipe[*schema.Message](10) go func() { defer sw.Close() sw.Send(schema.AssistantMessage("chunk1", nil), nil) sw.Send(schema.AssistantMessage("chunk2", nil), nil) sw.Send(nil, errRetryAble) }() return sr, nil } func (m *streamErrorModelNoRetry) WithTools(_ []*schema.ToolInfo) (model.ToolCallingChatModel, error) { return m, nil } func TestSequentialWorkflow_NonRetryAbleStreamError_StopsFlow(t *testing.T) { ctx := context.Background() nonRetryModel := &nonRetryAbleStreamErrorModel{} agentA, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "AgentA", Description: "Agent A that emits non-retryable stream error", Instruction: "You are agent A.", Model: nonRetryModel, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) }, }, }) assert.NoError(t, err) capturingModel := &inputCapturingModel{} agentB, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "AgentB", Description: "Agent B that captures input", Instruction: "You are agent B.", Model: capturingModel, }) assert.NoError(t, err) sequentialAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{ Name: "SequentialAgent", Description: "Sequential agent A->B", SubAgents: []Agent{agentA, agentB}, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } ctx, _ = initRunCtx(ctx, sequentialAgent.Name(ctx), input) iterator := sequentialAgent.Run(ctx, input) var streamErrFound bool var finalErrEvent *AgentEvent for { event, ok := iterator.Next() if !ok { break } if event.Err != nil { finalErrEvent = event } if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming { sr := event.Output.MessageOutput.MessageStream for { _, err := sr.Recv() if err == io.EOF { break } if err != nil { streamErrFound = true assert.True(t, errors.Is(err, errNonRetryAble), "Stream error should be the original error") break } } } } assert.True(t, streamErrFound, "End-user should receive stream error") assert.NotNil(t, finalErrEvent, "Should receive a final error event") assert.True(t, errors.Is(finalErrEvent.Err, errNonRetryAble), "Final error should be the non-retryable error") assert.Equal(t, 0, len(capturingModel.capturedInputs), "Agent B should NOT be called due to error") } func TestSequentialWorkflow_NoRetryConfig_StreamError_StopsFlow(t *testing.T) { ctx := context.Background() noRetryModel := &streamErrorModelNoRetry{} agentA, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "AgentA", Description: "Agent A without retry config that emits stream error", Instruction: "You are agent A.", Model: noRetryModel, }) assert.NoError(t, err) capturingModel := &inputCapturingModel{} agentB, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "AgentB", Description: "Agent B that captures input", Instruction: "You are agent B.", Model: capturingModel, }) assert.NoError(t, err) sequentialAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{ Name: "SequentialAgent", Description: "Sequential agent A->B", SubAgents: []Agent{agentA, agentB}, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } ctx, _ = initRunCtx(ctx, sequentialAgent.Name(ctx), input) iterator := sequentialAgent.Run(ctx, input) var streamErrFound bool var finalErrEvent *AgentEvent for { event, ok := iterator.Next() if !ok { break } if event.Err != nil { finalErrEvent = event } if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming { sr := event.Output.MessageOutput.MessageStream for { _, err := sr.Recv() if err == io.EOF { break } if err != nil { streamErrFound = true assert.True(t, errors.Is(err, errRetryAble), "Stream error should be the original error") break } } } } assert.True(t, streamErrFound, "End-user should receive stream error") assert.NotNil(t, finalErrEvent, "Should receive a final error event") assert.True(t, errors.Is(finalErrEvent.Err, errRetryAble), "Final error should be the original error") assert.Equal(t, 0, len(capturingModel.capturedInputs), "Agent B should NOT be called due to error") assert.Equal(t, int32(1), atomic.LoadInt32(&noRetryModel.callCount), "Model should only be called once (no retry)") } // failThenToolCallStreamModel is a ChatModel that: // - First Stream() call: yields a partial chunk then fails with a retryable error mid-stream. // - Second Stream() call (retry): yields a tool-call message (success). // - Third Generate() call (after tool result): yields a final assistant message. // // This exercises the path where the eventSenderModel copies the first stream, // wraps its error as WillRetryError, and sends it as an event to the session. // The retryModelWrapper then retries, gets a clean stream with a tool call, // the tool interrupts, and checkpoint save needs to gob-encode the session // (which still contains the unconsumed WillRetryError event stream). type failThenToolCallStreamModel struct { streamCallCount int32 genCallCount int32 } func (m *failThenToolCallStreamModel) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) { atomic.AddInt32(&m.genCallCount, 1) return schema.AssistantMessage("final answer", nil), nil } func (m *failThenToolCallStreamModel) Stream(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) { count := atomic.AddInt32(&m.streamCallCount, 1) sr, sw := schema.Pipe[*schema.Message](10) go func() { defer sw.Close() if count == 1 { // First call: yield a partial chunk then fail. sw.Send(schema.AssistantMessage("partial", nil), nil) sw.Send(nil, errRetryAble) return } // Second call (retry): yield a tool-call message. sw.Send(schema.AssistantMessage("", []schema.ToolCall{{ ID: "call-1", Function: schema.FunctionCall{ Name: "interrupt_tool", Arguments: `{}`, }, }}), nil) }() return sr, nil } func (m *failThenToolCallStreamModel) WithTools(_ []*schema.ToolInfo) (model.ToolCallingChatModel, error) { return m, nil } // interruptToolForRetryTest is a tool that always interrupts. type interruptToolForRetryTest struct{} func (t *interruptToolForRetryTest) Info(_ context.Context) (*schema.ToolInfo, error) { return &schema.ToolInfo{ Name: "interrupt_tool", Desc: "tool that interrupts", ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{ "input": {Type: "string"}, }), }, nil } func (t *interruptToolForRetryTest) InvokableRun(ctx context.Context, _ string, _ ...tool.Option) (string, error) { return "", tool.Interrupt(ctx, "interrupted by tool") } // TestCheckpointSave_WillRetryError_StreamNotConsumed verifies that checkpoint // saving succeeds when the session contains an event with an unconsumed stream // that ends with WillRetryError. // // Scenario: // 1. ChatModelAgent with retry (MaxRetries=1) and a tool that always interrupts // 2. Model.Stream() #1 yields "partial" then errRetryAble mid-stream // → eventSenderModel copies the stream, wraps the error as WillRetryError, // sends the event to the session (stream NOT consumed by anyone yet) // → retryModelWrapper detects error on its copy, retries // 3. Model.Stream() #2 succeeds with a tool-call message // 4. Tool executes → interrupts // 5. Runner.handleIter sees the interrupt → saveCheckPoint → gob encodes runSession // 6. The session has the WillRetryError event with an unconsumed stream // → agentEventWrapper.GobEncode proactively consumes the stream via // getMessageFromWrappedEvent, so MessageVariant.GobEncode sees an error-free // array and succeeds func TestCheckpointSave_WillRetryError_StreamNotConsumed(t *testing.T) { ctx := context.Background() mdl := &failThenToolCallStreamModel{} itool := &interruptToolForRetryTest{} agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "TestAgent", Description: "Agent for checkpoint stream error test", Instruction: "You are a test agent.", Model: mdl, ToolsConfig: ToolsConfig{ ToolsNodeConfig: compose.ToolsNodeConfig{ Tools: []tool.BaseTool{itool}, }, }, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 1, IsRetryAble: func(_ context.Context, err error) bool { return errors.Is(err, errRetryAble) }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) store := newMyStore() runner := NewRunner(ctx, RunnerConfig{ Agent: agent, EnableStreaming: true, CheckPointStore: store, }) iter := runner.Run(ctx, []Message{schema.UserMessage("hello")}, WithCheckPointID("ckpt-1"), ) for { event, ok := iter.Next() if !ok { break } if event.Err != nil { t.Logf("event error: %v", event.Err) } } // Verify the checkpoint was saved successfully. _, exists, _ := store.Get(ctx, "ckpt-1") assert.True(t, exists, "checkpoint should be saved successfully; "+ "if this fails, the WillRetryError stream in the session caused gob encoding to fail") // Sanity: the model should have been called twice for Stream (fail + retry). assert.Equal(t, int32(2), atomic.LoadInt32(&mdl.streamCallCount), "model should be called twice: first fail, then retry success") } func TestChatModelAgentRetry_ShouldRetry_RejectMessage_Stream(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { count := atomic.AddInt32(&callCount, 1) r, w := schema.Pipe[*schema.Message](1) go func() { if count < 2 { _ = w.Send(schema.AssistantMessage("bad stream content", nil), nil) } else { _ = w.Send(schema.AssistantMessage("good stream content", nil), nil) } w.Close() }() return r, nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "ShouldRetryStreamTestAgent", Description: "Test ShouldRetry message rejection in stream mode", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{Retry: true} } if retryCtx.OutputMessage != nil && strings.Contains(retryCtx.OutputMessage.Content, "bad") { return &RetryDecision{Retry: true} } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) events, _ := drainStreamingAgentEvents(t, iterator) var foundGoodContent bool for _, e := range events { if e.StreamContent == "good stream content" { foundGoodContent = true } } require.True(t, foundGoodContent, "should have received good stream content") assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) } func TestShouldRetry_Generate(t *testing.T) { t.Run("RetryContext_Fields", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) if count < 2 { return schema.AssistantMessage("bad", nil), nil } return schema.AssistantMessage("good", nil), nil }).Times(2) var capturedContexts []*RetryContext agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryContextFieldsAgent", Description: "Test that RetryContext fields are correctly populated", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { capturedContexts = append(capturedContexts, retryCtx) if retryCtx.OutputMessage != nil && retryCtx.OutputMessage.Content == "bad" { return &RetryDecision{Retry: true} } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) for { event, ok := iterator.Next() if !ok { break } _ = event } assert.Len(t, capturedContexts, 2, "ShouldRetry should be called twice") assert.Equal(t, 1, capturedContexts[0].RetryAttempt) assert.Len(t, capturedContexts[0].InputMessages, 2) assert.True(t, len(capturedContexts[0].Options) > 0, "should have default options") assert.Equal(t, "bad", capturedContexts[0].OutputMessage.Content) assert.Nil(t, capturedContexts[0].Err) assert.Equal(t, 2, capturedContexts[1].RetryAttempt) assert.Equal(t, "good", capturedContexts[1].OutputMessage.Content) assert.Nil(t, capturedContexts[1].Err) }) t.Run("RewriteError_OnMessage", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(schema.AssistantMessage("unrecoverable bad message", nil), nil).Times(1) fatalErr := errors.New("fatal: unrecoverable model output") agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RewriteErrorTestAgent", Description: "Test ShouldRetry RewriteError on message", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.OutputMessage != nil && strings.Contains(retryCtx.OutputMessage.Content, "unrecoverable") { return &RetryDecision{ Retry: false, RewriteError: fatalErr, } } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) events := drainAgentEvents(t, iterator) require.NotEmpty(t, events) foundErr := false for _, e := range events { if e.Err != nil && errors.Is(e.Err, fatalErr) { foundErr = true } } require.True(t, foundErr, "should have received the fatal rewrite error") }) t.Run("RewriteError_OnError", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) origErr := errors.New("original transient error") cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, origErr).Times(1) wrappedErr := errors.New("wrapped: original transient error with more context") agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RewriteErrorOnErrorTestAgent", Description: "Test ShouldRetry RewriteError replacing original error", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{ Retry: false, RewriteError: wrappedErr, } } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) events := drainAgentEvents(t, iterator) require.NotEmpty(t, events) foundErr := false for _, e := range events { if e.Err != nil && errors.Is(e.Err, wrappedErr) { foundErr = true } } require.True(t, foundErr, "should have received the wrapped rewrite error") }) t.Run("AdditionalOptions", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 var capturedOpts [][]model.Option cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) capturedOpts = append(capturedOpts, opts) if count < 2 { return nil, errRetryAble } return schema.AssistantMessage("success", nil), nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "AdditionalOptionsTestAgent", Description: "Test ShouldRetry AdditionalOptions", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{ Retry: true, AdditionalOptions: []model.Option{model.WithMaxTokens(8192)}, } } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.Nil(t, event.Err) assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) assert.Equal(t, 2, len(capturedOpts)) assert.Equal(t, len(capturedOpts[0])+1, len(capturedOpts[1])) }) t.Run("ModifiedInputMessages_NoPersist", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 var capturedInputs [][]*schema.Message cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) inputCopy := make([]*schema.Message, len(input)) copy(inputCopy, input) capturedInputs = append(capturedInputs, inputCopy) if count < 2 { return nil, errRetryAble } return schema.AssistantMessage("success", nil), nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "ModifiedInputNoPersistAgent", Description: "Test ShouldRetry ModifiedInputMessages without persistence", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{ Retry: true, ModifiedInputMessages: []*schema.Message{ schema.SystemMessage("compressed instruction"), schema.UserMessage("Hello"), }, PersistModifiedInputMessages: false, } } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.Nil(t, event.Err) assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) assert.Equal(t, 2, len(capturedInputs)) assert.Equal(t, "compressed instruction", capturedInputs[1][0].Content, "second call should use modified input") }) t.Run("Backoff", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) if count < 2 { return nil, errRetryAble } return schema.AssistantMessage("success", nil), nil }).Times(2) customBackoff := 50 * time.Millisecond agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "BackoffTestAgent", Description: "Test ShouldRetry custom Backoff in decision", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{ Retry: true, Backoff: customBackoff, } } return &RetryDecision{Retry: false} }, }, }) assert.NoError(t, err) start := time.Now() input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) event, ok := iterator.Next() assert.True(t, ok) assert.NotNil(t, event) assert.Nil(t, event.Err) elapsed := time.Since(start) assert.True(t, elapsed >= customBackoff && elapsed < customBackoff+200*time.Millisecond, "expected backoff ~%v, got %v", customBackoff, elapsed) assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) }) t.Run("SuppressFlag_Rejected_NoEvent", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) if count == 1 { return schema.AssistantMessage("bad", nil), nil } return schema.AssistantMessage("good", nil), nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "SuppressRejected", Description: "Test suppress flag rejects first then accepts", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 1, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.OutputMessage != nil && retryCtx.OutputMessage.Content == "bad" { return &RetryDecision{Retry: true} } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) var msgEvents []*AgentEvent for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil { msgEvents = append(msgEvents, event) } } assert.Equal(t, 1, len(msgEvents), "should have exactly 1 message event (suppressed rejected)") assert.Equal(t, "good", msgEvents[0].Output.MessageOutput.Message.Content) }) t.Run("SuppressFlag_AllRejected_NoEvents", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(schema.AssistantMessage("always bad", nil), nil).Times(3) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "SuppressAllRejected", Description: "Test suppress flag all rejected no events", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { return &RetryDecision{Retry: true} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) events := drainAgentEvents(t, iterator) var msgEventCount int var foundExhaustedErr bool for _, e := range events { if e.Output != nil && e.Output.MessageOutput != nil { msgEventCount++ } if e.Err != nil && errors.Is(e.Err, ErrExceedMaxRetries) { foundExhaustedErr = true } } assert.Equal(t, 0, msgEventCount, "no message events should be emitted when all are rejected") require.True(t, foundExhaustedErr, "final event should have RetryExhaustedError") }) t.Run("SuppressFlag_Accepted_FirstAttempt", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(schema.AssistantMessage("perfect", nil), nil).Times(1) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "SuppressAcceptedFirst", Description: "Test suppress flag accepted first attempt", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) var msgEvents []*AgentEvent for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil { msgEvents = append(msgEvents, event) } } assert.Equal(t, 1, len(msgEvents), "should have exactly 1 event") assert.Equal(t, "perfect", msgEvents[0].Output.MessageOutput.Message.Content) }) t.Run("ContextCanceled_DuringSleep", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { atomic.AddInt32(&callCount, 1) return nil, errors.New("transient") }).Times(1) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "ContextCancelDuringSleep", Description: "Test context cancellation during backoff sleep", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 5, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { return &RetryDecision{Retry: true} }, BackoffFunc: func(_ context.Context, _ int) time.Duration { return 10 * time.Second }, }, }) require.NoError(t, err) go func() { time.Sleep(50 * time.Millisecond) cancel() }() start := time.Now() iterator := agent.Run(ctx, &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, }) events := drainAgentEvents(t, iterator) elapsed := time.Since(start) require.True(t, elapsed < 2*time.Second, "should not block for full backoff; elapsed: %v", elapsed) assert.Equal(t, int32(1), atomic.LoadInt32(&callCount)) var foundCtxErr bool for _, e := range events { if e.Err != nil && errors.Is(e.Err, context.Canceled) { foundCtxErr = true } } require.True(t, foundCtxErr, "should have received context.Canceled error") }) } func TestShouldRetry_Stream(t *testing.T) { t.Run("ErrorRetry", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) streamErr := errors.New("stream unavailable") var callCount int32 cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { count := atomic.AddInt32(&callCount, 1) if count < 2 { return nil, streamErr } r, w := schema.Pipe[*schema.Message](1) go func() { _ = w.Send(schema.AssistantMessage("recovered stream", nil), nil) w.Close() }() return r, nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamErrorRetryAgent", Description: "Test ShouldRetry when Stream returns error (nil stream)", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{Retry: true} } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) events, _ := drainStreamingAgentEvents(t, iterator) var foundContent bool for _, e := range events { if e.StreamContent == "recovered stream" { foundContent = true } } require.True(t, foundContent, "should have received recovered stream content after error retry") assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) }) t.Run("ErrorRewrite", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) streamErr := errors.New("model overloaded") cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, streamErr).Times(1) fatalErr := errors.New("fatal: model overloaded, aborting") agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamErrorRewriteAgent", Description: "Test ShouldRetry RewriteError when Stream returns error", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil && strings.Contains(retryCtx.Err.Error(), "overloaded") { return &RetryDecision{ Retry: false, RewriteError: fatalErr, } } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) events := drainAgentEvents(t, iterator) require.NotEmpty(t, events) foundErr := false for _, e := range events { if e.Err != nil && errors.Is(e.Err, fatalErr) { foundErr = true } } require.True(t, foundErr, "should have received the fatal rewrite error from stream") }) t.Run("RewriteError_OnMessage", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { r, w := schema.Pipe[*schema.Message](1) go func() { _ = w.Send(schema.AssistantMessage("hallucinated garbage output", nil), nil) w.Close() }() return r, nil }).Times(1) fatalErr := errors.New("fatal: hallucinated output detected") agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamRewriteOnMessageAgent", Description: "Test ShouldRetry RewriteError on successful stream with bad content", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.OutputMessage != nil && strings.Contains(retryCtx.OutputMessage.Content, "hallucinated") { return &RetryDecision{ Retry: false, RewriteError: fatalErr, } } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) events := drainAgentEvents(t, iterator) require.NotEmpty(t, events) foundErr := false for _, e := range events { if e.Err != nil && errors.Is(e.Err, fatalErr) { foundErr = true } } require.True(t, foundErr, "should have received fatal rewrite error from stream message inspection") }) t.Run("PartialStreamError", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) partialErr := errors.New("connection reset mid-stream") var callCount int32 cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { count := atomic.AddInt32(&callCount, 1) r, w := schema.Pipe[*schema.Message](1) go func() { _ = w.Send(schema.AssistantMessage("partial chunk", nil), nil) if count < 2 { w.Send(nil, partialErr) } else { _ = w.Send(schema.AssistantMessage(" complete", nil), nil) w.Close() } }() return r, nil }).Times(2) var capturedContexts []*RetryContext agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamPartialErrorAgent", Description: "Test ShouldRetry when stream has partial content then error", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { capturedContexts = append(capturedContexts, retryCtx) if retryCtx.Err != nil { return &RetryDecision{Retry: true} } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil { mo := event.Output.MessageOutput if mo.IsStreaming && mo.MessageStream != nil { for { _, err := mo.MessageStream.Recv() if err != nil { break } } } } } assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) assert.Equal(t, 2, len(capturedContexts)) assert.NotNil(t, capturedContexts[0].Err, "first attempt should have stream error") assert.NotNil(t, capturedContexts[0].OutputMessage, "first attempt should have partial message despite error") assert.Equal(t, "partial chunk", capturedContexts[0].OutputMessage.Content) }) t.Run("ModifiedInputsAndOptions_WithPersist", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 var capturedInputs [][]*schema.Message var capturedOptLens []int cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { count := atomic.AddInt32(&callCount, 1) inputCopy := make([]*schema.Message, len(input)) copy(inputCopy, input) capturedInputs = append(capturedInputs, inputCopy) capturedOptLens = append(capturedOptLens, len(opts)) r, w := schema.Pipe[*schema.Message](1) go func() { if count < 2 { _ = w.Send(schema.AssistantMessage("too long response exceeds limit", nil), nil) } else { _ = w.Send(schema.AssistantMessage("good response", nil), nil) } w.Close() }() return r, nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamModifiedInputsPersistAgent", Description: "Test ShouldRetry with ModifiedInputMessages (persist) and AdditionalOptions in stream mode", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.OutputMessage != nil && strings.Contains(retryCtx.OutputMessage.Content, "too long") { return &RetryDecision{ Retry: true, ModifiedInputMessages: []*schema.Message{ schema.SystemMessage("compressed instruction"), schema.UserMessage("summarized history"), }, PersistModifiedInputMessages: true, AdditionalOptions: []model.Option{model.WithMaxTokens(16384)}, } } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) events, _ := drainStreamingAgentEvents(t, iterator) var foundGood bool for _, e := range events { if e.StreamContent == "good response" { foundGood = true } } require.True(t, foundGood, "should have received good response after retry with modified inputs") assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) assert.Equal(t, 2, len(capturedInputs)) assert.Equal(t, "compressed instruction", capturedInputs[1][0].Content, "second call should use modified input") assert.Equal(t, "summarized history", capturedInputs[1][1].Content) assert.Equal(t, capturedOptLens[0]+1, capturedOptLens[1]) }) t.Run("VerdictSignal_CleanStream_Rejected", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { count := atomic.AddInt32(&callCount, 1) if count == 1 { return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("bad", nil)}), nil } return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("good", nil)}), nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "VerdictCleanRejected", Description: "Test verdict signal on clean stream rejected", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 1, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.OutputMessage != nil && retryCtx.OutputMessage.Content == "bad" { return &RetryDecision{Retry: true} } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) var streamEvents []int for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil { mo := event.Output.MessageOutput if mo.IsStreaming && mo.MessageStream != nil { idx := len(streamEvents) streamEvents = append(streamEvents, idx) var lastErr error for { _, recvErr := mo.MessageStream.Recv() if recvErr != nil { lastErr = recvErr break } } if idx == 0 { var willRetryErr *WillRetryError assert.True(t, errors.As(lastErr, &willRetryErr), "first stream should end with WillRetryError") } else { assert.ErrorIs(t, lastErr, io.EOF, "second stream should end with io.EOF") } } } } assert.Equal(t, 2, len(streamEvents), "should have exactly 2 stream events") assert.Equal(t, int32(2), atomic.LoadInt32(&callCount)) }) t.Run("VerdictSignal_StreamError_Rejected", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) streamErr := errors.New("mid-stream error") var callCount int32 cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { count := atomic.AddInt32(&callCount, 1) if count == 1 { r, w := schema.Pipe[*schema.Message](1) go func() { _ = w.Send(schema.AssistantMessage("partial", nil), nil) w.Send(nil, streamErr) }() return r, nil } return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("good", nil)}), nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "VerdictStreamErrorRejected", Description: "Test verdict signal on stream error rejected", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 1, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{Retry: true} } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) var firstEventHasWillRetry bool var eventCount int for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil { mo := event.Output.MessageOutput if mo.IsStreaming && mo.MessageStream != nil { eventCount++ for { _, recvErr := mo.MessageStream.Recv() if recvErr != nil { if eventCount == 1 { var willRetryErr *WillRetryError if errors.As(recvErr, &willRetryErr) { firstEventHasWillRetry = true } } break } } } } } assert.True(t, firstEventHasWillRetry, "first event stream should end with WillRetryError via errWrapper path") assert.Equal(t, 2, eventCount, "should have 2 stream events") }) t.Run("VerdictSignal_Accepted_FirstAttempt", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("perfect", nil)}), nil }).Times(1) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "VerdictAcceptedFirst", Description: "Test verdict signal accepted first attempt", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) var eventCount int for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil { mo := event.Output.MessageOutput if mo.IsStreaming && mo.MessageStream != nil { eventCount++ var lastErr error for { _, recvErr := mo.MessageStream.Recv() if recvErr != nil { lastErr = recvErr break } } assert.ErrorIs(t, lastErr, io.EOF, "accepted stream should end with io.EOF") } } } assert.Equal(t, 1, eventCount, "should have exactly 1 event") }) t.Run("VerdictSignal_AllRejected_Exhausted", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("always bad", nil)}), nil }).Times(3) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "VerdictAllRejected", Description: "Test verdict signal all rejected exhausted", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { return &RetryDecision{Retry: true} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) events, streamTermErrs := drainStreamingAgentEvents(t, iterator) var willRetryCount int var foundExhaustedErr bool for _, e := range events { if e.Err != nil && errors.Is(e.Err, ErrExceedMaxRetries) { foundExhaustedErr = true } } for _, termErr := range streamTermErrs { var willRetryErr *WillRetryError if errors.As(termErr, &willRetryErr) { willRetryCount++ } } assert.Equal(t, 3, willRetryCount, "all 3 stream events should end with WillRetryError") require.True(t, foundExhaustedErr, "final error should be RetryExhaustedError") }) t.Run("ShouldRetry_Panics_VerdictStillSent", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("trigger panic", nil)}), nil }).Times(1) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "ShouldRetryPanicsAgent", Description: "Test that ShouldRetry panic sends verdict signal and does not deadlock", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 1, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { panic("deliberate panic in ShouldRetry") }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } done := make(chan struct{}) var events []agentEvent go func() { defer close(done) iterator := agent.Run(ctx, input) events = drainAgentEvents(t, iterator) }() select { case <-done: case <-time.After(5 * time.Second): t.Fatal("test deadlocked — verdict signal was not sent after ShouldRetry panic") } require.NotEmpty(t, events) var foundPanicErr bool for _, e := range events { if e.Err != nil && strings.Contains(e.Err.Error(), "panic") { foundPanicErr = true } } assert.True(t, foundPanicErr, "should have received a panic error event") }) } func TestErrStreamCanceled(t *testing.T) { t.Run("Stream_ShouldRetry_NeverRetried", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { r, w := schema.Pipe[*schema.Message](1) go func() { _ = w.Send(schema.AssistantMessage("partial", nil), nil) w.Send(nil, ErrStreamCanceled) }() return r, nil }).Times(1) var shouldRetryCalled int32 agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamCanceledShouldRetry", Description: "Test ErrStreamCanceled never retried with ShouldRetry", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { atomic.AddInt32(&shouldRetryCalled, 1) return &RetryDecision{Retry: true} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil { mo := event.Output.MessageOutput if mo.IsStreaming && mo.MessageStream != nil { for { _, recvErr := mo.MessageStream.Recv() if recvErr != nil { break } } } } } assert.Equal(t, int32(0), atomic.LoadInt32(&shouldRetryCalled), "ShouldRetry should never be called for ErrStreamCanceled") }) t.Run("Stream_LegacyIsRetryAble_NeverRetried", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { r, w := schema.Pipe[*schema.Message](1) go func() { _ = w.Send(schema.AssistantMessage("partial", nil), nil) w.Send(nil, ErrStreamCanceled) }() return r, nil }).Times(1) var isRetryAbleCalled int32 agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamCanceledLegacy", Description: "Test ErrStreamCanceled never retried with legacy IsRetryAble", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, IsRetryAble: func(_ context.Context, err error) bool { atomic.AddInt32(&isRetryAbleCalled, 1) return true }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil { mo := event.Output.MessageOutput if mo.IsStreaming && mo.MessageStream != nil { for { _, recvErr := mo.MessageStream.Recv() if recvErr != nil { break } } } } } assert.Equal(t, int32(0), atomic.LoadInt32(&isRetryAbleCalled), "IsRetryAble should never be called for ErrStreamCanceled") }) t.Run("Generate_ShouldRetry_NeverRetried", func(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, ErrStreamCanceled).Times(1) var shouldRetryCalled int32 agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "GenCanceledShouldRetry", Description: "Test ErrStreamCanceled in Generate never retried", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { atomic.AddInt32(&shouldRetryCalled, 1) return &RetryDecision{Retry: true} }, BackoffFunc: instantBackoff, }, }) assert.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, } iterator := agent.Run(ctx, input) for { _, ok := iterator.Next() if !ok { break } } assert.Equal(t, int32(0), atomic.LoadInt32(&shouldRetryCalled), "ShouldRetry should never be called for ErrStreamCanceled") }) } func TestAttack_ShouldRetry_NilDecisionOnEveryCall(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(schema.AssistantMessage("ok", nil), nil).Times(1) var shouldRetryCalls int32 agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "NilDecisionAgent", Description: "ShouldRetry always returns nil — should accept on first call", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { atomic.AddInt32(&shouldRetryCalls, 1) return nil }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) iterator := agent.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("Hello")}}) events := drainAgentEvents(t, iterator) require.NotEmpty(t, events) assert.Equal(t, int32(1), atomic.LoadInt32(&shouldRetryCalls)) var foundOK bool for _, e := range events { if e.Output != nil && e.Output.MessageOutput != nil && e.Output.MessageOutput.Message != nil { if e.Output.MessageOutput.Message.Content == "ok" { foundOK = true } } } assert.True(t, foundOK, "nil decision should accept the message as-is") } func TestAttack_ShouldRetry_MaxRetriesZero_RejectFirstAttempt(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). Return(schema.AssistantMessage("bad", nil), nil).Times(1) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "MaxZeroRejectAgent", Description: "MaxRetries=0 with ShouldRetry rejecting — should exhaust immediately", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 0, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { return &RetryDecision{Retry: true} }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) iterator := agent.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("Hello")}}) events := drainAgentEvents(t, iterator) var foundExhausted bool for _, e := range events { if e.Err != nil { var exhaustedErr *RetryExhaustedError if errors.As(e.Err, &exhaustedErr) { foundExhausted = true } } } assert.True(t, foundExhausted, "MaxRetries=0 with Retry:true should produce RetryExhaustedError") } func TestAttack_ShouldRetry_RetryTrueWithRewriteError_IgnoresRewrite(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) var callCount int32 cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) if count == 1 { return nil, errors.New("transient") } return schema.AssistantMessage("success", nil), nil }).Times(2) rewriteErr := errors.New("this should be ignored") agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "RetryTrueRewriteAgent", Description: "Retry=true with RewriteError should ignore the rewrite", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 3, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{Retry: true, RewriteError: rewriteErr} } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) iterator := agent.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("Hello")}}) events := drainAgentEvents(t, iterator) var foundSuccess bool for _, e := range events { if e.Err != nil && errors.Is(e.Err, rewriteErr) { t.Fatal("RewriteError should be ignored when Retry=true") } if e.Output != nil && e.Output.MessageOutput != nil && e.Output.MessageOutput.Message != nil { if e.Output.MessageOutput.Message.Content == "success" { foundSuccess = true } } } assert.True(t, foundSuccess, "should eventually succeed after retry, ignoring RewriteError") } func TestAttack_ShouldRetry_OptionsAccumulateAcrossRetries(t *testing.T) { ctx := context.Background() var capturedOpts [][]model.Option var callCount int32 ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) { count := atomic.AddInt32(&callCount, 1) capturedOpts = append(capturedOpts, opts) if count <= 2 { return nil, errors.New("needs retry") } return schema.AssistantMessage("done", nil), nil }).Times(3) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "OptsAccumulateAgent", Description: "Verify options accumulate across retries", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 5, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{ Retry: true, AdditionalOptions: []model.Option{model.WithMaxTokens(100 * retryCtx.RetryAttempt)}, } } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) iterator := agent.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("Hello")}}) drainAgentEvents(t, iterator) require.Len(t, capturedOpts, 3) assert.True(t, len(capturedOpts[1]) > len(capturedOpts[0]), "second call should have more options than first (accumulated AdditionalOptions)") assert.True(t, len(capturedOpts[2]) > len(capturedOpts[1]), "third call should have more options than second (accumulated AdditionalOptions)") } func TestAttack_ShouldRetry_Stream_NilDecisionAccepts(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("stream ok", nil)}), nil }).Times(1) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamNilDecisionAgent", Description: "ShouldRetry returns nil in stream mode — should accept", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { return nil }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } iterator := agent.Run(ctx, input) events, streamTermErrs := drainStreamingAgentEvents(t, iterator) var foundStreamContent bool for _, e := range events { if e.StreamContent == "stream ok" { foundStreamContent = true } } assert.True(t, foundStreamContent, "nil decision should accept the stream") for _, termErr := range streamTermErrs { assert.Equal(t, io.EOF, termErr, "stream should terminate with clean EOF, not error") } } func TestAttack_ShouldRetry_Stream_MaxRetriesZero_Exhausted(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("rejected", nil)}), nil }).Times(1) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamMaxZeroAgent", Description: "Stream mode with MaxRetries=0 rejecting — should exhaust immediately", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 0, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { return &RetryDecision{Retry: true} }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } done := make(chan struct{}) var events []agentEvent go func() { defer close(done) iterator := agent.Run(ctx, input) events, _ = drainStreamingAgentEvents(t, iterator) }() select { case <-done: case <-time.After(10 * time.Second): t.Fatal("test deadlocked — Stream MaxRetries=0 with reject should not hang") } var foundExhausted bool for _, e := range events { if e.Err != nil { var exhaustedErr *RetryExhaustedError if errors.As(e.Err, &exhaustedErr) { foundExhausted = true } } } assert.True(t, foundExhausted, "MaxRetries=0 stream reject should produce RetryExhaustedError") } func TestAttack_ShouldRetry_Stream_RewriteErrorOnCleanStream(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("looks good but bad", nil)}), nil }).Times(1) fatalErr := errors.New("fatal: content policy violation") agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "StreamRewriteCleanAgent", Description: "Stream returns cleanly but ShouldRetry rewrites to error", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 2, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { return &RetryDecision{Retry: false, RewriteError: fatalErr} }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } done := make(chan struct{}) var events []agentEvent go func() { defer close(done) iterator := agent.Run(ctx, input) events, _ = drainStreamingAgentEvents(t, iterator) }() select { case <-done: case <-time.After(10 * time.Second): t.Fatal("test deadlocked") } var foundFatal bool for _, e := range events { if e.Err != nil && errors.Is(e.Err, fatalErr) { foundFatal = true } } assert.True(t, foundFatal, "RewriteError on clean stream should propagate the fatal error") } func TestAttack_ShouldRetry_ConcatMessagesFails_EmptyStream(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { r, w := schema.Pipe[*schema.Message](1) w.Close() return r, nil }).Times(1) var capturedCtx *RetryContext agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "EmptyStreamAgent", Description: "Stream returns zero chunks — both OutputMessage and Err should be nil", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 1, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { capturedCtx = retryCtx return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } done := make(chan struct{}) go func() { defer close(done) iterator := agent.Run(ctx, input) drainStreamingAgentEvents(t, iterator) }() select { case <-done: case <-time.After(10 * time.Second): t.Fatal("test deadlocked on empty stream") } require.NotNil(t, capturedCtx) assert.NotNil(t, capturedCtx.OutputMessage, "empty stream should have non-nil OutputMessage from ConcatMessages") assert.Nil(t, capturedCtx.Err, "empty stream should have nil Err") } func TestAttack_ShouldRetry_Stream_MidStreamError_VerdictDoubleRead(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) defer ctrl.Finish() cm := mockModel.NewMockToolCallingChatModel(ctrl) midStreamErr := errors.New("mid-stream transient error") cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { r, w := schema.Pipe[*schema.Message](1) go func() { defer w.Close() _ = w.Send(schema.AssistantMessage("chunk1", nil), nil) _ = w.Send(nil, midStreamErr) }() return r, nil }).Times(2) agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "DoubleReadBugAgent", Description: "Reproduces signal.ch double-read when event stream hits mid-stream error then EOF", Instruction: "You are a helpful assistant.", Model: cm, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 1, ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{Retry: true} } return &RetryDecision{Retry: false} }, BackoffFunc: instantBackoff, }, }) require.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("Hello")}, EnableStreaming: true, } done := make(chan struct{}) go func() { defer close(done) iterator := agent.Run(ctx, input) for { event, ok := iterator.Next() if !ok { break } if event.Output != nil && event.Output.MessageOutput != nil { mo := event.Output.MessageOutput if mo.IsStreaming && mo.MessageStream != nil { for { _, recvErr := mo.MessageStream.Recv() if recvErr == io.EOF { break } } } } } }() select { case <-done: case <-time.After(10 * time.Second): t.Fatal("goroutine leak: onEOF blocked on signal.ch after errWrapper already drained the verdict") } } type rejectReasonTestModel struct { streamFn func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) } func (m *rejectReasonTestModel) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) { return schema.AssistantMessage("generated", nil), nil } func (m *rejectReasonTestModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) { return m.streamFn(ctx, input, opts...) } func TestRejectReason_StreamPath(t *testing.T) { ctx := context.Background() type rejectInfo struct { Reason string Attempt int } streamErr := errors.New("bad output") var streamCallCount int32 m := &rejectReasonTestModel{ streamFn: func(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) { n := atomic.AddInt32(&streamCallCount, 1) if n == 1 { return streamWithMidError( []*schema.Message{schema.AssistantMessage("rejected chunk", nil)}, streamErr, ), nil } sr, sw := schema.Pipe[*schema.Message](1) go func() { defer sw.Close() sw.Send(schema.AssistantMessage("accepted", nil), nil) }() return sr, nil }, } agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{ Name: "reject-reason-agent", Description: "test reject reason", Instruction: "test", Model: m, ModelRetryConfig: &ModelRetryConfig{ MaxRetries: 1, ShouldRetry: func(_ context.Context, retryCtx *RetryContext) *RetryDecision { if retryCtx.Err != nil { return &RetryDecision{ Retry: true, RejectReason: rejectInfo{ Reason: "output quality too low", Attempt: retryCtx.RetryAttempt, }, } } return nil }, BackoffFunc: func(_ context.Context, _ int) time.Duration { return time.Millisecond }, }, }) require.NoError(t, err) input := &AgentInput{ Messages: []Message{schema.UserMessage("hello")}, EnableStreaming: true, } ctx, _ = initRunCtx(ctx, agent.Name(ctx), input) iter := agent.Run(ctx, input) var capturedRejectReasons []any var finalContent string for { ev, ok := iter.Next() if !ok { break } if ev.Err != nil { continue } if ev.Output != nil && ev.Output.MessageOutput != nil && ev.Output.MessageOutput.IsStreaming { sr := ev.Output.MessageOutput.MessageStream for { chunk, recvErr := sr.Recv() if recvErr != nil { var willRetry *WillRetryError if errors.As(recvErr, &willRetry) { capturedRejectReasons = append(capturedRejectReasons, willRetry.RejectReason()) } break } if chunk != nil { finalContent = chunk.Content } } } } assert.Contains(t, finalContent, "accepted") require.NotEmpty(t, capturedRejectReasons, "should have at least one WillRetryError with RejectReason from stream Recv()") for _, reason := range capturedRejectReasons { require.NotNil(t, reason) ri, ok := reason.(rejectInfo) require.True(t, ok, "RejectReason should be rejectInfo type, got %T", reason) assert.Equal(t, "output quality too low", ri.Reason) assert.Equal(t, 1, ri.Attempt) } } func TestWillRetryError_RejectReason(t *testing.T) { t.Run("nil when not set", func(t *testing.T) { wrErr := &WillRetryError{ErrStr: "test", RetryAttempt: 1, err: errors.New("test")} assert.Nil(t, wrErr.RejectReason(), "RejectReason should be nil when not set") }) t.Run("returns value when set", func(t *testing.T) { reason := map[string]string{"key": "value"} wrErr := &WillRetryError{ ErrStr: "rejected", RetryAttempt: 2, rejectReason: reason, err: errors.New("inner"), } assert.Equal(t, reason, wrErr.RejectReason()) assert.Equal(t, "rejected", wrErr.Error()) assert.Equal(t, 2, wrErr.RetryAttempt) }) }