diff --git a/src/handleTestResults.ts b/src/handleTestResults.ts index 2055a174..79412e2c 100644 --- a/src/handleTestResults.ts +++ b/src/handleTestResults.ts @@ -18,6 +18,7 @@ import { stripVTControlCharacters } from 'node:util'; import { writeFile, mkdir } from 'node:fs/promises'; import { AgentTestResultsResponse, + AgentforceStudioTestCaseResult, AgentforceStudioTestResultsResponse, convertTestResultsToFormat, humanFriendlyName, @@ -102,17 +103,106 @@ function parseScorerResponse(raw: string): ParsedScorerResponse { } } -function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { +type TestCaseInput = { name: string; value: unknown }; + +// AgentforceStudioTestCaseResult doesn't yet declare `inputs` in @salesforce/agents, +// but the field is present on the wire — see PR #481 review discussion. +type AgentforceStudioTestCaseResultWithInputs = AgentforceStudioTestCaseResult & { inputs?: unknown }; + +function getTestCaseInputs(testCase: AgentforceStudioTestCaseResultWithInputs): TestCaseInput[] | undefined { + const inputs = testCase.inputs; + if (!Array.isArray(inputs)) { + return undefined; + } + const valid = inputs.filter( + (i): i is TestCaseInput => + typeof i === 'object' && + i !== null && + typeof (i as TestCaseInput).name === 'string' && + (i as TestCaseInput).value !== null && + (i as TestCaseInput).value !== undefined + ); + return valid.length > 0 ? valid : undefined; +} + +// Strips VT/ANSI escape sequences from untrusted API data before it's interpolated into +// titleLines, so remote data can't inject raw terminal escapes on the ux.log (non --output-dir) path. +// stripVTControlCharacters doesn't touch plain newlines, so those are collapsed separately. +// It also doesn't touch bare C0 control characters (e.g. a lone \r without a trailing \n), +// which could otherwise be used to visually overwrite already-rendered terminal output, so +// any remaining control characters are stripped outright. +function sanitizeForDisplay(value: string): string { + return ( + stripVTControlCharacters(value) + .replace(/\s*[\n\r\v\f]+\s*/g, ' ') + // eslint-disable-next-line no-control-regex -- intentionally stripping raw C0/DEL control chars, not matching them incidentally + .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '') + ); +} + +function formatInputsLine(inputs: TestCaseInput[]): string { + const shown = inputs.slice(0, 3); + const remaining = inputs.length - shown.length; + const pairs = shown.map((i) => `${sanitizeForDisplay(i.name)} = "${sanitizeForDisplay(String(i.value))}"`).join(', '); + return remaining > 0 ? `${pairs} (+${remaining} more)` : pairs; +} + +type ParsedSubjectResponse = { + userInput?: string; + performance?: { latency?: { duration?: number } }; + tokenUsage?: { completion?: number; prompt?: { total?: number }; total?: number }; +}; + +function parseSubjectResponse(raw: string): ParsedSubjectResponse { + try { + return JSON.parse(raw) as ParsedSubjectResponse; + } catch { + return {}; + } +} + +function formatMetricsLine(parsed: ParsedSubjectResponse): string | undefined { + const parts: string[] = []; + const latencyMs = parsed.performance?.latency?.duration; + if (typeof latencyMs === 'number') { + parts.push(`${ansis.dim('Latency')}: ${latencyMs}ms`); + } + const tokenUsage = parsed.tokenUsage; + const hasTokens = + tokenUsage !== undefined && + (typeof tokenUsage.completion === 'number' || + typeof tokenUsage.prompt?.total === 'number' || + typeof tokenUsage.total === 'number'); + if (hasTokens) { + const tokensIn = tokenUsage?.prompt?.total ?? 0; + const tokensOut = tokenUsage?.completion ?? 0; + const tokensTotal = tokenUsage?.total ?? 0; + parts.push(`${ansis.dim('Tokens')}: ${tokensIn} in / ${tokensOut} out / ${tokensTotal} total`); + } + return parts.length > 0 ? parts.join(' | ') : undefined; +} + +export function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { const ux = new Ux(); const tables: string[] = []; for (const testCase of results.testCases) { - let userInput = ''; - try { - const parsed = JSON.parse(testCase.subjectResponse) as { userInput?: string }; - userInput = parsed.userInput ?? ''; - } catch { - // ignore + const inputs = getTestCaseInputs(testCase as AgentforceStudioTestCaseResultWithInputs); + const parsedSubjectResponse = parseSubjectResponse(testCase.subjectResponse); + + const titleLines = [ansis.bold(`Test Case #${testCase.testNumber}`)]; + if (inputs) { + titleLines.push(`${ansis.dim('Inputs')}: ${formatInputsLine(inputs)}`); + } else { + const userInput = parsedSubjectResponse.userInput ?? ''; + if (userInput) { + titleLines.push(`${ansis.dim('User Input')}: ${sanitizeForDisplay(userInput)}`); + } + } + + const metricsLine = formatMetricsLine(parsedSubjectResponse); + if (metricsLine) { + titleLines.push(metricsLine); } const scorerRows = testCase.testScorerResults.map((scorer) => { @@ -126,15 +216,23 @@ function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsRespons }; }); + // Expected/Actual are a paired unit: show both if either has data on any row for this + // test case, otherwise drop both — never show just one. + const hasExpectedOrActual = scorerRows.some((row) => row.expected !== '' || row.actual !== ''); + tables.push( ux.makeTable({ - title: `${ansis.bold(`Test Case #${testCase.testNumber}`)}\n${ansis.dim('User Input')}: ${userInput}`, + title: titleLines.join('\n'), overflow: 'wrap', columns: [ { key: 'scorer', name: 'Scorer' }, { key: 'result', name: 'Result' }, - { key: 'expected', name: 'Expected', width: '25%' }, - { key: 'actual', name: 'Actual', width: '25%' }, + ...(hasExpectedOrActual + ? [ + { key: 'expected', name: 'Expected', width: '25%' } as const, + { key: 'actual', name: 'Actual', width: '25%' } as const, + ] + : []), { key: 'reasoning', name: 'Reasoning', width: '35%' }, ], data: scorerRows, @@ -217,7 +315,10 @@ function tapFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse) return `TAP version 13\n1..${expectationCount}\n${lines.join('\n')}`; } -function convertAgentforceStudioTestResultsToFormat(results: AgentforceStudioTestResultsResponse, format: 'json' | 'junit' | 'tap'): string { +function convertAgentforceStudioTestResultsToFormat( + results: AgentforceStudioTestResultsResponse, + format: 'json' | 'junit' | 'tap' +): string { switch (format) { case 'json': return JSON.stringify(results, null, 2); @@ -392,9 +493,24 @@ export async function handleTestResults({ if (!isLegacyResponse(results)) { const ngtFormatConfig = { human: { ext: 'txt', label: 'human-readable', get: () => humanFormatAgentforceStudio(results), strip: true }, - json: { ext: 'json', label: 'JSON', get: () => convertAgentforceStudioTestResultsToFormat(results, 'json'), strip: false }, - junit: { ext: 'xml', label: 'JUnit', get: () => convertAgentforceStudioTestResultsToFormat(results, 'junit'), strip: false }, - tap: { ext: 'txt', label: 'TAP', get: () => convertAgentforceStudioTestResultsToFormat(results, 'tap'), strip: false }, + json: { + ext: 'json', + label: 'JSON', + get: () => convertAgentforceStudioTestResultsToFormat(results, 'json'), + strip: false, + }, + junit: { + ext: 'xml', + label: 'JUnit', + get: () => convertAgentforceStudioTestResultsToFormat(results, 'junit'), + strip: false, + }, + tap: { + ext: 'txt', + label: 'TAP', + get: () => convertAgentforceStudioTestResultsToFormat(results, 'tap'), + strip: false, + }, } as const; const cfg = ngtFormatConfig[format]; const formatted = cfg.get(); diff --git a/test/handleTestResults.test.ts b/test/handleTestResults.test.ts index 795269d7..059bf7fd 100644 --- a/test/handleTestResults.test.ts +++ b/test/handleTestResults.test.ts @@ -14,9 +14,11 @@ * limitations under the License. */ import { readFile } from 'node:fs/promises'; +import { stripVTControlCharacters } from 'node:util'; import { expect, config } from 'chai'; -import { AgentTestResultsResponse } from '@salesforce/agents'; -import { humanFormat, readableTime, truncate } from '../src/handleTestResults.js'; +import { AgentTestResultsResponse, AgentforceStudioTestResultsResponse } from '@salesforce/agents'; +import ansis from 'ansis'; +import { humanFormat, humanFormatAgentforceStudio, readableTime, truncate } from '../src/handleTestResults.js'; config.truncateThreshold = 0; @@ -112,3 +114,175 @@ describe('metric calculations', () => { expect(output).to.include('Metric Pass % 0.00%'); }); }); + +describe('humanFormatAgentforceStudio - inputs line', () => { + it('renders Inputs line from testCase.inputs, using the raw field name as-is', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs: account = "Acme", notes = "what is kafka"'); + }); + + it('falls back to User Input when testCase.inputs is absent but subjectResponse.userInput exists', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/legacy-user-input-fallback.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('User Input: What is the account status?'); + expect(output).to.not.include('Inputs:'); + }); + + it('omits the inputs line entirely when neither inputs nor userInput is present', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('Inputs:'); + expect(output).to.not.include('User Input:'); + }); + + it('truncates to the first 3 inputs and appends a "+N more" suffix', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/many-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs: account = "Acme", region = "ANZ", tier = "Gold" (+2 more)'); + }); + + it('renders all inputs when values are a mix of string and non-string primitives', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/mixed-type-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs: account = "Acme", priority = "5", active = "true"'); + expect(output).to.not.include('more)'); + }); + + it('renders an Inputs line (not a User Input fallback) when every input value is non-string', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/all-non-string-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs:'); + expect(output).to.not.include('User Input:'); + }); +}); + +describe('humanFormatAgentforceStudio - sanitizing untrusted display data', () => { + it('strips ANSI escape sequences from input values before rendering, leaving ansis coloring intact', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/ansi-escape-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = humanFormatAgentforceStudio(input); + expect(output).to.not.include(''); + expect(output).to.not.include(''); + expect(output).to.include('account = "Acme"'); + expect(output).to.include(ansis.bold('Test Case #5')); + }); + + it('collapses embedded newlines in input values to a single space', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/newline-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('notes = "line one line two"'); + expect(output).to.not.include('line one\nline two'); + }); + + it('strips ANSI escape sequences from the User Input fallback before rendering', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/user-input-with-escapes.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = humanFormatAgentforceStudio(input); + expect(output).to.not.include(''); + expect(output).to.not.include(''); + expect(output).to.include(ansis.dim('User Input')); + expect(stripVTControlCharacters(output)).to.include('User Input: What is the account status?'); + }); + + it('collapses a bare carriage return (no newline) in an input value instead of letting it through raw', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/bare-cr-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('\r'); + expect(output).to.include('notes = "safe Inputs: EVIL"'); + }); + + it('collapses a bare carriage return (no newline) in the User Input fallback instead of letting it through raw', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/bare-cr-user-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('\r'); + expect(output).to.include('User Input: safe User Input: EVIL'); + }); + + it('strips stray BEL and backspace control characters from an input value', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/control-chars-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('\x07'); + expect(output).to.not.include('\x08'); + expect(output).to.include('notes = "badvalue"'); + }); +}); + +describe('humanFormatAgentforceStudio - latency/tokens line', () => { + it('renders combined Latency and Tokens line', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Latency: 842ms | Tokens: 156 in / 89 out / 245 total'); + }); + + it('renders Latency alone when tokenUsage is missing', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/latency-only.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Latency: 500ms'); + expect(output).to.not.include('Tokens:'); + }); + + it('renders Tokens alone when performance is missing', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/tokens-only.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Tokens: 30 in / 20 out / 50 total'); + expect(output).to.not.include('Latency:'); + }); + + it('omits the metrics line entirely when neither performance nor tokenUsage is present', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('Latency:'); + expect(output).to.not.include('Tokens:'); + }); +}); + +describe('humanFormatAgentforceStudio - Expected/Actual columns', () => { + it('omits the Expected and Actual columns when no scorer row has either value', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('Expected'); + expect(output).to.not.include('Actual'); + expect(output).to.include('Scorer'); + expect(output).to.include('Result'); + expect(output).to.include('Reasoning'); + }); + + it('shows both the Expected and Actual columns when a scorer row has either value', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-expected-actual.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Expected'); + expect(output).to.include('Actual'); + }); + + it('decides Expected/Actual visibility independently per test case', async () => { + const raw = await readFile( + './test/mocks/agentforce-studio-results/mixed-expected-actual-per-test-case.json', + 'utf8' + ); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + const [testCase9Section, testCase10Section] = output.split('Test Case #10'); + + expect(testCase9Section).to.include('Expected'); + expect(testCase9Section).to.include('Actual'); + expect(testCase10Section).to.not.include('Expected'); + expect(testCase10Section).to.not.include('Actual'); + }); +}); diff --git a/test/mocks/agentforce-studio-results/all-non-string-inputs.json b/test/mocks/agentforce-studio-results/all-non-string-inputs.json new file mode 100644 index 00000000..76b3469f --- /dev/null +++ b/test/mocks/agentforce-studio-results/all-non-string-inputs.json @@ -0,0 +1,19 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 4, + "inputs": [ + { "name": "priority", "value": 5 }, + { "name": "active", "value": true } + ], + "subjectResponse": "{\"text\":\"All non-string input types response.\",\"userInput\":\"What is the priority?\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.0,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/ansi-escape-input.json b/test/mocks/agentforce-studio-results/ansi-escape-input.json new file mode 100644 index 00000000..8ca25c46 --- /dev/null +++ b/test/mocks/agentforce-studio-results/ansi-escape-input.json @@ -0,0 +1,21 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 5, + "inputs": [ + { + "name": "account", + "value": "\u001b[31mAcme\u001b[0m" + } + ], + "subjectResponse": "{\"text\": \"Escape sequence in input value.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\": \"PASS\", \"score\": 4.0, \"reasoning\": \"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/bare-cr-input.json b/test/mocks/agentforce-studio-results/bare-cr-input.json new file mode 100644 index 00000000..5f00dbbd --- /dev/null +++ b/test/mocks/agentforce-studio-results/bare-cr-input.json @@ -0,0 +1,16 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 8, + "inputs": [{ "name": "notes", "value": "safe\rInputs: EVIL" }], + "subjectResponse": "{\"text\":\"Bare carriage return in input value.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.0,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/bare-cr-user-input.json b/test/mocks/agentforce-studio-results/bare-cr-user-input.json new file mode 100644 index 00000000..885fd31b --- /dev/null +++ b/test/mocks/agentforce-studio-results/bare-cr-user-input.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 9, + "subjectResponse": "{\"userInput\": \"safe\\rUser Input: EVIL\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\": \"PASS\", \"score\": 4.0, \"reasoning\": \"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/control-chars-input.json b/test/mocks/agentforce-studio-results/control-chars-input.json new file mode 100644 index 00000000..3146c96b --- /dev/null +++ b/test/mocks/agentforce-studio-results/control-chars-input.json @@ -0,0 +1,21 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 10, + "inputs": [ + { + "name": "notes", + "value": "bad\u0007\bvalue" + } + ], + "subjectResponse": "{\"text\": \"BEL and backspace in input value.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\": \"PASS\", \"score\": 4.0, \"reasoning\": \"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/latency-only.json b/test/mocks/agentforce-studio-results/latency-only.json new file mode 100644 index 00000000..fe58e5e9 --- /dev/null +++ b/test/mocks/agentforce-studio-results/latency-only.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Response with latency only.\",\"performance\":{\"latency\":{\"duration\":500}}}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.1,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/legacy-user-input-fallback.json b/test/mocks/agentforce-studio-results/legacy-user-input-fallback.json new file mode 100644 index 00000000..92bc5ddf --- /dev/null +++ b/test/mocks/agentforce-studio-results/legacy-user-input-fallback.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"userInput\":\"What is the account status?\",\"text\":\"The account is active.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.5,\"reasoning\":\"Clear.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/many-inputs.json b/test/mocks/agentforce-studio-results/many-inputs.json new file mode 100644 index 00000000..5a18ebd9 --- /dev/null +++ b/test/mocks/agentforce-studio-results/many-inputs.json @@ -0,0 +1,22 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 2, + "inputs": [ + { "name": "account", "value": "Acme" }, + { "name": "region", "value": "ANZ" }, + { "name": "tier", "value": "Gold" }, + { "name": "segment", "value": "Enterprise" }, + { "name": "priority", "value": "High" } + ], + "subjectResponse": "{\"text\":\"Multi-input response.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.2,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/mixed-expected-actual-per-test-case.json b/test/mocks/agentforce-studio-results/mixed-expected-actual-per-test-case.json new file mode 100644 index 00000000..d3aa6b33 --- /dev/null +++ b/test/mocks/agentforce-studio-results/mixed-expected-actual-per-test-case.json @@ -0,0 +1,25 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 9, + "subjectResponse": "{\"text\":\"Test case with expected/actual.\"}", + "testScorerResults": [ + { + "scorerName": "Output Validation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":5,\"reasoning\":\"Matches.\",\"expectedValue\":\"Expected text\",\"actualValue\":\"Actual text\"}" + } + ] + }, + { + "testNumber": 10, + "subjectResponse": "{\"text\":\"Test case without expected/actual.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.0,\"reasoning\":\"Coherent response.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/mixed-type-inputs.json b/test/mocks/agentforce-studio-results/mixed-type-inputs.json new file mode 100644 index 00000000..695aafdb --- /dev/null +++ b/test/mocks/agentforce-studio-results/mixed-type-inputs.json @@ -0,0 +1,20 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 3, + "inputs": [ + { "name": "account", "value": "Acme" }, + { "name": "priority", "value": 5 }, + { "name": "active", "value": true } + ], + "subjectResponse": "{\"text\":\"Mixed input types response.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.5,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/newline-input.json b/test/mocks/agentforce-studio-results/newline-input.json new file mode 100644 index 00000000..3f5571af --- /dev/null +++ b/test/mocks/agentforce-studio-results/newline-input.json @@ -0,0 +1,16 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 6, + "inputs": [{ "name": "notes", "value": "line one\nline two" }], + "subjectResponse": "{\"text\":\"Newline in input value.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.0,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/no-inputs-no-user-input.json b/test/mocks/agentforce-studio-results/no-inputs-no-user-input.json new file mode 100644 index 00000000..6e5b7200 --- /dev/null +++ b/test/mocks/agentforce-studio-results/no-inputs-no-user-input.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Some response with no metadata.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.0,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/tokens-only.json b/test/mocks/agentforce-studio-results/tokens-only.json new file mode 100644 index 00000000..12b2afc7 --- /dev/null +++ b/test/mocks/agentforce-studio-results/tokens-only.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Response with tokens only.\",\"tokenUsage\":{\"completion\":20,\"prompt\":{\"total\":30},\"total\":50}}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.3,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/user-input-with-escapes.json b/test/mocks/agentforce-studio-results/user-input-with-escapes.json new file mode 100644 index 00000000..82621eff --- /dev/null +++ b/test/mocks/agentforce-studio-results/user-input-with-escapes.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 7, + "subjectResponse": "{\"userInput\": \"\\u001b[31mWhat is the account status?\\u001b[0m\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\": \"PASS\", \"score\": 4.0, \"reasoning\": \"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/with-expected-actual.json b/test/mocks/agentforce-studio-results/with-expected-actual.json new file mode 100644 index 00000000..7033395c --- /dev/null +++ b/test/mocks/agentforce-studio-results/with-expected-actual.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 8, + "subjectResponse": "{\"text\":\"Response with expected/actual values.\"}", + "testScorerResults": [ + { + "scorerName": "Output Validation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":5,\"reasoning\":\"Matches.\",\"expectedValue\":\"Acme is a manufacturer.\",\"actualValue\":\"Acme is a manufacturer.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/with-inputs.json b/test/mocks/agentforce-studio-results/with-inputs.json new file mode 100644 index 00000000..13219175 --- /dev/null +++ b/test/mocks/agentforce-studio-results/with-inputs.json @@ -0,0 +1,19 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "inputs": [ + { "name": "account", "value": "Acme" }, + { "name": "notes", "value": "what is kafka" } + ], + "subjectResponse": "{\"text\":\"Acme is a manufacturing prospect.\",\"performance\":{\"latency\":{\"duration\":842}},\"tokenUsage\":{\"completion\":89,\"prompt\":{\"total\":156},\"total\":245}}", + "testScorerResults": [ + { + "scorerName": "Conciseness Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.7,\"reasoning\":\"Good.\"}" + } + ] + } + ] +}