[00139] Implement Rust Widget APIs for Chat, Audio, Camera, and Signature Inputs

e2e/app/index.html

@@ -846,13 +846,182 @@
           el.appendChild(select);
           break;
         }
+        case 'chat': {
+          if (node.streaming) el.setAttribute('data-streaming', 'true');
+          const log = document.createElement('div');
+          log.setAttribute('data-chat-log', 'true');
+          (node.children || []).forEach(child => {
+            log.appendChild(renderWidget(child));
+          });
+          el.appendChild(log);
+          const composer = document.createElement('textarea');
+          composer.setAttribute('data-chat-input', 'true');
+          if (node.placeholder) composer.placeholder = node.placeholder;
+          el.appendChild(composer);
+          (node.quickReplies || []).forEach(reply => {
+            const quick = document.createElement('button');
+            quick.setAttribute('data-quick-reply', reply);
+            quick.textContent = reply;
+            if (node.hasOnSend && node.id) {
+              // Quick replies reuse `send`: the payload is what typing the text would produce.
+              quick.addEventListener('click', () => sendEvent(node.id, 'send', { value: reply }));
+            }
+            el.appendChild(quick);
+          });
+          const send = document.createElement('button');
+          send.setAttribute('data-chat-send', 'true');
+          send.textContent = 'Send';
+          if (node.hasOnSend && node.id) {
+            send.addEventListener('click', () => {
+              sendEvent(node.id, 'send', { value: composer.value });
+              composer.value = '';
+            });
+          }
+          el.appendChild(send);
+          const cancel = document.createElement('button');
+          cancel.setAttribute('data-chat-cancel', 'true');
+          cancel.textContent = 'Cancel';
+          if (node.hasOnCancel && node.id) {
+            cancel.addEventListener('click', () => sendEvent(node.id, 'cancel', {}));
+          }
+          el.appendChild(cancel);
+          break;
+        }
+        case 'chat_message': {
+          el.setAttribute('data-sender', node.sender || '');
+          (node.children || []).forEach(child => {
+            el.appendChild(renderWidget(child));
+          });
+          break;
+        }
         case 'confetti': {
           el.setAttribute('data-trigger', node.trigger || 'auto');
           (node.children || []).forEach(child => {
             el.appendChild(renderWidget(child));
           });
           break;
         }
+        case 'chat_loading': {
+          el.setAttribute('data-chat-loading', 'true');
+          break;
+        }
+        case 'chat_status': {
+          const status = document.createElement('span');
+          status.setAttribute('data-chat-status', 'true');
+          status.textContent = node.text || '';
+          el.appendChild(status);
+          break;
+        }
+        case 'audio_input': {
+          if (node.mimeType) el.setAttribute('data-mime-type', node.mimeType);
+          if (node.recordingLabel) el.setAttribute('data-recording-label', node.recordingLabel);
+          if (node.uploadUrl) el.setAttribute('data-upload-url', node.uploadUrl);
+          el.setAttribute('data-chunk-interval', String(node.chunkInterval));
+          if (node.sampleRate) el.setAttribute('data-sample-rate', String(node.sampleRate));
+          if (node.showWaveform) {
+            const waveform = document.createElement('div');
+            waveform.setAttribute('data-waveform', 'true');
+            el.appendChild(waveform);
+          }
+          const record = document.createElement('button');
+          record.setAttribute('data-record', 'true');
+          record.textContent = node.label || 'Record';
+          if (node.disabled) record.disabled = true;
+          if (node.hasOnCapture && node.id) {
+            // No MediaRecorder and no getUserMedia: a fixed stub payload proves the
+            // round trip without asking the browser for a microphone.
+            record.addEventListener('click', () => {
+              sendEvent(node.id, 'capture', { value: 'data:audio/webm;base64,c3R1Yg==' });
+            });
+          }
+          if (node.hasOnFocus && node.id) {
+            record.addEventListener('focus', () => sendEvent(node.id, 'focus', {}));
+          }
+          if (node.hasOnBlur && node.id) {
+            record.addEventListener('blur', () => sendEvent(node.id, 'blur', {}));
+          }
+          el.appendChild(record);
+          if (node.invalid) {
+            const invalid = document.createElement('div');
+            invalid.setAttribute('data-invalid', 'true');
+            invalid.textContent = node.invalid;
+            el.appendChild(invalid);
+          }
+          break;
+        }
+        case 'camera_input': {
+          if (node.facingMode) el.setAttribute('data-facing-mode', node.facingMode);
+          if (node.captureMode) el.setAttribute('data-capture-mode', node.captureMode);
+          if (node.uploadUrl) el.setAttribute('data-upload-url', node.uploadUrl);
+          const preview = document.createElement('div');
+          preview.setAttribute('data-preview', 'true');
+          preview.textContent = node.placeholder || '';
+          el.appendChild(preview);
+          const shutter = document.createElement('button');
+          shutter.setAttribute('data-capture', 'true');
+          shutter.textContent = 'Capture';
+          if (node.disabled) shutter.disabled = true;
+          if (node.hasOnCapture && node.id) {
+            shutter.addEventListener('click', () => {
+              sendEvent(node.id, 'capture', { value: 'data:image/png;base64,c3R1Yg==' });
+            });
+          }
+          if (node.hasOnFocus && node.id) {
+            shutter.addEventListener('focus', () => sendEvent(node.id, 'focus', {}));
+          }
+          if (node.hasOnBlur && node.id) {
+            shutter.addEventListener('blur', () => sendEvent(node.id, 'blur', {}));
+          }
+          el.appendChild(shutter);
+          if (node.invalid) {
+            const invalid = document.createElement('div');
+            invalid.setAttribute('data-invalid', 'true');
+            invalid.textContent = node.invalid;
+            el.appendChild(invalid);
+          }
+          break;
+        }
+        case 'signature_input': {
+          const pad = document.createElement('canvas');
+          pad.setAttribute('data-signature-pad', 'true');
+          if (node.pen) pad.setAttribute('data-pen', node.pen);
+          if (node.background) pad.setAttribute('data-background', node.background);
+          if (node.penThickness) pad.setAttribute('data-pen-thickness', String(node.penThickness));
+          if (node.placeholder) pad.setAttribute('data-placeholder', node.placeholder);
+          if (node.value) pad.setAttribute('data-value', node.value);
+          el.appendChild(pad);
+          const sign = document.createElement('button');
+          sign.setAttribute('data-sign', 'true');
+          sign.textContent = 'Sign';
+          if (node.disabled) sign.disabled = true;
+          if (node.hasOnSign && node.id) {
+            // Drawing on a canvas is not what this harness is testing; the stub PNG is.
+            sign.addEventListener('click', () => {
+              sendEvent(node.id, 'sign', { value: 'data:image/png;base64,c3R1Yg==' });
+            });
+          }
+          if (node.hasOnFocus && node.id) {
+            sign.addEventListener('focus', () => sendEvent(node.id, 'focus', {}));
+          }
+          if (node.hasOnBlur && node.id) {
+            sign.addEventListener('blur', () => sendEvent(node.id, 'blur', {}));
+          }
+          el.appendChild(sign);
+          const clear = document.createElement('button');
+          clear.setAttribute('data-clear', 'true');
+          clear.textContent = 'Clear';
+          if (node.hasOnClear && node.id) {
+            clear.addEventListener('click', () => sendEvent(node.id, 'clear', {}));
+          }
+          el.appendChild(clear);
+          if (node.invalid) {
+            const invalid = document.createElement('div');
+            invalid.setAttribute('data-invalid', 'true');
+            invalid.textContent = node.invalid;
+            el.appendChild(invalid);
+          }
+          break;
+        }
         case 'animation': {
           el.setAttribute('data-animation-type', node.animationType || '');
           el.setAttribute('data-easing', node.easing || '');

e2e/tests/widgets/chat.spec.ts

@@ -0,0 +1,82 @@
+import { test, expect } from '@playwright/test';
+import { startHarness, navigateToHarness, stopHarness, type HarnessContext } from '../harness';
+
+let harness: HarnessContext;
+
+test.beforeAll(async () => {
+  harness = await startHarness('chat');
+});
+
+test.afterAll(() => {
+  stopHarness(harness);
+});
+
+test('chat renders its composer and streaming state', async ({ page }) => {
+  await navigateToHarness(page, harness.port);
+  const chat = page.locator('[data-widget-type="chat"]');
+  await expect(chat).toBeVisible();
+  await expect(chat).toHaveAttribute('data-streaming', 'true');
+  await expect(chat.locator('[data-chat-input]')).toHaveAttribute(
+    'placeholder',
+    'Ask something…',
+  );
+});
+
+test('chat renders one message per child with its sender', async ({ page }) => {
+  await navigateToHarness(page, harness.port);
+  // Seed message, plus the status and loading bubbles the app always appends.
+  const messages = page.locator('[data-widget-type="chat_message"]');
+  await expect(messages).toHaveCount(3);
+  await expect(messages.nth(0)).toHaveAttribute('data-sender', 'user');
+  await expect(messages.nth(0)).toContainText('Hello');
+  await expect(messages.nth(1)).toHaveAttribute('data-sender', 'assistant');
+});
+
+test('chat renders the loading and status indicators', async ({ page }) => {
+  await navigateToHarness(page, harness.port);
+  await expect(page.locator('[data-widget-type="chat_loading"]')).toHaveAttribute(
+    'data-chat-loading',
+    'true',
+  );
+  await expect(page.locator('[data-widget-type="chat_status"]')).toContainText('Searching…');
+});
+
+test('chat renders a button per quick reply', async ({ page }) => {
+  await navigateToHarness(page, harness.port);
+  const chat = page.locator('[data-widget-type="chat"]');
+  await expect(chat.locator('[data-quick-reply]')).toHaveCount(2);
+  await expect(chat.locator('[data-quick-reply="Yes"]')).toBeVisible();
+  await expect(chat.locator('[data-quick-reply="No"]')).toBeVisible();
+});
+
+test('sending a message appends it to the conversation', async ({ page }) => {
+  await navigateToHarness(page, harness.port);
+  await expect(page.locator('text=Messages: 1')).toBeVisible();
+
+  const chat = page.locator('[data-widget-type="chat"]');
+  await chat.locator('[data-chat-input]').fill('How are you?');
+  await chat.locator('[data-chat-send]').click();
+
+  // The server owns the message list, so a new bubble is proof the send arrived.
+  await expect(page.locator('text=Messages: 2')).toBeVisible({ timeout: 5000 });
+  await expect(page.locator('text=Last: How are you?')).toBeVisible();
+  await expect(page.locator('[data-widget-type="chat_message"]')).toHaveCount(4);
+});
+
+test('picking a quick reply sends it as a message', async ({ page }) => {
+  await navigateToHarness(page, harness.port);
+  await expect(page.locator('text=Messages: 1')).toBeVisible();
+
+  await page.locator('[data-quick-reply="Yes"]').click();
+
+  await expect(page.locator('text=Last: Yes')).toBeVisible({ timeout: 5000 });
+});
+
+test('cancelling a streaming response reaches the server', async ({ page }) => {
+  await navigateToHarness(page, harness.port);
+  await expect(page.locator('text=Cancelled: 0')).toBeVisible();
+
+  await page.locator('[data-chat-cancel]').click();
+
+  await expect(page.locator('text=Cancelled: 1')).toBeVisible({ timeout: 5000 });
+});

e2e/tests/widgets/media-inputs.spec.ts

@@ -0,0 +1,130 @@
+import { test, expect } from '@playwright/test';
+import { startHarness, navigateToHarness, stopHarness, type HarnessContext } from '../harness';
+
+// The renderer sends a fixed stub payload rather than driving MediaRecorder or
+// getUserMedia, so no device permissions are requested anywhere in this file.
+// What is under test is that the serialized props arrive and the capture events
+// round-trip, not that the browser can reach a microphone.
+
+test.describe('audio_input', () => {
+  let harness: HarnessContext;
+
+  test.beforeAll(async () => {
+    harness = await startHarness('audio_input');
+  });
+
+  test.afterAll(() => {
+    stopHarness(harness);
+  });
+
+  test('renders with its recording parameters', async ({ page }) => {
+    await navigateToHarness(page, harness.port);
+    const input = page.locator('[data-widget-type="audio_input"]');
+    await expect(input).toBeVisible();
+    await expect(input).toHaveAttribute('data-mime-type', 'audio/webm');
+    await expect(input).toHaveAttribute('data-recording-label', 'Stop');
+    await expect(input).toHaveAttribute('data-chunk-interval', '500');
+    await expect(input).toHaveAttribute('data-sample-rate', '44100');
+    await expect(input.locator('[data-waveform]')).toBeAttached();
+    await expect(input.locator('[data-record]')).toHaveText('Record');
+  });
+
+  test('sends no upload url when the app configures none', async ({ page }) => {
+    await navigateToHarness(page, harness.port);
+    const input = page.locator('[data-widget-type="audio_input"]');
+    await expect(input).not.toHaveAttribute('data-upload-url');
+  });
+
+  test('a capture reaches the server', async ({ page }) => {
+    await navigateToHarness(page, harness.port);
+    await expect(page.locator('text=Captured:')).toBeVisible();
+
+    await page.locator('[data-record]').click();
+
+    await expect(
+      page.locator('text=Captured: data:audio/webm;base64,c3R1Yg=='),
+    ).toBeVisible({ timeout: 5000 });
+  });
+
+  test('focusing the control reaches the server', async ({ page }) => {
+    await navigateToHarness(page, harness.port);
+    await expect(page.locator('text=Focuses: 0')).toBeVisible();
+
+    await page.locator('[data-record]').focus();
+
+    // Counted, not a boolean: the re-render this triggers replaces the button and
+    // fires a blur, which would flip a flag straight back.
+    await expect(page.locator('text=Focuses: 1')).toBeVisible({ timeout: 5000 });
+  });
+});
+
+test.describe('camera_input', () => {
+  let harness: HarnessContext;
+
+  test.beforeAll(async () => {
+    harness = await startHarness('camera_input');
+  });
+
+  test.afterAll(() => {
+    stopHarness(harness);
+  });
+
+  test('renders with its facing and capture modes', async ({ page }) => {
+    await navigateToHarness(page, harness.port);
+    const input = page.locator('[data-widget-type="camera_input"]');
+    await expect(input).toBeVisible();
+    // Lowercase on purpose: the real Ivy widget hands this to getUserMedia.
+    await expect(input).toHaveAttribute('data-facing-mode', 'environment');
+    await expect(input).toHaveAttribute('data-capture-mode', 'image');
+    await expect(input.locator('[data-preview]')).toContainText('Point at something');
+  });
+
+  test('a capture reaches the server', async ({ page }) => {
+    await navigateToHarness(page, harness.port);
+    await expect(page.locator('text=Captured:')).toBeVisible();
+
+    await page.locator('[data-capture]').click();
+
+    await expect(
+      page.locator('text=Captured: data:image/png;base64,c3R1Yg=='),
+    ).toBeVisible({ timeout: 5000 });
+  });
+});
+
+test.describe('signature_input', () => {
+  let harness: HarnessContext;
+
+  test.beforeAll(async () => {
+    harness = await startHarness('signature_input');
+  });
+
+  test.afterAll(() => {
+    stopHarness(harness);
+  });
+
+  test('renders a pad carrying the pen settings', async ({ page }) => {
+    await navigateToHarness(page, harness.port);
+    const input = page.locator('[data-widget-type="signature_input"]');
+    await expect(input).toBeVisible();
+    const pad = input.locator('canvas[data-signature-pad]');
+    await expect(pad).toBeAttached();
+    await expect(pad).toHaveAttribute('data-pen', 'primary');
+    await expect(pad).toHaveAttribute('data-pen-thickness', '2');
+    await expect(pad).toHaveAttribute('data-placeholder', 'Sign here');
+  });
+
+  test('signing and clearing both reach the server', async ({ page }) => {
+    await navigateToHarness(page, harness.port);
+    await expect(page.locator('text=Signature:')).toBeVisible();
+
+    await page.locator('[data-sign]').click();
+    await expect(
+      page.locator('text=Signature: data:image/png;base64,c3R1Yg=='),
+    ).toBeVisible({ timeout: 5000 });
+
+    await page.locator('[data-clear]').click();
+    await expect(
+      page.locator('text=Signature: data:image/png;base64,c3R1Yg=='),
+    ).toBeHidden({ timeout: 5000 });
+  });
+});