diff --git a/packages/cli/package.json b/packages/cli/package.json index 1c4c179e2..d0f53e41d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -22,7 +22,8 @@ "prepare:ai-context": "cross-env CHECKLY_SKIP_AUTH=1 CHECKLY_CLI_VERSION=99.0.0 ./bin/run import plan --root gen --debug-import-plan-input-file ./src/ai-context/context.fixtures.json && jiti ./scripts/prepare-ai-context.ts", "prepare:dist": "tsc --build", "prepare": "pnpm run clean && pnpm run prepare:dist && pnpm run prepare:ai-context", - "test": "pnpm pack && vitest --run", + "test": "pnpm pack && pnpm run test:types && vitest --run", + "test:types": "tsc -p tsconfig.type-tests.json --noEmit", "test:e2e": "pnpm pack && cross-env NODE_CONFIG_DIR=./e2e/config vitest --run -c ./vitest.config.e2e.mts", "test:e2e:local": "cross-env CHECKLY_BASE_URL=http://localhost:3000 CHECKLY_ENV=local pnpm run test:e2e", "watch": "tsc --watch" diff --git a/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts index 8e6875b2d..64bac47c1 100644 --- a/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts +++ b/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts @@ -116,6 +116,16 @@ describe('AgenticCheckCodegen', () => { expect(source).not.toContain('RetryStrategyBuilder') }) + it('should not emit intent because AgenticCheck does not support it', async () => { + const source = await renderResource(env, baseResource({ + intent: { + goal: 'Backend data that must not be exposed on this construct.', + }, + })) + + expect(source).not.toContain('intent:') + }) + it('should not emit `agentRuntime` when `agenticCheckData` is missing', async () => { const source = await renderResource(env, baseResource()) expect(source).not.toContain('agentRuntime') diff --git a/packages/cli/src/constructs/__tests__/api-check.spec.ts b/packages/cli/src/constructs/__tests__/api-check.spec.ts index b3588af34..ff4ffe3f8 100644 --- a/packages/cli/src/constructs/__tests__/api-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/api-check.spec.ts @@ -141,6 +141,44 @@ describe('ApiCheck', () => { })) }, DEFAULT_TEST_TIMEOUT) + it('should synthesize normalized intent from a packed fixture project', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-intent/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'dashboard-intent', + type: 'check', + member: true, + payload: expect.objectContaining({ + intent: { + goal: 'Verify that authenticated users can open the dashboard.', + constraints: [ + { + type: 'REQUIRED_OUTCOME', + statement: 'The dashboard displays the account overview.', + }, + { + type: 'MUST_PRESERVE', + statement: 'Do not weaken the authentication assertion.', + }, + ], + }, + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + it('should not synthesize default runtime', async () => { const output = await parseProject( fixt, diff --git a/packages/cli/src/constructs/__tests__/check-intent.spec.ts b/packages/cli/src/constructs/__tests__/check-intent.spec.ts new file mode 100644 index 000000000..e0d54abfc --- /dev/null +++ b/packages/cli/src/constructs/__tests__/check-intent.spec.ts @@ -0,0 +1,328 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { ApiCheck } from '../api-check.js' +import { CheckIntent, CheckIntentConstraintType } from '../check.js' +import { Diagnostics } from '../diagnostics.js' +import { DnsMonitor } from '../dns-monitor.js' +import { PlaywrightCheck } from '../playwright-check.js' +import { Project } from '../project.js' +import { Session } from '../session.js' +import { UrlMonitor } from '../url-monitor.js' + +const completeIntent: CheckIntent = { + goal: 'Verify that authenticated users can open the dashboard.', + constraints: [ + { + type: 'REQUIRED_OUTCOME', + statement: 'Authentication succeeds for a valid user.', + }, + { + type: 'REQUIRED_OUTCOME', + statement: 'The dashboard displays the account overview.', + }, + { + type: 'MUST_PRESERVE', + statement: 'Do not remove or weaken the authentication assertion.', + }, + { + type: 'MUST_PRESERVE', + statement: 'Do not replace the dashboard assertion with a generic page-load assertion.', + }, + ], +} + +const synthesizedCompleteIntent = { + goal: completeIntent.goal, + constraints: completeIntent.constraints, +} + +let nextLogicalId = 0 + +function apiCheck (intent?: CheckIntent | null): ApiCheck { + return new ApiCheck(`api-intent-${nextLogicalId++}`, { + name: 'Dashboard API', + intent, + request: { + method: 'GET', + url: 'https://example.com/api/dashboard', + }, + }) +} + +async function validateIntent (intent: unknown): Promise { + const diagnostics = new Diagnostics() + await apiCheck(intent as CheckIntent).validate(diagnostics) + return diagnostics +} + +function messages (diagnostics: Diagnostics): string[] { + return diagnostics.observations.map(observation => observation.message) +} + +function constraint (type: CheckIntentConstraintType, statement: string) { + return { type, statement } +} + +describe('check intent', () => { + beforeEach(() => { + nextLogicalId = 0 + Session.project = new Project('intent-project', { + name: 'Intent Project', + repoUrl: 'https://github.com/checkly/checkly-cli', + }) + }) + + afterEach(() => { + Session.reset() + }) + + describe('synthesis', () => { + it('normalizes omitted intent sections to empty arrays', () => { + const synthesized = apiCheck({ + goal: ' Verify that authenticated users can open the dashboard. ', + }).synthesize() + + expect(synthesized.intent).toEqual({ + goal: 'Verify that authenticated users can open the dashboard.', + constraints: [], + }) + }) + + it('synthesizes complete structured intent', () => { + const check = apiCheck(completeIntent) + + expect(check.intent).toBe(completeIntent) + expect(check.synthesize()).toMatchObject({ + intent: synthesizedCompleteIntent, + }) + }) + + it('synthesizes intent on runtime checks, monitors, and Playwright checks', () => { + const monitor = new UrlMonitor('url-intent', { + name: 'Dashboard URL', + intent: completeIntent, + request: { + url: 'https://example.com/dashboard', + }, + }) + const playwright = new PlaywrightCheck('playwright-intent', { + name: 'Dashboard browser flow', + intent: completeIntent, + playwrightConfigPath: '/tmp/playwright.config.ts', + }) + + expect(apiCheck(completeIntent).synthesize()).toHaveProperty('intent', synthesizedCompleteIntent) + expect(monitor.intent).toBe(completeIntent) + expect(monitor.synthesize()).toHaveProperty('intent', synthesizedCompleteIntent) + expect(playwright.intent).toBe(completeIntent) + expect(playwright.synthesize()).toHaveProperty('intent', synthesizedCompleteIntent) + }) + + it('omits undefined intent so deployments do not take ownership of existing intent', () => { + expect(apiCheck().synthesize()).not.toHaveProperty('intent') + }) + + it('synthesizes null to explicitly clear intent', () => { + expect(apiCheck(null).synthesize()).toHaveProperty('intent', null) + }) + + it('uses reassigned runtime-check intent for validation and synthesis', async () => { + const check = apiCheck(completeIntent) + check.intent = { goal: ' ' } + + const diagnostics = new Diagnostics() + await check.validate(diagnostics) + + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining('The intent goal must not be blank.'), + ])) + + check.intent = { goal: ' Verify the replacement dashboard flow. ' } + expect(check.synthesize()).toHaveProperty('intent', { + goal: 'Verify the replacement dashboard flow.', + constraints: [], + }) + + check.intent = null + expect(check.synthesize()).toHaveProperty('intent', null) + }) + + it('uses reassigned monitor intent when synthesizing an explicit clear', () => { + const monitor = new DnsMonitor('dns-intent-reassignment', { + name: 'Dashboard DNS', + intent: completeIntent, + request: { + recordType: 'A', + query: 'example.com', + }, + }) + + monitor.intent = null + + expect(monitor.synthesize()).toHaveProperty('intent', null) + }) + }) + + describe('validation', () => { + it('accepts a one-character goal and rejects a blank goal', async () => { + expect((await validateIntent({ goal: 'x' })).isFatal()).toBe(false) + + const diagnostics = await validateIntent({ goal: ' \n\t ' }) + expect(diagnostics.isFatal()).toBe(true) + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining('The intent goal must not be blank.'), + ])) + }) + + it('accepts a 2,000-character goal and rejects a 2,001-character goal', async () => { + expect((await validateIntent({ goal: 'g'.repeat(2_000) })).isFatal()).toBe(false) + + const diagnostics = await validateIntent({ goal: 'g'.repeat(2_001) }) + expect(diagnostics.isFatal()).toBe(true) + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining('The intent goal must be at most 2000 characters after trimming, got 2001.'), + ])) + }) + + it('accepts 20 required-outcome constraints and rejects 21', async () => { + expect((await validateIntent({ + goal: 'Verify the dashboard.', + constraints: Array.from( + { length: 20 }, + (_, index) => constraint('REQUIRED_OUTCOME', `Outcome ${index}`), + ), + })).isFatal()).toBe(false) + + const diagnostics = await validateIntent({ + goal: 'Verify the dashboard.', + constraints: Array.from( + { length: 21 }, + (_, index) => constraint('REQUIRED_OUTCOME', `Outcome ${index}`), + ), + }) + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining('may contain at most 20 required-outcome constraints, got 21.'), + ])) + }) + + it('accepts 20 must-preserve constraints and rejects 21', async () => { + expect((await validateIntent({ + goal: 'Verify the dashboard.', + constraints: Array.from( + { length: 20 }, + (_, index) => constraint('MUST_PRESERVE', `Guardrail ${index}`), + ), + })).isFatal()).toBe(false) + + const diagnostics = await validateIntent({ + goal: 'Verify the dashboard.', + constraints: Array.from( + { length: 21 }, + (_, index) => constraint('MUST_PRESERVE', `Guardrail ${index}`), + ), + }) + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining('may contain at most 20 must-preserve constraints, got 21.'), + ])) + }) + + it.each([ + ['required-outcome constraint statement', 'REQUIRED_OUTCOME'], + ['must-preserve constraint statement', 'MUST_PRESERVE'], + ] as const)('rejects a blank %s', async (label, type) => { + const diagnostics = await validateIntent({ + goal: 'Verify the dashboard.', + constraints: [constraint(type, ' ')], + }) + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining(`The intent ${label} must not be blank.`), + ])) + }) + + it.each([ + ['required-outcome constraint statement', 'REQUIRED_OUTCOME'], + ['must-preserve constraint statement', 'MUST_PRESERVE'], + ] as const)('accepts a 1,000-character %s and rejects 1,001 characters', async (label, type) => { + expect((await validateIntent({ + goal: 'Verify the dashboard.', + constraints: [constraint(type, 's'.repeat(1_000))], + })).isFatal()).toBe(false) + + const diagnostics = await validateIntent({ + goal: 'Verify the dashboard.', + constraints: [constraint(type, 's'.repeat(1_001))], + }) + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining(`The intent ${label} must be at most 1000 characters after trimming, got 1001.`), + ])) + }) + + it('rejects unknown intent and constraint fields instead of silently discarding them', async () => { + const diagnostics = await validateIntent({ + goal: 'Verify the dashboard.', + assertion: 'The dashboard is visible.', + constraints: [{ + type: 'REQUIRED_OUTCOME', + statement: 'The dashboard loads.', + priority: 'high', + }], + }) + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining('"intent" contains unknown field "assertion".'), + expect.stringContaining('"intent.constraints[0]" contains unknown field "priority".'), + ])) + }) + + it('rejects non-array constraints and non-object constraint entries', async () => { + const nonArrayDiagnostics = await validateIntent({ + goal: 'Verify the dashboard.', + constraints: 'The dashboard loads.', + }) + const nonObjectDiagnostics = await validateIntent({ + goal: 'Verify the dashboard.', + constraints: ['The dashboard loads.'], + }) + + expect(messages(nonArrayDiagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining('"intent.constraints" must be an array of constraint objects.'), + ])) + expect(messages(nonObjectDiagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining('"intent.constraints[0]" must be a constraint object.'), + ])) + }) + + it('rejects missing or unsupported constraint types', async () => { + const diagnostics = await validateIntent({ + goal: 'Verify the dashboard.', + constraints: [ + { statement: 'The dashboard loads.' }, + { type: 'NICE_TO_HAVE', statement: 'The dashboard loads quickly.' }, + ], + }) + + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining( + 'The intent constraint type must be "REQUIRED_OUTCOME" or "MUST_PRESERVE", got undefined.', + ), + expect.stringContaining( + 'The intent constraint type must be "REQUIRED_OUTCOME" or "MUST_PRESERVE", got "NICE_TO_HAVE".', + ), + ])) + }) + + it('rejects missing or non-string constraint statements', async () => { + const diagnostics = await validateIntent({ + goal: 'Verify the dashboard.', + constraints: [ + { type: 'REQUIRED_OUTCOME' }, + { type: 'MUST_PRESERVE', statement: 42 }, + ], + }) + + expect(messages(diagnostics)).toEqual(expect.arrayContaining([ + expect.stringContaining('The intent required-outcome constraint statement must be a string.'), + expect.stringContaining('The intent must-preserve constraint statement must be a string.'), + ])) + }) + }) +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-intent/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-intent/checkly.config.js new file mode 100644 index 000000000..7c3fd1553 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-intent/checkly.config.js @@ -0,0 +1,9 @@ +import { defineConfig } from 'checkly' + +export default defineConfig({ + projectName: 'Check Intent Fixture', + logicalId: 'check-intent-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-intent/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-intent/test.check.js new file mode 100644 index 000000000..88fa7e7f0 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-intent/test.check.js @@ -0,0 +1,22 @@ +import { ApiCheck } from 'checkly/constructs' + +new ApiCheck('dashboard-intent', { + name: 'Dashboard intent', + intent: { + goal: 'Verify that authenticated users can open the dashboard.', + constraints: [ + { + type: 'REQUIRED_OUTCOME', + statement: 'The dashboard displays the account overview.', + }, + { + type: 'MUST_PRESERVE', + statement: 'Do not weaken the authentication assertion.', + }, + ], + }, + request: { + method: 'GET', + url: 'https://example.com/api/dashboard', + }, +}) diff --git a/packages/cli/src/constructs/__tests__/uptime-monitor-codegen.spec.ts b/packages/cli/src/constructs/__tests__/uptime-monitor-codegen.spec.ts index 74b39baf9..f3f572b51 100644 --- a/packages/cli/src/constructs/__tests__/uptime-monitor-codegen.spec.ts +++ b/packages/cli/src/constructs/__tests__/uptime-monitor-codegen.spec.ts @@ -109,6 +109,65 @@ describe('GrpcMonitorCodegen', () => { expect(source).toContain('responseTime()') }) + it('emits complete structured intent in stable property order', async () => { + const source = await renderResource(env, p => new GrpcMonitorCodegen(p), resource({ + intent: { + goal: 'Verify that the gRPC health service is available.', + constraints: [ + { type: 'REQUIRED_OUTCOME', statement: 'The health RPC returns a serving response.' }, + { type: 'MUST_PRESERVE', statement: 'Do not weaken the serving-status assertion.' }, + ], + }, + })) + + expect(source).toContain(`intent: { + goal: 'Verify that the gRPC health service is available.', + constraints: [ + { + type: 'REQUIRED_OUTCOME', + statement: 'The health RPC returns a serving response.', + }, + { + type: 'MUST_PRESERVE', + statement: 'Do not weaken the serving-status assertion.', + }, + ], + }`) + expect(source.indexOf('name:')).toBeLessThan(source.indexOf('intent:')) + expect(source.indexOf('intent:')).toBeLessThan(source.indexOf('request:')) + }) + + it('supports stored-shape intent from import plans created before the backend cutover', async () => { + const source = await renderResource(env, p => new GrpcMonitorCodegen(p), resource({ + intent: { + goal: 'Verify that the gRPC health service is available.', + requiredOutcomes: ['The health RPC returns a serving response.'], + mustPreserve: ['Do not weaken the serving-status assertion.'], + }, + })) + + expect(source).toContain(`constraints: [ + { + type: 'REQUIRED_OUTCOME', + statement: 'The health RPC returns a serving response.', + }, + { + type: 'MUST_PRESERVE', + statement: 'Do not weaken the serving-status assertion.', + }, + ]`) + }) + + it('omits intent when the backend resource has no intent', async () => { + const source = await renderResource(env, p => new GrpcMonitorCodegen(p), resource()) + expect(source).not.toContain('intent:') + }) + + it('omits intent when the backend resource returns null for absent intent', async () => { + const source = await renderResource(env, p => new GrpcMonitorCodegen(p), resource({ intent: null })) + expect(source).not.toContain('intent:') + }) + it('describes the resource by name', () => { const program = new Program({ rootDirectory: env.rootDirectory, @@ -166,6 +225,16 @@ describe('SslMonitorCodegen', () => { expect(source).not.toContain('maxResponseTimeMs:') }) + it('does not emit intent because SslMonitor does not support it', async () => { + const source = await renderResource(env, p => new SslMonitorCodegen(p), resource({ + intent: { + goal: 'Backend data that must not be exposed on this construct.', + }, + })) + + expect(source).not.toContain('intent:') + }) + it('emits assertions through SslAssertionBuilder', async () => { const source = await renderResource(env, p => new SslMonitorCodegen(p), resource({ request: { diff --git a/packages/cli/src/constructs/agentic-check-codegen.ts b/packages/cli/src/constructs/agentic-check-codegen.ts index 80a630aa4..6b335ca87 100644 --- a/packages/cli/src/constructs/agentic-check-codegen.ts +++ b/packages/cli/src/constructs/agentic-check-codegen.ts @@ -52,6 +52,7 @@ export class AgenticCheckCodegen extends Codegen { buildCheckProps(this.program, file, builder, resource, context, { skipRetryStrategy: true, + skipIntent: true, }) }) }) diff --git a/packages/cli/src/constructs/check-codegen.ts b/packages/cli/src/constructs/check-codegen.ts index 2204b2857..029019c0b 100644 --- a/packages/cli/src/constructs/check-codegen.ts +++ b/packages/cli/src/constructs/check-codegen.ts @@ -21,11 +21,30 @@ import { GrpcMonitorCodegen, GrpcMonitorResource } from './grpc-monitor-codegen. import { SslMonitorCodegen, SslMonitorResource } from './ssl-monitor-codegen.js' import { TracerouteMonitorCodegen, TracerouteMonitorResource } from './traceroute-monitor-codegen.js' +/** + * Intent shape returned by the checks resource API. + * + * Construct code generation adapts this wire representation to typed intent + * constraints so exported resources round-trip through Monitoring as Code. + */ +export interface CheckIntentResource { + goal: string + constraints?: Array<{ + type: 'REQUIRED_OUTCOME' | 'MUST_PRESERVE' + statement: string + }> + /** Compatibility with import plans created before construct-shaped intent was deployed. */ + requiredOutcomes?: string[] + /** Compatibility with import plans created before construct-shaped intent was deployed. */ + mustPreserve?: string[] +} + export interface CheckResource { id: string checkType: string name: string description?: string | null + intent?: CheckIntentResource | null activated?: boolean muted?: boolean // Handled by the backend which creates the appropriate retryStrategy. @@ -60,6 +79,11 @@ export interface BuildCheckPropsOptions { * an explicit flag. */ skipRetryStrategy?: boolean + + /** + * Skip emitting the `intent` property for constructs that do not support it. + */ + skipIntent?: boolean } export function buildCheckProps ( @@ -76,6 +100,28 @@ export function buildCheckProps ( builder.string('description', resource.description) } + if (!options.skipIntent && resource.intent != null) { + const intent = resource.intent + builder.object('intent', builder => { + builder.string('goal', intent.goal) + + const constraints = intent.constraints ?? [ + ...(intent.requiredOutcomes ?? []).map(statement => ({ type: 'REQUIRED_OUTCOME' as const, statement })), + ...(intent.mustPreserve ?? []).map(statement => ({ type: 'MUST_PRESERVE' as const, statement })), + ] + if (constraints.length > 0) { + builder.array('constraints', builder => { + for (const constraint of constraints) { + builder.object(builder => { + builder.string('type', constraint.type) + builder.string('statement', constraint.statement) + }) + } + }) + } + }) + } + if (resource.activated !== undefined) { builder.boolean('activated', resource.activated) } diff --git a/packages/cli/src/constructs/check.ts b/packages/cli/src/constructs/check.ts index 13e31cc92..9ba68fbff 100644 --- a/packages/cli/src/constructs/check.ts +++ b/packages/cli/src/constructs/check.ts @@ -50,6 +50,88 @@ export type CheckRetryStrategy = | SingleRetryRetryStrategy | NoRetriesRetryStrategy +/** + * The kind of durable guidance expressed by a check intent constraint. + */ +export type CheckIntentConstraintType = 'REQUIRED_OUTCOME' | 'MUST_PRESERVE' + +/** + * A typed statement that refines a check's goal. + * + * New constraint types can be added without changing the overall intent shape. + */ +export interface CheckIntentConstraint { + /** + * How root cause analysis and check repair should interpret the statement. + */ + type: CheckIntentConstraintType + + /** + * The durable guidance for this constraint. + * Leading and trailing whitespace is removed during synthesis. + * + * @minLength 1 + * @maxLength 1000 + */ + statement: string +} + +/** + * Durable guidance describing what a check is intended to verify. + * + * Intent is used by Checkly's root cause analysis and check repair features. + * It is separate from the check description and from executable assertions. + */ +export interface CheckIntent { + /** + * The user-visible outcome the check should verify. + * Leading and trailing whitespace is removed during synthesis. + * + * @minLength 1 + * @maxLength 2000 + */ + goal: string + + /** + * Typed outcomes and guardrails that refine the goal. + * Omitted values are synthesized as empty backend constraint sections. + * At most 20 constraints of each supported type may be provided. + */ + constraints?: CheckIntentConstraint[] +} + +/** + * Intent authoring properties shared by checks and monitors that support RCA + * and check repair. + */ +export interface CheckIntentProps { + /** + * Durable guidance for root cause analysis and check repair. + * + * - Omit this property to leave an existing backend-authored intent unchanged. + * - Provide an object to set or update intent. + * - Set it to `null` to explicitly clear intent. + * + * @example + * ```typescript + * intent: { + * goal: 'Verify that authenticated users can open the dashboard.', + * constraints: [ + * { + * type: 'REQUIRED_OUTCOME', + * statement: 'Authentication succeeds for a valid user.', + * }, + * { + * type: 'MUST_PRESERVE', + * statement: 'Do not remove or weaken the authentication assertion.', + * }, + * ], + * } + * ``` + */ + intent?: CheckIntent | null +} + /** * Base configuration properties for all check types. * These properties are inherited by ApiCheck, BrowserCheck, and other check types. @@ -276,6 +358,7 @@ export abstract class Check extends Construct { runParallel?: boolean triggerIncident?: IncidentTrigger __checkFilePath?: string // internal variable to filter by check file name from the CLI + #intent?: CheckIntent | null static readonly __checklyType = 'check' @@ -349,10 +432,169 @@ export abstract class Check extends Construct { return false } + protected get checkIntent (): CheckIntent | null | undefined { + return this.#intent + } + + protected set checkIntent (intent: CheckIntent | null | undefined) { + this.#intent = intent + } + + protected validateIntent (diagnostics: Diagnostics): void { + if (this.#intent === undefined || this.#intent === null) { + return + } + + if (typeof this.#intent !== 'object' || Array.isArray(this.#intent)) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'intent', + new Error('"intent" must be an object or null.'), + )) + return + } + + const intent = this.#intent as unknown as Record + const supportedFields = new Set(['goal', 'constraints']) + for (const field of Object.keys(intent)) { + if (!supportedFields.has(field)) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'intent', + new Error( + `"intent" contains unknown field "${field}". ` + + 'Supported fields are "goal" and "constraints".', + ), + )) + } + } + + this.validateIntentStatement(diagnostics, 'intent.goal', 'goal', intent.goal, 2_000) + this.validateIntentConstraints(diagnostics, intent.constraints) + } + + private validateIntentConstraints ( + diagnostics: Diagnostics, + value: unknown, + ): void { + if (value === undefined) { + return + } + + if (!Array.isArray(value)) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'intent.constraints', + new Error('"intent.constraints" must be an array of constraint objects.'), + )) + return + } + + const counts: Record = { + REQUIRED_OUTCOME: 0, + MUST_PRESERVE: 0, + } + + for (const [index, constraint] of value.entries()) { + const property = `intent.constraints[${index}]` + if (typeof constraint !== 'object' || constraint === null || Array.isArray(constraint)) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + property, + new Error(`"${property}" must be a constraint object.`), + )) + continue + } + + const fields = constraint as Record + const supportedFields = new Set(['type', 'statement']) + for (const field of Object.keys(fields)) { + if (!supportedFields.has(field)) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + property, + new Error( + `"${property}" contains unknown field "${field}". ` + + 'Supported fields are "type" and "statement".', + ), + )) + } + } + + const type = fields.type + if (type !== 'REQUIRED_OUTCOME' && type !== 'MUST_PRESERVE') { + diagnostics.add(new InvalidPropertyValueDiagnostic( + `${property}.type`, + new Error( + `The intent constraint type must be "REQUIRED_OUTCOME" or "MUST_PRESERVE", got ${JSON.stringify(type)}.`, + ), + )) + } else { + counts[type] += 1 + } + + const statementLabel = type === 'REQUIRED_OUTCOME' + ? 'required-outcome constraint statement' + : type === 'MUST_PRESERVE' + ? 'must-preserve constraint statement' + : 'constraint statement' + this.validateIntentStatement( + diagnostics, + `${property}.statement`, + statementLabel, + fields.statement, + 1_000, + ) + } + + const constraintLabels: Record = { + REQUIRED_OUTCOME: 'required-outcome constraints', + MUST_PRESERVE: 'must-preserve constraints', + } + for (const type of ['REQUIRED_OUTCOME', 'MUST_PRESERVE'] as const) { + if (counts[type] > 20) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'intent.constraints', + new Error( + `"intent.constraints" may contain at most 20 ${constraintLabels[type]}, got ${counts[type]}.`, + ), + )) + } + } + } + + private validateIntentStatement ( + diagnostics: Diagnostics, + property: string, + label: string, + value: unknown, + maximumLength: number, + ): void { + if (typeof value !== 'string') { + diagnostics.add(new InvalidPropertyValueDiagnostic( + property, + new Error(`The intent ${label} must be a string.`), + )) + return + } + + const trimmed = value.trim() + if (trimmed.length === 0) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + property, + new Error(`The intent ${label} must not be blank.`), + )) + } else if (trimmed.length > maximumLength) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + property, + new Error( + `The intent ${label} must be at most ${maximumLength} characters after trimming, ` + + `got ${trimmed.length}.`, + ), + )) + } + } + async validate (diagnostics: Diagnostics): Promise { await super.validate(diagnostics) await this.validateDoubleCheck(diagnostics) await this.validateRetryStrategyOnlyOn(diagnostics) + this.validateIntent(diagnostics) } protected configDefaultsGetter (props: CheckProps): ConfigDefaultsGetter { @@ -445,9 +687,24 @@ export abstract class Check extends Construct { } })() + const intent = this.#intent === undefined + ? {} + : { + intent: this.#intent === null + ? null + : { + goal: this.#intent.goal.trim(), + constraints: (this.#intent.constraints ?? []).map(constraint => ({ + type: constraint.type, + statement: constraint.statement.trim(), + })), + }, + } + return { name: this.name, ...(this.description != null && { description: this.description }), + ...intent, activated: this.activated, muted: this.muted, shouldFail: this.shouldFail, @@ -480,7 +737,7 @@ export abstract class Check extends Construct { } } -export interface RuntimeCheckProps extends CheckProps { +export interface RuntimeCheckProps extends CheckProps, CheckIntentProps { /** * The runtime version, i.e. fixed set of runtime dependencies, used to execute this check. * @@ -509,9 +766,18 @@ export abstract class RuntimeCheck extends Check { runtimeId?: string environmentVariables?: EnvironmentVariable[] + get intent (): CheckIntent | null | undefined { + return this.checkIntent + } + + set intent (intent: CheckIntent | null | undefined) { + this.checkIntent = intent + } + protected constructor (logicalId: string, props: RuntimeCheckProps) { super(logicalId, props) const config = this.applyConfigDefaults(props) + this.intent = props.intent this.runtimeId = config.runtimeId this.environmentVariables = config.environmentVariables ?? [] } diff --git a/packages/cli/src/constructs/dns-monitor.ts b/packages/cli/src/constructs/dns-monitor.ts index 2fcd43e06..55a5144b2 100644 --- a/packages/cli/src/constructs/dns-monitor.ts +++ b/packages/cli/src/constructs/dns-monitor.ts @@ -5,8 +5,9 @@ import { validateResponseTimes } from './internal/common-diagnostics.js' import { DnsRequest } from './dns-request.js' import { RequiredPropertyDiagnostic } from './construct-diagnostics.js' import { responseTimeLimits } from './internal/account-features.js' +import { CheckIntent, CheckIntentProps } from './check.js' -export interface DnsMonitorProps extends MonitorProps { +export interface DnsMonitorProps extends MonitorProps, CheckIntentProps { /** * Determines the request that the monitor is going to run. */ @@ -53,6 +54,14 @@ export class DnsMonitor extends Monitor { degradedResponseTime?: number maxResponseTime?: number + get intent (): CheckIntent | null | undefined { + return this.checkIntent + } + + set intent (intent: CheckIntent | null | undefined) { + this.checkIntent = intent + } + /** * Constructs the DNS Monitor instance * @@ -65,6 +74,7 @@ export class DnsMonitor extends Monitor { constructor (logicalId: string, props: DnsMonitorProps) { super(logicalId, props) + this.intent = props.intent this.request = props.request this.degradedResponseTime = props.degradedResponseTime this.maxResponseTime = props.maxResponseTime diff --git a/packages/cli/src/constructs/grpc-monitor.ts b/packages/cli/src/constructs/grpc-monitor.ts index a197fb7df..78fcb0262 100644 --- a/packages/cli/src/constructs/grpc-monitor.ts +++ b/packages/cli/src/constructs/grpc-monitor.ts @@ -5,8 +5,9 @@ import { validateResponseTimes } from './internal/common-diagnostics.js' import { validateGrpcAssertion } from './grpc-assertion-validation.js' import { GrpcRequest } from './grpc-request.js' import { responseTimeLimits } from './internal/account-features.js' +import { CheckIntent, CheckIntentProps } from './check.js' -export interface GrpcMonitorProps extends MonitorProps { +export interface GrpcMonitorProps extends MonitorProps, CheckIntentProps { /** * Determines the request that the monitor is going to run. */ @@ -49,6 +50,14 @@ export class GrpcMonitor extends Monitor { degradedResponseTime?: number maxResponseTime?: number + get intent (): CheckIntent | null | undefined { + return this.checkIntent + } + + set intent (intent: CheckIntent | null | undefined) { + this.checkIntent = intent + } + /** * Constructs the gRPC Monitor instance * @@ -61,6 +70,7 @@ export class GrpcMonitor extends Monitor { constructor (logicalId: string, props: GrpcMonitorProps) { super(logicalId, props) + this.intent = props.intent this.request = props.request this.degradedResponseTime = props.degradedResponseTime this.maxResponseTime = props.maxResponseTime diff --git a/packages/cli/src/constructs/heartbeat-monitor-codegen.ts b/packages/cli/src/constructs/heartbeat-monitor-codegen.ts index 1ec6ee29a..41820f59a 100644 --- a/packages/cli/src/constructs/heartbeat-monitor-codegen.ts +++ b/packages/cli/src/constructs/heartbeat-monitor-codegen.ts @@ -32,7 +32,9 @@ export class HeartbeatMonitorCodegen extends Codegen { builder.number('grace', resource.heartbeat.grace) builder.string('graceUnit', resource.heartbeat.graceUnit) - buildMonitorProps(this.program, file, builder, resource, context) + buildMonitorProps(this.program, file, builder, resource, context, { + skipIntent: true, + }) }) }) })) diff --git a/packages/cli/src/constructs/icmp-monitor.ts b/packages/cli/src/constructs/icmp-monitor.ts index b1ebe595c..3ed85210d 100644 --- a/packages/cli/src/constructs/icmp-monitor.ts +++ b/packages/cli/src/constructs/icmp-monitor.ts @@ -2,8 +2,9 @@ import { Monitor, MonitorProps } from './monitor.js' import { Session } from './session.js' import { Diagnostics } from './diagnostics.js' import { IcmpRequest } from './icmp-request.js' +import { CheckIntent, CheckIntentProps } from './check.js' -export interface IcmpMonitorProps extends MonitorProps { +export interface IcmpMonitorProps extends MonitorProps, CheckIntentProps { /** * Determines the request that the monitor is going to run. */ @@ -46,6 +47,14 @@ export class IcmpMonitor extends Monitor { degradedPacketLossThreshold?: number maxPacketLossThreshold?: number + get intent (): CheckIntent | null | undefined { + return this.checkIntent + } + + set intent (intent: CheckIntent | null | undefined) { + this.checkIntent = intent + } + /** * Constructs the ICMP Monitor instance * @@ -58,6 +67,7 @@ export class IcmpMonitor extends Monitor { constructor (logicalId: string, props: IcmpMonitorProps) { super(logicalId, props) + this.intent = props.intent this.request = props.request this.degradedPacketLossThreshold = props.degradedPacketLossThreshold this.maxPacketLossThreshold = props.maxPacketLossThreshold diff --git a/packages/cli/src/constructs/ssl-monitor-codegen.ts b/packages/cli/src/constructs/ssl-monitor-codegen.ts index 876aae0ff..65ab1ba6f 100644 --- a/packages/cli/src/constructs/ssl-monitor-codegen.ts +++ b/packages/cli/src/constructs/ssl-monitor-codegen.ts @@ -83,7 +83,9 @@ export class SslMonitorCodegen extends Codegen { builder.number('maxResponseTime', resource.maxResponseTime) } - buildMonitorProps(this.program, file, builder, resource, context) + buildMonitorProps(this.program, file, builder, resource, context, { + skipIntent: true, + }) builder.value('request', valueForSslRequest(this.program, file, context, constructRequest)) }) diff --git a/packages/cli/src/constructs/tcp-monitor.ts b/packages/cli/src/constructs/tcp-monitor.ts index c821eed02..09ead1f50 100644 --- a/packages/cli/src/constructs/tcp-monitor.ts +++ b/packages/cli/src/constructs/tcp-monitor.ts @@ -4,6 +4,7 @@ import { Session } from './session.js' import { Assertion as CoreAssertion, NumericAssertionBuilder, GeneralAssertionBuilder } from './internal/assertion.js' import { Diagnostics } from './diagnostics.js' import { responseTimeLimits } from './internal/account-features.js' +import { CheckIntent, CheckIntentProps } from './check.js' import { validateResponseTimes } from './internal/common-diagnostics.js' type TcpAssertionSource = 'RESPONSE_DATA' | 'RESPONSE_TIME' @@ -89,7 +90,7 @@ export interface TcpRequest { data?: string } -export interface TcpMonitorProps extends MonitorProps { +export interface TcpMonitorProps extends MonitorProps, CheckIntentProps { /** * Determines the request that the check is going to run. */ @@ -131,6 +132,14 @@ export class TcpMonitor extends Monitor { degradedResponseTime?: number maxResponseTime?: number + get intent (): CheckIntent | null | undefined { + return this.checkIntent + } + + set intent (intent: CheckIntent | null | undefined) { + this.checkIntent = intent + } + /** * Constructs the TCP Monitor instance * @@ -143,6 +152,7 @@ export class TcpMonitor extends Monitor { constructor (logicalId: string, props: TcpMonitorProps) { super(logicalId, props) + this.intent = props.intent this.request = props.request this.degradedResponseTime = props.degradedResponseTime this.maxResponseTime = props.maxResponseTime diff --git a/packages/cli/src/constructs/traceroute-monitor-codegen.ts b/packages/cli/src/constructs/traceroute-monitor-codegen.ts index 1c704c860..e9178035c 100644 --- a/packages/cli/src/constructs/traceroute-monitor-codegen.ts +++ b/packages/cli/src/constructs/traceroute-monitor-codegen.ts @@ -40,7 +40,9 @@ export class TracerouteMonitorCodegen extends Codegen builder.number('maxResponseTime', resource.maxResponseTime) } - buildMonitorProps(this.program, file, builder, resource, context) + buildMonitorProps(this.program, file, builder, resource, context, { + skipIntent: true, + }) builder.value('request', valueForTracerouteRequest(this.program, file, context, resource.request)) }) diff --git a/packages/cli/src/constructs/url-monitor.ts b/packages/cli/src/constructs/url-monitor.ts index 6e64d3624..549e20162 100644 --- a/packages/cli/src/constructs/url-monitor.ts +++ b/packages/cli/src/constructs/url-monitor.ts @@ -2,6 +2,7 @@ import { Diagnostics } from './diagnostics.js' import { responseTimeLimits } from './internal/account-features.js' import { validateResponseTimes } from './internal/common-diagnostics.js' import { Monitor, MonitorProps } from './monitor.js' +import { CheckIntent, CheckIntentProps } from './check.js' import { Session } from './session.js' import { UrlRequest } from './url-request.js' @@ -9,7 +10,7 @@ import { UrlRequest } from './url-request.js' * Configuration properties for UrlMonitor. * Extends MonitorProps with URL-specific settings. */ -export interface UrlMonitorProps extends MonitorProps { +export interface UrlMonitorProps extends MonitorProps, CheckIntentProps { /** * Determines the request that the monitor is going to run. * Defines the URL and validation rules for the HTTP check. @@ -108,6 +109,10 @@ export class UrlMonitor extends Monitor { readonly degradedResponseTime?: number readonly maxResponseTime?: number + get intent (): CheckIntent | null | undefined { + return this.checkIntent + } + /** * Constructs the URL Monitor instance * @@ -120,6 +125,7 @@ export class UrlMonitor extends Monitor { constructor (logicalId: string, props: UrlMonitorProps) { super(logicalId, props) + this.checkIntent = props.intent this.request = props.request this.degradedResponseTime = props.degradedResponseTime this.maxResponseTime = props.maxResponseTime diff --git a/packages/cli/tsconfig.type-tests.json b/packages/cli/tsconfig.type-tests.json new file mode 100644 index 000000000..dfbff71ca --- /dev/null +++ b/packages/cli/tsconfig.type-tests.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": false, + "incremental": false, + "noEmit": true, + "outDir": "./dist-type-tests", + "rootDir": ".", + "sourceMap": false + }, + "include": [ + "type-tests/**/*" + ], + "exclude": [] +} diff --git a/packages/cli/type-tests/check-intent.ts b/packages/cli/type-tests/check-intent.ts new file mode 100644 index 000000000..3c2def375 --- /dev/null +++ b/packages/cli/type-tests/check-intent.ts @@ -0,0 +1,83 @@ +import type { + AgenticCheckProps, + ApiCheckProps, + BrowserCheckProps, + CheckIntent, + DnsMonitorProps, + GrpcMonitorProps, + HeartbeatMonitorProps, + IcmpMonitorProps, + MultiStepCheckProps, + PlaywrightCheckProps, + SslMonitorProps, + TcpMonitorProps, + TracerouteMonitorProps, + UrlMonitorProps, +} from '../src/constructs/index.js' + +type HasIntent = 'intent' extends keyof Props ? true : false + +type IntentExposure = { + api: HasIntent + browser: HasIntent + multiStep: HasIntent + url: HasIntent + dns: HasIntent + icmp: HasIntent + tcp: HasIntent + grpc: HasIntent + playwright: HasIntent + agentic: HasIntent + heartbeat: HasIntent + ssl: HasIntent + traceroute: HasIntent +} + +export const intentExposure: IntentExposure = { + api: true, + browser: true, + multiStep: true, + url: true, + dns: true, + icmp: true, + tcp: true, + grpc: true, + playwright: true, + agentic: false, + heartbeat: false, + ssl: false, + traceroute: false, +} + +export const goalOnlyIntent: CheckIntent = { + goal: 'Verify that authenticated users can open the dashboard.', +} + +export const completeIntent: CheckIntent = { + goal: 'Verify that authenticated users can open the dashboard.', + constraints: [ + { + type: 'REQUIRED_OUTCOME', + statement: 'The dashboard displays the account overview.', + }, + { + type: 'MUST_PRESERVE', + statement: 'Do not weaken the authentication assertion.', + }, + ], +} + +export const oldIntentShapeIsNotExposed: CheckIntent = { + goal: 'Verify that authenticated users can open the dashboard.', + // @ts-expect-error CheckIntent exposes typed constraints, not backend wire fields. + requiredOutcomes: ['The dashboard displays the account overview.'], +} + +export const unsupportedConstraintTypeIsRejected: CheckIntent = { + goal: 'Verify that authenticated users can open the dashboard.', + constraints: [{ + // @ts-expect-error New constraint types require explicit CLI and backend support. + type: 'NICE_TO_HAVE', + statement: 'The dashboard loads quickly.', + }], +}