ci.yml

name: ci

# Parallel quality gates (D-16). No `needs:` between jobs so branch
# protection on `main` can require each independently:
#   validate:      CI-01: gh skill publish --dry-run + claude plugin validate (marketplace root)
#                  + marketplace.json/plugins/ sync + version bumps on release PRs
#                  + post-merge version backstop on pushes to main
#   link-check:    CI-02: the tools/check-links.sh sweep
#   compile-check: EVAL-02 compile the baked JCSMP samples verbatim
#   trigger-evals: lightweight skill-triggering check; self-skips green without a key
on:
  # Deliberately no `edited` type: running on title/body edits either re-runs
  # the paid eval matrix for a no-code change or, if jobs are gated, lets the
  # edit supersede a red required check (skipped counts as success). Base
  # retargeting to main is instead covered by the post-merge backstop below
  # plus branch protection "require branches to be up to date" on main.
  pull_request:
  push:
    branches: [main]
  workflow_dispatch:        # manual run

# The workflow only reads the repo. `gh skill publish` WITHOUT --dry-run
# creates a real GitHub release; read-only permissions make a dropped or
# typo'd flag structurally harmless.
permissions:
  contents: read

# Pin the Claude Code CLI install to an exact version so CI never auto-pulls a
# fresh npm publish into a job that can hold the ANTHROPIC_API_KEY secret.
# Defined once so the two install steps cannot drift; bump deliberately.
env:
  CLAUDE_CODE_VERSION: 2.1.231

# Supersede stacked runs on the same ref: each trigger-evals leg spends real
# API budget, so consecutive PR pushes cancel the previous run instead of
# queuing. Pushes to main group by SHA instead, so a backstop run is never
# cancelled (or pending-replaced) by the next merge, which would leave its
# pushed range unchecked and grey instead of red.
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }}
  cancel-in-progress: true

jobs:
  # CI-01: validate every skill against the Agent Skills spec via the gh CLI's
  # built-in publish command in dry-run mode (Public Preview since gh 2.90;
  # the runner image's preinstalled gh is recent enough). It discovers
  # plugins/{scope}/skills/*/SKILL.md on its own, so new skills are covered
  # automatically. Spec violations (naming rules, name/directory match,
  # missing description) exit non-zero; recommended-field notes such as a
  # missing license are warnings only. Also validate the Claude
  # plugin/marketplace manifests.
  validate:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
        with:
          fetch-depth: 0    # version check: merge-base + `git show <base>:` need real history
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: 24
      - name: Install Claude Code CLI
        run: npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}"
      - name: Validate skills against the Agent Skills spec (gh skill publish dry run)
        env:
          GH_TOKEN: ${{ github.token }}
        run: gh skill publish --dry-run
      # Validate the Claude plugin/marketplace manifests from the marketplace root (".").
      - name: claude plugin validate (marketplace root)
        run: claude plugin validate . --strict
      # Every marketplace entry must resolve to a real plugin dir and every
      # plugin dir must be listed (pure tree check, runs on every event).
      - name: Marketplace sync check
        run: ./tools/check-plugin-versions.sh --sync-only
      # Version bumps happen only in release PRs: fork dev -> upstream
      # SolaceProducts main. github.base_ref is the PR's target branch in
      # whichever repo hosts the PR, so this step fires on those upstream
      # release PRs (the enforcement point) and on the fork's post-release
      # main sync (where it passes, the bump already landed). It never runs
      # on PRs into dev, and base_ref is empty on push and workflow_dispatch,
      # so it self-skips there too. Claude Code caches installed plugins by
      # the plugin.json version, so an unbumped release never reaches users.
      # origin/<base>, not the event's base.sha: the latter goes stale when
      # the base branch advances after the last PR event, silently widening
      # the diff to other PRs' commits.
      - name: Check plugin version bumps (release PRs)
        if: github.base_ref == 'main'
        run: ./tools/check-plugin-versions.sh "origin/${{ github.base_ref }}"
      # Post-merge backstop: merging one release PR does not re-run another's
      # checks (synchronize fires only on head pushes), so a second PR can
      # merge on a stale green with a version main already ships. Re-checking
      # the pushed merge against event.before (the pre-push tip) turns that
      # red instead of silent. Prevention needs branch protection (up-to-date
      # branches or a merge queue); `!created` skips the branch-creation push,
      # where event.before is the all-zeros SHA.
      - name: Check plugin version bumps (post-merge backstop)
        if: github.event_name == 'push' && !github.event.created
        run: ./tools/check-plugin-versions.sh "${{ github.event.before }}"

  # CI-02: sweep the plugins/ tree for broken and soft-404 documentation links.
  link-check:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
      - name: Link-check the plugins tree
        run: ./tools/check-links.sh plugins

  # EVAL-02: compile the baked JCSMP samples verbatim against the live-resolved
  # sol-jcsmp. Pure compile gate, no API budget and no credential.
  compile-check:
    runs-on: ubuntu-latest
    timeout-minutes: 25
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
      - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
        with:
          distribution: temurin
          java-version: 11
          cache: maven
      - name: Compile baked samples (EVAL-02 hard gate)
        run: ./plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/compile.sh

  # Lightweight skill-triggering check. For every case in each plugin's
  # evals/trigger-evals.json, run the prompt with only that plugin loaded and
  # assert whether the target skill fired (best-of-3 per case); a leg passes
  # when at least 90% of its cases match their expectation and no must-pass
  # case fails. Runs once per model (the latest haiku and sonnet) as
  # independent checks, so a weak skill on one model is visible on its own
  # leg. Self-skips GREEN when the ANTHROPIC_API_KEY secret is absent (HAS_KEY
  # guard), so a missing key (including fork PRs, which receive no secrets)
  # never turns the check red; the warning step keeps the skip visible in the
  # run summary. The secret is legal in the job env but illegal in a job-level
  # if, so the guard reads HAS_KEY at step level and the real key is scoped to
  # the run step only.
  trigger-evals:
    runs-on: ubuntu-latest
    timeout-minutes: 120                  # per model leg; 57 cases x 3 runs (~50 min observed)
    strategy:
      fail-fast: false                    # each model is an independent check
      matrix:
        model: [claude-haiku-4-5, claude-sonnet-5]
    env:
      HAS_KEY: ${{ secrets.ANTHROPIC_API_KEY != '' }}
    steps:
      - name: Warn when skipped (no ANTHROPIC_API_KEY)
        if: env.HAS_KEY != 'true'
        run: echo "::warning::trigger-evals self-skipped, the ANTHROPIC_API_KEY secret is not configured; this green check asserted nothing"
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        if: env.HAS_KEY == 'true'
        with:
          node-version: 24
      - name: Install Claude Code CLI
        if: env.HAS_KEY == 'true'
        run: npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}"
      - name: Run trigger evals for ${{ matrix.model }}
        if: env.HAS_KEY == 'true'
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          TRIGGER_EVAL_WORKDIR: ${{ runner.temp }}/trigger-evals
        run: ./tools/run-trigger-evals.sh --model ${{ matrix.model }}
      # The runner keeps transcripts and per-run stderr only on failure; the
      # runner VM is destroyed, so upload them where they can be inspected.
      - name: Upload transcripts on failure
        if: failure()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: trigger-eval-transcripts-${{ matrix.model }}
          path: ${{ runner.temp }}/trigger-evals
          if-no-files-found: ignore