chore: Assert changeset has conventional commit message at the top

.agents/skills/create-changeset/SKILL.md

@@ -0,0 +1,42 @@
+---
+name: create-changeset
+description: Create or update Changesets for this repository with a Conventional Commit summary aligned verbatim to the pull request title. Use whenever asked to add, create, edit, or fix a changeset.
+---
+
+# Create a Changeset
+
+Inspect the diff to identify every publishable package affected and choose the appropriate semantic version bump for each package.
+Use `pnpm changeset` when creating the changeset unless the existing task requires editing a specific changeset file.
+
+## Summary
+
+The first non-empty line after the changeset frontmatter must follow Conventional Commits 1.0.0:
+
+```text
+<type>[optional scope][optional !]: <description>
+```
+
+Use `feat` for a feature and `fix` for a bug fix.
+Other meaningful types such as `docs`, `test`, `refactor`, `build`, `ci`, and `chore` are allowed.
+Use `!` immediately before the colon for a breaking change.
+
+Check whether the current branch already has a pull request with:
+
+```bash
+gh pr view --json title --jq .title
+```
+
+If a pull request exists and its title is a valid, accurate Conventional Commit message, use that title verbatim as the changeset's first summary line.
+If its title is not conventional or does not accurately describe the release, do not create a conflicting summary; explain what needs to change before continuing.
+
+If no pull request exists, derive one concise Conventional Commit summary from the diff.
+Use the exact same summary for every changeset created for the branch, and preserve it verbatim as the title when the pull request is later created.
+
+Additional release-note detail may follow on later lines.
+Do not put introductory prose, a Markdown heading, or a list marker before the Conventional Commit summary.
+
+After creating or editing the changeset, run:
+
+```bash
+node scripts/release/validate-changesets.mjs
+```

.agents/skills/create-pull-request/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: create-pull-request
+description: Create or update GitHub pull requests for this repository with Conventional Commit titles aligned to their changeset summaries. Use whenever asked to open, create, draft, or retitle a pull request.
+---
+
+# Create a Pull Request
+
+Inspect the branch diff and any changeset files added or modified by the branch before choosing the pull request title.
+Use the GitHub CLI for GitHub operations.
+
+## Title
+
+The pull request title must follow Conventional Commits 1.0.0:
+
+```text
+<type>[optional scope][optional !]: <description>
+```
+
+Use `feat` for a feature and `fix` for a bug fix.
+Other meaningful types such as `docs`, `test`, `refactor`, `build`, `ci`, and `chore` are allowed.
+Use `!` immediately before the colon for a breaking change.
+Keep the description concise and representative of the entire pull request.
+
+If the branch contains a changeset whose first summary line accurately describes the pull request, reuse that line verbatim as the pull request title.
+When the branch contains multiple changesets, prefer one shared Conventional Commit summary for all of them and the pull request title.
+If intentionally distinct changeset summaries cannot share an accurate title, preserve their meaning and call out the mismatch rather than silently rewriting release notes.
+
+If no changeset exists, derive a Conventional Commit title from the complete branch diff.
+Reuse that exact title if a changeset is subsequently created for the same pull request.
+
+Before creating or updating the pull request, run:
+
+```bash
+node scripts/release/validate-changesets.mjs
+```
+
+Do not create the pull request with a non-conforming title.
+If the user supplied a non-conforming title, preserve its meaning while converting it to the required format and state the resulting title.

.github/workflows/changeset.yaml

@@ -29,5 +29,9 @@ jobs:
           node-version-file: .tool-versions
       - name: Fetch pull request base ref
         run: git fetch origin "${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
+      - name: Validate changeset summaries
+        run: |
+          node --test scripts/release/validate-changesets.test.mjs
+          node scripts/release/validate-changesets.mjs
       - name: Enforce changeset requirement for publishable package changes
         run: node scripts/release/enforce-changeset.mjs

scripts/release/validate-changesets.mjs

@@ -0,0 +1,80 @@
+import { readdirSync, readFileSync } from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+const CONVENTIONAL_COMMIT_SUMMARY =
+  /^[\p{L}\p{N}][\p{L}\p{N}-]*(?:\([^\s()]+\))?!?: \S.*$/u;
+
+export function getChangesetSummary(contents) {
+  const lines = contents.replace(/^\uFEFF/, "").split(/\r?\n/);
+  if (lines[0] !== "---") {
+    return undefined;
+  }
+
+  const endOfFrontmatter = lines.indexOf("---", 1);
+  if (endOfFrontmatter === -1) {
+    return undefined;
+  }
+
+  return lines
+    .slice(endOfFrontmatter + 1)
+    .find((line) => line.trim().length > 0)
+    ?.trimEnd();
+}
+
+export function isConventionalCommitSummary(summary) {
+  return CONVENTIONAL_COMMIT_SUMMARY.test(summary);
+}
+
+function main() {
+  const changesetDir = path.resolve(".changeset");
+  const changesetFiles = readdirSync(changesetDir, { withFileTypes: true })
+    .filter(
+      (entry) =>
+        entry.isFile() &&
+        entry.name.endsWith(".md") &&
+        entry.name.toLowerCase() !== "readme.md",
+    )
+    .map((entry) => path.join(changesetDir, entry.name))
+    .sort();
+
+  const errors = [];
+  for (const changesetFile of changesetFiles) {
+    const summary = getChangesetSummary(readFileSync(changesetFile, "utf8"));
+    const relativePath = path.relative(process.cwd(), changesetFile);
+
+    if (summary === undefined) {
+      errors.push(
+        `${relativePath} does not have a summary after its frontmatter`,
+      );
+    } else if (!isConventionalCommitSummary(summary)) {
+      errors.push(
+        `${relativePath} must start with a Conventional Commit message; found ${JSON.stringify(summary)}`,
+      );
+    }
+  }
+
+  if (errors.length > 0) {
+    console.error("Changeset validation failed:\n");
+    for (const error of errors) {
+      console.error(`- ${error}`);
+    }
+    console.error(
+      "\nExpected: <type>[optional scope][optional !]: <description>",
+    );
+    console.error("See https://www.conventionalcommits.org/en/v1.0.0/");
+    process.exitCode = 1;
+    return;
+  }
+
+  console.log(
+    `Validated Conventional Commit summaries in ${changesetFiles.length} changeset files.`,
+  );
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
+) {
+  main();
+}

scripts/release/validate-changesets.test.mjs

@@ -0,0 +1,61 @@
+import assert from "node:assert/strict";
+import { describe, test } from "node:test";
+
+import {
+  getChangesetSummary,
+  isConventionalCommitSummary,
+} from "./validate-changesets.mjs";
+
+describe("getChangesetSummary", () => {
+  test("returns the first non-empty line after frontmatter", () => {
+    assert.equal(
+      getChangesetSummary(`---
+"braintrust": minor
+---
+
+feat(js)!: add a feature
+
+More detail.
+`),
+      "feat(js)!: add a feature",
+    );
+  });
+
+  test("rejects missing frontmatter or summary", () => {
+    assert.equal(getChangesetSummary("feat: add a feature\n"), undefined);
+    assert.equal(
+      getChangesetSummary(`---
+"braintrust": patch
+---
+`),
+      undefined,
+    );
+  });
+});
+
+describe("isConventionalCommitSummary", () => {
+  test("accepts valid Conventional Commit summaries", () => {
+    for (const summary of [
+      "feat: add a feature",
+      "fix(js): repair batch uploads",
+      "refactor(parser)!: replace the parser",
+      "REVERT: restore the previous behavior",
+    ]) {
+      assert.equal(isConventionalCommitSummary(summary), true, summary);
+    }
+  });
+
+  test("rejects invalid Conventional Commit summaries", () => {
+    for (const summary of [
+      "Add a feature",
+      "feat add a feature",
+      "feat:add a feature",
+      "feat:  add a feature",
+      "feat(): add a feature",
+      "feat(scope with spaces): add a feature",
+      "feat: ",
+    ]) {
+      assert.equal(isConventionalCommitSummary(summary), false, summary);
+    }
+  });
+});