From 70436869c09d12e978bc8a2a99676a6e23ce2c8b Mon Sep 17 00:00:00 2001 From: Lokesh Chandra Date: Wed, 5 Aug 2026 14:08:37 +0530 Subject: [PATCH] fix: detect shrinkwrap siblings by workspace, not @bitgo/ prefix Ticket: WCI-1818 --- scripts/generate-bitgo-shrinkwrap.ts | 142 +++++++++++++++++++++------ scripts/set-umbrella-publishable.ts | 61 ++++++++++++ 2 files changed, 174 insertions(+), 29 deletions(-) create mode 100644 scripts/set-umbrella-publishable.ts diff --git a/scripts/generate-bitgo-shrinkwrap.ts b/scripts/generate-bitgo-shrinkwrap.ts index 6fe0a18e6c..fc33a20599 100644 --- a/scripts/generate-bitgo-shrinkwrap.ts +++ b/scripts/generate-bitgo-shrinkwrap.ts @@ -9,21 +9,29 @@ * BITGO_GENERATE_SHRINKWRAP=true (set by the release workflow) — otherwise a plain * local/offline `npm pack` would force a network install of the full dependency tree. * - * `@bitgo/*` siblings are resolved as part of the same tree as everything else. - * `lerna publish` publishes packages in dependency-topological order, so by the - * time bitgo (which depends on every sibling) is packed, the sibling versions it - * references are already live on the registry. Resolving them here — rather than - * excluding them and patching their names back into the shrinkwrap metadata after - * the fact — is what makes the generated `packages["node_modules/@bitgo/..."]` - * entries (version/resolved/integrity) actually present, which is what npm uses to - * populate node_modules for consumers. A shrinkwrap that lists a dependency in - * `packages[''].dependencies` without a matching resolved `packages[...]` entry is - * silently dropped from the install by npm rather than falling back to normal - * resolution — that's what an earlier version of this script did, which broke - * `npm install bitgo` for every consumer (siblings never landed in node_modules). - * If a sibling version isn't resolvable yet, the `npm install` below fails loudly - * and the release fails — which is correct: better a failed release than a - * silently broken shrinkwrap. + * Workspace siblings are resolved as part of the same tree as everything else, not + * stripped out. Resolving them here — rather than excluding them and patching their + * names back into the shrinkwrap metadata after the fact — is what makes the + * generated `packages["node_modules/"]` entries (version/resolved/integrity) + * actually present, which is what npm uses to populate node_modules for consumers. + * A shrinkwrap that lists a dependency in `packages[''].dependencies` without a + * matching resolved `packages[...]` entry is silently dropped from the install by + * npm rather than falling back to normal resolution — that's what an earlier version + * of this script did, which broke `npm install bitgo` for every consumer (siblings + * never landed in node_modules). + * + * This assumes sibling versions are already live on the registry when this script + * runs. That is NOT true of a single combined `lerna publish` — lerna runs every + * package's lifecycle hooks (bitgo's `prepack` included) before uploading any of + * them, so bitgo's siblings are not yet published at the point this script tries to + * resolve them. The release workflow is responsible for publishing siblings in a + * separate, earlier pass (with bitgo held back via `set-umbrella-publishable.ts`) + * before invoking a second pass that packs bitgo with generation enabled. If a + * sibling version genuinely isn't resolvable (wrong pass ordering, a sibling publish + * that itself failed, etc.), the `npm install` below fails loudly and the release + * fails — which is correct: better a failed release than a silently broken + * shrinkwrap. A short retry accounts for ordinary registry propagation lag right + * after a sibling publish; see `npmInstallWithRetry`. * * `npm shrinkwrap` isn't workspace-aware and modules/bitgo/.npmrc sets * `package-lock=false`, so generation happens in an isolated temp copy outside the @@ -37,6 +45,64 @@ import execa from 'execa'; const rootDir = path.resolve(__dirname, '..'); const bitgoDir = path.join(rootDir, 'modules/bitgo'); +const modulesDir = path.join(rootDir, 'modules'); + +/** + * Names of every package in the `modules/*` workspace, read from each package's own + * (possibly rescoped) package.json — e.g. `@bitgo/sdk-core` normally, `@bitgo-beta/sdk-core` + * on beta/alpha channels after `prepare-release.ts` re-scopes the workspace. Reading + * the current on-disk names, rather than hardcoding a scope prefix, is what makes + * sibling detection work regardless of which scope is active: `prepare-release.ts` + * rewrites both a module's own `name` and bitgo's `dependencies` keys to the same + * target scope, so intersecting bitgo's dependencies against this set matches + * correctly on every channel. A hardcoded `@bitgo/` prefix check matches nothing on + * alpha/beta (where everything is `@bitgo-beta/*`), which would make the safety check + * below silently verify zero siblings instead of catching a real problem. + */ +function getWorkspacePackageNames(): Set { + const names = new Set(); + for (const entry of fs.readdirSync(modulesDir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const pkgPath = path.join(modulesDir, entry.name, 'package.json'); + if (!fs.existsSync(pkgPath)) continue; + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')); + if (typeof pkg.name === 'string') names.add(pkg.name); + } + return names; +} + +/** + * Runs `npm install --package-lock-only --ignore-scripts`, retrying a bounded number + * of times if the failure looks like a sibling that was *just* published not having + * propagated to the registry yet (ETARGET/E404) — the two-phase publish flow runs + * this immediately after the sibling-publish pass completes, so a brief propagation + * lag is expected occasionally, not a sign of a real problem. Any other failure (or + * exhausting the retries) is rethrown as-is. + */ +async function npmInstallWithRetry(cwd: string, attempts = 5, delayMs = 5000): Promise { + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + const result = await execa('npm', ['install', '--package-lock-only', '--ignore-scripts'], { cwd }); + if (result.stdout) process.stdout.write(result.stdout + '\n'); + if (result.stderr) process.stderr.write(result.stderr + '\n'); + return; + } catch (e) { + const err = e as execa.ExecaError; + if (err.stdout) process.stdout.write(err.stdout + '\n'); + if (err.stderr) process.stderr.write(err.stderr + '\n'); + const output = `${err.stdout ?? ''}\n${err.stderr ?? ''}\n${err.message ?? ''}`; + const looksLikePropagationLag = /\bETARGET\b/.test(output) || /\bE404\b/.test(output); + if (attempt === attempts || !looksLikePropagationLag) { + throw e; + } + console.log( + `npm install failed with what looks like a registry propagation delay ` + + `(attempt ${attempt}/${attempts}) — retrying in ${delayMs}ms.` + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } +} async function main() { if (process.env.BITGO_GENERATE_SHRINKWRAP !== 'true') { @@ -55,10 +121,19 @@ async function main() { console.log(`Generating npm-shrinkwrap.json for bitgo@${bitgoPackageJson.version} in ${tempDir}`); try { - const siblingNames = Object.keys(bitgoPackageJson.dependencies ?? {}).filter((name) => name.startsWith('@bitgo/')); + const workspacePackageNames = getWorkspacePackageNames(); + const siblingNames = Object.keys(bitgoPackageJson.dependencies ?? {}).filter((name) => + workspacePackageNames.has(name) + ); if (siblingNames.length > 0) { - console.log(`Resolving ${siblingNames.length} @bitgo/* siblings as part of the shrinkwrap:`); + console.log(`Resolving ${siblingNames.length} workspace siblings as part of the shrinkwrap:`); siblingNames.forEach((name) => console.log(` - ${name}`)); + } else { + console.log( + 'No workspace siblings found among bitgo dependencies — double check this is expected ' + + '(e.g. an intentionally sibling-free release), since a detection bug here would silently ' + + 'skip the safety check below.' + ); } const isolatedPackageJson: Record = { ...bitgoPackageJson }; @@ -78,7 +153,7 @@ async function main() { fs.writeFileSync(path.join(tempDir, 'package.json'), JSON.stringify(isolatedPackageJson, null, 2) + '\n'); - await execa('npm', ['install', '--package-lock-only', '--ignore-scripts'], { cwd: tempDir, stdio: 'inherit' }); + await npmInstallWithRetry(tempDir); await execa('npm', ['shrinkwrap'], { cwd: tempDir, stdio: 'inherit' }); const shrinkwrapPath = path.join(tempDir, 'npm-shrinkwrap.json'); @@ -88,19 +163,28 @@ async function main() { const shrinkwrap = JSON.parse(fs.readFileSync(shrinkwrapPath, 'utf-8')); - // Every @bitgo/* sibling must have a resolved node_modules entry — that's what - // npm actually installs from. A sibling present only in `packages[''].dependencies` - // (or missing entirely) would be silently skipped by consumers' installs. - const resolvedPackageNames = new Set( - Object.keys(shrinkwrap.packages ?? {}) - .filter((key) => key.startsWith('node_modules/')) - .map((key) => key.slice('node_modules/'.length)) - ); - const unresolvedSiblings = siblingNames.filter((name) => !resolvedPackageNames.has(name)); + // Every workspace sibling must have a fully-resolved node_modules entry — that's + // what npm actually installs from. A sibling present only in + // `packages[''].dependencies` (or missing entirely, or present but missing + // version/resolved/integrity) would be silently skipped or under-specified in + // consumers' installs. Checking only for key presence previously let this pass + // for entries npm still couldn't act on; require the actual install-relevant + // fields to be strings, not just truthy. + const packages = (shrinkwrap.packages ?? {}) as Record>; + const unresolvedSiblings = siblingNames.filter((name) => { + const entry = packages[`node_modules/${name}`]; + return ( + !entry || + typeof entry.version !== 'string' || + typeof entry.resolved !== 'string' || + typeof entry.integrity !== 'string' + ); + }); if (unresolvedSiblings.length > 0) { throw new Error( - `The following @bitgo/* siblings have no resolved node_modules entry in the generated ` + - `shrinkwrap and would be silently missing from consumers' installs: ${unresolvedSiblings.join(', ')}` + `The following workspace siblings have no fully-resolved (version + resolved + integrity) ` + + `node_modules entry in the generated shrinkwrap and would be silently missing or ` + + `under-specified in consumers' installs: ${unresolvedSiblings.join(', ')}` ); } diff --git a/scripts/set-umbrella-publishable.ts b/scripts/set-umbrella-publishable.ts new file mode 100644 index 0000000000..1963393906 --- /dev/null +++ b/scripts/set-umbrella-publishable.ts @@ -0,0 +1,61 @@ +/** + * Toggles `private` on modules/bitgo/package.json so the release workflow can hold + * the `bitgo` umbrella package out of a `lerna publish` pass without permanently + * marking it private in the repo. + * + * Why a scripted toggle instead of lerna's `--include-private ` (which lets + * a named private package publish "by temporarily removing the private property + * from the package manifest" on its own): that would require `bitgo` to be + * permanently `private: true` in the committed manifest, and three separate checks + * in the release pipeline enumerate non-private packages to verify they exist/were + * published — a permanently-private bitgo would silently stop being covered by all + * three: + * - the pre-publish existence check (trusted publishing depends on it) + * - recovery verification + * - beta verification / recovery auto-retry + * + * Flipping the flag off, running pass 1 (siblings only — bitgo is skipped because + * lerna filters private packages before packing), then flipping it back on before + * pass 2 (bitgo only) keeps every one of those checks seeing a normal, publishable + * package by the time they run. Callers MUST run the "restore" invocation (`true`) + * under `always()` in the workflow so a failed or cancelled pass 1 cannot leave the + * manifest flipped — this script does not track or restore state on its own, it + * just sets the field to whatever you tell it. + * + * Usage: + * npx tsx scripts/set-umbrella-publishable.ts false # hold back for pass 1 + * npx tsx scripts/set-umbrella-publishable.ts true # restore for pass 2 + */ + +import fs from 'fs'; +import path from 'path'; + +const bitgoPackageJsonPath = path.resolve(__dirname, '..', 'modules', 'bitgo', 'package.json'); + +function parseArg(argv: string[]): boolean { + const raw = argv[2]; + if (raw === 'true') return true; + if (raw === 'false') return false; + throw new Error(`Expected a single argument "true" or "false", got: ${JSON.stringify(raw)}`); +} + +function main(): void { + const publishable = parseArg(process.argv); + const original = fs.readFileSync(bitgoPackageJsonPath, 'utf-8'); + const pkg = JSON.parse(original); + + if (publishable) { + delete pkg.private; + } else { + pkg.private = true; + } + + fs.writeFileSync(bitgoPackageJsonPath, JSON.stringify(pkg, null, 2) + '\n'); + console.log( + publishable + ? `Restored modules/bitgo/package.json to publishable (removed "private").` + : `Marked modules/bitgo/package.json as private — it will be skipped by the next lerna publish pass.` + ); +} + +main();