diff --git a/backend/src/osspckgs/migrations/V1784905809__repo_well_known_files.sql b/backend/src/osspckgs/migrations/V1784905809__repo_well_known_files.sql new file mode 100644 index 0000000000..a771cbc66a --- /dev/null +++ b/backend/src/osspckgs/migrations/V1784905809__repo_well_known_files.sql @@ -0,0 +1,22 @@ +-- Well-known files inventory per repo, populated by the github-repos-enricher. +CREATE TABLE IF NOT EXISTS repo_well_known_files ( + id bigserial PRIMARY KEY, + repo_id bigint NOT NULL REFERENCES repos (id) ON DELETE CASCADE, + file_type text NOT NULL, -- security | contributing | governance | maintainers | code_of_conduct | readme + directory text NOT NULL, -- root | .github | docs + path text NOT NULL, + blob_oid text NOT NULL, + first_seen_at timestamptz NOT NULL DEFAULT NOW(), + -- observation time, not commit time; bumped on content change, disappearance, reappearance + change_detected_at timestamptz NOT NULL DEFAULT NOW(), + checked_at timestamptz NOT NULL, + deleted_at timestamptz, + UNIQUE (repo_id, path) +); + +CREATE INDEX IF NOT EXISTS repo_well_known_files_change_detected_at_idx + ON repo_well_known_files (change_detected_at); + +CREATE INDEX IF NOT EXISTS repo_well_known_files_file_type_idx + ON repo_well_known_files (file_type) + WHERE deleted_at IS NULL; diff --git a/services/apps/packages_worker/src/enricher/fetchLightRepo.ts b/services/apps/packages_worker/src/enricher/fetchLightRepo.ts index 601306c6e9..5f7a6852d1 100644 --- a/services/apps/packages_worker/src/enricher/fetchLightRepo.ts +++ b/services/apps/packages_worker/src/enricher/fetchLightRepo.ts @@ -1,6 +1,12 @@ import { getServiceChildLogger } from '@crowd/logging' import { FetchError, LightRepoResult } from './types' +import { + RepoTrees, + TreeEntryNode, + classifyWellKnownFiles, + deriveSecurityFileEnabled, +} from './wellKnownFiles' const log = getServiceChildLogger('fetch-light-repo') @@ -24,6 +30,15 @@ const REPO_QUERY = ` createdAt isSecurityPolicyEnabled defaultBranchRef { name } + rootTree: object(expression: "HEAD:") { + ... on Tree { entries { name type oid } } + } + githubTree: object(expression: "HEAD:.github") { + ... on Tree { entries { name type oid } } + } + docsTree: object(expression: "HEAD:docs") { + ... on Tree { entries { name type oid } } + } } } ` @@ -34,67 +49,6 @@ export function parseGithubUrl(url: string): { owner: string; name: string } { return { owner: match[1], name: match[2] } } -// community/profile API doesn't reliably return files.security — use Contents API instead. -async function fetchSecurityFileEnabled( - url: string, - owner: string, - name: string, - token: string, - timeoutMs: number, -): Promise { - const headers = { Authorization: `bearer ${token}`, Accept: 'application/vnd.github+json' } - const check = async (path: string): Promise => { - const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), timeoutMs) - try { - const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${name}/contents/${path}`, { - headers, - signal: controller.signal, - }) - if (response.status === 200) return true - if (response.status === 404) return false - if (response.status === 403) { - const body = await response.text() - if (body.toLowerCase().includes('rate limit')) { - // REST secondary limits send retry-after; primary limits send x-ratelimit-reset - const retryAfterSec = parseInt(response.headers.get('retry-after') ?? '0', 10) - const resetSec = parseInt(response.headers.get('x-ratelimit-reset') ?? '0', 10) - const resetMs = retryAfterSec - ? Date.now() + retryAfterSec * 1000 - : resetSec - ? resetSec * 1000 + 5_000 - : Date.now() + 65_000 - throw new FetchError('RATE_LIMIT', `Contents API rate limited on ${path}`, resetMs) - } - } - throw new Error(`Unexpected status ${response.status} for ${path}`) - } finally { - clearTimeout(timeoutId) - } - } - - try { - const [root, dotGithub] = await Promise.all([ - check('SECURITY.md'), - check('.github/SECURITY.md'), - ]) - return root || dotGithub - } catch (err) { - // Rate limits propagate so the caller can park the installation and requeue the repo - if (err instanceof FetchError && err.kind === 'RATE_LIMIT') throw err - log.warn( - { - url, - errName: (err as Error).name, - errMsg: (err as Error).message, - errStack: (err as Error).stack, - }, - 'Security file check failed — securityFileEnabled will be null', - ) - return null - } -} - interface BranchProtection { enabled: boolean | null requiredReviews: number | null @@ -214,6 +168,9 @@ interface RepoGraphqlResponse { createdAt: string isSecurityPolicyEnabled: boolean defaultBranchRef: { name: string } | null + rootTree: { entries?: TreeEntryNode[] } | null + githubTree: { entries?: TreeEntryNode[] } | null + docsTree: { entries?: TreeEntryNode[] } | null } | null } errors?: Array<{ type?: string; message?: string }> @@ -262,10 +219,7 @@ export async function fetchLightRepo( if (response.status === 404) throw new FetchError('NOT_FOUND', `404 for ${url}`) if (response.status >= 500) throw new FetchError('TRANSIENT', `${response.status} for ${url}`) - const [json, securityFileEnabled] = await Promise.all([ - response.json() as Promise, - fetchSecurityFileEnabled(url, owner, name, token, timeoutMs), - ]) + const json = (await response.json()) as RepoGraphqlResponse if (json.errors?.length) { const err = json.errors[0] @@ -280,6 +234,12 @@ export async function fetchLightRepo( const repo = json.data?.repository if (!repo) throw new FetchError('NOT_FOUND', `No repository data for ${url}`) + const trees: RepoTrees = { + root: repo.rootTree?.entries ?? null, + github: repo.githubTree?.entries ?? null, + docs: repo.docsTree?.entries ?? null, + } + const defaultBranch = repo.defaultBranchRef?.name ?? null const branchProtection = defaultBranch ? await fetchBranchProtection(url, owner, name, defaultBranch, token, timeoutMs) @@ -305,7 +265,8 @@ export async function fetchLightRepo( isFork: repo.isFork ?? null, createdAt: repo.createdAt ?? null, securityPolicyEnabled: repo.isSecurityPolicyEnabled ?? null, - securityFileEnabled, + securityFileEnabled: deriveSecurityFileEnabled(trees), + wellKnownFiles: classifyWellKnownFiles(trees), branchProtectionEnabled: branchProtection.enabled, branchProtectionRequiredReviews: branchProtection.requiredReviews, branchProtectionRequiresStatusChecks: branchProtection.requiresStatusChecks, diff --git a/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts b/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts index bb68a380b9..50be98540a 100644 --- a/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts +++ b/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts @@ -7,9 +7,15 @@ import { fetchActivitySnapshot } from './fetchActivitySnapshot' import { fetchLightRepo, parseGithubUrl } from './fetchLightRepo' import { GithubAppConfig, getInstallationToken } from './githubAppAuth' import { InstallationPool } from './installationPool' -import { FetchError, LightRepoResult, RepoActivitySnapshot } from './types' +import { + FetchError, + LightRepoResult, + RepoActivitySnapshot, + RepoWellKnownFilesUpdate, +} from './types' import { bulkUpdateEnrichedRepos, markReposSkipped } from './updateEnrichedRepos' import { bulkUpsertRepoActivitySnapshot } from './updateRepoActivitySnapshot' +import { bulkUpsertRepoWellKnownFiles } from './updateWellKnownFiles' const log = getServiceChildLogger('github-repos-enricher') @@ -67,6 +73,7 @@ class WriteBuffer { private results: LightRepoResult[] = [] private snapshots: RepoActivitySnapshot[] = [] private skipUrls: string[] = [] + private wellKnownFiles: RepoWellKnownFilesUpdate[] = [] private lastFlushAt = Date.now() private flushing = false private flushFailures = 0 @@ -81,6 +88,10 @@ class WriteBuffer { this.snapshots.push(snapshot) } + addWellKnownFiles(update: RepoWellKnownFilesUpdate): void { + this.wellKnownFiles.push(update) + } + addSkip(url: string): void { this.skipUrls.push(url) } @@ -94,22 +105,34 @@ class WriteBuffer { ) } - private clearBatch(resultCount: number, snapshotCount: number, skipCount: number): void { + private clearBatch( + resultCount: number, + snapshotCount: number, + skipCount: number, + wellKnownFilesCount: number, + ): void { this.results.splice(0, resultCount) this.snapshots.splice(0, snapshotCount) this.skipUrls.splice(0, skipCount) + this.wellKnownFiles.splice(0, wellKnownFilesCount) this.flushFailures = 0 } async flush(): Promise { this.lastFlushAt = Date.now() - if (this.results.length === 0 && this.snapshots.length === 0 && this.skipUrls.length === 0) { + if ( + this.results.length === 0 && + this.snapshots.length === 0 && + this.skipUrls.length === 0 && + this.wellKnownFiles.length === 0 + ) { return 0 } const batch = [...this.results] const snapshotBatch = [...this.snapshots] const skips = [...this.skipUrls] + const wellKnownFilesBatch = [...this.wellKnownFiles] this.flushing = true try { // The snapshot upsert also updates repos rows — run in one transaction to avoid @@ -118,8 +141,9 @@ class WriteBuffer { await bulkUpdateEnrichedRepos(tx, batch) await markReposSkipped(tx, skips) await bulkUpsertRepoActivitySnapshot(tx, snapshotBatch) + await bulkUpsertRepoWellKnownFiles(tx, wellKnownFilesBatch) }) - this.clearBatch(batch.length, snapshotBatch.length, skips.length) + this.clearBatch(batch.length, snapshotBatch.length, skips.length, wellKnownFilesBatch.length) return batch.length } catch (err) { this.flushFailures++ @@ -139,7 +163,13 @@ class WriteBuffer { ? 'Flush failed repeatedly — dropping batch, repos will be re-enriched next sweep' : 'Flush failed — will retry on next cycle', ) - if (dropBatch) this.clearBatch(batch.length, snapshotBatch.length, skips.length) + if (dropBatch) + this.clearBatch( + batch.length, + snapshotBatch.length, + skips.length, + wellKnownFilesBatch.length, + ) return 0 } finally { this.flushing = false @@ -360,6 +390,11 @@ async function processRepo(row: RepoRow, ctx: WorkerContext): Promise { if (outcome.kind === 'success') { metrics.totalFetched++ writeBuffer.add(outcome.data) + writeBuffer.addWellKnownFiles({ + repoId: row.id, + checkedAt: new Date().toISOString(), + files: outcome.data.wellKnownFiles, + }) pool.parkIfBudgetLow( installationId, outcome.data.rateLimit?.remaining, diff --git a/services/apps/packages_worker/src/enricher/types.ts b/services/apps/packages_worker/src/enricher/types.ts index 89b8262058..f2b296635b 100644 --- a/services/apps/packages_worker/src/enricher/types.ts +++ b/services/apps/packages_worker/src/enricher/types.ts @@ -1,3 +1,5 @@ +import { WellKnownFileEntry } from './wellKnownFiles' + export interface LightRepoResult { url: string host: 'github' @@ -17,6 +19,7 @@ export interface LightRepoResult { createdAt: string | null securityPolicyEnabled: boolean | null securityFileEnabled: boolean | null + wellKnownFiles: WellKnownFileEntry[] branchProtectionEnabled: boolean | null branchProtectionRequiredReviews: number | null branchProtectionRequiresStatusChecks: boolean | null @@ -29,6 +32,12 @@ export interface LightRepoResult { } | null } +export interface RepoWellKnownFilesUpdate { + repoId: string + checkedAt: string + files: WellKnownFileEntry[] +} + export interface RepoActivitySnapshot { repoId: string snapshotAt: string diff --git a/services/apps/packages_worker/src/enricher/updateWellKnownFiles.ts b/services/apps/packages_worker/src/enricher/updateWellKnownFiles.ts new file mode 100644 index 0000000000..4020f79b71 --- /dev/null +++ b/services/apps/packages_worker/src/enricher/updateWellKnownFiles.ts @@ -0,0 +1,63 @@ +import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { RepoWellKnownFilesUpdate } from './types' + +export async function bulkUpsertRepoWellKnownFiles( + qx: QueryExecutor, + updates: RepoWellKnownFilesUpdate[], +): Promise { + if (updates.length === 0) return + + // dedupe by repoId: ON CONFLICT DO UPDATE cannot affect the same row twice in one statement + const byRepo = new Map(updates.map((u) => [u.repoId, u])) + const batch = [...byRepo.values()] + + await qx.result( + ` + WITH input AS ( + SELECT + (j->>'repoId')::bigint AS repo_id, + (j->>'checkedAt')::timestamptz AS checked_at, + j->'files' AS files + FROM jsonb_array_elements($1::jsonb) j + ), + found AS ( + SELECT + i.repo_id, + f->>'fileType' AS file_type, + f->>'directory' AS directory, + f->>'path' AS path, + f->>'blobOid' AS blob_oid, + i.checked_at + FROM input i, jsonb_array_elements(i.files) f + ), + soft_deleted AS ( + UPDATE repo_well_known_files w + SET deleted_at = i.checked_at, + change_detected_at = i.checked_at + FROM input i + WHERE w.repo_id = i.repo_id + AND w.deleted_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM found f WHERE f.repo_id = w.repo_id AND f.path = w.path + ) + ) + INSERT INTO repo_well_known_files (repo_id, file_type, directory, path, blob_oid, checked_at, change_detected_at) + SELECT repo_id, file_type, directory, path, blob_oid, checked_at, checked_at + FROM found + ON CONFLICT (repo_id, path) DO UPDATE SET + blob_oid = EXCLUDED.blob_oid, + file_type = EXCLUDED.file_type, + directory = EXCLUDED.directory, + checked_at = EXCLUDED.checked_at, + deleted_at = NULL, + change_detected_at = CASE + WHEN repo_well_known_files.blob_oid <> EXCLUDED.blob_oid + OR repo_well_known_files.deleted_at IS NOT NULL + THEN EXCLUDED.checked_at + ELSE repo_well_known_files.change_detected_at + END + `, + [JSON.stringify(batch)], + ) +} diff --git a/services/apps/packages_worker/src/enricher/wellKnownFiles.ts b/services/apps/packages_worker/src/enricher/wellKnownFiles.ts new file mode 100644 index 0000000000..95e34712cf --- /dev/null +++ b/services/apps/packages_worker/src/enricher/wellKnownFiles.ts @@ -0,0 +1,72 @@ +export type WellKnownFileType = + | 'security' + | 'contributing' + | 'governance' + | 'maintainers' + | 'code_of_conduct' + | 'readme' + +export type WellKnownDirectory = 'root' | '.github' | 'docs' + +export interface TreeEntryNode { + name: string + type: string + oid: string +} + +export interface WellKnownFileEntry { + fileType: WellKnownFileType + directory: WellKnownDirectory + path: string + blobOid: string +} + +export interface RepoTrees { + root: TreeEntryNode[] | null + github: TreeEntryNode[] | null + docs: TreeEntryNode[] | null +} + +const STEM_TO_TYPE: Record = { + README: 'readme', + SECURITY: 'security', + CONTRIBUTING: 'contributing', + GOVERNANCE: 'governance', + MAINTAINERS: 'maintainers', + CODEOWNERS: 'maintainers', + CODE_OF_CONDUCT: 'code_of_conduct', +} + +function stemOf(filename: string): string { + return filename + .replace(/\.[^.]+$/, '') + .toUpperCase() + .replace(/-/g, '_') +} + +const DIRECTORIES: Array<{ key: keyof RepoTrees; directory: WellKnownDirectory; prefix: string }> = + [ + { key: 'root', directory: 'root', prefix: '' }, + { key: 'github', directory: '.github', prefix: '.github/' }, + { key: 'docs', directory: 'docs', prefix: 'docs/' }, + ] + +export function classifyWellKnownFiles(trees: RepoTrees): WellKnownFileEntry[] { + const result: WellKnownFileEntry[] = [] + for (const { key, directory, prefix } of DIRECTORIES) { + for (const entry of trees[key] ?? []) { + if (entry.type !== 'blob') continue + const fileType = STEM_TO_TYPE[stemOf(entry.name)] + if (!fileType) continue + result.push({ fileType, directory, path: prefix + entry.name, blobOid: entry.oid }) + } + } + return result +} + +// Mirrors the retired REST probe so repos.security_file_enabled semantics don't shift +export function deriveSecurityFileEnabled(trees: RepoTrees): boolean { + return [...(trees.root ?? []), ...(trees.github ?? [])].some( + (entry) => entry.type === 'blob' && entry.name.toUpperCase() === 'SECURITY.MD', + ) +}