diff --git a/AGENTS.md b/AGENTS.md index 85137bb..d1a5ae0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,7 @@ src/ conftest.py # Test env defaults and database reset fixture alembic/ # Database migrations deploy/proxy/ # The secrets proxy: a mitmproxy addon, mounted into the official image -api-tests/ # Playwright black-box API tests +api-tests/ # Playwright black-box API tests, and the acceptance kit for the secrets path docs/ # Architecture, networking, deploy, add-a-provider Dockerfile # Single image: API + cron commands + migrations ``` diff --git a/api-tests/acceptance/.gitignore b/api-tests/acceptance/.gitignore new file mode 100644 index 0000000..3b50e3c --- /dev/null +++ b/api-tests/acceptance/.gitignore @@ -0,0 +1,3 @@ +# Token files. Never in the repository. +*.json +*.token diff --git a/api-tests/acceptance/README.md b/api-tests/acceptance/README.md new file mode 100644 index 0000000..e1566bd --- /dev/null +++ b/api-tests/acceptance/README.md @@ -0,0 +1,102 @@ +# Acceptance from inside the box + +Black-box checks of the secrets path, run by hand against a deployment. One +run covers one provider. The check creates a host with an issuer-backed +`anthropic` secret and an issuer-backed `github` secret. Then it works from +inside the box over SSH, and deletes the host at the end. + +Not in CI. A run needs a subscription token, an installation token for a +private repository, and a box image with `claude`, `git`, and `gh`. + +## What passes + +- `POST /hosts` answers 201 and echoes no secret. The host becomes active. +- The issuer is asked once for the github secret, and not before its first + use. On docker-sbx the API asks at provisioning, and the exchange asks again + when it first sees the host. +- The anthropic secret lives 70 seconds in this run, so the exchange asks for + it again and again. The outage step needs that. +- A plain session sees both placeholders. +- `claude -p` answers `ok`. +- `gh api` and `gh pr list` work on the private repository. +- `git clone`, a push of a throwaway branch, and its delete work. +- The placeholder sent straight to Anthropic, with the proxy bypassed, is + refused with 401. +- A wrong placeholder is refused: with 403 by the exchange, or with 401 by + Anthropic on docker-sbx, where nothing swaps it. +- While the issuer fails, the last value keeps the box working, and the + exchange kept asking the issuer in the meantime. +- A restart of the exchange costs one fetch of the github secret, and + `claude -p` and `gh` answer again. +- On docker, a restarted box comes back with the same git setup. +- On docker-sbx, every secret is scoped to the sandbox, and the value files + are 0600 beside the workspaces. Teardown leaves no secret in + `sbx secret ls`, no value file, and no workspace. +- `DELETE /hosts` answers 204. + +## The issuer + +`issuer.py` stands in for the service that mints tokens. It answers +`GET /mint/` with the JSON in `.json` next to it, behind the +bearer in `ISSUER_BEARER`. Write the token files yourself, with mode 0600, +and delete them when the run ends. Git ignores them. + +```bash +umask 077 +printf '{"value": "%s"}' "$SUBSCRIPTION_TOKEN" > api-tests/acceptance/anthropic.json +printf '{"value": "%s"}' "$INSTALLATION_TOKEN" > api-tests/acceptance/github.json +ISSUER_BEARER= python3 api-tests/acceptance/issuer.py 8791 +``` + +An `expires_at` in a file is optional. Without it the secret's refresh +interval sets the lifetime. The API takes issuer URLs over HTTPS only, so put +the stub behind a TLS front that the exchange trusts. Caddy with `tls +internal` and `reverse_proxy 127.0.0.1:8791` does it on one host. + +The check drives the stub through two control paths, behind the same bearer. +`GET /fetches` counts the mint requests per service. `POST /fail` with +`{"failing": true}` makes every mint answer 500 until `{"failing": false}`. + +## Run + +```bash +SERVICE_URL=http://localhost:8780 \ +SERVICE_TOKEN= \ +PROVIDER=docker \ +ISSUER_URL=https://localhost:8443 \ +ISSUER_BEARER= \ +ISSUER_CONTROL_URL=http://127.0.0.1:8791 \ +GITHUB_REPO=/ \ +RESTART_EXCHANGE='docker compose restart exchange' \ +python3 api-tests/acceptance/check.py +``` + +- `PROVIDER` is `docker`, `docker-sbx`, or `exe`. +- `ISSUER_URL` is the issuer as the exchange reaches it. The check appends + `/mint/anthropic` and `/mint/github`. +- `ISSUER_CONTROL_URL` is the issuer as the check reaches it, for the two + control paths. It defaults to `ISSUER_URL`. +- `RESTART_EXCHANGE` is a shell command that restarts the exchange process. +- `HOST_IMAGE` names the box image. `images/local` has `git` and `gh`. Add + Claude Code on top of it for this run. +- `SSH_KEY` is the account key for a provider whose boxes take it, such as + exe. +- `SBX_WORKSPACE_ROOT` turns on the docker-sbx checks. Run the check on the + sbx host then, where `sbx` and the workspace root are. +- `HOST_ACTIVE_TIMEOUT` is the wait for an active host, in seconds. The + default is 600. `POST /hosts` provisions before it answers, so the request + waits that long too. + +A box image for docker with Claude Code: + +```Dockerfile +FROM drukbox/sandbox:local +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && npm install -g @anthropic-ai/claude-code \ + && rm -rf /var/lib/apt/lists/* +``` + +The check prints one line per step and a summary. It exits 1 when a step +fails. It deletes its host in every case. The host carries a one hour lease +in case the run dies. The throwaway branch is named after the host. diff --git a/api-tests/acceptance/check.py b/api-tests/acceptance/check.py new file mode 100644 index 0000000..a7121b9 --- /dev/null +++ b/api-tests/acceptance/check.py @@ -0,0 +1,364 @@ +"""Acceptance from inside the box, one provider per run. README.md says what to set.""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + + +def get_setting(name: str) -> str: + return os.environ.get(name) or sys.exit(f"set {name}") + + +SERVICE_URL = get_setting("SERVICE_URL").rstrip("/") +SERVICE_TOKEN = get_setting("SERVICE_TOKEN") +PROVIDER = get_setting("PROVIDER") +ISSUER_URL = get_setting("ISSUER_URL").rstrip("/") +ISSUER_BEARER = get_setting("ISSUER_BEARER") +ISSUER_CONTROL_URL = os.environ.get("ISSUER_CONTROL_URL", ISSUER_URL).rstrip("/") +GITHUB_REPO = get_setting("GITHUB_REPO") +RESTART_EXCHANGE = get_setting("RESTART_EXCHANGE") +HOST_ACTIVE_TIMEOUT = int(os.environ.get("HOST_ACTIVE_TIMEOUT", "600")) +HOST_IMAGE = os.environ.get("HOST_IMAGE", "") +SSH_KEY = get_setting("SSH_KEY") if PROVIDER == "exe" else "" +SBX_WORKSPACE_ROOT = os.environ.get("SBX_WORKSPACE_ROOT", "") +PROVIDER_NEEDS_VALUE = PROVIDER == "docker-sbx" + +# 70 seconds: a refresh and an outage in one run. An hour: one fetch, one more at a restart. +LIFETIMES = {"anthropic": "70s", "github": "1h"} +GIT = "GIT_TERMINAL_PROMPT=0 git -c user.name=drukbox -c user.email=drukbox@example.invalid" +PROMPT = "Reply with exactly the word ok and nothing else." +MODELS_URL = "https://api.anthropic.com/v1/models?limit=1" +results: list[tuple[str, bool, str]] = [] + + +def call_api( + method: str, path: str, body: dict | None = None, timeout: int = 600 +) -> tuple[int, dict]: + request = urllib.request.Request( + f"{SERVICE_URL}{path}", + method=method, + data=json.dumps(body).encode() if body else None, + headers={"Authorization": f"Bearer {SERVICE_TOKEN}", "Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.status, (json.load(response) if response.status != 204 else {}) + except urllib.error.HTTPError as error: + print( + f" {method} {path} answered {error.code}: " + f"{error.read().decode(errors='replace')[:300]}" + ) + return error.code, {} + + +def call_issuer(path: str, body: dict | None = None) -> dict: + request = urllib.request.Request( + f"{ISSUER_CONTROL_URL}{path}", + method="POST" if body else "GET", + data=json.dumps(body).encode() if body else None, + headers={"Authorization": f"Bearer {ISSUER_BEARER}", "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + +def count_fetches(service: str) -> int: + return call_issuer("/fetches")["services"].get(service, 0) - at_start.get(service, 0) + + +def check(name: str, ok: bool, detail: str = "") -> None: + results.append((name, ok, detail)) + print(f" [{'ok' if ok else 'FAIL'}] {name} {detail}") + + +def run_on_host(command: str) -> str: + done = subprocess.run( + command, shell=True, capture_output=True, text=True, errors="replace", timeout=600 + ) + return (done.stdout + done.stderr).strip() + + +def run_in_box(host: dict, script: str) -> str: + with tempfile.TemporaryDirectory() as directory: + key = f"{directory}/key" + known = f"{directory}/known_hosts" + with open(key, "w") as file: + os.fchmod(file.fileno(), 0o600) + file.write(host["private_key"] or pathlib.Path(SSH_KEY).read_text()) + with open(known, "w") as file: + file.write(host["known_hosts"]) + command = [ + "ssh", + "-i", + key, + "-o", + f"UserKnownHostsFile={known}", + "-o", + "StrictHostKeyChecking=yes", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=60", + "-p", + str(host["external_ssh_port"]), + f"{host['ssh_username']}@{host['external_ssh_host']}", + script, + ] + done = subprocess.run( + command, capture_output=True, text=True, errors="replace", timeout=600 + ) + return done.stdout.strip() + ( + f"\n[stderr] {done.stderr.strip()[-300:]}" if done.returncode else "" + ) + + +def call_claude(host: dict) -> str: + script = ( + f"export ANTHROPIC_API_KEY=; timeout 120 claude -p {json.dumps(PROMPT)}" + " --output-format text &1 | tail -1" + ) + return run_in_box(host, script) + + +def get_status(host: dict, curl: str) -> str: + return run_in_box( + host, + f"curl -s -m 25 -o /dev/null -w '%{{http_code}}' {curl} -H 'anthropic-version: 2023-06-01'", + ) + + +def make_entry(service: str) -> dict: + return { + "issuer": { + "url": f"{ISSUER_URL}/mint/{service}", + "headers": {"Authorization": f"Bearer {ISSUER_BEARER}"}, + "refresh": LIFETIMES[service], + } + } + + +print(f"== {PROVIDER}") +at_start = call_issuer("/fetches")["services"] +# A lease, so a run that dies leaves no permanent host. +expires_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() + 3600)) +body = { + "provider": PROVIDER, + "expires_at": expires_at, + "secrets": {"anthropic": make_entry("anthropic"), "github": make_entry("github")}, +} +if HOST_IMAGE: + body["image"] = HOST_IMAGE +status, host = call_api("POST", "/hosts", body, timeout=HOST_ACTIVE_TIMEOUT) +check( + "POST /hosts with two issuer-backed secrets answers 201, and echoes no secret", + status == 201 and "secrets" not in host, + str(status), +) +if not host: + sys.exit(1) +name = host["name"] +branch = f"drukbox-acceptance/{name}" +try: + deadline = time.time() + HOST_ACTIVE_TIMEOUT + while host["status"] not in ("active", "error") and time.time() < deadline: + time.sleep(5) + host = {**host, **call_api("GET", f"/hosts/{host['id']}")[1]} + check( + "the host becomes active", + host["status"] == "active", + f"{name} {host['status']} {host.get('last_error') or ''}", + ) + if PROVIDER_NEEDS_VALUE: + time.sleep(8) + check( + "the issuer was asked for each secret at provisioning, and again at first sight", + count_fetches("github") == 2 and count_fetches("anthropic") >= 2, + f"github={count_fetches('github')} anthropic={count_fetches('anthropic')}", + ) + else: + check( + "the issuer was not asked at boot", + count_fetches("github") + count_fetches("anthropic") == 0, + ) + + env = run_in_box( + host, 'echo "$ANTHROPIC_AUTH_TOKEN" | cut -c1-12; echo "$GH_TOKEN" | cut -c1-12' + ) + check( + "a plain session sees both placeholders", env.count("drk.") == 2, env.replace("\n", " | ") + ) + tools = run_in_box(host, "command -v claude git gh | wc -l") + check("claude, git, and gh are in the box", tools == "3", tools[-40:]) + + if SBX_WORKSPACE_ROOT: + listing = run_on_host(f"sbx secret ls --sandbox {name}") + check( + "sbx holds the github service secret and a custom secret for api.anthropic.com", + "github" in listing and "api.anthropic.com" in listing, + listing.replace("\n", " | ")[-120:], + ) + rows = [ + re.split(r"\s{2,}", line.strip()) for line in run_on_host("sbx secret ls").splitlines() + ] + check( + "no sbx secret is global", + not [row for row in rows if len(row) >= 4 and row[0] == "global"], + ) + modes = { + service: stat.S_IMODE(os.stat(f"{SBX_WORKSPACE_ROOT}/secrets/{name}/{service}").st_mode) + for service in ("anthropic", "github") + } + check( + "the value files are 0600 beside the workspaces", + all(mode == 0o600 for mode in modes.values()), + " ".join(f"{service}={mode:o}" for service, mode in modes.items()), + ) + + answer = call_claude(host) + check("claude -p answers ok", answer.endswith("ok"), answer[-60:]) + check( + "gh api reaches the private repository", + run_in_box(host, f"gh api repos/{GITHUB_REPO} --jq .full_name 2>&1 | tail -1") + == GITHUB_REPO, + ) + check( + "gh pr list works", + run_in_box(host, f"gh pr list -R {GITHUB_REPO} --limit 3 >/dev/null 2>&1 && echo listed") + == "listed", + ) + check( + "the issuer was asked once for the github secret", + count_fetches("github") == (2 if PROVIDER_NEEDS_VALUE else 1), + f"github={count_fetches('github')}", + ) + + clone = run_in_box( + host, + f"rm -rf /tmp/repo && {GIT} clone -q https://github.com/{GITHUB_REPO}.git /tmp/repo 2>&1" + " && echo cloned", + ) + check("git clone of the private repository", clone.endswith("cloned"), clone[-100:]) + push = run_in_box( + host, + f"cd /tmp/repo && {GIT} checkout -q -b {branch} && date > acceptance.txt" + f" && {GIT} add acceptance.txt && {GIT} commit -q -m acceptance" + f" && {GIT} push -q origin {branch} 2>&1 && echo pushed", + ) + check("git push of a throwaway branch", push.endswith("pushed"), push[-100:]) + delete = run_in_box( + host, f"cd /tmp/repo && {GIT} push -q origin --delete {branch} 2>&1 && echo deleted" + ) + check("the throwaway branch is deleted", delete.endswith("deleted"), delete[-100:]) + + direct = get_status( + host, f"--noproxy '*' {MODELS_URL} -H \"Authorization: Bearer $ANTHROPIC_AUTH_TOKEN\"" + ) + check("the placeholder sent straight to Anthropic is refused with 401", direct == "401", direct) + wrong = get_status( + host, f'{MODELS_URL} -H "Authorization: Bearer ${{ANTHROPIC_AUTH_TOKEN%??}}xx"' + ) + check( + "a wrong placeholder is refused" + + ("" if PROVIDER_NEEDS_VALUE else " with 403 by the exchange"), + wrong == ("401" if PROVIDER_NEEDS_VALUE else "403"), + wrong, + ) + + # A fresh value first, then the issuer fails while that value nears its end. + answer = call_claude(host) + asked = count_fetches("anthropic") + call_issuer("/fail", {"failing": True}) + try: + time.sleep(20) + answer = call_claude(host) + time.sleep(2) + check( + "while the issuer fails, the last value keeps the box working", + answer.endswith("ok") and count_fetches("anthropic") > asked, + f"{answer[-40:]} failed attempts={count_fetches('anthropic') - asked}", + ) + finally: + call_issuer("/fail", {"failing": False}) + + asked = count_fetches("github") + restarted = subprocess.run( + RESTART_EXCHANGE, shell=True, capture_output=True, text=True, timeout=300 + ) + check( + "the exchange restarts", + restarted.returncode == 0, + (restarted.stdout + restarted.stderr).strip()[-100:], + ) + time.sleep(10) + answer = call_claude(host) + check("after the restart claude -p answers ok again", answer.endswith("ok"), answer[-60:]) + check( + "after the restart gh still works", + run_in_box(host, f"gh api repos/{GITHUB_REPO} --jq .full_name 2>&1 | tail -1") + == GITHUB_REPO, + ) + check( + "the restart cost one fetch of the github secret", + count_fetches("github") - asked == 1, + f"github={count_fetches('github') - asked}", + ) + + if PROVIDER == "docker" and shutil.which("docker"): + # The entrypoint runs again at a restart: sshd comes back, the git setup stays single. + restarted = subprocess.run( + ["docker", "restart", name], capture_output=True, text=True, timeout=120 + ) + time.sleep(3) + port = ( + subprocess.run(["docker", "port", name, "22"], capture_output=True, text=True) + .stdout.strip() + .rsplit(":", 1)[-1] + ) + host = { + **host, + "external_ssh_port": int(port), + "known_hosts": host["known_hosts"].replace( + f"]:{host['external_ssh_port']} ", f"]:{port} " + ), + } + helpers = run_in_box( + host, + "git config --system --get-all credential.https://github.com.helper | wc -l;" + f" gh api repos/{GITHUB_REPO} --jq .full_name 2>&1 | tail -1", + ) + check( + "a restarted box comes back with the same git setup and gh still works", + restarted.returncode == 0 and helpers.startswith("2") and GITHUB_REPO in helpers, + helpers.replace("\n", " | ")[-80:], + ) +finally: + status, _ = call_api("DELETE", f"/hosts/{host['id']}") + check("DELETE /hosts answers 204", status == 204, str(status)) + if SBX_WORKSPACE_ROOT: + check( + "teardown leaves no secret in sbx secret ls", + "No secrets found" in run_on_host(f"sbx secret ls --sandbox {name}"), + ) + check( + "teardown leaves no value file and no workspace", + not os.path.exists(f"{SBX_WORKSPACE_ROOT}/secrets/{name}") + and not os.path.exists(f"{SBX_WORKSPACE_ROOT}/{name}"), + ) + +failed = [check_name for check_name, ok, _ in results if not ok] +print( + f"== {PROVIDER}: {len(results) - len(failed)}/{len(results)} passed" + + (f", failed: {', '.join(failed)}" if failed else "") +) +sys.exit(1 if failed else 0) diff --git a/api-tests/acceptance/issuer.py b/api-tests/acceptance/issuer.py new file mode 100644 index 0000000..2df3d23 --- /dev/null +++ b/api-tests/acceptance/issuer.py @@ -0,0 +1,71 @@ +"""A dummy issuer: GET /mint/ answers the JSON in .json from DIRECTORY. + +usage: ISSUER_BEARER= issuer.py PORT [DIRECTORY] + +Control, behind the same bearer: GET /fetches counts the mint requests per +service. POST /fail with {"failing": true} makes every mint answer 500 until +{"failing": false}. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +PORT = int(sys.argv[1]) +BEARER = os.environ.get("ISSUER_BEARER") or sys.exit("set ISSUER_BEARER") +DIRECTORY = sys.argv[2] if len(sys.argv) > 2 else os.path.dirname(os.path.abspath(__file__)) +state: dict = {"fetches": 0, "services": {}, "failing": False} + + +class Issuer(BaseHTTPRequestHandler): + def do_GET(self) -> None: + if not self.is_authorized(): + return self.answer(403) + if self.path == "/fetches": + return self.answer(200, {"fetches": state["fetches"], "services": state["services"]}) + if match := re.fullmatch(r"/mint/([a-z0-9-]+)", self.path): + return self.mint(match.group(1)) + return self.answer(404) + + def do_POST(self) -> None: + if not self.is_authorized(): + return self.answer(403) + if self.path != "/fail": + return self.answer(404) + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0")) or b"{}")) + state["failing"] = bool(body.get("failing")) + print(f"failing={state['failing']}", flush=True) + return self.answer(200, {"failing": state["failing"]}) + + def mint(self, service: str) -> None: + state["fetches"] += 1 + state["services"][service] = state["services"].get(service, 0) + 1 + print(f"fetch #{state['fetches']} {service} failing={state['failing']}", flush=True) + if state["failing"]: + return self.answer(500) + try: + with open(os.path.join(DIRECTORY, f"{service}.json")) as file: + return self.answer(200, json.load(file)) + except FileNotFoundError: + print(f"no {service}.json in {DIRECTORY}", flush=True) + return self.answer(404) + + def is_authorized(self) -> bool: + return self.headers.get("Authorization") == f"Bearer {BEARER}" + + def answer(self, status: int, body: dict | None = None) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.end_headers() + if body: + self.wfile.write(json.dumps(body).encode()) + + def log_message(self, *args: object) -> None: + pass + + +HTTPServer(("127.0.0.1", PORT), Issuer).serve_forever() diff --git a/docs/deploy.md b/docs/deploy.md index 941f8e9..724076a 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -406,6 +406,10 @@ infrastructure only): SERVICE_URL=http://localhost:8780 SERVICE_TOKEN=... npm --prefix api-tests test ``` +The secrets path has its own check, run by hand from inside a box. It runs +`claude -p`, `git push`, and `gh` through one provider's secrets, with a dummy +issuer. `api-tests/acceptance/README.md` says what to set and what passes. + ## Configuration reference Core, required: