feat(backend): report the backend's memory footprint as Sentry gauges

AGENTS.md

@@ -113,6 +113,7 @@ Mirrored verbatim from Expo's official [`expo/skills`](https://github.com/expo/s
 - [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — process model, IPC channels and framing, boot handshake, lifecycle state machines, error handling, Sentry observability, alternatives considered.
 - [`docs/BUILD.md`](docs/BUILD.md) — the `build-backend.ts` pipeline, the versioned addon-filename scheme, runtime addon loading, the native-modules source-of-truth model.
 - [`docs/TESTING.md`](docs/TESTING.md) — the seven test layers, the workflows, the merge queue and required checks, the e2e device suite, the secrets/trust boundary.
+- [`docs/BENCHMARKING.md`](docs/BENCHMARKING.md) — measuring the backend's memory footprint: the Sentry gauges that answer the question across the fleet, what they measure and why, including the consent-tier reasoning.
 - [`CONTRIBUTING.md`](./CONTRIBUTING.md) — setup, every `npm run` script, per-layer test commands, commit/PR/release conventions.
 
 Open work is tracked in [GitHub issues](https://github.com/digidem/comapeo-core-react-native/issues).

backend/index.js

@@ -7,6 +7,7 @@ import { ComapeoRpc } from "./lib/comapeo-rpc.js";
 import { flushCompileCacheAfterBoot } from "./lib/compile-cache.js";
 import { createComapeo } from "./lib/create-comapeo.js";
 import { createMapServer } from "./lib/create-map-server.js";
+import { memorySnapshot } from "./lib/memory-snapshot.js";
 import { SimpleRpcServer } from "./lib/simple-rpc.js";
 import * as sentry from "./lib/sentry.js";
 import * as metrics from "./lib/metrics.js";
@@ -16,6 +17,12 @@ import { observeSyncSessions } from "./lib/sync-observer.js";
 // when Sentry is off (the metrics layer never got its SDK).
 const MEMORY_SAMPLE_INTERVAL_MS = 60_000;
 
+// One extra sample this long after `ready`, so a short-lived process still
+// reports its boot footprint. `ready` lands ~1.8s in and `VmHWM` stops
+// climbing at ~2s, so 3s captures the peak and still reports inside the
+// window a killed process survives.
+const BOOT_MEMORY_SAMPLE_DELAY_MS = 3_000;
+
 // `KEEP_THESE_FROM_BACKEND` in `scripts/build-backend.ts` mirrors this
 // directory into the on-device bundle.
 const MIGRATIONS_FOLDER_PATH = fileURLToPath(
@@ -333,11 +340,22 @@ async function withPhase(phase, fn) {
  * iOS suspends the app, a background suspension is correctly NOT counted as delay.
  */
 function startMemorySampler() {
+  // Boot sample. The log line is unconditional — one line per launch, no PII,
+  // and it is what `scripts/benchmark-boot.sh` reads to compare builds on a
+  // device that may have telemetry switched off entirely. The metric emission
+  // is still gated, by `backendMemorySample` itself.
+  const bootTimer = setTimeout(() => {
+    const snapshot = memorySnapshot();
+    console.log(`[comapeo.memory] boot ${JSON.stringify(snapshot)}`);
+    metrics.backendMemorySample(snapshot, "boot");
+  }, BOOT_MEMORY_SAMPLE_DELAY_MS);
+  bootTimer.unref?.();
+
   if (!metrics.isEnabled()) return;
   const eld = monitorEventLoopDelay({ resolution: 10 });
   eld.enable();
   const timer = setInterval(() => {
-    metrics.backendMemorySample();
+    metrics.backendMemorySample(memorySnapshot(), "interval");
     metrics.eventLoopDelaySample(eld.max / 1e6);
     eld.reset();
   }, MEMORY_SAMPLE_INTERVAL_MS);

backend/lib/memory-snapshot.js

@@ -0,0 +1,131 @@
+// V8 heap statistics plus, where `/proc` exists (Android only), whole-process
+// memory. What each number means and why it is the one reported:
+// docs/BENCHMARKING.md.
+
+import fs from "node:fs";
+import v8 from "node:v8";
+
+const PROC_SELF_STATUS = "/proc/self/status";
+
+/**
+ * nodejs-mobile revision (`process.versions.mobile`, e.g. `24.19.0-0`). Shared
+ * so the Sentry `nodejs_mobile` event tag and the `runtime` metric attribute
+ * always join on the same value.
+ *
+ * @param {Record<string, string | undefined>} [versions]
+ */
+export function runtimeVersion(versions = process.versions) {
+  return versions.mobile ?? "unknown";
+}
+
+/**
+ * @typedef {{
+ *   rssBytes: number,
+ *   peakRssBytes: number,
+ *   anonBytes: number,
+ *   fileBytes: number,
+ *   swapBytes: number,
+ * }} ProcessMemory
+ *
+ * @typedef {{
+ *   usedBytes: number,
+ *   physicalBytes: number,
+ *   totalBytes: number,
+ *   limitBytes: number,
+ *   externalBytes: number,
+ * }} HeapStats
+ *
+ * @typedef {{
+ *   runtime: string,
+ *   heap: HeapStats,
+ *   process: ProcessMemory | null,
+ * }} MemorySnapshot
+ */
+
+/** `/proc` fields we care about, all reported in kB. */
+const PROC_FIELDS = {
+  VmRSS: "rssBytes",
+  VmHWM: "peakRssBytes",
+  RssAnon: "anonBytes",
+  RssFile: "fileBytes",
+  VmSwap: "swapBytes",
+};
+
+/**
+ * Parses the `Key: <n> kB` lines of `/proc/<pid>/status` into bytes.
+ *
+ * @param {string} text Contents of a `/proc/<pid>/status` file.
+ * @returns {Record<string, number>} Bytes, keyed by the names in `PROC_FIELDS`.
+ */
+export function parseProcStatus(text) {
+  /** @type {Record<string, number>} */
+  const out = {};
+  for (const line of text.split("\n")) {
+    const match = /^(\w+):\s+(\d+)\s*kB$/.exec(line);
+    if (!match) continue;
+    const key = PROC_FIELDS[/** @type {keyof typeof PROC_FIELDS} */ (match[1])];
+    if (key) out[key] = Number(match[2]) * 1024;
+  }
+  return out;
+}
+
+/**
+ * Whole-process memory from `/proc/<pid>/status` text, or `null` when the
+ * meaningful fields are absent.
+ *
+ * @param {string} text Contents of a `/proc/<pid>/status` file.
+ * @returns {ProcessMemory | null}
+ */
+export function processMemoryFrom(text) {
+  const parsed = parseProcStatus(text);
+  // VmRSS and VmHWM are the two that carry the meaning; without them the
+  // sample is not worth emitting.
+  if (parsed.rssBytes === undefined || parsed.peakRssBytes === undefined) {
+    return null;
+  }
+  return {
+    rssBytes: parsed.rssBytes,
+    peakRssBytes: parsed.peakRssBytes,
+    anonBytes: parsed.anonBytes ?? 0,
+    fileBytes: parsed.fileBytes ?? 0,
+    swapBytes: parsed.swapBytes ?? 0,
+  };
+}
+
+/**
+ * The V8 heap fields we report, in bytes.
+ *
+ * @param {v8.HeapInfo} heap
+ * @returns {HeapStats}
+ */
+export function heapStatsFrom(heap) {
+  return {
+    usedBytes: heap.used_heap_size,
+    physicalBytes: heap.total_physical_size,
+    totalBytes: heap.total_heap_size,
+    limitBytes: heap.heap_size_limit,
+    externalBytes: heap.external_memory ?? 0,
+  };
+}
+
+/**
+ * One combined snapshot, `runtime` included so the log line names the
+ * nodejs-mobile build it measured. `process` is `null` where `/proc` is
+ * unavailable (iOS) or unreadable — best-effort by design: this is telemetry,
+ * never a boot dependency.
+ *
+ * @returns {MemorySnapshot}
+ */
+export function memorySnapshot() {
+  let process_ = null;
+  try {
+    process_ = processMemoryFrom(fs.readFileSync(PROC_SELF_STATUS, "utf8"));
+  } catch {
+    // No `/proc` here.
+  }
+  return {
+    runtime: runtimeVersion(),
+    heap: heapStatsFrom(v8.getHeapStatistics()),
+    process: process_,
+  };
+}

backend/lib/memory-snapshot.test.mjs

@@ -0,0 +1,117 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  heapStatsFrom,
+  memorySnapshot,
+  parseProcStatus,
+  processMemoryFrom,
+  runtimeVersion,
+} from "./memory-snapshot.js";
+
+const PROC_STATUS = `Name:\tnode
+Umask:\t0077
+State:\tS (sleeping)
+Tgid:\t14704
+VmPeak:\t26003584 kB
+VmSize:\t25937544 kB
+VmLck:\t       0 kB
+VmHWM:\t  261000 kB
+VmRSS:\t  192600 kB
+RssAnon:\t   92500 kB
+RssFile:\t   99100 kB
+RssShmem:\t     580 kB
+VmSwap:\t   26500 kB
+Threads:\t14
+`;
+
+/** @type {import("node:v8").HeapInfo} */
+const HEAP = {
+  total_heap_size: 35_233_792,
+  total_heap_size_executable: 2_838_528,
+  total_physical_size: 27_484_160,
+  total_available_size: 551_480_616,
+  used_heap_size: 16_598_748,
+  heap_size_limit: 569_114_624,
+  malloced_memory: 548_908,
+  peak_malloced_memory: 28_204_576,
+  does_zap_garbage: 0,
+  number_of_native_contexts: 1,
+  number_of_detached_contexts: 0,
+  total_global_handles_size: 8_192,
+  used_global_handles_size: 4_096,
+  external_memory: 10_081_653,
+};
+
+test("parseProcStatus converts the kB fields we care about to bytes", () => {
+  const parsed = parseProcStatus(PROC_STATUS);
+  assert.equal(parsed.peakRssBytes, 261_000 * 1024);
+  assert.equal(parsed.rssBytes, 192_600 * 1024);
+  assert.equal(parsed.anonBytes, 92_500 * 1024);
+  assert.equal(parsed.fileBytes, 99_100 * 1024);
+  assert.equal(parsed.swapBytes, 26_500 * 1024);
+});
+
+test("parseProcStatus ignores fields outside the allowlist", () => {
+  const parsed = parseProcStatus(PROC_STATUS);
+  assert.equal(parsed.VmPeak, undefined);
+  assert.equal(parsed.VmSize, undefined);
+  assert.equal(Object.keys(parsed).length, 5);
+});
+
+test("processMemoryFrom maps a full status file", () => {
+  assert.deepEqual(processMemoryFrom(PROC_STATUS), {
+    rssBytes: 192_600 * 1024,
+    peakRssBytes: 261_000 * 1024,
+    anonBytes: 92_500 * 1024,
+    fileBytes: 99_100 * 1024,
+    swapBytes: 26_500 * 1024,
+  });
+});
+
+test("processMemoryFrom returns null when the meaningful fields are missing", () => {
+  assert.equal(processMemoryFrom("Name:\tnode\nThreads:\t3\n"), null);
+});
+
+test("processMemoryFrom tolerates a status file without the optional fields", () => {
+  assert.deepEqual(processMemoryFrom("VmHWM:\t  100 kB\nVmRSS:\t   80 kB\n"), {
+    rssBytes: 80 * 1024,
+    peakRssBytes: 100 * 1024,
+    anonBytes: 0,
+    fileBytes: 0,
+    swapBytes: 0,
+  });
+});
+
+test("heapStatsFrom maps the V8 fields we report", () => {
+  assert.deepEqual(heapStatsFrom(HEAP), {
+    usedBytes: 16_598_748,
+    physicalBytes: 27_484_160,
+    totalBytes: 35_233_792,
+    limitBytes: 569_114_624,
+    externalBytes: 10_081_653,
+  });
+});
+
+test("runtimeVersion reads the nodejs-mobile revision", () => {
+  assert.equal(runtimeVersion({ mobile: "24.19.0-0" }), "24.19.0-0");
+  assert.equal(runtimeVersion({}), "unknown");
+});
+
+// The real production path, end to end. `/proc/self/status` genuinely exists
+// on Linux and genuinely doesn't on macOS/iOS, so between dev machines and CI
+// both sides of the platform gate run for real.
+test("memorySnapshot returns a well-formed snapshot of this process", () => {
+  const snapshot = memorySnapshot();
+  assert.equal(typeof snapshot.runtime, "string");
+  assert.ok(snapshot.heap.usedBytes > 0);
+  assert.ok(snapshot.heap.physicalBytes > 0);
+  assert.ok(snapshot.heap.limitBytes > 0);
+  if (process.platform === "linux") {
+    assert.ok(snapshot.process !== null);
+    assert.ok(snapshot.process.rssBytes > 0);
+    assert.ok(snapshot.process.peakRssBytes >= snapshot.process.rssBytes);
+  } else {
+    assert.equal(snapshot.process, null);
+  }
+});

backend/lib/metrics.js

@@ -18,6 +18,7 @@
 // so the chunk stays unloaded when Sentry is off.
 
 import { isForbiddenMetric } from "../before-send.js";
+import { runtimeVersion } from "./memory-snapshot.js";
 
 /** @type {typeof import("@sentry/node-core") | null} */
 let Sentry = null;
@@ -26,6 +27,7 @@ let Sentry = null;
  *   platform: string,
  *   deviceClass: string,
  *   osMajor: string,
+ *   runtime: string,
  *   applicationUsageData: boolean,
  * } | null}
  */
@@ -46,6 +48,9 @@ export function init(args) {
     platform: args.platform,
     deviceClass: args.deviceClass,
     osMajor: args.osMajor,
+    // Derived, not injected: sharing `runtimeVersion()` with the
+    // `nodejs_mobile` event tag is what lets events and metrics join.
+    runtime: runtimeVersion(),
     applicationUsageData: args.applicationUsageData,
   };
 }
@@ -67,6 +72,9 @@ function defaultTags() {
   return { platform: config?.platform ?? "unknown" };
 }
 
+// Mirrored by `asMetricAttributes()` in android's `DeviceTags.kt` and
+// `ios/DeviceTags.swift`; the names must stay in lock-step or native and
+// backend metrics stop joining.
 function deviceTags() {
   return {
     device_class: config?.deviceClass ?? "unknown",
@@ -197,19 +205,35 @@ export function syncSession(outcome, ms, peersBucket, bytesBucket) {
   }
 }
 
-// ── Backend health (60s sampler) ────────────────────────────────
+// ── Backend health (boot sample, then 60s sampler) ──────────────
 
 /**
- * Heap-used gauge. `rss` is intentionally omitted: node's `rss` is the whole
- * OS process, which on iOS is the entire app (node runs in-process), so it
- * wouldn't measure "the backend". `heapUsed` is the V8 JS heap and is
- * meaningful on both platforms.
+ * Backend footprint gauges, at the diagnostic tier. What each number measures,
+ * why the RSS pair is Android-only and why this tier: docs/BENCHMARKING.md.
+ *
+ * @param {import("./memory-snapshot.js").MemorySnapshot} snapshot
+ * @param {"boot" | "interval"} sample Which of the two call sites emitted it —
+ *   without it the boot samples are indistinguishable from the 60s series and
+ *   a population change reads as a footprint change.
  */
-export function backendMemorySample() {
-  const metrics = api();
-  if (!metrics) return;
-  const mem = process.memoryUsage();
-  gauge("comapeo.backend.heap_used_bytes", mem.heapUsed, "byte", {});
+export function backendMemorySample(snapshot, sample) {
+  if (!api()) return;
+  const attrs = {
+    ...deviceTags(),
+    runtime: config?.runtime ?? "unknown",
+    sample,
+  };
+  const values = {
+    "comapeo.backend.heap_used_bytes": snapshot.heap.usedBytes,
+    "comapeo.backend.heap_physical_bytes": snapshot.heap.physicalBytes,
+    ...(snapshot.process && {
+      "comapeo.backend.rss_bytes": snapshot.process.rssBytes,
+      "comapeo.backend.rss_peak_bytes": snapshot.process.peakRssBytes,
+    }),
+  };
+  for (const [name, value] of Object.entries(values)) {
+    gauge(name, value, "byte", attrs);
+  }
 }
 
 /**