Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ Research/
# NEVER committed. The tiny committed test fixture (*.fixture.csv) is kept.
showcase/server/data/dbip-city-lite.*
!showcase/server/data/dbip-city-lite.fixture.csv
!showcase/server/data/dbip-city-lite.ipv6.fixture.csv
.claude/worktrees/
BUG-REPORT-*.md
dist/skill/
Expand Down
8 changes: 5 additions & 3 deletions fly.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@ primary_region = 'sjc'
NODE_ENV = 'production'
# Worldwide IP->region dataset (DB-IP IP-to-City Lite, generated by
# showcase/server/scripts/refresh-dbip-dataset.mjs). Lives on the mounted
# /data volume (NOT baked into the image). Safe to set before the file exists:
# ip-geo.js degrades to 'unknown' until it is uploaded, then restart the
# machine so the lazy loader re-reads. See showcase/server/data/README.md.
# /data volume (NOT baked into the image). Safe to set before the files exist:
# ip-geo.js degrades to 'unknown' until they are uploaded, then restart the
# machine so the lazy loader re-reads. IPv4 + sibling IPv6. See
# showcase/server/data/README.md.
DBIP_DATASET_PATH = '/data/dbip-city-lite.csv'
DBIP_IPV6_DATASET_PATH = '/data/dbip-city-lite.ipv6.csv'

[http_service]
internal_port = 3847
Expand Down
2 changes: 1 addition & 1 deletion package.json

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions showcase/angular/src/app/core/stats/fsb-telemetry.types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,12 @@ export interface FSBTelemetryHeadline {
popular_agents: Array<{ label: string; uniq: number }>;
/** Latest day's coarse region aggregate with a k>=5 floor. */
popular_regions: Array<{ label: string; uniq: number }>;
/**
* Last-known coarse region per install across the retained 365-day rollups,
* k>=5 floored. Anonymous (country / US-state labels only). Powers the globe
* so location survives the 7-day raw-event wipe.
*/
users_by_region_365d?: Array<{ label: string; uniq: number }>;
/** Compatibility alias for avg_agents_per_reporting_user. */
avg_agents_per_user: number;
/** active_agents_now / active_agents_reporting_users_now. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,12 @@
appearance: none;
-webkit-appearance: none;
-moz-appearance: none;
background: transparent;
/* Blink ignores a fully transparent select background and keeps UA Field
* (white unless color-scheme is dark). --bg-card is opaque and matches the footer. */
background-color: var(--bg-card);
border: 0;
color: inherit;
color-scheme: inherit;
font: inherit;
padding: 0.25rem 1.25rem 0.25rem 0.5rem;
cursor: pointer;
Expand Down
15 changes: 10 additions & 5 deletions showcase/angular/src/app/pages/stats/stats-page.component.ts
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ export class StatsPageComponent implements OnInit, OnDestroy {
}

get accessibleGlobeData(): readonly AccessibleDatum[] {
return (this.latestFsbHeadline?.popular_regions ?? []).map((item) => ({
return this.globeRegionList.map((item) => ({
label: this.displayLabel(item.label),
value: this.fmtNum(item.uniq),
}));
Expand Down Expand Up @@ -415,8 +415,13 @@ export class StatsPageComponent implements OnInit, OnDestroy {
// an explicit "still gathering data" message when this is false rather
// than silently showing a globe with no nodes.
get hasPlottableRegions(): boolean {
const regions = this.latestFsbHeadline?.popular_regions ?? [];
return regions.some((r) => regionCentroid(r.label) !== null);
return this.globeRegionList.some((r) => regionCentroid(r.label) !== null);
}

private get globeRegionList(): readonly { label: string; uniq: number }[] {
const persistent = this.latestFsbHeadline?.users_by_region_365d;
if (persistent && persistent.length > 0) return persistent;
return this.latestFsbHeadline?.popular_regions ?? [];
}

get fanItemsLeft(): readonly FanItem[] {
Expand Down Expand Up @@ -793,7 +798,7 @@ export class StatsPageComponent implements OnInit, OnDestroy {

if (this.selectedView === 'fsb-active-now') {
const key = JSON.stringify({
regions: this.latestFsbHeadline?.popular_regions ?? [],
regions: this.globeRegionList,
reducedMotion: this.prefersReducedMotion,
theme: typeof document === 'undefined'
? ''
Expand Down Expand Up @@ -888,7 +893,7 @@ export class StatsPageComponent implements OnInit, OnDestroy {
// moderate jitter radius since we only have a single centroid per label,
// not a real distribution.
private buildGlobeRegions(): GlobeRegion[] {
const list = this.latestFsbHeadline?.popular_regions ?? [];
const list = this.globeRegionList;
const regions: GlobeRegion[] = [];
for (const { label, uniq } of list) {
const centroid = regionCentroid(label);
Expand Down
6 changes: 5 additions & 1 deletion showcase/angular/src/styles.scss
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
@import url('https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.6.0/css/all.min.css');
@import url('https://unpkg.com/@phosphor-icons/web@2.1.1/src/regular/style.css');

/* Shared showcase foundation: tokens, reset, typography, and utilities only. */
/* Shared showcase foundation: tokens, reset, typography, and utilities only.
* color-scheme is required so native form controls (the footer <select> especially)
* use a dark Field palette instead of light-on-light in Blink/Gecko. */
:root {
color-scheme: dark;
--primary: #ff6b35;
--primary-hover: #e55a2b;
--primary-gradient: linear-gradient(135deg, #ff6b35, #ff8c42);
Expand Down Expand Up @@ -45,6 +48,7 @@
}

[data-theme="light"] {
color-scheme: light;
--bg-body: #ffffff;
--bg-card: #f8f9fa;
--bg-elevated: #f1f5f9;
Expand Down
65 changes: 42 additions & 23 deletions showcase/server/data/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ live third-party geo API.**

| File | Committed? | Purpose |
| --- | --- | --- |
| `dbip-city-lite.csv` | **NO** (gitignored) | The real **worldwide** production artifact (~70 MB; ~2.1 M merged IPv4 ranges, all countries, state/subdivision granularity). Generated by the refresh script; never committed. In production it lives on the Fly `/data` volume, pointed at by `DBIP_DATASET_PATH` (see below). |
| `dbip-city-lite.fixture.csv` | **YES** | Tiny deterministic fixture (a handful of reserved IPv4 ranges) for offline geo + aggregation tests. |
| `dbip-city-lite.csv` | **NO** (gitignored) | The real **worldwide** production IPv4 artifact (~70 MB; ~2.1 M merged ranges). Generated by the refresh script; never committed. In production it lives on the Fly `/data` volume, pointed at by `DBIP_DATASET_PATH`. |
| `dbip-city-lite.ipv6.csv` | **NO** (gitignored) | Sibling IPv6 artifact (`start_hi,start_lo,end_hi,end_lo,country,subdivision`). Generate **off the 256 MB Fly VM** (the source CSV is ~685 MB). Pointed at by `DBIP_IPV6_DATASET_PATH`. |
| `dbip-city-lite.fixture.csv` | **YES** | Tiny deterministic IPv4 fixture for offline geo + aggregation tests. |
| `dbip-city-lite.ipv6.fixture.csv` | **YES** | Tiny IPv6 fixture (`2001:db8::/32`, `2405:201::/32`) so native IPv6 (Jio-like) lookups are tested offline. |
| `README.md` | YES | This file. |

`.gitignore` ignores `showcase/server/data/dbip-city-lite.*` **except**
Expand All @@ -28,27 +30,37 @@ ships.

## Dataset format

`ip-geo.js` reads a compact, sorted range table (`#`/blank lines ignored):
`ip-geo.js` reads two compact, sorted range tables (`#`/blank lines ignored):

IPv4 (`dbip-city-lite.csv`):

```
start_ip_int,end_ip_int,country,subdivision
```

where `start_ip_int`/`end_ip_int` are the inclusive **uint32** forms of the
IPv4 range bounds, sorted ascending by `start_ip_int` (the module binary-searches
on that key). The lookup is **coarse IPv4-only**: IPv6 or unparseable input
resolves to `'unknown'`.
IPv6 (`dbip-city-lite.ipv6.csv`):

```
start_hi,start_lo,end_hi,end_lo,country,subdivision
```

`start_ip_int`/`end_ip_int` are inclusive **uint32** IPv4 bounds. IPv6 bounds are
inclusive **uint64** halves of the 128-bit address (big-endian). Each file is
sorted ascending by start. IPv4-mapped IPv6 (`::ffff:a.b.c.d`) unwraps to IPv4
before lookup. Native IPv6 uses the sibling table. Unparseable input, Fly 6PN
ULA, and CIDR `/56` rate-limit keys resolve to `'unknown'`.

## Dataset path / env var

`ip-geo.js` resolves its dataset from:

```
process.env.DBIP_DATASET_PATH || <this dir>/dbip-city-lite.csv
process.env.DBIP_IPV6_DATASET_PATH || sibling of the IPv4 path (*.ipv6.csv / *.ipv6.fixture.csv)
```

Set `DBIP_DATASET_PATH` to point at the fixture in tests, or at an alternate
production path in deployment.
Set `DBIP_DATASET_PATH` to point at the fixture in tests (the IPv6 sibling is
resolved automatically), or set both paths in deployment.

## Graceful degradation (hard requirement)

Expand All @@ -67,36 +79,43 @@ curl -L -o /tmp/dbip-city-lite.csv.gz \
https://download.db-ip.com/free/dbip-city-lite-YYYY-MM.csv.gz # ~85 MB gz
gunzip -k /tmp/dbip-city-lite.csv.gz # ~685 MB

# 2. Transform -> compact, sorted, IPv4-only, ADJACENT-SAME-REGION-MERGED table:
# 2. Transform -> compact, sorted, ADJACENT-SAME-REGION-MERGED IPv4 + IPv6 tables.
# Do this on a machine with RAM to spare (not the 256 MB Fly VM):
node showcase/server/scripts/refresh-dbip-dataset.mjs --in /tmp/dbip-city-lite.csv
# (optional: --out <path>; defaults to DBIP_DATASET_PATH or <this dir>/dbip-city-lite.csv)
# (optional: --out <path> --ipv6-out <path>; defaults to DBIP_DATASET_PATH /
# a sibling *.ipv6.csv, or <this dir>/dbip-city-lite.csv and .ipv6.csv)
```

The transform is worldwide (all countries) and emits IPv4 ranges only; it merges
consecutive same-`(country, subdivision)` adjacent ranges (a ~40%+ row reduction)
so the output stays small. A 2026-06 run produced ~2.1 M ranges / ~70 MB from
~3.7 M raw IPv4 rows. Running the script with no `--in` prints the download URL +
the format spec (and the DB-IP attribution) and exits non-zero.
The transform is worldwide (all countries). It merges consecutive
same-`(country, subdivision)` adjacent ranges so the output stays small. A 2026-06
IPv4 run produced ~2.1 M ranges / ~70 MB from ~3.7 M raw IPv4 rows. IPv6 is
written to the sibling file. Running the script with no `--in` prints the
download URL + the format spec (and the DB-IP attribution) and exits non-zero.

The loader reads this file in 1 MiB chunks into compact typed arrays (no
whole-file Buffer/string), so even the worldwide dataset loads with a ~25–30 MB
footprint — comfortably within the 256 MB Fly VM, and robust as the dataset grows.

## Production deployment (Fly `/data` volume)

`DBIP_DATASET_PATH = '/data/dbip-city-lite.csv'` is set in `fly.toml`. The dataset
is **not** baked into the Docker image (it would bloat every deploy); it lives on
the existing `fsb_data` volume mounted at `/data`. To deploy/refresh:
`DBIP_DATASET_PATH = '/data/dbip-city-lite.csv'` and
`DBIP_IPV6_DATASET_PATH = '/data/dbip-city-lite.ipv6.csv'` are set in `fly.toml`.
The datasets are **not** baked into the Docker image (they would bloat every
deploy); they live on the existing `fsb_data` volume mounted at `/data`. To
deploy/refresh:

```bash
# After generating the file (above), deploy the app (env var already set):
# After generating both files (above), deploy the app (env vars already set):
fly deploy

# Upload the generated dataset (written to this dir by the refresh script) to the volume:
fly ssh sftp shell -a fsb-server # then: put showcase/server/data/dbip-city-lite.csv /data/dbip-city-lite.csv
# Upload the generated datasets to the volume:
fly ssh sftp shell -a fsb-server
# then:
# put showcase/server/data/dbip-city-lite.csv /data/dbip-city-lite.csv
# put showcase/server/data/dbip-city-lite.ipv6.csv /data/dbip-city-lite.ipv6.csv

# The lazy loader caches a "no dataset" result on first miss, so restart the
# machine once after the upload so it re-reads the now-present file:
# machine once after the upload so it re-reads the now-present files:
fly machine restart -a fsb-server
```

Expand Down
12 changes: 12 additions & 0 deletions showcase/server/data/dbip-city-lite.ipv6.fixture.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# dbip-city-lite.ipv6.fixture.csv -- TINY committed IPv6 test fixture.
#
# Format: start_hi,start_lo,end_hi,end_lo,country,subdivision
# hi/lo are inclusive uint64 halves of the 128-bit IPv6 address (big-endian).
# Sorted ascending by (start_hi, start_lo).
#
# 2001:db8::/32 -> AU/Victoria (docs prefix; not a real geo)
# 2405:201::/32 -> IN/Maharashtra (synthetic stand-in for Indian IPv6 / Jio)
#
# Attribution for the real dataset: IP Geolocation by DB-IP (https://db-ip.com), CC-BY-4.0.
2306139568115548160,0,2306139572410515455,18446744073709551615,AU,Victoria
2595482963567181824,0,2595482967862149119,18446744073709551615,IN,Maharashtra
Loading
Loading