diff --git a/dsm_client/deterministic_state_machine/dsm/src/ccb/decode.rs b/dsm_client/deterministic_state_machine/dsm/src/ccb/decode.rs index ec8dfcb3e..f2caee37f 100644 --- a/dsm_client/deterministic_state_machine/dsm/src/ccb/decode.rs +++ b/dsm_client/deterministic_state_machine/dsm/src/ccb/decode.rs @@ -302,13 +302,20 @@ pub fn decode_vault_state(bytes: &[u8]) -> Result { // 14 StorageSet. c.envelope(class::STORAGE_SET, StorageSetMembers::SCHEMA)?; let member_count = c.u32()?; - let mut members: Vec> = Vec::new(); + let mut entries: Vec<(Vec, [u8; 32])> = Vec::new(); for _ in 0..member_count { let len = c.u32()? as usize; - members.push(c.take(len)?.to_vec()); + let member_id = c.take(len)?.to_vec(); + // The incarnation is part of the entry, not a trailing array: a + // truncated stream fails here rather than producing a set whose + // members have lost their incarnations. + entries.push((member_id, c.digest32()?)); } - let member_refs: Vec<&[u8]> = members.iter().map(|m| m.as_slice()).collect(); - let storage_set = StorageSetMembers::new(&member_refs).map_err(invalid)?; + let entry_refs: Vec<(&[u8], [u8; 32])> = entries + .iter() + .map(|(id, inc)| (id.as_slice(), *inc)) + .collect(); + let storage_set = StorageSetMembers::new(&entry_refs).map_err(invalid)?; let quorum = c.u32()?; // 15 diff --git a/dsm_client/deterministic_state_machine/dsm/src/ccb/mod.rs b/dsm_client/deterministic_state_machine/dsm/src/ccb/mod.rs index 700e4c5db..e1176c706 100644 --- a/dsm_client/deterministic_state_machine/dsm/src/ccb/mod.rs +++ b/dsm_client/deterministic_state_machine/dsm/src/ccb/mod.rs @@ -55,8 +55,8 @@ pub use devtree::{ }; pub use genesis::{genesis_v3_commitment, sigalg, GenesisParamsV3}; pub use state::{ - EncumbranceClaim, EncumbranceSet, FeePolicy, MarketPolicy, ReleasePolicy, StorageSetMembers, - VaultStateV2, + EncumbranceClaim, EncumbranceSet, FeePolicy, MarketPolicy, ReleasePolicy, StorageSetEntry, + StorageSetMembers, VaultStateV2, }; /// Object-class discriminants, from the single namespace of registry §3. @@ -201,7 +201,17 @@ pub mod schema { pub const BURNED: &[(u16, u16)] = &[ (super::class::VAULT_STATE_V2, 1), (super::class::VAULT_STATE_V2, 2), + // Schema 3 nested `0x0002` at schema 2, i.e. a storage set of bare + // member ids. The register-incarnation cut makes field 14 a set of + // `(member_id, register_incarnation_id)` pairs, and §2.7 nests by + // complete CCB — so the enclosing bytes differ even though the field + // list did not move. + (super::class::VAULT_STATE_V2, 3), (super::class::STORAGE_SET, 1), + // A set of bare member ids says only WHICH NODES a vault trusts. A + // member that rebuilt its register still satisfied it, which is the + // ambiguity schema 3 removes. + (super::class::STORAGE_SET, 2), (super::class::ENCUMBRANCE_CLAIM, 1), (super::class::ENCUMBRANCE_SET, 1), ]; @@ -255,6 +265,10 @@ pub enum CcbError { DuplicateSetElement { class: u16 }, /// A storage set had no members, or a member id was empty. EmptyStorageSetOrMember, + /// A storage-set entry carried an all-zero register incarnation — the + /// value a member has before it has established one. Committing it would + /// bind a vault to an incarnation the member had not yet decided. + ZeroRegisterIncarnation, /// `token_a_policy_commit` was not strictly less than `token_b`. TokenPairNotStrictlyOrdered, /// `fee_bps` was at or above the denominator. @@ -336,6 +350,11 @@ impl core::fmt::Display for CcbError { CcbError::EmptyStorageSetOrMember => { write!(f, "storage set: at least one member, and no empty member id") } + CcbError::ZeroRegisterIncarnation => write!( + f, + "storage set: a member's register incarnation is all zero, which is the \ + value it has before establishing one" + ), CcbError::TokenPairNotStrictlyOrdered => write!( f, "market policy: token_a must be strictly less than token_b; \ @@ -560,11 +579,15 @@ pub fn parent_state_commitment_for_successor_of( /// `storage_set_id = H_dom(DSM/storage-set, CCB(S))`. /// -/// An ordinary CCB object under an ordinary domain. Both halves changed with -/// the cut: the frozen envelope-less layout became `0x0002` schema 2, and the -/// `DSM/storage-set/v1` tag that named it is burned in favour of the -/// normative `DSM/storage-set`. Set ids therefore differ from the deployed -/// ones, which is the reprovision rather than a regression. +/// An ordinary CCB object under an ordinary domain, over the canonical +/// ordered list of `(member_id, register_incarnation_id)` pairs. +/// +/// The id is therefore not merely *which nodes* a vault trusts, but *which +/// durable register histories on those nodes*. That is the whole authority +/// commitment: a member that rebuilt its register is a different entry, so it +/// resolves to a different set id and cannot serve the vault that committed +/// the old one. Set ids differ from schema-2 ones, which is the reprovision +/// rather than a regression. pub fn storage_set_id(members: &StorageSetMembers) -> Result<[u8; 32], CcbError> { let body = members.encode()?; let mut h: Hasher = dsm_domain_hasher(TAG_DSM_STORAGE_SET); @@ -609,7 +632,7 @@ mod dlv_policy_digest_tests { iteration_budget: None, parent_state_commitment: [0; 32], owner_authority_transition_digest: [0; 32], - storage_set: StorageSetMembers::new(&[&[9u8; 32][..]]).expect("set"), + storage_set: StorageSetMembers::new(&[(&[9u8; 32][..], [0xE1; 32])]).expect("set"), quorum: 1, }; let sa = diff --git a/dsm_client/deterministic_state_machine/dsm/src/ccb/state.rs b/dsm_client/deterministic_state_machine/dsm/src/ccb/state.rs index e78678f21..a2e796fa3 100644 --- a/dsm_client/deterministic_state_machine/dsm/src/ccb/state.rs +++ b/dsm_client/deterministic_state_machine/dsm/src/ccb/state.rs @@ -7,60 +7,127 @@ use super::{ push_u32, push_u64, CcbError, CcbObject, FEE_DENOMINATOR, }; -/// `0x0002` schema 2 — the committed storage set, an ordinary CCB object. +/// One committed set entry: a member, and the register incarnation that +/// member was serving when the vault committed this set. +/// +/// The pair is ONE authority fact — "this member, in this register +/// incarnation" — so it is one object rather than two index-aligned arrays. +/// `member_id` stays independently readable, because a resolver still needs +/// it to find the member's endpoint; `register_incarnation_id` stays +/// independently verifiable, because every read requires the responding +/// member to echo the exact value committed here. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageSetEntry { + member_id: Vec, + register_incarnation_id: [u8; 32], +} + +impl StorageSetEntry { + pub fn member_id(&self) -> &[u8] { + &self.member_id + } + + pub fn register_incarnation_id(&self) -> [u8; 32] { + self.register_incarnation_id + } +} + +/// `0x0002` schema 3 — the committed storage set, an ordinary CCB object. /// /// Schema 1 froze an envelope-less layout because deployed anchors committed -/// set ids under it. The state-identity cut deletes those anchors, so the -/// exception is gone and this class carries the §2.1 envelope like every -/// other. The special-case warning that used to live here is deleted with the -/// special case. +/// set ids under it. The state-identity cut deleted those anchors, so schema 2 +/// carried the §2.1 envelope like every other class. +/// +/// Schema 3 changes WHAT is committed, not just how. A set of bare node ids +/// says only *which nodes*; a member that lost and rebuilt its register still +/// satisfies it, and can then assert emptiness for a cell the real incarnation +/// once held — an undetectable substitution, because owning the node identity +/// was the whole test. An entry is now the pair, so the set id commits to +/// *which durable register histories on those nodes*, and a rebuilt member +/// cannot impersonate continuity merely by still holding its identity key. +/// +/// Set ids therefore differ from schema-2 ones. That is the reprovision, and +/// it is the point: an ambiguous authority encoding is not worth preserving. #[derive(Debug, Clone, PartialEq, Eq)] pub struct StorageSetMembers { - members: Vec>, + entries: Vec, } impl CcbObject for StorageSetMembers { const CLASS: u16 = class::STORAGE_SET; - const SCHEMA: u16 = 2; + const SCHEMA: u16 = 3; } impl StorageSetMembers { - /// Sorts the ids and refuses an empty set, an empty id, or a duplicate. + /// Sorts by MEMBER ID and refuses an empty set, an empty member id, a + /// duplicate member id, or a zero incarnation. + /// + /// Sorting is by `member_id` alone, never by the pair: sorting on the + /// whole entry would let one member appear twice under two incarnations + /// and still produce a strictly ascending list, which is exactly the + /// ambiguity this schema exists to remove. A duplicate member id is + /// therefore refused REGARDLESS of incarnation. /// - /// Sorting is canonicalization the format calls for; refusing duplicates - /// is not. A duplicate is a producer bug, and collapsing it would map two - /// logical inputs onto one encoding. - pub fn new(member_ids: &[&[u8]]) -> Result { - if member_ids.is_empty() || member_ids.iter().any(|id| id.is_empty()) { + /// An all-zero incarnation is refused because that is the value a member + /// has before it has ever established one; committing it would bind a + /// vault to "whatever this node had not yet decided". + pub fn new(entries: &[(&[u8], [u8; 32])]) -> Result { + if entries.is_empty() || entries.iter().any(|(id, _)| id.is_empty()) { return Err(CcbError::EmptyStorageSetOrMember); } - let mut members: Vec> = member_ids.iter().map(|id| id.to_vec()).collect(); - members.sort_unstable(); - if members.windows(2).any(|w| w[0] == w[1]) { + if entries.iter().any(|(_, inc)| inc == &[0u8; 32]) { + return Err(CcbError::ZeroRegisterIncarnation); + } + let mut entries: Vec = entries + .iter() + .map(|(id, inc)| StorageSetEntry { + member_id: id.to_vec(), + register_incarnation_id: *inc, + }) + .collect(); + entries.sort_by(|a, b| a.member_id.cmp(&b.member_id)); + if entries.windows(2).any(|w| w[0].member_id == w[1].member_id) { return Err(CcbError::DuplicateSetElement { class: class::STORAGE_SET, }); } - Ok(Self { members }) + Ok(Self { entries }) } pub fn len(&self) -> usize { - self.members.len() + self.entries.len() } pub fn is_empty(&self) -> bool { - self.members.is_empty() + self.entries.is_empty() + } + + /// The committed entries, ascending by member id. + pub fn entries(&self) -> &[StorageSetEntry] { + &self.entries } - /// `envelope ‖ u32_be(count) ‖ for each id in ascending byte order: - /// u32_be(len) ‖ id`. + /// The incarnation this set commits for `member_id`, if it is a member. + /// + /// A reader resolves an endpoint by member id and then requires THIS + /// value back from whatever answers there. + pub fn register_incarnation_of(&self, member_id: &[u8]) -> Option<[u8; 32]> { + self.entries + .iter() + .find(|e| e.member_id == member_id) + .map(|e| e.register_incarnation_id) + } + + /// `envelope ‖ u32_be(count) ‖ for each entry in ascending member-id + /// order: u32_be(len) ‖ member_id ‖ register_incarnation_id`. pub fn encode(&self) -> Result, CcbError> { let mut out = Vec::new(); push_envelope::(&mut out); - let count = u32::try_from(self.members.len()).map_err(|_| CcbError::LengthOverflow)?; + let count = u32::try_from(self.entries.len()).map_err(|_| CcbError::LengthOverflow)?; push_u32(&mut out, count); - for id in &self.members { - push_bytes(&mut out, id)?; + for e in &self.entries { + push_bytes(&mut out, &e.member_id)?; + push_digest32(&mut out, &e.register_incarnation_id); } Ok(out) } @@ -290,10 +357,13 @@ pub struct VaultStateV2 { impl CcbObject for VaultStateV2 { const CLASS: u16 = class::VAULT_STATE_V2; - /// Schema 3. Schema 2 named field 13 but nested `0x0002`/`0x0005` at - /// schema 1; §2.7 nests by complete CCB, so its bytes differ from these - /// despite an identical field list. - const SCHEMA: u16 = 3; + /// Schema 4. The field list is unchanged from schema 3; field 14 now + /// nests `0x0002` at schema 3 (the storage set carries each member's + /// register incarnation), and §2.7 nests by complete CCB including the + /// nested schema version — so a nested bump propagates upward whether or + /// not this object's own fields moved. Schemas 1 and 2 are burned for the + /// same reason. + const SCHEMA: u16 = 4; } impl VaultStateV2 { diff --git a/dsm_client/deterministic_state_machine/dsm/src/economic/lineage.rs b/dsm_client/deterministic_state_machine/dsm/src/economic/lineage.rs index 941467983..ab9276d0d 100644 --- a/dsm_client/deterministic_state_machine/dsm/src/economic/lineage.rs +++ b/dsm_client/deterministic_state_machine/dsm/src/economic/lineage.rs @@ -559,16 +559,31 @@ pub fn advance_validated( // The canonical register set for the claimant's network, resolved // FAIL-CLOSED: an unknown network refuses rather than defaulting, and a // winning claim naming any other set is foreign whatever its bytes say. - let canonical_set = crate::economic::register::resolve_root_register_profile(network_id) - .map_err(|e| { + let profile = + crate::economic::register::resolve_root_register_profile(network_id).map_err(|e| { EconomicValidationError::Provenance(ProvenanceError::FaucetWinnerInvalid(match e { crate::economic::register::RegisterResolutionError::UnknownNetwork { .. } => { "no register profile for the claimant's network" } _ => "register profile not derivable", })) - })? - .storage_set_id; + })?; + // The set id is a function of `(member_id, register_incarnation_id)` + // pairs, so it is re-derived from what the resolver offers and refused + // unless the membership is exactly this network's. The resolver supplies + // candidates; this is where they stop being taken on trust. + let candidate = resolver + .root_register_candidate_set(network_id) + .map_err(|_| { + EconomicValidationError::Provenance(ProvenanceError::FaucetWinnerInvalid( + "the network's register set could not be resolved", + )) + })?; + let canonical_set = profile.derive_set_id(&candidate).map_err(|_| { + EconomicValidationError::Provenance(ProvenanceError::FaucetWinnerInvalid( + "resolved register membership is not the network's canonical membership", + )) + })?; let ctx = ProvenanceContext { genesis, device_id, diff --git a/dsm_client/deterministic_state_machine/dsm/src/economic/peer_lineage.rs b/dsm_client/deterministic_state_machine/dsm/src/economic/peer_lineage.rs index caa043b0c..431bee8bc 100644 --- a/dsm_client/deterministic_state_machine/dsm/src/economic/peer_lineage.rs +++ b/dsm_client/deterministic_state_machine/dsm/src/economic/peer_lineage.rs @@ -86,6 +86,12 @@ pub trait PeerEvidenceFetcher { storage_set: &crate::ccb::StorageSetMembers, quorum: u32, ) -> crate::economic::cell_observation::CellObservation; + /// The network's root-register set as the local catalog resolves it — + /// CANDIDATE entries the caller must re-derive and check, never authority. + fn root_register_candidate_set( + &self, + network_id: &[u8], + ) -> Result; /// Exact immutable bytes at `addr` under `namespace`. fn immutable( &self, @@ -171,6 +177,13 @@ impl ProvenanceResolver for WalkingResolver<'_> { .settlement_slot_observation(vault_id, parent_sequence, storage_set, quorum) } + fn root_register_candidate_set( + &self, + network_id: &[u8], + ) -> Result { + self.fetcher.root_register_candidate_set(network_id) + } + fn immutable_evidence( &self, namespace: TaggedHashDomain<'static>, @@ -364,7 +377,16 @@ fn walk_positions( // canonical set — never sourced from transfer metadata or contacts. let profile = resolve_for_trader(&facts.network_id, expected_network_id) .map_err(|e| invalid(format!("peer network refused: {e}")))?; - if body.root_register_storage_set_id != profile.storage_set_id { + // The id is re-derived from the resolved `(member, incarnation)` + // pairs, and `derive_set_id` refuses a candidate whose membership is + // not this network's. A member that rebuilt its register is a + // different entry, so a claim written under the old incarnation no + // longer names the set this network resolves to. + let candidate = fetcher.root_register_candidate_set(&facts.network_id)?; + let expected_set_id = profile + .derive_set_id(&candidate) + .map_err(|e| invalid(format!("peer register set refused: {e}")))?; + if body.root_register_storage_set_id != expected_set_id { return Err(invalid( "claim binds a register set that is not the canonical set of the peer's \ committed network", diff --git a/dsm_client/deterministic_state_machine/dsm/src/economic/provenance.rs b/dsm_client/deterministic_state_machine/dsm/src/economic/provenance.rs index c3b37df4d..09e98be06 100644 --- a/dsm_client/deterministic_state_machine/dsm/src/economic/provenance.rs +++ b/dsm_client/deterministic_state_machine/dsm/src/economic/provenance.rs @@ -203,6 +203,21 @@ pub trait ProvenanceResolver { quorum: u32, ) -> crate::economic::cell_observation::CellObservation; + /// The network's root-register set as the local catalog resolves it. + /// + /// CANDIDATE entries, never authority. A set id is now a function of + /// `(member_id, register_incarnation_id)` pairs, and an incarnation is a + /// runtime fact a member generates once — so the pairs cannot be a + /// constant and must come from somewhere. This is that somewhere, and it + /// is deliberately named a candidate: the caller re-derives the id from + /// these entries and refuses any membership that is not the network's + /// canonical list, so a catalog that offers the wrong set is caught + /// rather than believed. + fn root_register_candidate_set( + &self, + network_id: &[u8], + ) -> Result; + /// Exact immutable bytes at `addr` under `namespace` — evidence the /// verifier itself checks (the resolver supplies bytes, never verdicts, /// so the acyclicity and verification stay in the verifier's hands). diff --git a/dsm_client/deterministic_state_machine/dsm/src/economic/register.rs b/dsm_client/deterministic_state_machine/dsm/src/economic/register.rs index 8ff8d1a1b..1e8b89a33 100644 --- a/dsm_client/deterministic_state_machine/dsm/src/economic/register.rs +++ b/dsm_client/deterministic_state_machine/dsm/src/economic/register.rs @@ -93,11 +93,41 @@ pub fn economic_root_register_key( /// one network's register into another's. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RootRegisterProfile { - pub storage_set_id: [u8; 32], pub quorum: u32, pub members: Vec>, } +impl RootRegisterProfile { + /// Re-derive this profile's set id from CANDIDATE entries, refusing any + /// candidate whose membership is not exactly this network's. + /// + /// The set id is a function of `(member_id, register_incarnation_id)` + /// pairs, and a member's incarnation is a runtime fact — a value it + /// generates once and cannot re-derive from its identity key — so it + /// cannot be a constant here. A resolver supplies the candidate pairs; + /// this function is what stops the resolver from being believed. + /// + /// A catalog entry is NEVER accepted merely for existing: the candidate's + /// member ids must equal this network's canonical members exactly, and + /// only then is the id computed from the canonical encoding of the pairs. + /// Endpoints are transport metadata and are not inputs. + pub fn derive_set_id( + &self, + candidate: &StorageSetMembers, + ) -> Result<[u8; 32], RegisterResolutionError> { + let mut want: Vec<&[u8]> = self.members.iter().map(|m| m.as_slice()).collect(); + want.sort_unstable(); + let got: Vec<&[u8]> = candidate.entries().iter().map(|e| e.member_id()).collect(); + if got != want { + return Err(RegisterResolutionError::MembershipNotCanonical { + expected: self.members.clone(), + got: got.iter().map(|m| m.to_vec()).collect(), + }); + } + storage_set_id(candidate).map_err(RegisterResolutionError::ProfileNotDerivable) + } +} + /// Why a register could not be resolved. Every variant is **fail-closed**: /// there is no default register and no fallback set. #[derive(Debug, Clone, PartialEq, Eq)] @@ -106,6 +136,12 @@ pub enum RegisterResolutionError { /// with a default — a default register is a register an attacker can /// steer traffic into. UnknownNetwork { network_id: Vec }, + /// A resolver offered a candidate set whose membership is not this + /// network's. The catalog resolves a set; it never chooses one. + MembershipNotCanonical { + expected: Vec>, + got: Vec>, + }, /// The trader's committed network is not the one being settled against. NetworkMismatch { claimed: Vec, expected: Vec }, /// The profile resolved, but its set id could not be re-derived from the @@ -122,6 +158,18 @@ impl core::fmt::Display for RegisterResolutionError { is one an attacker can steer traffic into", String::from_utf8_lossy(network_id) ), + Self::MembershipNotCanonical { expected, got } => write!( + f, + "resolved root-register membership {:?} is not this network's canonical \ + membership {:?} — the catalog resolves a set, it never chooses one", + got.iter() + .map(|m| String::from_utf8_lossy(m)) + .collect::>(), + expected + .iter() + .map(|m| String::from_utf8_lossy(m)) + .collect::>() + ), Self::NetworkMismatch { claimed, expected } => write!( f, "trader genesis commits network {:?} but this is network {:?} — a genesis \ @@ -160,12 +208,8 @@ pub fn resolve_root_register_profile( network_id: network_id.to_vec(), }); } - let set = StorageSetMembers::new(&BETA_MEMBERS) - .map_err(RegisterResolutionError::ProfileNotDerivable)?; let members: Vec> = BETA_MEMBERS.iter().map(|m| m.to_vec()).collect(); - let id = storage_set_id(&set).map_err(RegisterResolutionError::ProfileNotDerivable)?; Ok(RootRegisterProfile { - storage_set_id: id, // Req 6.13's fixed three-member profile. Read from the DLV profile // module rather than restated, so the threshold has one home. quorum: crate::dlv::beta_storage_profile::SOFI_BETA_QUORUM, diff --git a/dsm_client/deterministic_state_machine/dsm/src/types/device_state.rs b/dsm_client/deterministic_state_machine/dsm/src/types/device_state.rs index a5f353654..5e08b3e52 100644 --- a/dsm_client/deterministic_state_machine/dsm/src/types/device_state.rs +++ b/dsm_client/deterministic_state_machine/dsm/src/types/device_state.rs @@ -3188,7 +3188,8 @@ impl DeviceState { iteration_budget: None, parent_state_commitment: [0u8; 32], owner_authority_transition_digest: [0u8; 32], - storage_set: crate::ccb::StorageSetMembers::new(&[b"test-node"]).map_err(ccb)?, + storage_set: crate::ccb::StorageSetMembers::new(&[(&b"test-node"[..], [0xD1; 32])]) + .map_err(ccb)?, quorum: 1, }) } diff --git a/dsm_client/deterministic_state_machine/dsm/tests/authority_resolver_obligations.rs b/dsm_client/deterministic_state_machine/dsm/tests/authority_resolver_obligations.rs index e302a6e23..86f5e52bf 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/authority_resolver_obligations.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/authority_resolver_obligations.rs @@ -453,7 +453,12 @@ fn two_individually_valid_halves_do_not_join() { iteration_budget: None, parent_state_commitment: [0x44; 32], owner_authority_transition_digest: position, - storage_set: StorageSetMembers::new(&[b"n1", b"n2", b"n3"]).expect("fixture"), + storage_set: StorageSetMembers::new(&[ + (&b"n1"[..], [0xB1; 32]), + (&b"n2"[..], [0xB2; 32]), + (&b"n3"[..], [0xB3; 32]), + ]) + .expect("fixture"), quorum: 4, }; let vn_bytes = state.encode().expect("fixture"); diff --git a/dsm_client/deterministic_state_machine/dsm/tests/ccb_conformance.rs b/dsm_client/deterministic_state_machine/dsm/tests/ccb_conformance.rs index 4bdf5f9d1..a4662bb29 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/ccb_conformance.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/ccb_conformance.rs @@ -52,13 +52,15 @@ mod indep { [u32be(v.len() as u32), v.to_vec()].concat() } - /// §5.2 frozen layout: no envelope, count then bare length-prefixed ids in - /// ascending raw-byte order. - pub fn storage_set(mut ids: Vec>) -> Vec { - ids.sort(); - let mut out = [envelope(0x0002, 2), u32be(ids.len() as u32)].concat(); - for id in ids { + /// Schema 3: envelope, count, then for each entry in ascending MEMBER-ID + /// order a length-prefixed member id followed by its 32-byte register + /// incarnation. Sorted by member id only — never by the pair. + pub fn storage_set(mut entries: Vec<(Vec, [u8; 32])>) -> Vec { + entries.sort_by(|a, b| a.0.cmp(&b.0)); + let mut out = [envelope(0x0002, 3), u32be(entries.len() as u32)].concat(); + for (id, incarnation) in entries { out.extend(bytes_field(&id)); + out.extend_from_slice(&incarnation); } out } @@ -132,7 +134,7 @@ mod indep { Some(v) => [vec![0x01], u64be(v)].concat(), }; [ - envelope(0x0001, 3), + envelope(0x0001, 4), g_o.to_vec(), d_o.to_vec(), vault_id.to_vec(), @@ -272,12 +274,15 @@ fn parse_vault_state(bytes: &[u8]) -> ParsedVaultState { // from the registry alone that this field starts at a count — the // discriminant says so, like every other nested member. assert_eq!(c.u16(), 0x0002, "field 14 must be a StorageSet envelope"); - assert_eq!(c.u16(), 2, "storage-set schema 1 is burned"); + assert_eq!(c.u16(), 3, "storage-set schemas 1 and 2 are burned"); let member_count = c.u32(); let mut storage_members = Vec::new(); for _ in 0..member_count { let len = c.u32() as usize; storage_members.push(c.take(len).to_vec()); + // The incarnation is part of the entry; skipping it here would leave + // the cursor mid-entry and field 15 would not read as `q`. + let _incarnation = c.take(32); } let quorum = c.u32(); // 15 @@ -306,12 +311,16 @@ fn d(b: u8) -> [u8; 32] { const VAULT_ID: [u8; 32] = [0x7E; 32]; const TOKEN_A: [u8; 32] = [0x11; 32]; const TOKEN_B: [u8; 32] = [0x22; 32]; -const MEMBERS: [&[u8]; 5] = [ - b"dsm-node-3", - b"dsm-node-1", - b"dsm-node-5", - b"dsm-node-2", - b"dsm-node-4", +/// Deliberately unsorted, and the incarnations deliberately do NOT ascend +/// with the member ids: the canonical order is by MEMBER ID alone, so a test +/// whose incarnations happened to sort the same way would not detect an +/// encoder that sorted by the pair. +const MEMBERS: [(&[u8], [u8; 32]); 5] = [ + (b"dsm-node-3", [0x31; 32]), + (b"dsm-node-1", [0x95; 32]), + (b"dsm-node-5", [0x07; 32]), + (b"dsm-node-2", [0xF2; 32]), + (b"dsm-node-4", [0x64; 32]), ]; fn beta_set() -> StorageSetMembers { @@ -359,7 +368,12 @@ fn indep_state_bytes( beta, h_n, d(0xA3), - indep::storage_set(MEMBERS.iter().map(|m| m.to_vec()).collect()), + indep::storage_set( + MEMBERS + .iter() + .map(|(id, inc)| (id.to_vec(), *inc)) + .collect(), + ), 4, ) } @@ -452,7 +466,7 @@ fn the_storage_set_nests_with_an_envelope_and_still_ends_exactly_at_the_quorum_f "the layout must consume exactly its bytes" ); assert_eq!(parsed.class, 0x0001); - assert_eq!(parsed.schema, 3, "schemas 1 and 2 are burned"); + assert_eq!(parsed.schema, 4, "schemas 1, 2 and 3 are burned"); assert_eq!(parsed.generation, 7); assert_eq!(parsed.reserve_a, 4_242); assert_eq!(parsed.reserve_b, 8_888); @@ -465,7 +479,7 @@ fn the_storage_set_nests_with_an_envelope_and_still_ends_exactly_at_the_quorum_f "field 15 must read as q, not as set bytes" ); - let mut expected: Vec> = MEMBERS.iter().map(|m| m.to_vec()).collect(); + let mut expected: Vec> = MEMBERS.iter().map(|(id, _)| id.to_vec()).collect(); expected.sort(); assert_eq!(parsed.storage_members, expected); @@ -521,16 +535,20 @@ fn the_storage_set_id_is_the_ccb_construction_and_not_the_burned_one() { let members = beta_set(); let via_ccb = storage_set_id(&members).expect("id"); - let mut ids: Vec> = MEMBERS.iter().map(|m| m.to_vec()).collect(); - ids.sort(); + let mut entries: Vec<(Vec, [u8; 32])> = MEMBERS + .iter() + .map(|(id, inc)| (id.to_vec(), *inc)) + .collect(); + entries.sort_by(|a, b| a.0.cmp(&b.0)); - // New: H(DSM/storage-set ‖ 0x00 ‖ envelope ‖ count ‖ (len ‖ id)*). + // New: H(DSM/storage-set ‖ 0x00 ‖ envelope ‖ count ‖ (len ‖ id ‖ inc)*). let mut preimage = b"DSM/storage-set".to_vec(); preimage.push(0x00); - preimage.extend(indep::envelope(0x0002, 2)); - preimage.extend(indep::u32be(ids.len() as u32)); - for id in &ids { + preimage.extend(indep::envelope(0x0002, 3)); + preimage.extend(indep::u32be(entries.len() as u32)); + for (id, inc) in &entries { preimage.extend(indep::bytes_field(id)); + preimage.extend_from_slice(inc); } let expected: [u8; 32] = *blake3::hash(&preimage).as_bytes(); assert_eq!(via_ccb, expected, "the CCB construction fixes these bytes"); @@ -538,8 +556,8 @@ fn the_storage_set_id_is_the_ccb_construction_and_not_the_burned_one() { // Burned: the old tag over the envelope-less layout. let mut burned = b"DSM/storage-set/v1".to_vec(); burned.push(0x00); - burned.extend(indep::u32be(ids.len() as u32)); - for id in &ids { + burned.extend(indep::u32be(entries.len() as u32)); + for (id, _) in &entries { burned.extend(indep::bytes_field(id)); } let burned_id: [u8; 32] = *blake3::hash(&burned).as_bytes(); @@ -547,6 +565,22 @@ fn the_storage_set_id_is_the_ccb_construction_and_not_the_burned_one() { via_ccb, burned_id, "equality here would mean the frozen layout survived the cut" ); + + // Schema 2 is burned too, and it is the near miss that matters: the same + // members without their incarnations. Equality here would mean the + // incarnation is not actually an input to the id. + let mut ids_only = b"DSM/storage-set".to_vec(); + ids_only.push(0x00); + ids_only.extend(indep::envelope(0x0002, 2)); + ids_only.extend(indep::u32be(entries.len() as u32)); + for (id, _) in &entries { + ids_only.extend(indep::bytes_field(id)); + } + assert_ne!( + via_ccb, + *blake3::hash(&ids_only).as_bytes(), + "the incarnation must be an input, not decoration" + ); } /// Every live schema matches the registry, and none is burned. @@ -583,8 +617,8 @@ fn live_schemas_match_the_registry_and_none_is_burned() { // Registry §3, the live column. let expected: &[(u16, u16)] = &[ - (0x0001, 3), - (0x0002, 2), + (0x0001, 4), + (0x0002, 3), (0x0004, 2), (0x0005, 2), (0x0007, 1), @@ -625,7 +659,7 @@ fn a_nested_schema_bump_changes_the_enclosing_encoding() { // The same fields, with field 14 written at the burned storage-set schema // and nothing else changed. let mut forged = produced.clone(); - let needle = indep::envelope(0x0002, 2); + let needle = indep::envelope(0x0002, 3); let at = forged .windows(needle.len()) .position(|w| w == needle.as_slice()) @@ -736,14 +770,24 @@ fn invalid_inputs_are_refused_rather_than_normalized() { "one below is the legal maximum" ); assert!( - StorageSetMembers::new(&[b"a", b"a"]).is_err(), + StorageSetMembers::new(&[(&b"a"[..], [1; 32]), (&b"a"[..], [1; 32])]).is_err(), "a duplicate member is refused, not collapsed" ); + assert!( + StorageSetMembers::new(&[(&b"a"[..], [1; 32]), (&b"a"[..], [2; 32])]).is_err(), + "one member under two incarnations is refused REGARDLESS of incarnation — \ + sorting by the pair would have let this through as strictly ascending" + ); assert!(StorageSetMembers::new(&[]).is_err(), "an empty set"); assert!( - StorageSetMembers::new(&[b""]).is_err(), + StorageSetMembers::new(&[(&b""[..], [1; 32])]).is_err(), "an empty member id" ); + assert!( + StorageSetMembers::new(&[(&b"a"[..], [0; 32])]).is_err(), + "an all-zero incarnation is the value a member has before it has \ + established one, and is not committable" + ); let claim = EncumbranceClaim { parent_binding: d(0x01), @@ -802,7 +846,12 @@ fn a_populated_encumbrance_set_agrees_and_is_ordered_by_element_encoding() { None, h0, d(0xA3), - indep::storage_set(MEMBERS.iter().map(|m| m.to_vec()).collect()), + indep::storage_set( + MEMBERS + .iter() + .map(|(id, inc)| (id.to_vec(), *inc)) + .collect(), + ), 4, ); assert_eq!(produced, expected, "populated set must match byte for byte"); diff --git a/dsm_client/deterministic_state_machine/dsm/tests/economic_admission_lifecycle.rs b/dsm_client/deterministic_state_machine/dsm/tests/economic_admission_lifecycle.rs index 867f87449..da0ffb6c2 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/economic_admission_lifecycle.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/economic_admission_lifecycle.rs @@ -217,7 +217,8 @@ const SUBSTRATE_ADDR: [u8; 32] = [0xA4; 32]; fn canonical_set_id() -> [u8; 32] { dsm::economic::register::resolve_root_register_profile(b"dsm-testnet") .expect("beta profile") - .storage_set_id + .derive_set_id(&beta_candidate_set()) + .expect("canonical membership") } struct FaucetFixture { @@ -298,6 +299,13 @@ struct OneTicket { envelope: Vec, } impl ProvenanceResolver for OneTicket { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, _g: &[u8; 32], @@ -695,6 +703,13 @@ struct MarketRooted { } impl ProvenanceResolver for MarketRooted { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, _peer_genesis: &[u8; 32], @@ -955,3 +970,18 @@ fn policy_bytes_that_do_not_hash_to_the_leg_are_refused() { "the refusal is the hash binding, got: {msg}" ); } + +/// The beta fleet as a catalog resolves it: the network's canonical member +/// ids paired with the register incarnations those members are serving. +/// +/// A set id is a function of `(member_id, register_incarnation_id)` pairs, so +/// a fixture cannot state one as a constant — it derives it the same way +/// production does, from candidate entries the profile then checks. +fn beta_candidate_set() -> dsm::ccb::StorageSetMembers { + dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"dsm-node-3"[..], [0xC3; 32]), + ]) + .expect("beta candidate set") +} diff --git a/dsm_client/deterministic_state_machine/dsm/tests/economic_authorized_issuance.rs b/dsm_client/deterministic_state_machine/dsm/tests/economic_authorized_issuance.rs index f98db6350..037856cef 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/economic_authorized_issuance.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/economic_authorized_issuance.rs @@ -116,6 +116,13 @@ struct IssuanceResolver { } impl ProvenanceResolver for IssuanceResolver { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, _g: &[u8; 32], @@ -554,3 +561,18 @@ fn the_same_fixture_with_a_named_second_signer_verifies() { verify_transition_provenance(&fx.witness, &resolver(&fx), &ctx_for(&fx.op)) .expect("two named signers meet the threshold"); } + +/// The beta fleet as a catalog resolves it: the network's canonical member +/// ids paired with the register incarnations those members are serving. +/// +/// A set id is a function of `(member_id, register_incarnation_id)` pairs, so +/// a fixture cannot state one as a constant — it derives it the same way +/// production does, from candidate entries the profile then checks. +fn beta_candidate_set() -> dsm::ccb::StorageSetMembers { + dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"dsm-node-3"[..], [0xC3; 32]), + ]) + .expect("beta candidate set") +} diff --git a/dsm_client/deterministic_state_machine/dsm/tests/economic_dlv_owner_apply_provenance.rs b/dsm_client/deterministic_state_machine/dsm/tests/economic_dlv_owner_apply_provenance.rs index fdbd7112f..7a11e9ef1 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/economic_dlv_owner_apply_provenance.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/economic_dlv_owner_apply_provenance.rs @@ -179,6 +179,13 @@ struct ApplyResolver { } impl ProvenanceResolver for ApplyResolver { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, peer_genesis: &[u8; 32], @@ -484,6 +491,14 @@ fn an_unresolvable_trader_lineage_fails_closed() { let ctx = ctx_for(&fx.apply); struct Nothing; impl ProvenanceResolver for Nothing { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result + { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, _g: &[u8; 32], @@ -531,3 +546,18 @@ fn an_unresolvable_trader_lineage_fails_closed() { other => panic!("an outage must fail closed as Incomplete, got {other:?}"), } } + +/// The beta fleet as a catalog resolves it: the network's canonical member +/// ids paired with the register incarnations those members are serving. +/// +/// A set id is a function of `(member_id, register_incarnation_id)` pairs, so +/// a fixture cannot state one as a constant — it derives it the same way +/// production does, from candidate entries the profile then checks. +fn beta_candidate_set() -> dsm::ccb::StorageSetMembers { + dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"dsm-node-3"[..], [0xC3; 32]), + ]) + .expect("beta candidate set") +} diff --git a/dsm_client/deterministic_state_machine/dsm/tests/economic_dlv_settle_provenance.rs b/dsm_client/deterministic_state_machine/dsm/tests/economic_dlv_settle_provenance.rs index 87b563a0c..c24f078dc 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/economic_dlv_settle_provenance.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/economic_dlv_settle_provenance.rs @@ -268,8 +268,12 @@ fn fixture() -> Fixture { iteration_budget: None, parent_state_commitment: [0x33; 32], owner_authority_transition_digest: ow.t0_digest, - storage_set: StorageSetMembers::new(&[b"dsm-node-1", b"dsm-node-2", b"dsm-node-3"]) - .expect("set"), + storage_set: StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"dsm-node-3"[..], [0xC3; 32]), + ]) + .expect("set"), quorum: 2, }; let c_n = vault_state_commitment(&vn).expect("c_n"); @@ -479,6 +483,13 @@ impl SettleResolver { } impl ProvenanceResolver for SettleResolver { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, peer_genesis: &[u8; 32], @@ -638,6 +649,14 @@ fn an_unobservable_slot_cell_is_retryable_and_a_divergent_one_is_quarantined() { observation: CellObservation, } impl ProvenanceResolver for Observing<'_> { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result + { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, g: &[u8; 32], @@ -1426,3 +1445,18 @@ fn an_over_paying_trade_is_refused_by_re_simulation_alone() { other => panic!("expected the re-sim refusal, got {other:?}"), } } + +/// The beta fleet as a catalog resolves it: the network's canonical member +/// ids paired with the register incarnations those members are serving. +/// +/// A set id is a function of `(member_id, register_incarnation_id)` pairs, so +/// a fixture cannot state one as a constant — it derives it the same way +/// production does, from candidate entries the profile then checks. +fn beta_candidate_set() -> dsm::ccb::StorageSetMembers { + dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"dsm-node-3"[..], [0xC3; 32]), + ]) + .expect("beta candidate set") +} diff --git a/dsm_client/deterministic_state_machine/dsm/tests/economic_lineage_register.rs b/dsm_client/deterministic_state_machine/dsm/tests/economic_lineage_register.rs index 20a454d94..3ea89c988 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/economic_lineage_register.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/economic_lineage_register.rs @@ -58,12 +58,72 @@ fn the_beta_register_resolves_to_the_three_member_fleet_at_q_two() { let p = resolve_root_register_profile(b"dsm-testnet").expect("known network"); assert_eq!(p.members.len(), 3); assert_eq!(p.quorum, 2); - // The set id is a re-derivation over the members, not a constant somebody - // typed — so a member list that drifts changes the id rather than silently + // The set id is a re-derivation over `(member, incarnation)` pairs, not a + // constant somebody typed — so a member list that drifts, or a member + // that rebuilt its register, changes the id rather than silently // resolving the old register. - let members: Vec<&[u8]> = p.members.iter().map(|m| m.as_slice()).collect(); - let set = dsm::ccb::StorageSetMembers::new(&members).expect("valid set"); - assert_eq!(p.storage_set_id, dsm::ccb::storage_set_id(&set).unwrap()); + let candidate = beta_candidate_set(); + assert_eq!( + p.derive_set_id(&candidate).expect("canonical membership"), + dsm::ccb::storage_set_id(&candidate).unwrap() + ); +} + +/// THE CATALOG RESOLVES A SET; IT NEVER CHOOSES ONE. A candidate whose +/// membership is not this network's is refused before any id is computed, +/// so a local catalog cannot steer a claim into a register of its own. +#[test] +fn a_candidate_whose_membership_is_not_the_networks_is_refused() { + let p = resolve_root_register_profile(b"dsm-testnet").expect("known network"); + + let impostor = dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"attacker-node"[..], [0xC3; 32]), + ]) + .expect("well-formed but wrong"); + match p.derive_set_id(&impostor) { + Err(RegisterResolutionError::MembershipNotCanonical { .. }) => {} + other => panic!("a foreign membership must be refused, got {other:?}"), + } + + // A SHORT set is refused too: a quorum argument over two of the three + // members is not this network's register. + let short = dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + ]) + .expect("well-formed but short"); + assert!( + p.derive_set_id(&short).is_err(), + "a subset of the members is not the set" + ); +} + +/// The incarnation is an INPUT to the id, which is the whole point: the same +/// three nodes, one of them having rebuilt its register, resolve to a +/// different set and therefore cannot serve a claim bound to the old one. +#[test] +fn one_member_rebuilding_its_register_changes_the_set_id() { + let p = resolve_root_register_profile(b"dsm-testnet").expect("known network"); + let before = p.derive_set_id(&beta_candidate_set()).expect("canonical"); + + let rebuilt = dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + // node-3 lost its register and generated a new incarnation. Same + // node, same identity key, different durable history. + (&b"dsm-node-3"[..], [0x99; 32]), + ]) + .expect("canonical membership, new incarnation"); + let after = p + .derive_set_id(&rebuilt) + .expect("membership is still canonical"); + + assert_ne!( + before, after, + "a rebuilt register must not resolve to the set it used to serve" + ); } #[test] @@ -117,7 +177,8 @@ fn a_signed_claim_round_trips_and_a_tampered_one_does_not() { let (pk, sk) = keypair(); let set = resolve_root_register_profile(b"dsm-testnet") .unwrap() - .storage_set_id; + .derive_set_id(&beta_candidate_set()) + .expect("canonical membership"); let b = body(&pk, set); let envelope = sign_economic_root_claim(&b, &sk).expect("signable"); @@ -153,7 +214,8 @@ fn a_claim_signed_for_one_position_does_not_verify_at_another() { let (pk, sk) = keypair(); let set = resolve_root_register_profile(b"dsm-testnet") .unwrap() - .storage_set_id; + .derive_set_id(&beta_candidate_set()) + .expect("canonical membership"); let at7 = body(&pk, set); let envelope = sign_economic_root_claim(&at7, &sk).expect("signable"); let verified = decode_and_verify_economic_root_claim(&envelope).expect("verifies"); @@ -172,7 +234,8 @@ fn a_member_refuses_a_claim_that_is_not_the_callers() { let (pk, sk) = keypair(); let set = resolve_root_register_profile(b"dsm-testnet") .unwrap() - .storage_set_id; + .derive_set_id(&beta_candidate_set()) + .expect("canonical membership"); let envelope = sign_economic_root_claim(&body(&pk, set), &sk).expect("signable"); let claim = decode_and_verify_economic_root_claim(&envelope).expect("verifies"); @@ -263,7 +326,8 @@ fn registering_an_arbitrary_root_yields_nothing_validated() { admission_manifest_addr: [0xDD; 32], storage_set_id: resolve_root_register_profile(b"dsm-testnet") .unwrap() - .storage_set_id, + .derive_set_id(&beta_candidate_set()) + .expect("canonical membership"), }; assert_eq!( registered.register_key(), @@ -294,7 +358,8 @@ fn a_decodable_but_noncanonical_envelope_is_refused() { let (pk, sk) = keypair(); let set = resolve_root_register_profile(b"dsm-testnet") .unwrap() - .storage_set_id; + .derive_set_id(&beta_candidate_set()) + .expect("canonical membership"); let envelope = sign_economic_root_claim(&body(&pk, set), &sk).expect("signable"); assert!(decode_and_verify_economic_root_claim(&envelope).is_ok()); @@ -310,3 +375,14 @@ fn a_decodable_but_noncanonical_envelope_is_refused() { other => panic!("a non-canonical envelope must be refused, got {other:?}"), } } + +/// The beta fleet as a catalog resolves it: the network's canonical member +/// ids paired with the register incarnations those members are serving. +fn beta_candidate_set() -> dsm::ccb::StorageSetMembers { + dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"dsm-node-3"[..], [0xC3; 32]), + ]) + .expect("beta candidate set") +} diff --git a/dsm_client/deterministic_state_machine/dsm/tests/economic_peer_evidence.rs b/dsm_client/deterministic_state_machine/dsm/tests/economic_peer_evidence.rs index 36f77dc4e..dd3cd84a0 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/economic_peer_evidence.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/economic_peer_evidence.rs @@ -362,6 +362,13 @@ struct OnePeer { vpt: ValidatedPeerTransition, } impl ProvenanceResolver for OnePeer { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, _g: &[u8; 32], @@ -585,6 +592,14 @@ fn the_addr_checked_acceptance_bytes_must_hash_to_the_descriptor_address() { vpt: ValidatedPeerTransition, } impl ProvenanceResolver for WrongBytes { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result + { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, _g: &[u8; 32], @@ -648,3 +663,18 @@ fn the_addr_checked_acceptance_bytes_must_hash_to_the_descriptor_address() { } let _ = acceptance_evidence_addr(b"anchor the helper in this file"); } + +/// The beta fleet as a catalog resolves it: the network's canonical member +/// ids paired with the register incarnations those members are serving. +/// +/// A set id is a function of `(member_id, register_incarnation_id)` pairs, so +/// a fixture cannot state one as a constant — it derives it the same way +/// production does, from candidate entries the profile then checks. +fn beta_candidate_set() -> dsm::ccb::StorageSetMembers { + dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"dsm-node-3"[..], [0xC3; 32]), + ]) + .expect("beta candidate set") +} diff --git a/dsm_client/deterministic_state_machine/dsm/tests/economic_provenance_semantics.rs b/dsm_client/deterministic_state_machine/dsm/tests/economic_provenance_semantics.rs index 5ba0bfe53..a404a8f5f 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/economic_provenance_semantics.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/economic_provenance_semantics.rs @@ -33,6 +33,13 @@ const VAULT: [u8; 32] = [0xCC; 32]; struct NoPeers; impl ProvenanceResolver for NoPeers { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, _g: &[u8; 32], @@ -377,3 +384,18 @@ fn a_transition_with_no_credits_needs_no_provenance() { .is_empty() ); } + +/// The beta fleet as a catalog resolves it: the network's canonical member +/// ids paired with the register incarnations those members are serving. +/// +/// A set id is a function of `(member_id, register_incarnation_id)` pairs, so +/// a fixture cannot state one as a constant — it derives it the same way +/// production does, from candidate entries the profile then checks. +fn beta_candidate_set() -> dsm::ccb::StorageSetMembers { + dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"dsm-node-3"[..], [0xC3; 32]), + ]) + .expect("beta candidate set") +} diff --git a/dsm_client/deterministic_state_machine/dsm/tests/era_faucet_wire.rs b/dsm_client/deterministic_state_machine/dsm/tests/era_faucet_wire.rs index 7c98bcc4f..36bfca1de 100644 --- a/dsm_client/deterministic_state_machine/dsm/tests/era_faucet_wire.rs +++ b/dsm_client/deterministic_state_machine/dsm/tests/era_faucet_wire.rs @@ -122,6 +122,13 @@ struct OneTicket { envelope: Vec, } impl ProvenanceResolver for OneTicket { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, _g: &[u8; 32], @@ -363,6 +370,14 @@ fn no_quorum_winner_fails_closed_and_out_of_range_is_refused() { let fx = fixture(1); struct Nothing; impl ProvenanceResolver for Nothing { + fn root_register_candidate_set( + &self, + _network_id: &[u8], + ) -> Result + { + Ok(crate::beta_candidate_set()) + } + fn validated_peer_transition( &self, _g: &[u8; 32], @@ -612,3 +627,18 @@ fn the_ticket_model_has_no_shared_state_idioms() { ); } } + +/// The beta fleet as a catalog resolves it: the network's canonical member +/// ids paired with the register incarnations those members are serving. +/// +/// A set id is a function of `(member_id, register_incarnation_id)` pairs, so +/// a fixture cannot state one as a constant — it derives it the same way +/// production does, from candidate entries the profile then checks. +fn beta_candidate_set() -> dsm::ccb::StorageSetMembers { + dsm::ccb::StorageSetMembers::new(&[ + (&b"dsm-node-1"[..], [0xC1; 32]), + (&b"dsm-node-2"[..], [0xC2; 32]), + (&b"dsm-node-3"[..], [0xC3; 32]), + ]) + .expect("beta candidate set") +} diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/economic_fixtures.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/economic_fixtures.rs index 1080f8858..75d734cac 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/economic_fixtures.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/economic_fixtures.rs @@ -66,8 +66,10 @@ pub fn install_canonical_fleet() -> FleetGuard { "protocol = \"http\"\nlan_ip = \"127.0.0.1\"\nallow_localhost = true\nports = [8080]\n", ); for i in 1..=3 { + let inc = fixture_register_incarnation(&format!("dsm-node-{i}")); cfg.push_str(&format!( - "\n[[nodes]]\nname = \"dsm-node-{i}\"\nendpoint = \"http://127.0.0.1:808{i}\"\n" + "\n[[nodes]]\nname = \"dsm-node-{i}\"\nendpoint = \"http://127.0.0.1:808{i}\"\n\ + register_incarnation = \"{inc}\"\n" )); } std::fs::write(&cfg_path, cfg).expect("write env config"); @@ -377,3 +379,23 @@ pub fn pending_state( .and_then(|h| h.pending_economic_admission().cloned()) .map(|p| p.state) } + +/// A test fleet member's register incarnation, derived from its name so every +/// fixture and every assertion agree without threading the value. +/// +/// Production derives nothing: a node's incarnation is random at its first +/// init and lives only in its own database. This stands in for "the value +/// that node reported to whoever wrote the catalog". +pub fn fixture_register_incarnation(member_id: &str) -> String { + crate::util::text_id::encode_base32_crockford(&fixture_register_incarnation_bytes(member_id)) +} + +/// The same value as bytes, for fixtures that build the COMMITTED set. +/// +/// One derivation for both sides on purpose: a committed set whose +/// incarnations differ from the catalog's resolves to a different set id, and +/// the vault's own storage set stops being resolvable — which is correct +/// behaviour and a useless test failure. +pub fn fixture_register_incarnation_bytes(member_id: &str) -> [u8; 32] { + *blake3::hash(format!("dsm-test-incarnation/{member_id}").as_bytes()).as_bytes() +} diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/artifact_republish.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/artifact_republish.rs index 07fbf3565..f2ff0369a 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/artifact_republish.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/artifact_republish.rs @@ -381,7 +381,12 @@ mod tests { #[serial] async fn an_unresolvable_frozen_set_is_held_and_sent_nowhere() { let _cat = init(); - let foreign = crate::sdk::storage_set::compute_storage_set_id(&["c", "d", "e"]).unwrap(); + let foreign = crate::sdk::storage_set::compute_storage_set_id(&[ + ("c", [0xCC; 32]), + ("d", [0xDD; 32]), + ("e", [0xEE; 32]), + ]) + .unwrap(); let d = freeze(&foreign, "sofi/foreign/latest", b"bytes"); assert_eq!(republish_unpublished_artifacts().await.unwrap(), 0); let row = fpa::get_artifact("sofi/foreign/latest", &d) diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/dlv_routes.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/dlv_routes.rs index 2a92fc4de..d1875099c 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/dlv_routes.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/dlv_routes.rs @@ -3360,12 +3360,12 @@ fn build_vault_publication_artifacts( device's catalog", ) })?; - let member_ids: Vec<&[u8]> = set + let entries: Vec<(&[u8], [u8; 32])> = set .members() .iter() - .map(|m| m.member_id.as_bytes()) + .map(|m| (m.member_id.as_bytes(), m.register_incarnation_id)) .collect(); - let storage_set = StorageSetMembers::new(&member_ids) + let storage_set = StorageSetMembers::new(&entries) .map_err(|e| DsmError::invalid_operation(format!("vault publication: set members: {e}")))?; if dsm::ccb::storage_set_id(&storage_set) .map_err(|e| DsmError::invalid_operation(format!("vault publication: set id: {e}")))? diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/faucet_flow_tests.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/faucet_flow_tests.rs index 3b05a6f4c..6ff2e040b 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/faucet_flow_tests.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/faucet_flow_tests.rs @@ -98,8 +98,10 @@ pub(crate) fn install_canonical_fleet() -> FleetGuard { "protocol = \"http\"\nlan_ip = \"127.0.0.1\"\nallow_localhost = true\nports = [8080]\n", ); for i in 1..=3 { + let inc = crate::economic_fixtures::fixture_register_incarnation(&format!("dsm-node-{i}")); cfg.push_str(&format!( - "\n[[nodes]]\nname = \"dsm-node-{i}\"\nendpoint = \"http://127.0.0.1:808{i}\"\n" + "\n[[nodes]]\nname = \"dsm-node-{i}\"\nendpoint = \"http://127.0.0.1:808{i}\"\n\ + register_incarnation = \"{inc}\"\n" )); } std::fs::write(&cfg_path, cfg).expect("write env config"); @@ -128,7 +130,14 @@ fn canonical_set() -> StorageSet { let profile = dsm::economic::register::resolve_root_register_profile(NETWORK).expect("profile"); StorageSetCatalog::from_env_config() .expect("catalog") - .resolve(&profile.storage_set_id) + .sets() + .iter() + .find(|s| { + crate::sdk::storage_set::as_ccb_members(s) + .ok() + .and_then(|m| profile.derive_set_id(&m).ok()) + .is_some() + }) .cloned() .expect("canonical set resolvable in test mode") } diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/sender_admission_tests.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/sender_admission_tests.rs index f13aa7910..7f602347f 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/sender_admission_tests.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/handlers/sender_admission_tests.rs @@ -74,7 +74,14 @@ async fn an_admitted_burn_advances_the_lineage_and_is_foreign_walkable() { dsm::economic::register::resolve_root_register_profile(NETWORK).expect("profile"); let set = crate::sdk::storage_set::StorageSetCatalog::from_env_config() .expect("catalog") - .resolve(&profile.storage_set_id) + .sets() + .iter() + .find(|s| { + crate::sdk::storage_set::as_ccb_members(s) + .ok() + .and_then(|m| profile.derive_set_id(&m).ok()) + .is_some() + }) .cloned() .expect("canonical set"); let resolver = crate::sdk::economic_registers::LiveRegisterResolver { @@ -655,7 +662,14 @@ async fn token_routes_admit_an_authorized_mint_that_is_foreign_walkable() { dsm::economic::register::resolve_root_register_profile(NETWORK).expect("profile"); let set = crate::sdk::storage_set::StorageSetCatalog::from_env_config() .expect("catalog") - .resolve(&profile.storage_set_id) + .sets() + .iter() + .find(|s| { + crate::sdk::storage_set::as_ccb_members(s) + .ok() + .and_then(|m| profile.derive_set_id(&m).ok()) + .is_some() + }) .cloned() .expect("canonical set"); let resolver = crate::sdk::economic_registers::LiveRegisterResolver { diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/ingress.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/ingress.rs index c1864ad6e..840a73518 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/ingress.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/ingress.rs @@ -1072,6 +1072,7 @@ allow_localhost = true [[nodes]] name = "test-1" endpoint = "http://127.0.0.1:8080" +register_incarnation = "BHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE0" "#; std::fs::write(&path, body).expect("write env config"); path.to_string_lossy().to_string() diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/network.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/network.rs index 10c8341f5..67527102c 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/network.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/network.rs @@ -95,6 +95,16 @@ pub struct EnvConfig { pub struct NodeConfig { pub name: String, pub endpoint: String, // e.g., "http://10.0.0.5:8080" + /// The member's durable register incarnation, Base32-Crockford over 32 + /// bytes, as that node reports it. + /// + /// REQUIRED, deliberately. A storage-set id is a function of + /// `(member_id, register_incarnation_id)` pairs, so a catalog that cannot + /// state a member's incarnation cannot resolve any set it belongs to — + /// and defaulting the field would resolve every set to whatever the + /// default hashed to. A config written before this field existed fails to + /// load, which is the reprovision. + pub register_incarnation: String, } pub struct NetworkConfigLoader; @@ -176,14 +186,23 @@ impl NetworkConfigLoader { nodes: vec![ NodeConfig { name: "test-1".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford( + &[0xC1u8; 32], + ), endpoint: "http://127.0.0.1:8080".into(), }, NodeConfig { name: "test-2".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford( + &[0xC2u8; 32], + ), endpoint: "http://127.0.0.1:8081".into(), }, NodeConfig { name: "test-3".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford( + &[0xC3u8; 32], + ), endpoint: "http://127.0.0.1:8082".into(), }, ], @@ -427,6 +446,12 @@ impl NodeRegistry { nodes.push(NodeConfig { name, endpoint: endpoint.to_string(), + // DISCOVERY IS NOT AUTHORITY. A node found at runtime has no + // known register incarnation, and inventing one would let + // discovery mint storage-set membership. The empty string fails + // `from_env_config` closed, so such a node can carry transport + // and never contribute to a set id. + register_incarnation: String::new(), }); log::info!( "NodeRegistry: added endpoint {}, total={}", @@ -683,10 +708,12 @@ ports = [8080, 8081] [[nodes]] name = "node-a" endpoint = "http://10.0.0.1:8080" +register_incarnation = "BHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE0" [[nodes]] name = "node-b" endpoint = "http://10.0.0.2:8081" +register_incarnation = "BHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE0" "# .to_string() } @@ -710,6 +737,7 @@ lan_ip = "" [[nodes]] name = "n1" endpoint = "http://1.2.3.4:80" +register_incarnation = "BHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE0" "#; let cfg = parse_env_config_toml(toml).unwrap(); assert_eq!(cfg.protocol, "http"); @@ -745,6 +773,7 @@ dbtc_dust_floor_sats = 1000 [[nodes]] name = "n1" endpoint = "http://10.0.0.5:9090" +register_incarnation = "BHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE5RQ2WBHE0" "#; let cfg = parse_env_config_toml(toml).unwrap(); assert_eq!( @@ -762,14 +791,17 @@ endpoint = "http://10.0.0.5:9090" let nodes = vec![ NodeConfig { name: "a".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://a".into(), }, NodeConfig { name: "b".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://b".into(), }, NodeConfig { name: "c".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://c".into(), }, ]; @@ -788,6 +820,7 @@ endpoint = "http://10.0.0.5:9090" fn node_registry_add_and_remove() { let nodes = vec![NodeConfig { name: "a".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://a".into(), }]; let reg = NodeRegistry::new(nodes, None); @@ -813,10 +846,12 @@ endpoint = "http://10.0.0.5:9090" let nodes = vec![ NodeConfig { name: "a".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://a".into(), }, NodeConfig { name: "b".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://b".into(), }, ]; @@ -835,6 +870,7 @@ endpoint = "http://10.0.0.5:9090" fn node_registry_clear_quarantine() { let nodes = vec![NodeConfig { name: "a".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://a".into(), }]; let reg = NodeRegistry::new(nodes, None); @@ -886,10 +922,12 @@ endpoint = "http://10.0.0.5:9090" let nodes = vec![ NodeConfig { name: "local".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://127.0.0.1:8080".into(), }, NodeConfig { name: "remote".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://10.0.0.5:9090".into(), }, ]; @@ -911,6 +949,7 @@ endpoint = "http://10.0.0.5:9090" ports: vec![8080], nodes: vec![NodeConfig { name: "n1".into(), + register_incarnation: crate::util::text_id::encode_base32_crockford(&[0x5C_u8; 32]), endpoint: "http://10.0.0.1:8080".into(), }], mpc_genesis_url: None, diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/economic_admission_flow.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/economic_admission_flow.rs index 83f6da150..17e0613ea 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/economic_admission_flow.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/economic_admission_flow.rs @@ -90,8 +90,20 @@ pub(crate) fn canonical_set(network_id: &[u8]) -> Result { .map_err(|e| storage_err("resolve root register", e))?; let catalog = StorageSetCatalog::from_env_config().map_err(|e| storage_err("load storage catalog", e))?; + // The set id is a function of `(member_id, register_incarnation_id)` + // pairs, so it cannot be asked for by name: the catalog offers candidates + // and `derive_set_id` refuses any whose membership is not this network's. + // A member that rebuilt its register therefore stops resolving here + // rather than silently serving the register it used to. catalog - .resolve(&profile.storage_set_id) + .sets() + .iter() + .find(|s| { + crate::sdk::storage_set::as_ccb_members(s) + .ok() + .and_then(|m| profile.derive_set_id(&m).ok()) + .is_some() + }) .cloned() .ok_or_else(|| { DsmError::storage( diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/economic_registers.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/economic_registers.rs index 16f69be3d..6feb40d53 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/economic_registers.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/economic_registers.rs @@ -342,6 +342,39 @@ impl LiveRegisterResolver<'_> { } impl dsm::economic::peer_lineage::PeerEvidenceFetcher for LiveRegisterResolver<'_> { + /// The network's root-register set as THIS device's catalog resolves it. + /// + /// Candidates, not authority: the caller re-derives the id from these + /// pairs and refuses a membership that is not the network's canonical + /// one, so a locally misconfigured or hostile catalog is caught rather + /// than believed. A member that rebuilt its register appears here under a + /// new incarnation and therefore changes the id it can serve. + fn root_register_candidate_set( + &self, + network_id: &[u8], + ) -> Result { + let profile = dsm::economic::register::resolve_root_register_profile(network_id) + .map_err(|e| PeerLineageFailure::Incomplete(e.to_string()))?; + let catalog = crate::sdk::storage_set::StorageSetCatalog::from_env_config() + .map_err(|e| PeerLineageFailure::Incomplete(e.to_string()))?; + // The catalog holds sets, not networks: find the one whose membership + // IS this network's, and let `derive_set_id` be the thing that decides + // whether it really is. + let candidate = catalog + .sets() + .iter() + .find_map(|s| { + let members = crate::sdk::storage_set::as_ccb_members(s).ok()?; + profile.derive_set_id(&members).ok().map(|_| members) + }) + .ok_or_else(|| { + PeerLineageFailure::Incomplete( + "no configured storage set has this network's canonical membership".into(), + ) + })?; + Ok(candidate) + } + fn register_cell(&self, k_root: &[u8; 32]) -> Result>, PeerLineageFailure> { let k = *k_root; tokio::task::block_in_place(|| self.runtime.block_on(read_economic_root_cell(self.set, &k))) @@ -552,6 +585,15 @@ impl<'a> RecordingResolver<'a> { } impl dsm::economic::peer_lineage::PeerEvidenceFetcher for RecordingResolver<'_> { + fn root_register_candidate_set( + &self, + network_id: &[u8], + ) -> Result { + dsm::economic::peer_lineage::PeerEvidenceFetcher::root_register_candidate_set( + self.inner, network_id, + ) + } + fn register_cell(&self, k_root: &[u8; 32]) -> Result>, PeerLineageFailure> { dsm::economic::peer_lineage::PeerEvidenceFetcher::register_cell(self.inner, k_root) } @@ -615,6 +657,39 @@ impl dsm::economic::peer_lineage::PeerEvidenceFetcher for RecordingResolver<'_> } impl ProvenanceResolver for LiveRegisterResolver<'_> { + /// The network's root-register set as THIS device's catalog resolves it. + /// + /// Candidates, not authority: the caller re-derives the id from these + /// pairs and refuses a membership that is not the network's canonical + /// one, so a locally misconfigured or hostile catalog is caught rather + /// than believed. A member that rebuilt its register appears here under a + /// new incarnation and therefore changes the id it can serve. + fn root_register_candidate_set( + &self, + network_id: &[u8], + ) -> Result { + let profile = dsm::economic::register::resolve_root_register_profile(network_id) + .map_err(|e| PeerLineageFailure::Incomplete(e.to_string()))?; + let catalog = crate::sdk::storage_set::StorageSetCatalog::from_env_config() + .map_err(|e| PeerLineageFailure::Incomplete(e.to_string()))?; + // The catalog holds sets, not networks: find the one whose membership + // IS this network's, and let `derive_set_id` be the thing that decides + // whether it really is. + let candidate = catalog + .sets() + .iter() + .find_map(|s| { + let members = crate::sdk::storage_set::as_ccb_members(s).ok()?; + profile.derive_set_id(&members).ok().map(|_| members) + }) + .ok_or_else(|| { + PeerLineageFailure::Incomplete( + "no configured storage set has this network's canonical membership".into(), + ) + })?; + Ok(candidate) + } + fn validated_peer_transition( &self, peer_genesis: &[u8; 32], @@ -867,6 +942,7 @@ mod tests { StorageSet::new( (1..=3) .map(|i| StorageMember { + register_incarnation_id: [0xC0 | i as u8; 32], member_id: format!("dsm-node-{i}"), endpoint: format!("http://127.0.0.1:808{i}"), }) @@ -1008,17 +1084,17 @@ mod tests { #[test] fn a_committed_set_resolves_only_from_members_that_re_derive_its_id() { let members = dsm::ccb::StorageSetMembers::new(&[ - b"dsm-node-1".as_slice(), - b"dsm-node-2".as_slice(), - b"dsm-node-3".as_slice(), + (b"dsm-node-1".as_slice(), [0xC0; 32]), + (b"dsm-node-2".as_slice(), [0xC1; 32]), + (b"dsm-node-3".as_slice(), [0xC2; 32]), ]) .expect("members"); let committed_id = dsm::ccb::storage_set_id(&members).expect("id"); let foreign = dsm::ccb::StorageSetMembers::new(&[ - b"somebody-elses-node-1".as_slice(), - b"somebody-elses-node-2".as_slice(), - b"somebody-elses-node-3".as_slice(), + (b"somebody-elses-node-1".as_slice(), [0xC0; 32]), + (b"somebody-elses-node-2".as_slice(), [0xC1; 32]), + (b"somebody-elses-node-3".as_slice(), [0xC2; 32]), ]) .expect("members"); assert_ne!( diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/settlement_slot.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/settlement_slot.rs index f5707c315..4343f9768 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/settlement_slot.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/settlement_slot.rs @@ -417,6 +417,7 @@ mod tests { StorageSet::new( (1..=3) .map(|i| StorageMember { + register_incarnation_id: [0xC0 | i as u8; 32], member_id: format!("test-{i}"), endpoint: format!("http://127.0.0.1:808{i}"), }) diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_io.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_io.rs index 18e5d3213..ce164ff3d 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_io.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_io.rs @@ -422,6 +422,10 @@ pub mod fake_registers { cells: HashMap<(String, RegisterKind), HashMap, (Vec, [u8; 32])>>, failing: HashSet, echo_override: HashMap>, + /// What a member echoes as its REGISTER INCARNATION, when it is not + /// the one the set committed. `None` models a member that answers + /// without saying which register it is serving. + incarnation_override: HashMap>, } static STATE: Mutex> = Mutex::new(None); @@ -457,6 +461,19 @@ pub mod fake_registers { }) } + /// Model a member that is no longer serving the register the set + /// committed — a lost-and-rebuilt database, or a restore from a snapshot. + /// + /// The member keeps its id and answers perfectly honestly; what changed is + /// which durable register history is answering. `None` models a member + /// that will not say which register it is serving at all. + pub fn set_register_incarnation(member_id: &str, incarnation: Option<[u8; 32]>) { + with_state(|s| { + s.incarnation_override + .insert(member_id.to_string(), incarnation); + }) + } + pub fn ticket_key(faucet_id: &[u8; 32], ticket_index: u64) -> Vec { let mut k = faucet_id.to_vec(); k.extend_from_slice(&ticket_index.to_be_bytes()); @@ -683,7 +700,19 @@ pub mod fake_registers { .get(&member.member_id) .cloned() .unwrap_or_else(|| Some(member.member_id.clone())); - if echoed.as_deref() != Some(member.member_id.as_str()) { + // BOTH halves of the echo, exactly as the live client + // folds them (`storage_node_sdk::answer_counts_for`): a + // member that rebuilt its register still answers with its + // own id, so identity alone cannot tell it apart from the + // member the vault committed. + let echoed_incarnation = s + .incarnation_override + .get(&member.member_id) + .copied() + .unwrap_or(Some(member.register_incarnation_id)); + if echoed.as_deref() != Some(member.member_id.as_str()) + || echoed_incarnation != Some(member.register_incarnation_id) + { return MemberCellRead::Unavailable; } match s diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_node_sdk.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_node_sdk.rs index 0ff7d02b1..a656f4419 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_node_sdk.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_node_sdk.rs @@ -628,6 +628,46 @@ impl std::fmt::Display for StorageNodeError { impl std::error::Error for StorageNodeError {} +/// What a member said about ITSELF alongside a register answer. +/// +/// Both halves are required for the answer to count. `node_id` says which +/// member replied; `register_incarnation` says which durable register history +/// that member is serving. A vault commits the pair at birth, so a member +/// that still owns its identity key but rebuilt its register no longer +/// matches, and its answers stop counting. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct MemberEcho { + pub node_id: Option, + pub register_incarnation: Option<[u8; 32]>, +} + +/// Whether an answer can be counted for the member the set committed. +/// +/// BOTH halves must match, and a failure of either is `Unavailable` — never +/// an absence and never a value: +/// +/// | echoed | verdict | +/// |---|---| +/// | member A, incarnation X (the committed pair) | may count | +/// | member A, incarnation Y | unavailable — this is A, but not the register history the vault named | +/// | member B, incarnation X | unavailable — attribution failure | +/// | either half missing | unavailable — an answer about nobody | +/// +/// The second row is the one this exists for. A member that lost and rebuilt +/// its register can honestly report "nothing here" for a cell the committed +/// incarnation once held, and node identity alone cannot tell that apart from +/// the real member reporting the same thing. Counting it as emptiness is the +/// undetectable substitution. It is also NOT a forgery — the node is not +/// lying, it simply is no longer the member this vault committed — which is +/// why the verdict is `Unavailable` and not an invalid-credit finding. +pub fn answer_counts_for( + echoed: &MemberEcho, + member: &crate::sdk::storage_set::StorageMember, +) -> bool { + echoed.node_id.as_deref() == Some(member.member_id.as_str()) + && echoed.register_incarnation == Some(member.register_incarnation_id) +} + impl StorageNodeClient { pub async fn new(config: StorageNodeConfig) -> Result { let client = build_ca_aware_client(); @@ -996,27 +1036,33 @@ impl StorageNodeClient { /// members manufacture an emptiness fact, which is the one observation a /// forward lineage walk treats as terminal. /// - /// The echo is NORMATIVE: a response without it, or with an id other than - /// this member's, is uncountable — the caller folds that into - /// `Unavailable` rather than counting it either way. + /// The echo is NORMATIVE, and it has TWO halves: a response without the + /// member's own node id, or without the register incarnation that member + /// is serving, is uncountable — the caller folds either into + /// `Unavailable` rather than counting it as a value or as an absence. pub async fn get_register_cell( &self, path: &str, - ) -> ( - dsm::economic::cell_observation::MemberCellRead, - Option, - ) { + ) -> (dsm::economic::cell_observation::MemberCellRead, MemberEcho) { use dsm::economic::cell_observation::MemberCellRead; let url = format!("{base}{path}", base = self.node_info.url); let response = match self.client.get(&url).send().await { Ok(r) => r, - Err(_) => return (MemberCellRead::Unavailable, None), + Err(_) => return (MemberCellRead::Unavailable, MemberEcho::default()), + }; + let echoed = MemberEcho { + node_id: response + .headers() + .get("x-dsm-node-id") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()), + register_incarnation: response + .headers() + .get("x-dsm-register-incarnation") + .and_then(|v| v.to_str().ok()) + .and_then(crate::util::text_id::decode_base32_crockford) + .and_then(|raw| <[u8; 32]>::try_from(raw).ok()), }; - let echoed = response - .headers() - .get("x-dsm-node-id") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()); let status = response.status().as_u16(); let outcome = response .headers() @@ -1576,7 +1622,7 @@ impl StorageNodeSDK { None => MemberCellRead::Unavailable, Some(c) => { let (read, echoed) = c.get_register_cell(path).await; - if echoed.as_deref() == Some(member.member_id.as_str()) { + if answer_counts_for(&echoed, member) { read } else { MemberCellRead::Unavailable @@ -5510,3 +5556,58 @@ mod tests { assert_ne!(p1, p2); } } + +#[cfg(test)] +mod member_echo_tests { + #![allow(clippy::disallowed_methods)] // unwrap/expect acceptable in deterministic tests + use super::{answer_counts_for, MemberEcho}; + use crate::sdk::storage_set::StorageMember; + + fn committed() -> StorageMember { + StorageMember { + member_id: "dsm-node-1".into(), + register_incarnation_id: [0xC1; 32], + endpoint: "http://n1.example".into(), + } + } + + fn echo(node_id: Option<&str>, incarnation: Option<[u8; 32]>) -> MemberEcho { + MemberEcho { + node_id: node_id.map(|s| s.to_string()), + register_incarnation: incarnation, + } + } + + /// The whole matrix, one case per row of the contract. + #[test] + fn only_the_committed_member_in_its_committed_incarnation_counts() { + let m = committed(); + + assert!( + answer_counts_for(&echo(Some("dsm-node-1"), Some([0xC1; 32])), &m), + "the committed pair counts" + ); + + assert!( + !answer_counts_for(&echo(Some("dsm-node-1"), Some([0x99; 32])), &m), + "SAME NODE, REBUILT REGISTER: this is the substitution the pair \ + exists to catch — it must not count, in either direction" + ); + + assert!( + !answer_counts_for(&echo(Some("dsm-node-2"), Some([0xC1; 32])), &m), + "another member answering for this one is an attribution failure" + ); + + assert!( + !answer_counts_for(&echo(None, Some([0xC1; 32])), &m), + "an answer that names no member is an answer about nobody" + ); + assert!( + !answer_counts_for(&echo(Some("dsm-node-1"), None), &m), + "a member that will not say which register it is serving has not \ + answered the question that matters" + ); + assert!(!answer_counts_for(&echo(None, None), &m), "no echo at all"); + } +} diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_set.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_set.rs index daff2a3b1..b9b5e56d7 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_set.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/storage_set.rs @@ -30,11 +30,17 @@ use dsm::types::error::DsmError; -/// One member of a storage set: its protocol identity and its current -/// transport endpoint. `endpoint` is metadata; only `member_id` is hashed. +/// One member of a storage set: its protocol identity, the register +/// incarnation it is serving, and its current transport endpoint. +/// +/// `endpoint` is transport metadata and is NOT hashed. The pair +/// `(member_id, register_incarnation_id)` is: a member that lost and rebuilt +/// its register is a different entry, so it resolves to a different set id +/// and cannot serve a vault that committed the old one. #[derive(Debug, Clone, PartialEq, Eq)] pub struct StorageMember { pub member_id: String, + pub register_incarnation_id: [u8; 32], pub endpoint: String, } @@ -58,14 +64,32 @@ pub struct StorageSet { /// Validity conditions live in the encoder too: at least one member, no empty /// id, no duplicate. Duplicates are refused rather than collapsed, since /// collapsing would map two logical inputs onto one encoding. -pub fn compute_storage_set_id(member_ids: &[&str]) -> Result<[u8; 32], DsmError> { - let ids: Vec<&[u8]> = member_ids.iter().map(|s| s.as_bytes()).collect(); - let members = dsm::ccb::StorageSetMembers::new(&ids) +pub fn compute_storage_set_id(entries: &[(&str, [u8; 32])]) -> Result<[u8; 32], DsmError> { + let pairs: Vec<(&[u8], [u8; 32])> = entries + .iter() + .map(|(id, inc)| (id.as_bytes(), *inc)) + .collect(); + let members = dsm::ccb::StorageSetMembers::new(&pairs) .map_err(|e| DsmError::invalid_operation(format!("storage set: {e}")))?; dsm::ccb::storage_set_id(&members) .map_err(|e| DsmError::invalid_operation(format!("storage set: {e}"))) } +/// This set's members as the CCB object a verifier re-derives an id from. +/// +/// Handed to callers that must prove a resolved set IS the one an authority +/// named — they recompute the id from these pairs rather than believing the +/// catalog. +pub fn as_ccb_members(set: &StorageSet) -> Result { + let pairs: Vec<(&[u8], [u8; 32])> = set + .members() + .iter() + .map(|m| (m.member_id.as_bytes(), m.register_incarnation_id)) + .collect(); + dsm::ccb::StorageSetMembers::new(&pairs) + .map_err(|e| DsmError::invalid_operation(format!("storage set: {e}"))) +} + impl StorageSet { /// Build a set from members, validating distinctness on BOTH axes: unique /// member ids (the identity the quorum counts) and injective endpoints @@ -91,8 +115,11 @@ impl StorageSet { )); } } - let ids: Vec<&str> = members.iter().map(|m| m.member_id.as_str()).collect(); - let id = compute_storage_set_id(&ids)?; + let entries: Vec<(&str, [u8; 32])> = members + .iter() + .map(|m| (m.member_id.as_str(), m.register_incarnation_id)) + .collect(); + let id = compute_storage_set_id(&entries)?; Ok(Self { id, members }) } @@ -157,11 +184,31 @@ impl StorageSetCatalog { let members: Vec = env .nodes .into_iter() - .map(|n| StorageMember { - member_id: n.name, - endpoint: n.endpoint, + .map(|n| { + // Fail closed: a member whose incarnation the config cannot + // state is a member no set id can be derived over. Decoding + // the wrong width is the same failure as omitting it. + let raw = crate::util::text_id::decode_base32_crockford(&n.register_incarnation) + .ok_or_else(|| { + DsmError::invalid_operation(format!( + "storage set: member {:?} has a register_incarnation that is not \ + Base32-Crockford", + n.name + )) + })?; + let register_incarnation_id: [u8; 32] = raw.try_into().map_err(|_| { + DsmError::invalid_operation(format!( + "storage set: member {:?} has a register_incarnation that is not 32 bytes", + n.name + )) + })?; + Ok(StorageMember { + member_id: n.name, + register_incarnation_id, + endpoint: n.endpoint, + }) }) - .collect(); + .collect::>()?; let set = StorageSet::new(members)?; Self::new(vec![set]) } @@ -171,8 +218,12 @@ impl StorageSetCatalog { /// re-hashing the entry's member ids, not by trusting a stored id. pub fn resolve(&self, storage_set_id: &[u8; 32]) -> Option<&StorageSet> { self.sets.iter().find(|s| { - let ids: Vec<&str> = s.members().iter().map(|m| m.member_id.as_str()).collect(); - compute_storage_set_id(&ids).ok().as_ref() == Some(storage_set_id) + let entries: Vec<(&str, [u8; 32])> = s + .members() + .iter() + .map(|m| (m.member_id.as_str(), m.register_incarnation_id)) + .collect(); + compute_storage_set_id(&entries).ok().as_ref() == Some(storage_set_id) }) } @@ -196,29 +247,42 @@ mod tests { use super::*; fn m(id: &str, ep: &str) -> StorageMember { + // A distinct incarnation per member id, so a test never accidentally + // asserts over a set whose entries collide. + let mut inc = [0u8; 32]; + inc[..id.len().min(32)].copy_from_slice(&id.as_bytes()[..id.len().min(32)]); + inc[31] = 0xA7; StorageMember { member_id: id.to_string(), + register_incarnation_id: inc, endpoint: ep.to_string(), } } + fn e(id: &str) -> (&str, [u8; 32]) { + let mut inc = [0u8; 32]; + inc[..id.len().min(32)].copy_from_slice(&id.as_bytes()[..id.len().min(32)]); + inc[31] = 0xA7; + (id, inc) + } + #[test] fn set_id_is_order_independent_and_length_prefixed() { - let a = compute_storage_set_id(&["n1", "n2", "n3"]).unwrap(); - let b = compute_storage_set_id(&["n3", "n1", "n2"]).unwrap(); + let a = compute_storage_set_id(&[e("n1"), e("n2"), e("n3")]).unwrap(); + let b = compute_storage_set_id(&[e("n3"), e("n1"), e("n2")]).unwrap(); assert_eq!(a, b, "order-independent"); // Length-prefixing: ["ab","c"] and ["a","bc"] concatenate to the same // bytes; they must NOT hash the same. - let x = compute_storage_set_id(&["ab", "c"]).unwrap(); - let y = compute_storage_set_id(&["a", "bc"]).unwrap(); + let x = compute_storage_set_id(&[e("ab"), e("c")]).unwrap(); + let y = compute_storage_set_id(&[e("a"), e("bc")]).unwrap(); assert_ne!(x, y, "variable-length ids cannot be re-split"); assert!( - compute_storage_set_id(&["n1", "n1"]).is_err(), + compute_storage_set_id(&[e("n1"), e("n1")]).is_err(), "duplicate refused" ); assert!(compute_storage_set_id(&[]).is_err(), "empty refused"); assert!( - compute_storage_set_id(&["", "n1"]).is_err(), + compute_storage_set_id(&[e(""), e("n1")]).is_err(), "empty id refused" ); } @@ -252,7 +316,7 @@ mod tests { .unwrap(); let cat = StorageSetCatalog::new(vec![s.clone()]).unwrap(); assert!(cat.resolve(&s.id()).is_some()); - let foreign = compute_storage_set_id(&["c", "d", "e"]).unwrap(); + let foreign = compute_storage_set_id(&[e("c"), e("d"), e("e")]).unwrap(); assert!( cat.resolve(&foreign).is_none(), "an unknown set resolves to nothing" diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/vault_state_composition.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/vault_state_composition.rs index 36a006725..b36ea1c0f 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/vault_state_composition.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/sdk/vault_state_composition.rs @@ -829,8 +829,21 @@ mod tests { // through the local catalog and counts cells at THIS q — so the // fixture must commit the fleet these tests actually run against, // exactly as a real vault commits the fleet it was born under. - storage_set: StorageSetMembers::new(&[b"dsm-node-1", b"dsm-node-2", b"dsm-node-3"]) - .expect("set"), + storage_set: StorageSetMembers::new(&[ + ( + &b"dsm-node-1"[..], + crate::economic_fixtures::fixture_register_incarnation_bytes("dsm-node-1"), + ), + ( + &b"dsm-node-2"[..], + crate::economic_fixtures::fixture_register_incarnation_bytes("dsm-node-2"), + ), + ( + &b"dsm-node-3"[..], + crate::economic_fixtures::fixture_register_incarnation_bytes("dsm-node-3"), + ), + ]) + .expect("set"), quorum: 2, }; let ccb = state.encode().expect("encode"); diff --git a/dsm_client/deterministic_state_machine/dsm_sdk/src/test_support/fake_node.rs b/dsm_client/deterministic_state_machine/dsm_sdk/src/test_support/fake_node.rs index ca4675373..3ae5d38fc 100644 --- a/dsm_client/deterministic_state_machine/dsm_sdk/src/test_support/fake_node.rs +++ b/dsm_client/deterministic_state_machine/dsm_sdk/src/test_support/fake_node.rs @@ -400,8 +400,10 @@ pub fn point_env_config_at(endpoints: &[String]) { // never satisfy `resolve_root_register_profile` (economic admissions // fail closed on it). let n = i + 1; + let inc = crate::economic_fixtures::fixture_register_incarnation(&format!("dsm-node-{n}")); cfg_toml.push_str(&format!( - "\n[[nodes]]\nname = \"dsm-node-{n}\"\nendpoint = \"{ep}\"\n" + "\n[[nodes]]\nname = \"dsm-node-{n}\"\nendpoint = \"{ep}\"\n\ + register_incarnation = \"{inc}\"\n" )); } std::fs::write(&cfg_path, cfg_toml).expect("write env config"); diff --git a/dsm_storage_node/config/local-dev.toml b/dsm_storage_node/config/local-dev.toml index 94ac89202..bacf108f4 100644 --- a/dsm_storage_node/config/local-dev.toml +++ b/dsm_storage_node/config/local-dev.toml @@ -20,8 +20,21 @@ id = "local-dev-0" # settlement-slot claim register is INACTIVE on this node (every claim refused). # On the fleet, list all three members on every node; the set is immutable for # the lifetime of every vault born under it. -[storage_set] -members = ["local-dev-0"] +# The canonical storage set this node is a member of. +# +# Each member is its protocol id AND the register incarnation it is serving — +# the set id commits to `(id, register_incarnation)` pairs, so a member that +# lost and rebuilt its register is a different member and cannot serve a vault +# born under the old set. +# +# BOOTSTRAP, two steps, because an incarnation is generated by the node and +# cannot be chosen: start the node with this section commented out, read the +# `register incarnation for node ...` line it logs, then write it here. The +# node REFUSES to start if the value here is not the one its database holds. +# +# [[storage_set.members]] +# id = "local-dev-0" +# register_incarnation = "" [network] listen_addr = "127.0.0.1" diff --git a/dsm_storage_node/config/production.toml b/dsm_storage_node/config/production.toml index 332e40693..dae205cfd 100644 --- a/dsm_storage_node/config/production.toml +++ b/dsm_storage_node/config/production.toml @@ -5,9 +5,8 @@ # __LISTEN_ADDR__ — e.g. "0.0.0.0" # __PORT__ — e.g. 8080 # __DATABASE_URL__ — injected at deploy time from operator-provided settings -# __STORAGE_SET_MEMBERS__ — the canonical storage set's member ids, e.g. -# "us-west-1a-1", "us-west-1a-2", "us-west-1a-3" -# (must match the client env config's [[nodes]] names) +# +# The storage set is NOT a placeholder: see the two-phase bring-up below. [node] id = "__NODE_ID__" @@ -29,8 +28,35 @@ id = "__NODE_ID__" # * the register's non-equivocation must survive restart AND storage lifecycle — # restoring this node's database from a snapshot that predates a held claim, # or replacing the node without its register, is a SAFETY violation. -[storage_set] -members = ["__STORAGE_SET_MEMBERS__"] +# +# TWO-PHASE BRING-UP, and it cannot be collapsed into one. +# +# A member is `(id, register_incarnation)`. The incarnation is generated by the +# node itself on first boot, from the OS random source, into its own database — +# it is deliberately NOT derivable from the node id or the signing key, because +# a node that kept its key but lost its register MUST come back as a different +# member. So no generator can know these values in advance: +# +# Phase 1 — deploy every node with this section COMMENTED OUT. The register is +# inactive and every claim is refused, which is the correct state for +# a fleet that has not yet agreed on a set. Each node logs +# `register incarnation for node : ` at startup. +# Phase 2 — collect those values, write the full member list into EVERY node's +# config below and into the client env config's `[[nodes]]` entries, +# then restart. A node whose configured incarnation is not the one in +# its database REFUSES TO START, so a mistake here is loud. +# +# Re-running phase 1 on a node that has already minted an incarnation is safe: +# the value is write-once and is read back, never regenerated. +# +# [storage_set] +# [[storage_set.members]] +# id = "dsm-node-1" +# register_incarnation = "" +# +# [[storage_set.members]] +# id = "dsm-node-2" +# register_incarnation = "" [network] listen_addr = "__LISTEN_ADDR__" diff --git a/dsm_storage_node/deploy/generate_node_configs.sh b/dsm_storage_node/deploy/generate_node_configs.sh index 772900f13..0e7996527 100755 --- a/dsm_storage_node/deploy/generate_node_configs.sh +++ b/dsm_storage_node/deploy/generate_node_configs.sh @@ -84,21 +84,22 @@ openssl req -new -x509 -days 3650 -key "${CA_DIR}/ca.key" \ # Generate a random PostgreSQL password (shared across all nodes for simplicity) PG_PASS="$(openssl rand -base64 24 | tr -d '/+=' | head -c 32)" -# ----- The canonical storage set ----- -# Every node must be handed the SAME member list: the set id is derived by -# hashing the sorted ids, so a node configured with a different list computes a -# different id and refuses claims for the set its peers belong to. Built once, -# here, and substituted into every bundle unchanged. +# ----- The canonical storage set: PHASE 1 LEAVES IT UNSET ----- +# A set member is `(id, register_incarnation)`, and the incarnation is minted by +# the node itself on first boot into its own database — never derivable from the +# node id or its keys, because a node that kept its key but lost its register +# must come back as a DIFFERENT member. This generator therefore cannot know the +# member list, and inventing one would produce a fleet whose configured +# incarnations no node actually holds; every node would refuse to start. # -# These ids are also the client's contract: the member ids must equal the -# `[[nodes]] name` entries in the client env config, because that is what the -# client hashes to name the set a vault is born under. -STORAGE_SET_MEMBERS="" -for i in $(seq 1 "${N}"); do - [ -n "${STORAGE_SET_MEMBERS}" ] && STORAGE_SET_MEMBERS="${STORAGE_SET_MEMBERS}, " - STORAGE_SET_MEMBERS="${STORAGE_SET_MEMBERS}\"dsm-node-${i}\"" -done -echo "Canonical storage set (${N} members): ${STORAGE_SET_MEMBERS}" +# So these bundles ship with `[storage_set]` commented out (register INACTIVE, +# every claim refused — the correct state for a fleet that has not agreed on a +# set yet). Boot them, collect each node's logged +# `register incarnation for node : `, write the full member +# list into every node config AND the client env config, then restart. +echo "Storage set: NOT configured by this generator (phase 1)." +echo " Boot the fleet, collect each node's logged register incarnation, then" +echo " write [[storage_set.members]] into every node config and restart." # ----- Per-Node Bundles ----- for i in $(seq 1 "${N}"); do @@ -146,8 +147,7 @@ EXTEOF DB_URL="postgresql://postgres:5432/dsm_storage?user=dsm&password=${PG_PASS}" # Escape '&' in DB_URL so sed doesn't interpret it as backreference DB_URL_ESCAPED="${DB_URL//&/\\&}" - sed -e "s|members = \[\"__STORAGE_SET_MEMBERS__\"\]|members = [${STORAGE_SET_MEMBERS}]|g" \ - -e "s|__NODE_ID__|${NODE_ID}|g" \ + sed -e "s|__NODE_ID__|${NODE_ID}|g" \ -e "s|__LISTEN_ADDR__|0.0.0.0|g" \ -e "s|__PORT__|8080|g" \ -e "s|__DATABASE_URL__|${DB_URL_ESCAPED}|g" \ diff --git a/dsm_storage_node/src/api/economic/faucet_ticket.rs b/dsm_storage_node/src/api/economic/faucet_ticket.rs index 5ed82edd6..95b457185 100644 --- a/dsm_storage_node/src/api/economic/faucet_ticket.rs +++ b/dsm_storage_node/src/api/economic/faucet_ticket.rs @@ -268,8 +268,16 @@ mod tests { async fn endpoint_enforces_coordinates_attribution_and_writes_once() { let pool = Arc::new(db::create_pool(":memory:", true).expect("pool")); db::init_db(&pool).await.expect("init"); - let set = - crate::NodeStorageSet::new(vec!["n1".into(), "n2".into(), "n3".into()], "n1").unwrap(); + let set = crate::NodeStorageSet::new( + vec![ + ("n1".into(), [0xC1; 32]), + ("n2".into(), [0xC2; 32]), + ("n3".into(), [0xC3; 32]), + ], + "n1", + [0xC1; 32], + ) + .unwrap(); let state = state_with(pool.clone(), Some(set.clone()), true); let (pk, sk) = dsm::crypto::sphincs::generate_sphincs_keypair().unwrap(); @@ -369,8 +377,16 @@ mod tests { async fn poisoned_ticket_does_not_brick_the_faucet() { let pool = Arc::new(db::create_pool(":memory:", true).expect("pool")); db::init_db(&pool).await.expect("init"); - let set = - crate::NodeStorageSet::new(vec!["n1".into(), "n2".into(), "n3".into()], "n1").unwrap(); + let set = crate::NodeStorageSet::new( + vec![ + ("n1".into(), [0xC1; 32]), + ("n2".into(), [0xC2; 32]), + ("n3".into(), [0xC3; 32]), + ], + "n1", + [0xC1; 32], + ) + .unwrap(); let state = state_with(pool.clone(), Some(set.clone()), true); let canonical = era_faucet_id(NETWORK); @@ -416,8 +432,16 @@ mod tests { async fn unconfigured_node_refuses_rather_than_defaulting() { let pool = Arc::new(db::create_pool(":memory:", true).expect("pool")); db::init_db(&pool).await.expect("init"); - let set = - crate::NodeStorageSet::new(vec!["n1".into(), "n2".into(), "n3".into()], "n1").unwrap(); + let set = crate::NodeStorageSet::new( + vec![ + ("n1".into(), [0xC1; 32]), + ("n2".into(), [0xC2; 32]), + ("n3".into(), [0xC3; 32]), + ], + "n1", + [0xC1; 32], + ) + .unwrap(); let (pk, sk) = dsm::crypto::sphincs::generate_sphincs_keypair().unwrap(); let devid = [0x81; 32]; diff --git a/dsm_storage_node/src/api/economic/root_register.rs b/dsm_storage_node/src/api/economic/root_register.rs index f30389dfd..eecd3635c 100644 --- a/dsm_storage_node/src/api/economic/root_register.rs +++ b/dsm_storage_node/src/api/economic/root_register.rs @@ -230,8 +230,16 @@ mod tests { async fn endpoint_recomputes_the_cell_and_enforces_three_way_attribution() { let pool = Arc::new(db::create_pool(":memory:", true).expect("pool")); db::init_db(&pool).await.expect("init"); - let set = - crate::NodeStorageSet::new(vec!["n1".into(), "n2".into(), "n3".into()], "n1").unwrap(); + let set = crate::NodeStorageSet::new( + vec![ + ("n1".into(), [0xC1; 32]), + ("n2".into(), [0xC2; 32]), + ("n3".into(), [0xC3; 32]), + ], + "n1", + [0xC1; 32], + ) + .unwrap(); let state = test_state(pool.clone(), set.clone()); let (pk, sk) = dsm::crypto::sphincs::generate_sphincs_keypair().unwrap(); diff --git a/dsm_storage_node/src/api/vault/settlement_slot.rs b/dsm_storage_node/src/api/vault/settlement_slot.rs index 08a6974c3..d1d65aaf4 100644 --- a/dsm_storage_node/src/api/vault/settlement_slot.rs +++ b/dsm_storage_node/src/api/vault/settlement_slot.rs @@ -65,6 +65,14 @@ use dsm_sdk::util::text_id; const MAX_CLAIM_BYTES: usize = 160 * 1024; pub const OUTCOME_HEADER: &str = "x-dsm-slot-outcome"; +/// The register incarnation this node is serving, Base32-Crockford. +/// +/// Echoed on EVERY register read. A reader that committed a different +/// incarnation for this member counts the answer as `Unavailable` — never as +/// a value and never as an absence — because a rebuilt register can honestly +/// report "nothing here" for a cell the incarnation the vault committed once +/// held, and node identity alone cannot tell the two apart. +pub const INCARNATION_HEADER: &str = "x-dsm-register-incarnation"; pub const HELD_DIGEST_HEADER: &str = "x-dsm-slot-held-digest"; pub const CLAIM_DIGEST_HEADER: &str = "x-dsm-slot-digest"; @@ -164,6 +172,22 @@ pub async fn get_claim( if vault_id.len() != 32 { return StatusCode::BAD_REQUEST.into_response(); } + // The incarnation is stamped on EVERY answer this handler gives, held or + // absent alike. Stamping only the "held" case would leave the dangerous + // one — a rebuilt member reporting emptiness — indistinguishable from the + // real member reporting it. + let incarnation = state + .storage_set + .as_ref() + .map(|set| text_id::encode_base32_crockford(&set.own_incarnation)); + let stamp = |resp: &mut Response| { + if let Some(v) = incarnation + .as_deref() + .and_then(|s| HeaderValue::from_str(s).ok()) + { + resp.headers_mut().insert(INCARNATION_HEADER, v); + } + }; match db::get_settlement_slot_claim(&state.db_pool, &vault_id, parent_sequence).await { Ok(Some((bytes, digest))) => { let mut resp = (StatusCode::OK, bytes).into_response(); @@ -176,6 +200,7 @@ pub async fn get_claim( if let Ok(v) = HeaderValue::from_str(&text_id::encode_base32_crockford(&digest)) { resp.headers_mut().insert(CLAIM_DIGEST_HEADER, v); } + stamp(&mut resp); resp } // AN ABSENCE IS ASSERTED, NEVER INFERRED FROM A STATUS CODE. A bare @@ -189,6 +214,7 @@ pub async fn get_claim( let mut resp = StatusCode::NOT_FOUND.into_response(); resp.headers_mut() .insert(OUTCOME_HEADER, HeaderValue::from_static("absent")); + stamp(&mut resp); resp } Err(e) => { @@ -227,6 +253,86 @@ mod tests { } } + /// EVERY ANSWER CARRIES THE INCARNATION, held and absent alike. + /// + /// The absent case is the one that matters: a member that rebuilt its + /// register answers "nothing here" perfectly honestly, and a reader that + /// could not tell which register history said so would count that as + /// emptiness for a cell the committed incarnation once held. Stamping + /// only the `held` answer would leave exactly that case unmarked. + #[tokio::test] + async fn every_read_answer_names_the_register_incarnation_serving_it() { + use crate::replication::{ReplicationConfig, ReplicationManager}; + let vault = unique_key(0x55); + let pool = Arc::new(test_pool()); + db::init_db(&pool).await.expect("init"); + let rm = Arc::new( + ReplicationManager::new_for_tests( + ReplicationConfig { + replication_factor: 3, + gossip_interval_ticks: 100, + failure_timeout_ticks: 300, + gossip_fanout: 3, + max_concurrent_jobs: 10, + }, + "n1".to_string(), + "http://localhost:8080".to_string(), + ) + .expect("replication manager for tests"), + ); + let set = crate::NodeStorageSet::new( + vec![ + ("n1".into(), [0xC1; 32]), + ("n2".into(), [0xC2; 32]), + ("n3".into(), [0xC3; 32]), + ], + "n1", + [0xC1; 32], + ) + .unwrap(); + let expected = dsm_sdk::util::text_id::encode_base32_crockford(&[0xC1u8; 32]); + let state = Arc::new( + AppState::new("n1".into(), "127.0.0.1:1", None, pool, rm).with_storage_set(set), + ); + + // ABSENT: the cell has never been claimed. + let r = get_claim( + Extension(state.clone()), + axum::extract::Path((text_id::encode_base32_crockford(&vault), 9u64)), + ) + .await; + assert_eq!(r.status(), StatusCode::NOT_FOUND); + assert_eq!(r.headers()[OUTCOME_HEADER], "absent"); + assert_eq!( + r.headers()[INCARNATION_HEADER], + expected.as_str(), + "an absence must say WHICH register history is asserting it" + ); + + // HELD: the same stamp, on the other branch. + let claim = b"claim-bytes".to_vec(); + let digest = *blake3::hash(&claim).as_bytes(); + db::claim_settlement_slot( + &state.db_pool, + &vault, + 9, + &claim, + &digest, + b"pk", + &[0x6B; 32], + ) + .await + .expect("claim"); + let r = get_claim( + Extension(state.clone()), + axum::extract::Path((text_id::encode_base32_crockford(&vault), 9u64)), + ) + .await; + assert_eq!(r.status(), StatusCode::OK); + assert_eq!(r.headers()[OUTCOME_HEADER], "held"); + assert_eq!(r.headers()[INCARNATION_HEADER], expected.as_str()); + } + /// Endpoint semantics: attribution (body key == caller key), set /// enforcement, and the outcome header on accept / re-ack / refuse. #[tokio::test] @@ -249,8 +355,16 @@ mod tests { ) .expect("replication manager for tests"), ); - let set = - crate::NodeStorageSet::new(vec!["n1".into(), "n2".into(), "n3".into()], "n1").unwrap(); + let set = crate::NodeStorageSet::new( + vec![ + ("n1".into(), [0xC1; 32]), + ("n2".into(), [0xC2; 32]), + ("n3".into(), [0xC3; 32]), + ], + "n1", + [0xC1; 32], + ) + .unwrap(); let state = Arc::new( AppState::new("n1".into(), "127.0.0.1:1", None, pool, rm).with_storage_set(set.clone()), ); diff --git a/dsm_storage_node/src/db/pg.rs b/dsm_storage_node/src/db/pg.rs index 41e2ce487..56378925d 100644 --- a/dsm_storage_node/src/db/pg.rs +++ b/dsm_storage_node/src/db/pg.rs @@ -360,12 +360,53 @@ mod durable_posture_tests { } } +/// This node's durable REGISTER INCARNATION: the identity of its register +/// history, not of the node. +/// +/// Generated once, from the OS random source, and stored only here. It is +/// deliberately not derivable from the node's signing key, its id, or any +/// seed — a node that still holds its identity key but lost its register +/// database MUST come back with a different incarnation, because that is the +/// fact a vault needs to know. Otherwise a rebuilt member can assert "no +/// claim here" for a cell the real incarnation once held, and owning the +/// identity key would be the whole test. +/// +/// Write-once with a same-transaction read-back, like the registers it +/// speaks for: two racing callers agree on one value rather than each +/// minting one. +pub async fn register_incarnation(pool: &Pool) -> Result<[u8; 32]> { + let fresh: [u8; 32] = rand::random(); + let mut client = pool.get().await?; + let tx = begin_durable_write(&mut client).await?; + tx.execute( + "INSERT INTO register_incarnation (only_row, incarnation) VALUES (1, $1) + ON CONFLICT (only_row) DO NOTHING", + &[&fresh.as_slice()], + ) + .await?; + let row = tx + .query_one( + "SELECT incarnation FROM register_incarnation WHERE only_row = 1", + &[], + ) + .await?; + let held: Vec = row.get(0); + tx.commit().await?; + held.try_into() + .map_err(|_| anyhow!("stored register incarnation is not 32 bytes")) +} + /// Initialize database schema for storage node. pub async fn init_db(pool: &Pool) -> Result<()> { let client = pool.get().await?; client .batch_execute( - r#"CREATE TABLE IF NOT EXISTS dlv_slots ( + r#"CREATE TABLE IF NOT EXISTS register_incarnation ( + only_row SMALLINT PRIMARY KEY CHECK (only_row = 1), + incarnation BYTEA NOT NULL + ); + + CREATE TABLE IF NOT EXISTS dlv_slots ( dlv_id BYTEA PRIMARY KEY, capacity_bytes BIGINT NOT NULL, used_bytes BIGINT NOT NULL DEFAULT 0, diff --git a/dsm_storage_node/src/db/sqlite.rs b/dsm_storage_node/src/db/sqlite.rs index 8c97188c2..a564eb9b9 100644 --- a/dsm_storage_node/src/db/sqlite.rs +++ b/dsm_storage_node/src/db/sqlite.rs @@ -156,11 +156,53 @@ pub async fn require_durable_commit_posture(_pool: &DBPool) -> Result<()> { Ok(()) } +/// This node's durable REGISTER INCARNATION: the identity of its register +/// history, not of the node. +/// +/// Generated once, from the OS random source, and stored only here. It is +/// deliberately not derivable from the node's signing key, its id, or any +/// seed — a node that still holds its identity key but lost its register +/// database MUST come back with a different incarnation, because that is the +/// fact a vault needs to know. Otherwise a rebuilt member can assert "no +/// claim here" for a cell the real incarnation once held, and owning the +/// identity key would be the whole test. +/// +/// Write-once with a same-transaction read-back, like the registers it +/// speaks for: two racing callers agree on one value rather than each +/// minting one. +pub async fn register_incarnation(pool: &DBPool) -> Result<[u8; 32]> { + let fresh: [u8; 32] = rand::random(); + with_conn(pool, move |conn| { + conn.execute_batch("PRAGMA synchronous=FULL;")?; + let tx = conn.unchecked_transaction()?; + tx.execute( + "INSERT OR IGNORE INTO register_incarnation (only_row, incarnation) VALUES (1, ?1)", + params![fresh.to_vec()], + )?; + let held: Vec = tx.query_row( + "SELECT incarnation FROM register_incarnation WHERE only_row = 1", + [], + |row| row.get(0), + )?; + tx.commit()?; + let held: [u8; 32] = held + .try_into() + .map_err(|_| anyhow!("stored register incarnation is not 32 bytes"))?; + Ok(held) + }) + .await +} + /// Initialize database schema (SQLite version). pub async fn init_db(pool: &DBPool) -> Result<()> { with_conn(pool, |conn| { conn.execute_batch( - r#"CREATE TABLE IF NOT EXISTS dlv_slots ( + r#"CREATE TABLE IF NOT EXISTS register_incarnation ( + only_row INTEGER PRIMARY KEY CHECK (only_row = 1), + incarnation BLOB NOT NULL + ); + + CREATE TABLE IF NOT EXISTS dlv_slots ( dlv_id BLOB PRIMARY KEY, capacity_bytes INTEGER NOT NULL, used_bytes INTEGER NOT NULL DEFAULT 0, diff --git a/dsm_storage_node/src/lib.rs b/dsm_storage_node/src/lib.rs index b04491399..6f20b31a8 100644 --- a/dsm_storage_node/src/lib.rs +++ b/dsm_storage_node/src/lib.rs @@ -41,27 +41,65 @@ pub struct AppState { /// This node's view of the canonical storage set it belongs to. #[derive(Debug, Clone, PartialEq, Eq)] pub struct NodeStorageSet { - /// `dsm_sdk::sdk::storage_set::compute_storage_set_id` over `member_ids`. + /// `compute_storage_set_id` over the `(member_id, incarnation)` pairs. pub id: [u8; 32], - /// The configured protocol identities of every member (this node's own - /// `node.id` string must be among them). - pub member_ids: Vec, + /// Every member's configured protocol identity paired with the register + /// incarnation it is serving. This node's own `node.id` must be among + /// them, and its configured incarnation must be the one this node's + /// database actually holds. + pub members: Vec<(String, [u8; 32])>, + /// This node's own register incarnation — the value it echoes on every + /// register read so a reader can tell it apart from a rebuilt member + /// wearing the same node id. + pub own_incarnation: [u8; 32], } impl NodeStorageSet { - /// Build from configured member ids; refuses an empty set, duplicate ids, - /// or a set that does not contain `own_node_id` — a node that would - /// acknowledge claims for a set it is not a member of is misconfigured. - pub fn new(member_ids: Vec, own_node_id: &str) -> anyhow::Result { - if !member_ids.iter().any(|m| m == own_node_id) { + /// Build from configured members; refuses an empty set, duplicate ids, a + /// set that does not contain `own_node_id` — a node that would + /// acknowledge claims for a set it is not a member of is misconfigured — + /// and, decisively, a configured incarnation for THIS node that is not + /// the one its database holds. + /// + /// That last refusal is the point of the whole mechanism. A node that + /// lost and rebuilt its register comes back with a new incarnation; if it + /// were allowed to keep serving the configured old one it would be + /// asserting a register history it no longer has. Refusing at startup + /// makes the discontinuity loud, at the one moment an operator is looking, + /// instead of silent at read time. + pub fn new( + members: Vec<(String, [u8; 32])>, + own_node_id: &str, + own_incarnation: [u8; 32], + ) -> anyhow::Result { + let Some((_, configured_own)) = members.iter().find(|(m, _)| m == own_node_id) else { anyhow::bail!( "storage_set.members does not contain this node's own id {own_node_id:?}" ); + }; + if *configured_own != own_incarnation { + anyhow::bail!( + "storage_set.members lists a register incarnation for this node ({}) that is \ + not the one this node's database holds ({}) — this node's register was \ + rebuilt or restored, so it is no longer the member the configured set names", + dsm_sdk::util::text_id::encode_base32_crockford(configured_own), + dsm_sdk::util::text_id::encode_base32_crockford(&own_incarnation) + ); } - let refs: Vec<&str> = member_ids.iter().map(|s| s.as_str()).collect(); - let id = dsm_sdk::sdk::storage_set::compute_storage_set_id(&refs) + let entries: Vec<(&str, [u8; 32])> = + members.iter().map(|(m, i)| (m.as_str(), *i)).collect(); + let id = dsm_sdk::sdk::storage_set::compute_storage_set_id(&entries) .map_err(|e| anyhow::anyhow!("storage_set.members: {e}"))?; - Ok(Self { id, member_ids }) + Ok(Self { + id, + members, + own_incarnation, + }) + } + + /// The configured member ids, for logging and endpoint resolution. + pub fn member_ids(&self) -> impl Iterator { + self.members.iter().map(|(m, _)| m.as_str()) } } @@ -205,3 +243,77 @@ pub async fn build_app_for_tests() -> anyhow::Result { .merge(api::registry::core::create_router(state_arc.clone())) .layer(Extension(state_arc))) } + +#[cfg(test)] +mod storage_set_tests { + #![allow(clippy::disallowed_methods)] // unwrap/expect acceptable in deterministic tests + use super::NodeStorageSet; + + fn members() -> Vec<(String, [u8; 32])> { + vec![ + ("n1".into(), [0xC1; 32]), + ("n2".into(), [0xC2; 32]), + ("n3".into(), [0xC3; 32]), + ] + } + + /// THE REFUSAL THIS MECHANISM EXISTS FOR. + /// + /// A node that lost and rebuilt its register still owns its identity key + /// and its configured id, so every check that looks at identity alone + /// passes. What it no longer has is the register history the set names. + /// Startup is where that becomes loud: the configured incarnation is what + /// the set committed, this node's database is what it can still speak + /// for, and serving the set while those disagree would be asserting a + /// history it does not have. + #[test] + fn a_node_whose_register_was_rebuilt_refuses_to_serve_the_configured_set() { + let err = NodeStorageSet::new(members(), "n1", [0x99; 32]) + .expect_err("a rebuilt register must refuse the configured set"); + let text = err.to_string(); + assert!( + text.contains("rebuilt or restored"), + "the refusal must say WHY, got: {text}" + ); + + // The same node, still serving the incarnation the set committed, is + // fine — so the refusal is about the register history, not about + // being strict. + assert!(NodeStorageSet::new(members(), "n1", [0xC1; 32]).is_ok()); + } + + #[test] + fn a_set_that_does_not_name_this_node_is_refused() { + let err = NodeStorageSet::new(members(), "n4", [0xC4; 32]) + .expect_err("a node must be a member of the set it serves"); + assert!(err + .to_string() + .contains("does not contain this node's own id")); + } + + /// The incarnation is an INPUT to the id, not a label beside it: the same + /// three node ids under a different incarnation are a different set, so a + /// rebuilt member cannot resolve to the set it used to serve. + #[test] + fn one_members_incarnation_changes_the_whole_set_id() { + let before = NodeStorageSet::new(members(), "n1", [0xC1; 32]).unwrap(); + let mut rebuilt = members(); + rebuilt[2] = ("n3".into(), [0x77; 32]); + let after = NodeStorageSet::new(rebuilt, "n1", [0xC1; 32]).unwrap(); + assert_ne!( + before.id, after.id, + "a member's new register incarnation must change the set id" + ); + } + + /// Ordering is by MEMBER ID, never by the pair — so the id does not + /// depend on how the operator happened to list the members. + #[test] + fn the_set_id_does_not_depend_on_configuration_order() { + let a = NodeStorageSet::new(members(), "n1", [0xC1; 32]).unwrap(); + let mut reversed = members(); + reversed.reverse(); + let b = NodeStorageSet::new(reversed, "n1", [0xC1; 32]).unwrap(); + assert_eq!(a.id, b.id); + } +} diff --git a/dsm_storage_node/src/main.rs b/dsm_storage_node/src/main.rs index b041fc748..985cfb809 100644 --- a/dsm_storage_node/src/main.rs +++ b/dsm_storage_node/src/main.rs @@ -57,8 +57,8 @@ struct ServerConfig { hsts_max_age: Option, database_url: String, seed_peers: Vec, - /// `[storage_set] members` — configured member ids of this node's set. - storage_set_members: Vec, + /// `[[storage_set.members]]` — each member's id and register incarnation. + storage_set_members: Vec<(String, [u8; 32])>, /// The DSM network this node serves (`node.network_id`). Gates the ERA /// faucet-ticket register — its canonical identity is network-scoped, so /// no network means the register is inactive (fail closed, like an @@ -121,16 +121,53 @@ fn load_server_config(opts: &Opts) -> Result { .filter_map(|v| v.into_string().ok()) .collect(); - // The canonical storage set this node is a member of ([storage_set] - // members = ["id-1", "id-2", "id-3"]). Absent = the settlement-slot - // register is inactive (fail closed); present but not containing this - // node's own id = misconfiguration, refused at startup. - let storage_set_members: Vec = settings + // The canonical storage set this node is a member of: + // + // [[storage_set.members]] + // id = "dsm-node-1" + // register_incarnation = "" + // + // Absent = the settlement-slot register is inactive (fail closed); + // present but not containing this node's own id = misconfiguration, + // refused at startup. The incarnation is REQUIRED per member: a set id is + // a function of `(member_id, register_incarnation)` pairs, so a member + // whose incarnation the config cannot state is a member no set id can be + // derived over. A malformed entry refuses rather than defaulting, because + // a defaulted incarnation would resolve every set to whatever the default + // hashed to. + let storage_set_members: Vec<(String, [u8; 32])> = settings .get_array("storage_set.members") .unwrap_or_default() .into_iter() - .filter_map(|v| v.into_string().ok()) - .collect(); + .map(|v| { + let t = v + .into_table() + .map_err(|e| anyhow::anyhow!("[[storage_set.members]] is not a table: {e}"))?; + let id = t + .get("id") + .and_then(|v| v.clone().into_string().ok()) + .ok_or_else(|| anyhow::anyhow!("[[storage_set.members]] is missing `id`"))?; + let inc_text = t + .get("register_incarnation") + .and_then(|v| v.clone().into_string().ok()) + .ok_or_else(|| { + anyhow::anyhow!( + "[[storage_set.members]] {id:?} is missing `register_incarnation`" + ) + })?; + let raw = text_id::decode_base32_crockford(&inc_text).ok_or_else(|| { + anyhow::anyhow!( + "[[storage_set.members]] {id:?} register_incarnation is not Base32-Crockford" + ) + })?; + let inc: [u8; 32] = raw.try_into().map_err(|_| { + anyhow::anyhow!( + "[[storage_set.members]] {id:?} register_incarnation is not 32 bytes" + ) + })?; + Ok((id, inc)) + }) + .collect::>>()?; // The DSM network this node serves. NOT defaulted: the ERA faucet // identity is era_faucet_id(network_id), so a defaulted network would let @@ -483,14 +520,32 @@ async fn async_main() -> Result<()> { db_pool.clone(), replication_manager, ); + // ESTABLISHED AND LOGGED UNCONDITIONALLY, before any set is considered. + // + // The incarnation is a property of this node's register, not of its + // membership in a set, and an operator cannot write `[[storage_set.members]]` + // for this node without knowing the value. Establishing it only when a set + // is already configured would be a bootstrap that never starts: no set + // means no incarnation, no incarnation means no set can be written. + let own_incarnation = db::register_incarnation(&db_pool) + .await + .context("failed to establish this node's register incarnation")?; + log::info!( + "register incarnation for node {}: {}", + server_config.node_id, + text_id::encode_base32_crockford(&own_incarnation) + ); if !server_config.storage_set_members.is_empty() { + // Config states what the set COMMITTED; the database states what this + // node can still speak for. `NodeStorageSet` refuses when they differ. let set = dsm_storage_node::NodeStorageSet::new( server_config.storage_set_members.clone(), &server_config.node_id, + own_incarnation, )?; log::info!( "storage set configured: {} members, id={}", - set.member_ids.len(), + set.members.len(), text_id::encode_base32_crockford(&set.id) ); state = state.with_storage_set(set); diff --git a/dsm_storage_node/tests/economic_register_conformance.rs b/dsm_storage_node/tests/economic_register_conformance.rs index 30dbdcb01..0c60928c6 100644 --- a/dsm_storage_node/tests/economic_register_conformance.rs +++ b/dsm_storage_node/tests/economic_register_conformance.rs @@ -97,8 +97,13 @@ async fn real_member(id: &str, set_members: &[&str], network: Option<&[u8]>) -> ); let mut state = AppState::new(id.to_string(), &endpoint, None, pool.clone(), rm); if !set_members.is_empty() { - let ids: Vec = set_members.iter().map(|s| s.to_string()).collect(); - state = state.with_storage_set(NodeStorageSet::new(ids, id).expect("node set")); + let members: Vec<(String, [u8; 32])> = set_members + .iter() + .map(|s| (s.to_string(), member_incarnation(s))) + .collect(); + state = state.with_storage_set( + NodeStorageSet::new(members, id, member_incarnation(id)).expect("node set"), + ); } if let Some(n) = network { state = state.with_network_id(n.to_vec()); @@ -116,6 +121,16 @@ async fn real_member(id: &str, set_members: &[&str], network: Option<&[u8]>) -> } } +/// A member's register incarnation, derived from its id so the node side and +/// the client side agree without a fixture having to thread the value. +/// +/// Production derives nothing: a node's incarnation is random at first init +/// and stored only in its own database. This is a test standing in for "what +/// that node reported to the operator who wrote the catalog". +fn member_incarnation(member_id: &str) -> [u8; 32] { + *blake3::hash(format!("test-incarnation/{member_id}").as_bytes()).as_bytes() +} + /// The canonical three-member fleet, every member configured for the same set /// and network — what a client's catalog names as `dsm-node-1..3`. async fn fleet() -> Vec { @@ -133,6 +148,7 @@ fn client_set(members: &[RealMember]) -> StorageSet { .iter() .map(|m| StorageMember { member_id: m.id.clone(), + register_incarnation_id: member_incarnation(&m.id), endpoint: m.endpoint.clone(), }) .collect(), diff --git a/dsm_storage_node/tests/identity_milestone_e2e.rs b/dsm_storage_node/tests/identity_milestone_e2e.rs index 817002484..4a3a548a8 100644 --- a/dsm_storage_node/tests/identity_milestone_e2e.rs +++ b/dsm_storage_node/tests/identity_milestone_e2e.rs @@ -110,7 +110,14 @@ async fn the_milestone_path_works_end_to_end_with_no_legacy_anywhere() { iteration_budget: None, parent_state_commitment: dsm::ccb::genesis_parent_commitment(&[0x55; 32]), owner_authority_transition_digest: t0.digest(), - storage_set: StorageSetMembers::new(&[b"n1", b"n2", b"n3", b"n4", b"n5"]).expect("set"), + storage_set: StorageSetMembers::new(&[ + (&b"n1"[..], [0xE1; 32]), + (&b"n2"[..], [0xE2; 32]), + (&b"n3"[..], [0xE3; 32]), + (&b"n4"[..], [0xE4; 32]), + (&b"n5"[..], [0xE5; 32]), + ]) + .expect("set"), quorum: 4, }; let ccb = state.encode().expect("encodes");