diff --git a/Cargo.lock b/Cargo.lock index 5bedffc65..12241380c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3671,7 +3671,6 @@ dependencies = [ "flate2", "futures", "glob", - "handlebars", "hex", "hmac 0.13.0", "hybrid-array", @@ -3682,6 +3681,7 @@ dependencies = [ "ic-management-canister-types 0.7.1", "ic-utils", "icp-canister-interfaces", + "icp-deploy-canister", "icp-sync-plugin", "icrc-ledger-types", "indexmap", @@ -3771,6 +3771,7 @@ dependencies = [ "ic-utils", "icp", "icp-canister-interfaces", + "icp-deploy-canister", "icrc-ledger-types", "indicatif", "indoc", @@ -3814,6 +3815,46 @@ dependencies = [ "wslpath2", ] +[[package]] +name = "icp-deploy-canister" +version = "1.1.0" +dependencies = [ + "async-trait", + "bigdecimal", + "camino", + "camino-tempfile", + "candid", + "candid_parser", + "clap", + "dunce", + "futures", + "glob", + "handlebars", + "hex", + "ic-agent", + "ic-management-canister-types 0.7.1", + "icp-canister-interfaces", + "icp-sync-plugin", + "indexmap", + "indoc", + "itertools 0.14.0", + "jsonschema", + "num-bigint 0.4.6", + "num-integer", + "num-traits", + "pathdiff", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "sha2 0.11.0", + "snafu", + "strum 0.28.0", + "tokio", + "tracing", + "url", +] + [[package]] name = "icp-sync-plugin" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 946375dee..2c8842f3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ ic-management-canister-types = { version = "0.7.1" } ic-utils = { version = "0.47.0" } icp = { path = "crates/icp" } icp-canister-interfaces = { path = "crates/icp-canister-interfaces" } +icp-deploy-canister = { path = "crates/icp-deploy-canister" } icp-sync-plugin = { path = "crates/icp-sync-plugin" } ic-identity-hsm = "0.47.0" icrc-ledger-types = "0.1.10" diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index bddc01ea8..ed5610c60 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -43,6 +43,7 @@ ic-management-canister-types.workspace = true ic-utils.workspace = true icp-canister-interfaces.workspace = true icp = { workspace = true, features = ["clap"] } +icp-deploy-canister.workspace = true icrc-ledger-types.workspace = true indicatif.workspace = true itertools.workspace = true diff --git a/crates/icp-cli/src/commands/canister/install.rs b/crates/icp-cli/src/commands/canister/install.rs index 67efe9e9a..5eb35b5d1 100644 --- a/crates/icp-cli/src/commands/canister/install.rs +++ b/crates/icp-cli/src/commands/canister/install.rs @@ -10,14 +10,13 @@ use icp::fs; use icp::prelude::*; use tracing::{info, warn}; +use icp_deploy_canister::install_canister_resolved; + use crate::{ commands::args::{self, ArgsOpt}, operations::{ candid_compat::{CandidCompatibility, check_candid_compatibility}, - install::{ - WasmMemoryPersistenceOpt, install_canister, is_eop_canister, - resolve_install_mode_and_status, - }, + install::{WasmMemoryPersistenceOpt, is_eop_canister, resolve_install_mode_and_status}, }, }; @@ -184,16 +183,19 @@ pub(crate) async fn exec(ctx: &Context, args: &InstallArgs) -> Result<(), anyhow } } - install_canister( - &agent, - args.proxy, - &canister_id, + let wmp = args + .wasm_memory_persistence + .map(WasmMemoryPersistenceOpt::to_ic); + install_canister_resolved( &canister_display, + canister_id, &wasm, install_mode, status, init_args_bytes.as_deref(), - args.wasm_memory_persistence, + wmp, + &agent, + args.proxy, ) .await?; diff --git a/crates/icp-cli/src/commands/deploy.rs b/crates/icp-cli/src/commands/deploy.rs index 829824cf0..f10b5d9fa 100644 --- a/crates/icp-cli/src/commands/deploy.rs +++ b/crates/icp-cli/src/commands/deploy.rs @@ -479,17 +479,16 @@ pub(crate) async fn exec(ctx: &Context, args: &DeployArgs) -> Result<(), anyhow: .into_iter() .collect(); - let pkg_cache = ctx.dirs.package_cache()?; + let resolver = ctx.resource_resolver()?; sync_many( - ctx.syncer.clone(), agent.clone(), + resolver, sync_canisters, environment_selection.name().to_owned(), env.network.name.clone(), canister_ids, args.proxy, ctx.debug, - &pkg_cache, ) .await?; } diff --git a/crates/icp-cli/src/commands/sync.rs b/crates/icp-cli/src/commands/sync.rs index 7756a50ff..319bf83c1 100644 --- a/crates/icp-cli/src/commands/sync.rs +++ b/crates/icp-cli/src/commands/sync.rs @@ -8,8 +8,10 @@ use icp::identity::IdentitySelection; use std::collections::BTreeMap; use tracing::info; +use icp::Canister; + use crate::{ - operations::{proxy_management, sync::sync_many}, + operations::{binding_env_vars::set_binding_env_vars_many, proxy_management, sync::sync_many}, options::{EnvironmentOpt, IdentityOpt}, }; @@ -122,17 +124,33 @@ pub(crate) async fn exec(ctx: &Context, args: &SyncArgs) -> Result<(), anyhow::E .into_iter() .collect(); - let pkg_cache = ctx.dirs.package_cache()?; + // Apply the generated `PUBLIC_CANISTER_ID:*` environment variables before + // syncing. `deploy` does this, but standalone `icp sync` previously did not, + // so a synced canister could run against stale/absent binding ids. + let target_canisters: Vec<(Principal, Canister)> = sync_canisters + .iter() + .map(|(cid, _, info)| (*cid, info.clone())) + .collect(); + set_binding_env_vars_many( + agent.clone(), + args.proxy, + environment_selection.name(), + target_canisters, + canister_ids.clone(), + ctx.debug, + ) + .await?; + + let resolver = ctx.resource_resolver()?; sync_many( - ctx.syncer.clone(), agent, + resolver, sync_canisters, environment_selection.name().to_owned(), env.network.name.clone(), canister_ids, args.proxy, ctx.debug, - &pkg_cache, ) .await?; diff --git a/crates/icp-cli/src/operations/binding_env_vars.rs b/crates/icp-cli/src/operations/binding_env_vars.rs index a2118f193..cd8b8c50f 100644 --- a/crates/icp-cli/src/operations/binding_env_vars.rs +++ b/crates/icp-cli/src/operations/binding_env_vars.rs @@ -1,29 +1,15 @@ use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::{Agent, export::Principal}; -use ic_management_canister_types::{CanisterSettings, EnvironmentVariable, UpdateSettingsArgs}; use icp::Canister; +use icp_deploy_canister::{SyncCanisterError, apply_binding_env_vars}; use snafu::Snafu; use tracing::error; use crate::progress::{ProgressManager, ProgressManagerSettings}; -use super::proxy::UpdateOrProxyError; -use super::proxy_management; - -#[derive(Debug, Snafu)] -pub enum BindingEnvVarsOperationError { - #[snafu(display("Could not find canister id(s) for {} in environment '{environment}'. Make sure they are created first", canister_names.join(", ")))] - CanisterNotCreated { - environment: String, - canister_names: Vec, - }, - - #[snafu(transparent)] - UpdateOrProxy { source: UpdateOrProxyError }, -} - #[derive(Debug, Snafu)] #[snafu(display("Canister(s) {names:?} failed to update environment variables."))] pub struct SetBindingEnvVarsManyError { @@ -34,50 +20,15 @@ pub struct SetBindingEnvVarsManyError { struct BindingEnvVarsFailure { canister_name: String, canister_id: Principal, - error: BindingEnvVarsOperationError, -} - -pub(crate) async fn set_env_vars_for_canister( - agent: &Agent, - proxy: Option, - canister_id: &Principal, - canister_info: &Canister, - binding_vars: &[(String, String)], -) -> Result<(), BindingEnvVarsOperationError> { - let mut environment_variables = canister_info - .settings - .environment_variables - .to_owned() - .unwrap_or_default(); - - // inject the ids of the other canisters - for (k, v) in binding_vars.iter() { - environment_variables.insert(k.to_string(), v.to_string()); - } - - let environment_variables = environment_variables - .into_iter() - .map(|(name, value)| EnvironmentVariable { name, value }) - .collect::>(); - - proxy_management::update_settings( - agent, - proxy, - UpdateSettingsArgs { - canister_id: *canister_id, - settings: CanisterSettings { - environment_variables: Some(environment_variables), - ..Default::default() - }, - sender_canister_version: None, - }, - ) - .await?; - - Ok(()) + error: SyncCanisterError, } -/// Orchestrates setting environment variables for multiple canisters with progress tracking +/// Orchestrates setting environment variables for multiple canisters with progress tracking. +/// +/// The per-canister work (computing the generated `PUBLIC_CANISTER_ID:*` +/// bindings, merging with manifest env vars, and applying them) lives in +/// `icp_deploy_canister::apply_binding_env_vars`; this wrapper only adds the +/// missing-id precheck and progress display. pub(crate) async fn set_binding_env_vars_many( agent: Agent, proxy: Option, @@ -86,20 +37,14 @@ pub(crate) async fn set_binding_env_vars_many( canister_list: BTreeMap, debug: bool, ) -> Result<(), SetBindingEnvVarsManyError> { - // Check that all the canisters in this environment have an id - // We need to have all the ids to generate environment variables - // for the bindings + // Check that all the canisters in this environment have an id: we need all + // ids to generate the binding environment variables. let canisters_with_ids: HashSet<&String> = canister_list.keys().collect(); - let all_canister_names: Vec = target_canisters + let missing_canisters: Vec = target_canisters .iter() .map(|(_, info)| info.name.clone()) - .collect(); - - let missing_canisters: Vec = all_canister_names - .iter() - .filter(|c| !canisters_with_ids.contains(*c)) - .map(|c| c.to_string()) + .filter(|c| !canisters_with_ids.contains(c)) .collect(); if !missing_canisters.is_empty() { @@ -116,38 +61,22 @@ pub(crate) async fn set_binding_env_vars_many( .fail(); } + let canister_list = Arc::new(canister_list); + let mut futs = FuturesOrdered::new(); let progress_manager = ProgressManager::new(ProgressManagerSettings { hidden: debug }); for (cid, info) in target_canisters { let pb = progress_manager.create_progress_bar(&info.name); let canister_name = info.name.clone(); - - // Each canister receives only the ids it is wired to (its own project's - // canisters by their local names, plus any declared dependencies under - // their aliases), resolved to the ids that exist in this environment. - // A project without dependencies wires every canister to every sibling, - // reproducing the previous flat behavior. - let binding_vars: Vec<(String, String)> = info - .bindings - .iter() - .filter_map(|(env_name, referenced_key)| { - canister_list.get(referenced_key).map(|principal| { - ( - format!("PUBLIC_CANISTER_ID:{env_name}"), - principal.to_text(), - ) - }) - }) - .collect(); + let agent = agent.clone(); + let canister_list = canister_list.clone(); let settings_fn = { - let agent = agent.clone(); let pb = pb.clone(); - async move { pb.set_message("Updating environment variables..."); - set_env_vars_for_canister(&agent, proxy, &cid, &info, &binding_vars).await + apply_binding_env_vars(&info, cid, &canister_list, &agent, proxy).await } }; @@ -160,7 +89,6 @@ pub(crate) async fn set_binding_env_vars_many( ) .await; - // Map error to include canister context for deferred printing result.map_err(|error| BindingEnvVarsFailure { canister_name, canister_id: cid, diff --git a/crates/icp-cli/src/operations/install.rs b/crates/icp-cli/src/operations/install.rs index 24f90fd4e..d2891dc2b 100644 --- a/crates/icp-cli/src/operations/install.rs +++ b/crates/icp-cli/src/operations/install.rs @@ -1,15 +1,12 @@ -use candid::Encode; use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::{Agent, export::Principal}; use ic_management_canister_types::{ - CanisterId, CanisterIdRecord, CanisterInstallMode, CanisterStatusType, ChunkHash, - ClearChunkStoreArgs, InstallChunkedCodeArgs, InstallCodeArgs, UpgradeFlags, UploadChunkArgs, - WasmMemoryPersistence, + CanisterId, CanisterIdRecord, CanisterInstallMode, CanisterStatusType, WasmMemoryPersistence, }; -use sha2::{Digest, Sha256}; +use icp_deploy_canister::{InstallCanisterError, install_canister_resolved}; use snafu::{ResultExt, Snafu}; use std::sync::Arc; -use tracing::{debug, error, warn}; +use tracing::error; use crate::progress::{ProgressManager, ProgressManagerSettings}; @@ -28,7 +25,7 @@ pub enum WasmMemoryPersistenceOpt { } impl WasmMemoryPersistenceOpt { - fn to_ic(self) -> WasmMemoryPersistence { + pub(crate) fn to_ic(self) -> WasmMemoryPersistence { match self { WasmMemoryPersistenceOpt::Keep => WasmMemoryPersistence::Keep, WasmMemoryPersistenceOpt::Replace => WasmMemoryPersistence::Replace, @@ -44,43 +41,13 @@ pub(crate) async fn is_eop_canister(agent: &Agent, canister_id: &Principal) -> b .is_some() } -#[derive(Debug, Snafu)] -pub enum InstallOperationError { - #[snafu(display("Could not find build artifact for canister '{canister_name}'"))] - ArtifactNotFound { canister_name: String }, - - #[snafu(display("Failed to stop canister '{canister_name}' before upgrade"))] - StopCanister { - canister_name: String, - source: UpdateOrProxyError, - }, - - #[snafu(display("Failed to start canister '{canister_name}' after upgrade"))] - StartCanister { - canister_name: String, - source: UpdateOrProxyError, - }, - - #[snafu(transparent)] - UpdateOrProxy { source: UpdateOrProxyError }, -} - -#[derive(Debug, Snafu)] -#[snafu(display("Canister(s) {names:?} failed to install."))] -pub struct InstallManyError { - names: Vec, -} - -/// Holds error information from a failed canister install operation -struct InstallFailure { - canister_name: String, - canister_id: Principal, - error: InstallOperationError, -} - /// Resolve a mode string ("auto", "install", "reinstall", "upgrade") into /// a [`CanisterInstallMode`]. For "auto", queries `canister_status` to /// determine whether the canister already has code installed. +/// +/// Returns the resolved mode plus the current status; callers (deploy, the +/// candid-compat gate) need the resolved mode before installing, so resolution +/// happens here once and the result is handed to [`install_canister_resolved`]. pub(crate) async fn resolve_install_mode_and_status( agent: &Agent, proxy: Option, @@ -118,235 +85,66 @@ pub(crate) struct ResolveInstallModeError { source: UpdateOrProxyError, } -pub(crate) async fn install_canister( +#[derive(Debug, Snafu)] +pub(crate) enum InstallStoredError { + #[snafu(display("failed to read the built artifact for canister '{canister}'"))] + ReadArtifact { + canister: String, + source: icp::store_artifact::LookupArtifactError, + }, + + #[snafu(transparent)] + Install { source: InstallCanisterError }, +} + +/// Install one canister whose build artifact lives in the store, addressed by +/// its store key `canister_name`. The install-code/chunking/EOP logic lives in +/// `icp_deploy_canister::install_canister_resolved`; this reads the built wasm +/// from the artifact store and hands it to the agent-backed installer. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn install_stored_canister( agent: &Agent, proxy: Option, + artifacts: &dyn icp::store_artifact::Access, canister_id: &Principal, canister_name: &str, - wasm: &[u8], mode: CanisterInstallMode, status: CanisterStatusType, init_args: Option<&[u8]>, wasm_memory_persistence: Option, -) -> Result<(), InstallOperationError> { - let mode = match mode { - CanisterInstallMode::Upgrade(_) => { - // if this is a motoko canister using EOP we need to set additional options. - // If the caller supplied an explicit override, trust it (the CLI layer has - // already validated that it's an EOP canister); otherwise auto-detect and - // default to Keep. - let persistence = match wasm_memory_persistence { - Some(opt) => Some(opt.to_ic()), - None => is_eop_canister(agent, canister_id) - .await - .then_some(WasmMemoryPersistence::Keep), - }; - if let Some(persistence) = persistence { - CanisterInstallMode::Upgrade(Some(UpgradeFlags { - skip_pre_upgrade: None, - wasm_memory_persistence: Some(persistence), - })) - } else { - mode - } - } - _ => mode, - }; - - debug!( - "Install new canister code for {} with mode `{:?}`", - canister_name, mode - ); - - do_install_operation( - agent, - proxy, - canister_id, +) -> Result<(), InstallStoredError> { + let wasm = artifacts + .lookup(canister_name) + .await + .context(ReadArtifactSnafu { + canister: canister_name, + })?; + install_canister_resolved( canister_name, - wasm, + *canister_id, + &wasm, mode, status, init_args, + wasm_memory_persistence.map(WasmMemoryPersistenceOpt::to_ic), + agent, + proxy, ) - .await -} - -async fn do_install_operation( - agent: &Agent, - proxy: Option, - canister_id: &Principal, - canister_name: &str, - wasm: &[u8], - mode: CanisterInstallMode, - status: CanisterStatusType, - init_args: Option<&[u8]>, -) -> Result<(), InstallOperationError> { - // Threshold for chunked installation: 2 MB - // Raw install_code messages are limited to 2 MiB - const CHUNK_THRESHOLD: usize = 2 * 1024 * 1024; - - // Chunk size: 1 MB (spec limit is 1 MiB per chunk) - const CHUNK_SIZE: usize = 1024 * 1024; - - // Generous overhead for encoding, target canister ID, install mode, etc. - const ENCODING_OVERHEAD: usize = 500; - - let cid = CanisterId::from(*canister_id); - let arg = init_args - .map(|a| a.to_vec()) - .unwrap_or_else(|| Encode!().unwrap()); - - // Calculate total install message size - let total_install_size = wasm.len() + arg.len() + ENCODING_OVERHEAD; - - if total_install_size <= CHUNK_THRESHOLD { - // Small wasm: use regular install_code - debug!("Installing wasm for {canister_name} using install_code"); - - let install_args = InstallCodeArgs { - mode, - canister_id: cid, - wasm_module: wasm.to_vec(), - arg, - sender_canister_version: None, - }; - - stop_and_start_if_upgrade( - agent, - proxy, - canister_id, - canister_name, - mode, - status, - async { - proxy_management::install_code(agent, proxy, install_args).await?; - Ok(()) - }, - ) - .await?; - } else { - // Large wasm: use chunked installation - debug!("Installing wasm for {canister_name} using chunked installation"); - - // Clear any existing chunks to ensure a clean state - proxy_management::clear_chunk_store(agent, proxy, ClearChunkStoreArgs { canister_id: cid }) - .await?; - - // Split wasm into chunks and upload them - let chunks: Vec<&[u8]> = wasm.chunks(CHUNK_SIZE).collect(); - let mut chunk_hashes: Vec = Vec::new(); - - for (i, chunk) in chunks.iter().enumerate() { - debug!( - "Uploading chunk {}/{} ({} bytes)", - i + 1, - chunks.len(), - chunk.len() - ); - - let upload_args = UploadChunkArgs { - canister_id: cid, - chunk: chunk.to_vec(), - }; - - let chunk_hash = proxy_management::upload_chunk(agent, proxy, upload_args).await?; - - chunk_hashes.push(chunk_hash); - } - - // Compute SHA-256 hash of the entire wasm module - let mut hasher = Sha256::new(); - hasher.update(wasm); - let wasm_module_hash = hasher.finalize().to_vec(); - - debug!("Installing chunked code with {} chunks", chunk_hashes.len()); - - let chunked_args = InstallChunkedCodeArgs { - mode, - target_canister: cid, - store_canister: None, - chunk_hashes_list: chunk_hashes, - wasm_module_hash, - arg, - sender_canister_version: None, - }; - - let install_res = stop_and_start_if_upgrade( - agent, - proxy, - canister_id, - canister_name, - mode, - status, - async { - proxy_management::install_chunked_code(agent, proxy, chunked_args).await?; - Ok(()) - }, - ) - .await; - - // Clear chunk store after successful installation to free up storage - let clear_res = proxy_management::clear_chunk_store( - agent, - proxy, - ClearChunkStoreArgs { canister_id: cid }, - ) - .await - .map_err(InstallOperationError::from); - - if let Err(clear_error) = clear_res { - if let Err(install_error) = install_res { - warn!("Failed to clear chunk store after failed install: {clear_error}"); - return Err(install_error); - } else { - return Err(clear_error); - } - } - install_res?; - } - + .await?; Ok(()) } -async fn stop_and_start_if_upgrade( - agent: &Agent, - proxy: Option, - canister_id: &Principal, - canister_name: &str, - mode: CanisterInstallMode, - status: CanisterStatusType, - f: impl Future>, -) -> Result<(), InstallOperationError> { - let should_guard = matches!( - mode, - CanisterInstallMode::Upgrade(_) | CanisterInstallMode::Reinstall - ) && matches!(status, CanisterStatusType::Running); - let cid_record = CanisterIdRecord { - canister_id: CanisterId::from(*canister_id), - }; - // Stop the canister before proceeding - if should_guard { - proxy_management::stop_canister(agent, proxy, cid_record.clone()) - .await - .context(StopCanisterSnafu { canister_name })?; - } - // Install the canister - let install_result = f.await; - // Restart the canister whether or not the installation succeeded - if should_guard { - let start_result = proxy_management::start_canister(agent, proxy, cid_record).await; - if let Err(start_error) = start_result { - // If both install and start failed, report the install error since it's more likely to be the root cause - if let Err(install_error) = install_result { - warn!("Failed to start canister after failed upgrade: {start_error}"); - return Err(install_error); - } else { - return Err(start_error).context(StartCanisterSnafu { canister_name }); - } - } - } +#[derive(Debug, Snafu)] +#[snafu(display("Canister(s) {names:?} failed to install."))] +pub struct InstallManyError { + names: Vec, +} - install_result +/// Holds error information from a failed canister install operation +struct InstallFailure { + canister_name: String, + canister_id: Principal, + error: InstallStoredError, } /// Installs code to multiple canisters and displays progress bars. @@ -371,26 +169,19 @@ pub(crate) async fn install_many( for (name, cid, mode, status, init_args) in canisters { let pb = progress_manager.create_progress_bar(&name); let agent = agent.clone(); + let artifacts = artifacts.clone(); let install_fn = { let pb = pb.clone(); - let artifacts = artifacts.clone(); let name = name.clone(); async move { pb.set_message("Installing..."); - - let wasm = artifacts.lookup(&name).await.map_err(|_| { - InstallOperationError::ArtifactNotFound { - canister_name: name.clone(), - } - })?; - - install_canister( + install_stored_canister( &agent, proxy, + artifacts.as_ref(), &cid, &name, - &wasm, mode, status, init_args.as_deref(), diff --git a/crates/icp-cli/src/operations/proxy_management.rs b/crates/icp-cli/src/operations/proxy_management.rs index ffc7cac70..43ec4aa52 100644 --- a/crates/icp-cli/src/operations/proxy_management.rs +++ b/crates/icp-cli/src/operations/proxy_management.rs @@ -1,15 +1,14 @@ use candid::Principal; use ic_agent::Agent; use ic_management_canister_types::{ - CanisterIdRecord, CanisterStatusResult, ClearChunkStoreArgs, CreateCanisterArgs, - DeleteCanisterArgs, DeleteCanisterSnapshotArgs, FetchCanisterLogsArgs, FetchCanisterLogsResult, - InstallChunkedCodeArgs, InstallCodeArgs, ListCanisterSnapshotsArgs, - ListCanisterSnapshotsResult, LoadCanisterSnapshotArgs, ReadCanisterSnapshotDataArgs, - ReadCanisterSnapshotDataResult, ReadCanisterSnapshotMetadataArgs, + CanisterIdRecord, CanisterStatusResult, CreateCanisterArgs, DeleteCanisterArgs, + DeleteCanisterSnapshotArgs, FetchCanisterLogsArgs, FetchCanisterLogsResult, InstallCodeArgs, + ListCanisterSnapshotsArgs, ListCanisterSnapshotsResult, LoadCanisterSnapshotArgs, + ReadCanisterSnapshotDataArgs, ReadCanisterSnapshotDataResult, ReadCanisterSnapshotMetadataArgs, ReadCanisterSnapshotMetadataResult, StartCanisterArgs, StopCanisterArgs, TakeCanisterSnapshotArgs, TakeCanisterSnapshotResult, UpdateSettingsArgs, UploadCanisterSnapshotDataArgs, UploadCanisterSnapshotMetadataArgs, - UploadCanisterSnapshotMetadataResult, UploadChunkArgs, UploadChunkResult, + UploadCanisterSnapshotMetadataResult, }; use snafu::{ResultExt, Snafu}; @@ -144,61 +143,6 @@ pub async fn install_code( .await } -pub async fn install_chunked_code( - agent: &Agent, - proxy: Option, - args: InstallChunkedCodeArgs, -) -> Result<(), UpdateOrProxyError> { - let effective = args.target_canister; - update_or_proxy::<_, ()>( - agent, - Principal::management_canister(), - "install_chunked_code", - (args,), - proxy, - Some(effective), - 0, - ) - .await -} - -pub async fn upload_chunk( - agent: &Agent, - proxy: Option, - args: UploadChunkArgs, -) -> Result { - let effective = args.canister_id; - let (result,): (UploadChunkResult,) = update_or_proxy( - agent, - Principal::management_canister(), - "upload_chunk", - (args,), - proxy, - Some(effective), - 0, - ) - .await?; - Ok(result) -} - -pub async fn clear_chunk_store( - agent: &Agent, - proxy: Option, - args: ClearChunkStoreArgs, -) -> Result<(), UpdateOrProxyError> { - let effective = args.canister_id; - update_or_proxy::<_, ()>( - agent, - Principal::management_canister(), - "clear_chunk_store", - (args,), - proxy, - Some(effective), - 0, - ) - .await -} - #[derive(Debug, Snafu)] pub enum FetchCanisterLogsError { #[snafu(display("failed to encode call arguments: {source}"))] diff --git a/crates/icp-cli/src/operations/sync.rs b/crates/icp-cli/src/operations/sync.rs index 18c77efcc..c3f7bd967 100644 --- a/crates/icp-cli/src/operations/sync.rs +++ b/crates/icp-cli/src/operations/sync.rs @@ -1,15 +1,16 @@ +use async_trait::async_trait; use candid::Principal; use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::Agent; -use icp::{ - Canister, - canister::sync::{Params, Synchronize, SynchronizeError}, - package::PackageCache, - prelude::PathBuf, +use icp::{Canister, canister::recipe::RemoteResourceResolve, canister::script, prelude::PathBuf}; +use icp_deploy_canister::sync_exec::{ + ScriptInvocation, ScriptRunError, ScriptRunner, StepProgress, }; +use icp_deploy_canister::{SyncCanisterError, SyncStepContext, run_sync_steps}; use snafu::prelude::*; use std::collections::BTreeMap; use std::sync::Arc; +use tokio::sync::mpsc::Sender; use tracing::error; use crate::progress::{MultiStepProgressBar, ProgressManager, ProgressManagerSettings}; @@ -24,14 +25,61 @@ pub struct SyncOperationError { struct SyncFailure { canister_name: String, canister_id: Principal, - error: SynchronizeError, + error: SyncCanisterError, progress_output: Vec, } -/// Synchronizes a single canister using its configured sync steps +/// [`ScriptRunner`] backed by the host subprocess executor. Sync-step dispatch +/// and the `ICP_CLI_*` environment are assembled by `icp_deploy_canister`; this +/// only spawns the resolved command(s). +struct HostScriptRunner; + +#[async_trait] +impl ScriptRunner for HostScriptRunner { + async fn run_script( + &self, + invocation: ScriptInvocation, + stdio: Option>, + ) -> Result, ScriptRunError> { + let env_refs: Vec<(&str, &str)> = invocation + .env + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + script::execute_commands(&invocation.commands, &invocation.cwd, &env_refs, stdio) + .await + .map_err(|source| ScriptRunError { + source: Box::new(source), + })?; + // Persistent stderr is a sync-plugin feature only; script steps don't + // currently retain any output past the rolling step view. + Ok(vec![]) + } +} + +/// [`StepProgress`] that frames each sync step on the canister's multi-step +/// progress bar and streams the step's output lines to it. +struct BarStepProgress<'a> { + pb: &'a mut MultiStepProgressBar, +} + +#[async_trait] +impl StepProgress for BarStepProgress<'_> { + fn begin_step(&mut self, header: String) -> Option> { + Some(self.pb.begin_step(header)) + } + + async fn end_step(&mut self) { + self.pb.end_step().await; + } +} + +/// Synchronize a single canister's steps through the library, framing progress +/// on `pb`. Environment variables are applied separately by the caller. +#[allow(clippy::too_many_arguments)] async fn sync_canister( - syncer: &Arc, agent: &Agent, + resolver: &dyn RemoteResourceResolve, canister_path: PathBuf, canister_id: Principal, canister_info: &Canister, @@ -40,56 +88,38 @@ async fn sync_canister( canister_ids: &BTreeMap, proxy: Option, pb: &mut MultiStepProgressBar, - pkg_cache: &PackageCache, -) -> Result, SynchronizeError> { - let step_count = canister_info.sync.steps.len(); - let mut stderr_lines = Vec::new(); - - for (i, step) in canister_info.sync.steps.iter().enumerate() { - // Indicate to user the current step being executed - let current_step = i + 1; - let pb_hdr = format!("\nSyncing: {step} {current_step} of {step_count}"); - - let tx = pb.begin_step(pb_hdr); - - // Execute step - let sync_result = syncer - .sync( - step, - &Params { - path: canister_path.clone(), - cid: canister_id, - environment: environment.to_owned(), - network: network.to_owned(), - canister_ids: canister_ids.clone(), - proxy, - }, - agent, - Some(tx), - pkg_cache, - ) - .await; - - // Ensure background receiver drains all messages - pb.end_step().await; - - stderr_lines.extend(sync_result?); - } - - Ok(stderr_lines) +) -> Result, SyncCanisterError> { + let ctx = SyncStepContext { + canister_path, + canister_id, + environment: environment.to_owned(), + network: network.to_owned(), + canister_ids: canister_ids.clone(), + proxy, + }; + let mut progress = BarStepProgress { pb }; + run_sync_steps( + canister_info, + &ctx, + agent, + resolver, + &HostScriptRunner, + Some(&mut progress), + ) + .await } /// Orchestrates syncing multiple canisters with progress tracking +#[allow(clippy::too_many_arguments)] pub(crate) async fn sync_many( - syncer: Arc, agent: Agent, + resolver: Arc, canisters: Vec<(Principal, PathBuf, Canister)>, environment: String, network: String, canister_ids: BTreeMap, proxy: Option, debug: bool, - pkg_cache: &PackageCache, ) -> Result<(), SyncOperationError> { let mut futs = FuturesOrdered::new(); let progress_manager = ProgressManager::new(ProgressManagerSettings { hidden: debug }); @@ -99,16 +129,15 @@ pub(crate) async fn sync_many( let fut = { let agent = agent.clone(); - let syncer = syncer.clone(); + let resolver = resolver.clone(); let environment = environment.clone(); let network = network.clone(); let canister_ids = canister_ids.clone(); async move { - // Define the sync logic let sync_result = sync_canister( - &syncer, &agent, + resolver.as_ref(), canister_path, cid, &canister_info, @@ -117,7 +146,6 @@ pub(crate) async fn sync_many( &canister_ids, proxy, &mut pb, - pkg_cache, ) .await; diff --git a/crates/icp-deploy-canister/Cargo.toml b/crates/icp-deploy-canister/Cargo.toml new file mode 100644 index 000000000..503b67369 --- /dev/null +++ b/crates/icp-deploy-canister/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "icp-deploy-canister" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[features] +# Enables `clap::ValueEnum` derives on manifest enums used as CLI value types +# (e.g. `ArgsFormat`). Enabled transitively by `icp/clap`. +clap = ["dep:clap"] + +[dependencies] +async-trait.workspace = true +bigdecimal.workspace = true +camino.workspace = true +candid.workspace = true +clap = { workspace = true, optional = true } +candid_parser.workspace = true +dunce.workspace = true +futures.workspace = true +glob.workspace = true +handlebars.workspace = true +hex.workspace = true +ic-agent.workspace = true +ic-management-canister-types.workspace = true +icp-canister-interfaces.workspace = true +icp-sync-plugin.workspace = true +indexmap.workspace = true +itertools.workspace = true +num-bigint.workspace = true +num-integer.workspace = true +num-traits.workspace = true +pathdiff.workspace = true +schemars.workspace = true +serde.workspace = true +sha2.workspace = true +serde_json.workspace = true +serde_yaml.workspace = true +snafu.workspace = true +strum.workspace = true +tokio = { workspace = true, features = ["full"] } +tracing.workspace = true +url.workspace = true + +[dev-dependencies] +camino-tempfile.workspace = true +indoc.workspace = true +jsonschema.workspace = true diff --git a/crates/icp-deploy-canister/src/canister/mod.rs b/crates/icp-deploy-canister/src/canister/mod.rs new file mode 100644 index 000000000..978bf84f6 --- /dev/null +++ b/crates/icp-deploy-canister/src/canister/mod.rs @@ -0,0 +1,535 @@ +use std::collections::HashMap; + +use candid::{Nat, Principal}; +use ic_management_canister_types::{CanisterSettings, LogVisibility}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::parsers::{CyclesAmount, DurationAmount, MemoryAmount}; + +pub mod recipe; + +/// Controls who can read canister logs. +/// Supports both string format ("controllers", "public") and object format ({ allowed_viewers: [...] }). +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(untagged)] +pub enum LogVisibilityDef { + /// Simple string variants for controllers or public + Simple(LogVisibilitySimple), + /// Object format with allowed_viewers list + AllowedViewers { allowed_viewers: Vec }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum LogVisibilitySimple { + Controllers, + Public, +} + +impl<'de> Deserialize<'de> for LogVisibilityDef { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::{Error, MapAccess, Visitor}; + use std::fmt; + + struct LogVisibilityVisitor; + + impl<'de> Visitor<'de> for LogVisibilityVisitor { + type Value = LogVisibilityDef; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("'controllers', 'public', or object with 'allowed_viewers'") + } + + fn visit_str(self, value: &str) -> Result { + LogVisibilitySimple::deserialize( + serde::de::value::StrDeserializer::::new(value), + ) + .map(LogVisibilityDef::Simple) + .map_err(|_| { + E::custom(format!( + "unknown log_visibility value: '{}', expected 'controllers' or 'public'", + value + )) + }) + } + + fn visit_map(self, mut map: M) -> Result + where + M: MapAccess<'de>, + { + let mut allowed_viewers: Option> = None; + + while let Some(key) = map.next_key::()? { + match key.as_str() { + "allowed_viewers" => { + if allowed_viewers.is_some() { + return Err(Error::duplicate_field("allowed_viewers")); + } + allowed_viewers = Some(map.next_value()?); + } + _ => { + return Err(Error::unknown_field(&key, &["allowed_viewers"])); + } + } + } + + allowed_viewers + .map(|v| LogVisibilityDef::AllowedViewers { allowed_viewers: v }) + .ok_or_else(|| Error::missing_field("allowed_viewers")) + } + } + + deserializer.deserialize_any(LogVisibilityVisitor) + } +} + +impl JsonSchema for LogVisibilityDef { + fn schema_name() -> std::borrow::Cow<'static, str> { + std::borrow::Cow::Borrowed("LogVisibility") + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "description": "Controls who can read canister logs.", + "oneOf": [ + { + "type": "string", + "enum": ["controllers", "public"], + "description": "Simple log visibility: 'controllers' (only controllers can view) or 'public' (anyone can view)" + }, + { + "type": "object", + "properties": { + "allowed_viewers": { + "type": "array", + "items": { + "type": "string", + "description": "A principal ID that can view logs" + }, + "description": "List of principal IDs that can view canister logs" + } + }, + "required": ["allowed_viewers"], + "additionalProperties": false, + "description": "Specific principals that can view logs" + } + ] + }) + } +} + +impl From for LogVisibility { + fn from(value: LogVisibilityDef) -> Self { + match value { + LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) => { + LogVisibility::Controllers + } + LogVisibilityDef::Simple(LogVisibilitySimple::Public) => LogVisibility::Public, + LogVisibilityDef::AllowedViewers { allowed_viewers } => { + LogVisibility::AllowedViewers(allowed_viewers) + } + } + } +} + +/// A reference to a controller: either an explicit principal or a canister name in this project. +/// +/// During deserialization, principal text format is tried first; strings that don't parse as a +/// principal are treated as canister names. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ControllerRef { + /// An explicitly specified principal (e.g. "2vxsx-fae") + Principal(candid::Principal), + /// A canister name from the same project (e.g. "my_canister") + CanisterName(String), +} + +impl ControllerRef { + /// Resolve to a `Principal` using the provided ID mapping. + /// Returns `None` if this is a `CanisterName` not present in `ids`. + pub fn resolve(&self, ids: &crate::ids::IdMapping) -> Option { + match self { + ControllerRef::Principal(p) => Some(*p), + ControllerRef::CanisterName(name) => ids.get(name).copied(), + } + } + + /// If this is a `CanisterName`, returns the name; otherwise `None`. + pub fn canister_name(&self) -> Option<&str> { + match self { + ControllerRef::CanisterName(n) => Some(n), + ControllerRef::Principal(_) => None, + } + } +} + +/// Partition a slice of controller references into resolved principals and unresolved canister +/// names, using `ids` for name lookup. +pub fn resolve_controllers( + crefs: &[ControllerRef], + ids: &crate::ids::IdMapping, +) -> (Vec, Vec) { + let mut resolved = Vec::new(); + let mut unresolved = Vec::new(); + for cref in crefs { + match cref.resolve(ids) { + Some(p) => resolved.push(p), + None => { + if let Some(name) = cref.canister_name() { + unresolved.push(name.to_owned()); + } + } + } + } + (resolved, unresolved) +} + +impl schemars::JsonSchema for ControllerRef { + fn schema_name() -> std::borrow::Cow<'static, str> { + std::borrow::Cow::Borrowed("ControllerRef") + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "description": "A controller: either a principal text (e.g. '2vxsx-fae') or a canister name in this project (e.g. 'my_canister')" + }) + } +} + +/// Canister settings, such as compute and memory allocation. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct Settings { + /// Controls who can read canister logs. + #[serde(skip_serializing_if = "Option::is_none")] + pub log_visibility: Option, + + /// Compute allocation (0 to 100). Represents guaranteed compute capacity. + #[serde(skip_serializing_if = "Option::is_none")] + pub compute_allocation: Option, + + /// Memory allocation in bytes. If unset, memory is allocated dynamically. + /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_allocation: Option, + + /// Freezing threshold in seconds. Controls how long a canister can be inactive before being frozen. + /// Supports duration suffixes in YAML: s, m, h, d, w (e.g. "30d" or "4w"). + #[serde(skip_serializing_if = "Option::is_none")] + pub freezing_threshold: Option, + + /// Upper limit on cycles reserved for future resource payments. + /// Memory allocations that would push the reserved balance above this limit will fail. + /// Supports suffixes in YAML: k, m, b, t (e.g. "4t" or "4.3t"). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reserved_cycles_limit: Option, + + /// Wasm memory limit in bytes. Sets an upper bound for Wasm heap growth. + /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] + pub wasm_memory_limit: Option, + + /// Wasm memory threshold in bytes. Triggers a callback when exceeded. + /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] + pub wasm_memory_threshold: Option, + + /// Log memory limit in bytes (max 2 MiB). Oldest logs are purged when usage exceeds this value. + /// Supports suffixes in YAML: kb, kib, mb, mib (e.g. "2mib" or "256kib"). Canister default is 4096 bytes. + #[serde(skip_serializing_if = "Option::is_none")] + pub log_memory_limit: Option, + + /// Environment variables for the canister as key-value pairs. + /// These variables are accessible within the canister and can be used to configure + /// behavior without hardcoding values in the WASM module. + #[serde(skip_serializing_if = "Option::is_none")] + pub environment_variables: Option>, + + /// Controllers for this canister. Each entry is either a principal text + /// (e.g. "2vxsx-fae") or the name of another canister in this project. + /// Named canisters that do not yet exist will be set as controllers once created. + #[serde(default)] + pub controllers: Option>, +} + +impl From for CanisterSettings { + fn from(settings: Settings) -> Self { + CanisterSettings { + freezing_threshold: settings.freezing_threshold.map(|d| Nat::from(d.get())), + controllers: None, + reserved_cycles_limit: settings.reserved_cycles_limit.map(|c| Nat::from(c.get())), + log_visibility: settings.log_visibility.map(Into::into), + memory_allocation: settings.memory_allocation.map(|m| Nat::from(m.get())), + compute_allocation: settings.compute_allocation.map(Nat::from), + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn log_visibility_deserialize_controllers() { + let yaml = "controllers"; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) + ); + } + + #[test] + fn log_visibility_deserialize_public() { + let yaml = "public"; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + LogVisibilityDef::Simple(LogVisibilitySimple::Public) + ); + } + + #[test] + fn log_visibility_deserialize_allowed_viewers() { + let yaml = r#" +allowed_viewers: + - "aaaaa-aa" + - "2vxsx-fae" +"#; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + match result { + LogVisibilityDef::AllowedViewers { allowed_viewers } => { + assert_eq!(allowed_viewers.len(), 2); + assert_eq!( + allowed_viewers[0], + Principal::from_text("aaaaa-aa").unwrap() + ); + assert_eq!( + allowed_viewers[1], + Principal::from_text("2vxsx-fae").unwrap() + ); + } + _ => panic!("Expected AllowedViewers variant"), + } + } + + #[test] + fn log_visibility_deserialize_allowed_viewers_empty() { + let yaml = "allowed_viewers: []"; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + match result { + LogVisibilityDef::AllowedViewers { allowed_viewers } => { + assert!(allowed_viewers.is_empty()); + } + _ => panic!("Expected AllowedViewers variant"), + } + } + + #[test] + fn log_visibility_deserialize_invalid_string() { + let yaml = "invalid"; + let result: Result = serde_yaml::from_str(yaml); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("unknown log_visibility value")); + } + + #[test] + fn log_visibility_deserialize_invalid_field() { + let yaml = "unknown_field: []"; + let result: Result = serde_yaml::from_str(yaml); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("unknown field")); + } + + #[test] + fn log_visibility_serialize_controllers() { + let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); + let yaml = serde_yaml::to_string(&log_vis).unwrap(); + assert_eq!(yaml.trim(), "controllers"); + } + + #[test] + fn log_visibility_serialize_public() { + let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Public); + let yaml = serde_yaml::to_string(&log_vis).unwrap(); + assert_eq!(yaml.trim(), "public"); + } + + #[test] + fn log_visibility_serialize_allowed_viewers() { + let log_vis = LogVisibilityDef::AllowedViewers { + allowed_viewers: vec![ + Principal::from_text("aaaaa-aa").unwrap(), + Principal::from_text("2vxsx-fae").unwrap(), + ], + }; + let yaml = serde_yaml::to_string(&log_vis).unwrap(); + assert!(yaml.contains("allowed_viewers")); + assert!(yaml.contains("aaaaa-aa")); + assert!(yaml.contains("2vxsx-fae")); + } + + #[test] + fn settings_reserved_cycles_limit_parses_suffix() { + let yaml = "reserved_cycles_limit: 4.3t"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.reserved_cycles_limit.as_ref().map(|c| c.get()), + Some(4_300_000_000_000) + ); + } + + #[test] + fn settings_reserved_cycles_limit_parses_number() { + let yaml = "reserved_cycles_limit: 5000000000000"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.reserved_cycles_limit.as_ref().map(|c| c.get()), + Some(5_000_000_000_000) + ); + } + + #[test] + fn settings_memory_allocation_parses_suffix() { + let yaml = "memory_allocation: 4gib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.memory_allocation.as_ref().map(|m| m.get()), + Some(4 * 1024 * 1024 * 1024) + ); + } + + #[test] + fn settings_memory_allocation_parses_number() { + let yaml = "memory_allocation: 4294967296"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.memory_allocation.as_ref().map(|m| m.get()), + Some(4294967296) + ); + } + + #[test] + fn settings_wasm_memory_limit_parses_suffix() { + let yaml = "wasm_memory_limit: 1.5gib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.wasm_memory_limit.as_ref().map(|m| m.get()), + Some(1610612736) + ); + } + + #[test] + fn settings_log_memory_limit_parses_suffix() { + let yaml = "log_memory_limit: 256kib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.log_memory_limit.as_ref().map(|m| m.get()), + Some(256 * 1024) + ); + } + + #[test] + fn settings_log_memory_limit_parses_mib() { + let yaml = "log_memory_limit: 2mib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.log_memory_limit.as_ref().map(|m| m.get()), + Some(2 * 1024 * 1024) + ); + } + + #[test] + fn controller_ref_deserializes_principal() { + let yaml = "\"2vxsx-fae\""; + let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + ControllerRef::Principal(Principal::from_text("2vxsx-fae").unwrap()) + ); + } + + #[test] + fn controller_ref_deserializes_canister_name() { + let yaml = "\"my_canister\""; + let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + ControllerRef::CanisterName("my_canister".to_owned()) + ); + } + + #[test] + fn controller_ref_resolve_principal() { + let p = Principal::from_text("aaaaa-aa").unwrap(); + let cref = ControllerRef::Principal(p); + let ids = crate::ids::IdMapping::new(); + assert_eq!(cref.resolve(&ids), Some(p)); + } + + #[test] + fn controller_ref_resolve_canister_name_present() { + let p = Principal::from_text("aaaaa-aa").unwrap(); + let cref = ControllerRef::CanisterName("backend".to_owned()); + let mut ids = crate::ids::IdMapping::new(); + ids.insert("backend".to_owned(), p); + assert_eq!(cref.resolve(&ids), Some(p)); + } + + #[test] + fn controller_ref_resolve_canister_name_absent() { + let cref = ControllerRef::CanisterName("backend".to_owned()); + let ids = crate::ids::IdMapping::new(); + assert_eq!(cref.resolve(&ids), None); + } + + #[test] + fn settings_controllers_parses_mixed() { + let yaml = r#" +controllers: + - "aaaaa-aa" + - "my_other_canister" +"#; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + let controllers = settings.controllers.unwrap(); + assert_eq!(controllers.len(), 2); + assert_eq!( + controllers[0], + ControllerRef::Principal(Principal::from_text("aaaaa-aa").unwrap()) + ); + assert_eq!( + controllers[1], + ControllerRef::CanisterName("my_other_canister".to_owned()) + ); + } + + #[test] + fn log_visibility_conversion_to_ic_type() { + let controllers = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); + let ic_controllers: LogVisibility = controllers.into(); + assert!(matches!(ic_controllers, LogVisibility::Controllers)); + + let public = LogVisibilityDef::Simple(LogVisibilitySimple::Public); + let ic_public: LogVisibility = public.into(); + assert!(matches!(ic_public, LogVisibility::Public)); + + let viewers = LogVisibilityDef::AllowedViewers { + allowed_viewers: vec![Principal::from_text("aaaaa-aa").unwrap()], + }; + let ic_viewers: LogVisibility = viewers.into(); + match ic_viewers { + LogVisibility::AllowedViewers(v) => { + assert_eq!(v.len(), 1); + } + _ => panic!("Expected AllowedViewers"), + } + } +} diff --git a/crates/icp-deploy-canister/src/canister/recipe/mod.rs b/crates/icp-deploy-canister/src/canister/recipe/mod.rs new file mode 100644 index 000000000..dc824732d --- /dev/null +++ b/crates/icp-deploy-canister/src/canister/recipe/mod.rs @@ -0,0 +1,315 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use handlebars::{Context, Handlebars, Helper, HelperDef, HelperResult, Output, RenderContext}; +use serde::Deserialize; +use snafu::prelude::*; +use tokio::sync::mpsc::Sender; + +use crate::manifest::{ + adapter::prebuilt::SourceField, + canister::{BuildSteps, SyncSteps}, + recipe::{Recipe, RecipeType}, +}; +use crate::prelude::*; + +/// Context passed to a recipe resolver, describing the canister being built. +/// +/// Serializes to the shape injected into recipe templates under the `_` namespace: +/// +/// ```yaml +/// canister: +/// name: +/// ``` +pub struct RecipeContext { + pub canister_name: String, +} + +impl RecipeContext { + /// Builds the YAML value injected into recipe templates under the `_` namespace. + /// Constructing the mapping directly is infallible, unlike `serde` serialization. + pub fn to_yaml(&self) -> serde_yaml::Value { + use serde_yaml::{Mapping, Value}; + + let mut canister = Mapping::new(); + canister.insert("name".into(), Value::String(self.canister_name.clone())); + + let mut root = Mapping::new(); + root.insert("canister".into(), Value::Mapping(canister)); + + Value::Mapping(root) + } +} + +/// Fetches the remote resources a project references — recipe templates and +/// plugin wasms — retrieving them over HTTP and caching on disk as needed. +/// +/// The concrete resolver (which owns the HTTP client and the package cache) +/// lives in the host `icp` crate; this crate defines the interface, and renders +/// fetched recipe templates itself (see [`render_recipe`]), so that consolidation +/// and sync can call an injected resolver. +#[async_trait] +pub trait RemoteResourceResolve: Sync + Send { + /// Fetch a recipe's Handlebars template, returning its raw source. Callers + /// render it into build/sync steps with [`render_recipe`]. + async fn resolve_recipe(&self, recipe: &Recipe) -> Result; + + /// Resolve a plugin wasm `source` (relative to `base_dir`) to a local path, + /// verifying `sha256` and caching a remote download. `stdio` receives + /// progress lines. + async fn resolve_wasm( + &self, + source: &SourceField, + base_dir: &Path, + sha256: Option<&str>, + stdio: Option>, + ) -> Result; +} + +#[derive(Debug, Snafu)] +pub enum ResolveError { + /// The injected resolver failed. The concrete source (e.g. a fetch/cache + /// error from the host resolver) is boxed because this crate does not depend + /// on the resolver's implementation. + #[snafu(display("failed to fetch recipe template"))] + Resolve { + source: Box, + }, + + #[snafu(display("failed to resolve plugin wasm"))] + ResolveWasm { + source: Box, + }, +} + +/// A [`RemoteResourceResolve`] that always fails, for environments that don't +/// support remote recipe templates or plugin wasms. +pub struct NoResolve; + +#[derive(Debug, Snafu)] +#[snafu(display("remote recipes are not allowed"))] +pub struct NoResolveError; + +#[derive(Debug, Snafu)] +#[snafu(display("remote modules are not allowed"))] +pub struct NoResolveModuleError; + +#[async_trait] +impl RemoteResourceResolve for NoResolve { + async fn resolve_recipe(&self, recipe: &Recipe) -> Result { + match &recipe.recipe_type { + RecipeType::File(path) => { + crate::fs::read_to_string(path.as_ref()).map_err(|e| ResolveError::Resolve { + source: Box::new(e), + }) + } + _ => Err(ResolveError::Resolve { + source: Box::new(NoResolveError), + }), + } + } + + async fn resolve_wasm( + &self, + source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _stdio: Option>, + ) -> Result { + match source { + SourceField::Local(source) => Ok(source.path.clone()), + _ => Err(ResolveError::ResolveWasm { + source: Box::new(NoResolveModuleError), + }), + } + } +} + +#[derive(Debug, Snafu)] +pub enum RenderRecipeError { + #[snafu(display("recipe template for '{recipe}' failed to render"))] + Render { + source: handlebars::RenderError, + recipe: RecipeType, + }, + + #[snafu(display("recipe '{recipe}' did not render into a valid build/sync manifest"))] + Parse { + source: serde_yaml::Error, + recipe: RecipeType, + }, +} + +/// Render a recipe's Handlebars `template` into concrete build/sync steps. +/// +/// The template is rendered with the recipe's `configuration` plus the reserved +/// `_` namespace (the `_` key always overrides any user-supplied value), then the +/// resulting YAML is parsed. A recipe may only produce `build` and `sync`. +#[allow(clippy::result_large_err)] +pub fn render_recipe( + template: &str, + recipe: &Recipe, + recipe_context: &RecipeContext, +) -> Result<(BuildSteps, SyncSteps), RenderRecipeError> { + let mut reg = Handlebars::new(); + // The output is YAML, not HTML, so disable HTML escaping. + reg.register_escape_fn(handlebars::no_escape); + reg.register_helper("replace", Box::new(ReplaceHelper)); + // Reject unset template variables. + reg.set_strict_mode(true); + + // User-provided configuration plus the injected `_.*` variables. The `_` key + // is reserved and always overrides any user-supplied value. + let mut render_context: HashMap = recipe.configuration.clone(); + render_context.insert("_".to_string(), recipe_context.to_yaml()); + + let out = reg + .render_template(template, &render_context) + .context(RenderSnafu { + recipe: recipe.recipe_type.clone(), + })?; + + // Recipes can only render `build`/`sync`. + #[derive(Deserialize)] + struct BuildSyncHelper { + build: BuildSteps, + #[serde(default)] + sync: SyncSteps, + } + + let helper: BuildSyncHelper = serde_yaml::from_str(&out).context(ParseSnafu { + recipe: recipe.recipe_type.clone(), + })?; + Ok((helper.build, helper.sync)) +} + +/// Handlebars helper for string replacement operations. +/// Usage: `{{ replace "from" "to" value }}` +#[derive(Clone, Copy)] +struct ReplaceHelper; + +impl HelperDef for ReplaceHelper { + fn call<'reg: 'rc, 'rc>( + &self, + h: &Helper, + _: &'reg Handlebars<'reg>, + _: &Context, + _: &mut RenderContext<'reg, 'rc>, + out: &mut dyn Output, + ) -> HelperResult { + let (from, to) = (h.param(0).unwrap().render(), h.param(1).unwrap().render()); + let v = h.param(2).unwrap().render(); + out.write(&v.replace(&from, &to))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::canister::BuildStep; + + fn recipe(config: &[(&str, &str)]) -> Recipe { + Recipe { + recipe_type: RecipeType::File("recipe.hbs".to_owned()), + configuration: config + .iter() + .map(|(k, v)| ((*k).to_owned(), serde_yaml::Value::String((*v).to_owned()))) + .collect(), + sha256: None, + } + } + + fn ctx(name: &str) -> RecipeContext { + RecipeContext { + canister_name: name.to_owned(), + } + } + + /// The only build step's command, for a recipe that renders a single script step. + fn rendered_command(template: &str, recipe: &Recipe, context: &RecipeContext) -> String { + let (build, _sync) = render_recipe(template, recipe, context).unwrap(); + match &build.steps[0] { + BuildStep::Script(adapter) => adapter.command.as_vec()[0].clone(), + other => panic!("expected a script build step, got {other:?}"), + } + } + + /// Interpolated values are not HTML-escaped (the output is YAML): `=` and `&` + /// must survive. + #[test] + fn template_values_are_not_html_escaped() { + let template = indoc::indoc! {r#" + build: + steps: + - type: script + command: "{{ command }}" + "#}; + let r = recipe(&[("command", "SITE=https://example.com&foo=bar npm run build")]); + assert_eq!( + rendered_command(template, &r, &ctx("my-canister")), + "SITE=https://example.com&foo=bar npm run build" + ); + } + + /// The canister name is injected under the reserved `_` namespace. + #[test] + fn canister_name_is_injected() { + let template = indoc::indoc! {r#" + build: + steps: + - type: script + command: "build {{_.canister.name}}" + "#}; + assert_eq!( + rendered_command(template, &recipe(&[]), &ctx("my-canister")), + "build my-canister" + ); + } + + /// The `_` namespace works through the `replace` helper. + #[test] + fn canister_name_works_with_replace_helper() { + let template = indoc::indoc! {r#" + build: + steps: + - type: script + command: "cp {{ replace "-" "_" _.canister.name }}.wasm out.wasm" + "#}; + assert_eq!( + rendered_command(template, &recipe(&[]), &ctx("my-canister")), + "cp my_canister.wasm out.wasm" + ); + } + + /// User configuration cannot override the reserved `_` namespace. + #[test] + fn reserved_namespace_cannot_be_overridden_by_user_config() { + let template = indoc::indoc! {r#" + build: + steps: + - type: script + command: "build {{_.canister.name}}" + "#}; + let mut r = recipe(&[]); + r.configuration.insert( + "_".to_owned(), + serde_yaml::from_str("canister:\n name: user-override").unwrap(), + ); + assert_eq!( + rendered_command(template, &r, &ctx("real-name")), + "build real-name" + ); + } + + /// A template that renders to invalid build/sync YAML is a `Parse` error, + /// not a panic. + #[test] + fn invalid_rendered_yaml_is_a_parse_error() { + let template = "not: a valid build manifest\n"; + assert!(matches!( + render_recipe(template, &recipe(&[]), &ctx("c")), + Err(RenderRecipeError::Parse { .. }) + )); + } +} diff --git a/crates/icp-deploy-canister/src/deploy.rs b/crates/icp-deploy-canister/src/deploy.rs new file mode 100644 index 000000000..ee4a5879b --- /dev/null +++ b/crates/icp-deploy-canister/src/deploy.rs @@ -0,0 +1,1039 @@ +//! Canister installation, environment-variable wiring, syncing, and deploy +//! orchestration over an `ic-agent` `Agent`, reading built artifacts from disk +//! and running sync plugins in the sandboxed wasmtime engine. + +use std::collections::BTreeMap; + +use candid::utils::ArgumentEncoder; +use candid::{Encode, Nat, Principal}; +use ic_agent::Agent; +use ic_management_canister_types::{ + CanisterIdRecord, CanisterInstallMode, CanisterSettings, CanisterStatusResult, + CanisterStatusType, ChunkHash, ClearChunkStoreArgs, EnvironmentVariable, + InstallChunkedCodeArgs, InstallCodeArgs, UpdateSettingsArgs, UpgradeFlags, UploadChunkArgs, + WasmMemoryPersistence, +}; +use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; +use sha2::{Digest, Sha256}; +use snafu::prelude::*; + +use crate::{ + Canister, Project, + canister::recipe::RemoteResourceResolve, + ids::IdStore, + manifest::canister::SyncStep, + network::Configuration, + prelude::*, + sync_exec::{ScriptInvocation, ScriptRunner, StepProgress, SyncStepContext}, +}; + +/// EOP custom-section metadata key marking a Motoko enhanced-orthogonal-persistence canister. +const EOP_METADATA: &str = "enhanced-orthogonal-persistence"; + +/// Requested installation mode. `Auto` resolves to `Install` or `Upgrade` by +/// querying the canister's current status. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InstallMode { + Auto, + Install, + Reinstall, + Upgrade, +} + +// --------------------------------------------------------------------------- +// Management-canister transport +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum UpdateOrProxyError { + #[snafu(display("failed to encode proxy call arguments"))] + ProxyEncode { source: candid::Error }, + + #[snafu(display("direct update call failed"))] + DirectUpdateCall { source: ic_agent::AgentError }, + + #[snafu(display("proxy update call failed"))] + ProxyUpdateCall { source: ic_agent::AgentError }, + + #[snafu(display("failed to decode proxy canister response"))] + ProxyDecode { source: candid::Error }, + + #[snafu(display("proxy call failed: {message}"))] + ProxyCall { message: String }, +} + +/// Dispatch a canister update call, optionally routing through a proxy canister. +/// +/// With no proxy, makes a direct call, overriding the effective canister id when +/// given (required for management-canister calls, where it must be the target). +/// With a proxy, wraps the call in [`ProxyArgs`] for the proxy's `proxy` method. +async fn update_or_proxy_raw( + agent: &Agent, + canister_id: Principal, + method: &str, + arg: Vec, + proxy: Option, + effective_canister_id: Option, + cycles: u128, +) -> Result, UpdateOrProxyError> { + if let Some(proxy_cid) = proxy { + let proxy_args = ProxyArgs { + canister_id, + method: method.to_string(), + args: arg, + cycles: Nat::from(cycles), + }; + let proxy_arg_bytes = Encode!(&proxy_args).context(ProxyEncodeSnafu)?; + let proxy_res = agent + .update(&proxy_cid, "proxy") + .with_arg(proxy_arg_bytes) + .await + .context(ProxyUpdateCallSnafu)?; + let proxy_result: (ProxyResult,) = + candid::decode_args(&proxy_res).context(ProxyDecodeSnafu)?; + match proxy_result.0 { + ProxyResult::Ok(ok) => Ok(ok.result), + ProxyResult::Err(err) => ProxyCallSnafu { + message: err.format_error(), + } + .fail(), + } + } else { + let mut builder = agent.update(&canister_id, method).with_arg(arg); + if let Some(eid) = effective_canister_id { + builder = builder.with_effective_canister_id(eid); + } + builder.await.context(DirectUpdateCallSnafu) + } +} + +#[derive(Debug, Snafu)] +pub enum MgmtCallError { + #[snafu(display("failed to encode arguments for management method '{method}'"))] + Encode { + method: String, + source: candid::Error, + }, + + #[snafu(display("management call '{method}' failed"))] + Call { + method: String, + source: UpdateOrProxyError, + }, + + #[snafu(display("failed to decode reply from management method '{method}'"))] + Decode { + method: String, + source: candid::Error, + }, +} + +/// Encode + issue a management-canister update call and decode its reply. +/// +/// The management canister has no routing of its own, so the effective canister +/// id is the `target`. +async fn mgmt_call( + agent: &Agent, + proxy: Option, + method: &str, + target: Principal, + args: A, + cycles: u128, +) -> Result +where + A: ArgumentEncoder, + R: for<'a> candid::utils::ArgumentDecoder<'a>, +{ + let arg = candid::encode_args(args).context(EncodeSnafu { method })?; + let raw = update_or_proxy_raw( + agent, + Principal::management_canister(), + method, + arg, + proxy, + Some(target), + cycles, + ) + .await + .context(CallSnafu { method })?; + candid::decode_args(&raw).context(DecodeSnafu { method }) +} + +// --------------------------------------------------------------------------- +// Install +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum InstallCanisterError { + #[snafu(display("failed to read the built artifact for canister '{canister}'"))] + ReadArtifact { + canister: String, + source: crate::fs::IoError, + }, + + #[snafu(display("failed to query status of canister '{canister}'"))] + CanisterStatus { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to stop canister '{canister}' before upgrade"))] + StopCanister { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to start canister '{canister}' after upgrade"))] + StartCanister { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to clear the chunk store for canister '{canister}'"))] + ClearChunkStore { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to upload wasm chunk {index} for canister '{canister}'"))] + UploadChunk { + canister: String, + index: usize, + source: MgmtCallError, + }, + + #[snafu(display("failed to install code on canister '{canister}'"))] + InstallCode { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to install chunked code on canister '{canister}'"))] + InstallChunkedCode { + canister: String, + source: MgmtCallError, + }, +} + +/// Query `canister_status` and pick the concrete install mode for `Auto`. +async fn resolve_mode_and_status( + agent: &Agent, + proxy: Option, + canister_name: &str, + canister_id: Principal, + mode: InstallMode, +) -> Result<(CanisterInstallMode, CanisterStatusType), InstallCanisterError> { + let (status,): (CanisterStatusResult,) = mgmt_call( + agent, + proxy, + "canister_status", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(CanisterStatusSnafu { + canister: canister_name, + })?; + let install_mode = match mode { + InstallMode::Auto => { + if status.module_hash.is_some() { + CanisterInstallMode::Upgrade(None) + } else { + CanisterInstallMode::Install + } + } + InstallMode::Install => CanisterInstallMode::Install, + InstallMode::Reinstall => CanisterInstallMode::Reinstall, + InstallMode::Upgrade => CanisterInstallMode::Upgrade(None), + }; + Ok((install_mode, status.status)) +} + +/// Whether the canister exposes the `enhanced-orthogonal-persistence` metadata. +/// A read failure is treated as "absent", so a missing custom section never +/// aborts an install. +async fn is_eop_canister(agent: &Agent, canister_id: Principal) -> bool { + agent + .read_state_canister_metadata(canister_id, EOP_METADATA) + .await + .is_ok_and(|section| !section.is_empty()) +} + +/// Install (or upgrade/reinstall) a single, already-built canister. +/// +/// Resolves `Auto` mode and EOP-upgrade flags via `agent`. Large wasm is +/// installed via the chunk store. +#[allow(clippy::too_many_arguments)] +pub async fn install_canister( + canister_name: &str, + canister_id: Principal, + wasm: &[u8], + mode: InstallMode, + init_args: Option<&[u8]>, + wasm_memory_persistence: Option, + agent: &Agent, + proxy: Option, +) -> Result<(), InstallCanisterError> { + let (mode, status) = + resolve_mode_and_status(agent, proxy, canister_name, canister_id, mode).await?; + install_canister_resolved( + canister_name, + canister_id, + wasm, + mode, + status, + init_args, + wasm_memory_persistence, + agent, + proxy, + ) + .await +} + +/// Like [`install_canister`], but with the install mode and current status +/// already resolved by the caller. Callers that need the resolved mode/status +/// for their own logic first (e.g. a Candid-compatibility gate before install) +/// resolve once via [`resolve_install_mode_and_status`] and pass the result here, +/// avoiding a second `canister_status` call. +#[allow(clippy::too_many_arguments)] +pub async fn install_canister_resolved( + canister_name: &str, + canister_id: Principal, + wasm: &[u8], + mode: CanisterInstallMode, + status: CanisterStatusType, + init_args: Option<&[u8]>, + wasm_memory_persistence: Option, + agent: &Agent, + proxy: Option, +) -> Result<(), InstallCanisterError> { + // For EOP Motoko canisters an upgrade must set `wasm_memory_persistence`. + // Trust an explicit caller override; otherwise auto-detect and default to Keep. + let mode = match mode { + CanisterInstallMode::Upgrade(_) => { + let persistence = match wasm_memory_persistence { + Some(p) => Some(p), + None => is_eop_canister(agent, canister_id) + .await + .then_some(WasmMemoryPersistence::Keep), + }; + if let Some(persistence) = persistence { + CanisterInstallMode::Upgrade(Some(UpgradeFlags { + skip_pre_upgrade: None, + wasm_memory_persistence: Some(persistence), + })) + } else { + mode + } + } + other => other, + }; + + do_install( + agent, + proxy, + canister_name, + canister_id, + wasm, + mode, + status, + init_args, + ) + .await +} + +/// Resolve an [`InstallMode`] and the canister's current status via +/// `canister_status` (the resolution [`install_canister`] performs internally). +/// Exposed for callers that need the resolved mode before installing. +pub async fn resolve_install_mode_and_status( + agent: &Agent, + proxy: Option, + canister_name: &str, + canister_id: Principal, + mode: InstallMode, +) -> Result<(CanisterInstallMode, CanisterStatusType), InstallCanisterError> { + resolve_mode_and_status(agent, proxy, canister_name, canister_id, mode).await +} + +#[allow(clippy::too_many_arguments)] +async fn do_install( + agent: &Agent, + proxy: Option, + canister_name: &str, + canister_id: Principal, + wasm: &[u8], + mode: CanisterInstallMode, + status: CanisterStatusType, + init_args: Option<&[u8]>, +) -> Result<(), InstallCanisterError> { + // Raw install_code messages are limited to 2 MiB; larger wasm goes through + // the chunk store (spec limit is 1 MiB per chunk). + const CHUNK_THRESHOLD: usize = 2 * 1024 * 1024; + const CHUNK_SIZE: usize = 1024 * 1024; + // Generous overhead for encoding, target canister ID, install mode, etc. + const ENCODING_OVERHEAD: usize = 500; + + let arg = init_args + .map(|a| a.to_vec()) + .unwrap_or_else(|| candid::encode_args(()).expect("encoding empty args is infallible")); + + let total_install_size = wasm.len() + arg.len() + ENCODING_OVERHEAD; + + if total_install_size <= CHUNK_THRESHOLD { + let install_args = InstallCodeArgs { + mode, + canister_id, + wasm_module: wasm.to_vec(), + arg, + sender_canister_version: None, + }; + stop_and_start_if_needed( + agent, + proxy, + canister_name, + canister_id, + mode, + status, + async { + mgmt_call::<_, ()>( + agent, + proxy, + "install_code", + canister_id, + (install_args,), + 0, + ) + .await + .context(InstallCodeSnafu { + canister: canister_name, + }) + }, + ) + .await?; + } else { + // Clear any existing chunks to ensure a clean state. + clear_chunk_store(agent, proxy, canister_name, canister_id).await?; + + let chunks: Vec<&[u8]> = wasm.chunks(CHUNK_SIZE).collect(); + let mut chunk_hashes: Vec = Vec::new(); + for (i, chunk) in chunks.iter().enumerate() { + let (hash,): (ChunkHash,) = mgmt_call( + agent, + proxy, + "upload_chunk", + canister_id, + (UploadChunkArgs { + canister_id, + chunk: chunk.to_vec(), + },), + 0, + ) + .await + .context(UploadChunkSnafu { + canister: canister_name, + index: i, + })?; + chunk_hashes.push(hash); + } + + let wasm_module_hash = Sha256::digest(wasm).to_vec(); + let chunked_args = InstallChunkedCodeArgs { + mode, + target_canister: canister_id, + store_canister: None, + chunk_hashes_list: chunk_hashes, + wasm_module_hash, + arg, + sender_canister_version: None, + }; + + let install_res = stop_and_start_if_needed( + agent, + proxy, + canister_name, + canister_id, + mode, + status, + async { + mgmt_call::<_, ()>( + agent, + proxy, + "install_chunked_code", + canister_id, + (chunked_args,), + 0, + ) + .await + .context(InstallChunkedCodeSnafu { + canister: canister_name, + }) + }, + ) + .await; + + // Always clear the chunk store afterwards to free storage. If the install + // failed, report that error in preference to a clear-store failure. + let clear_res = clear_chunk_store(agent, proxy, canister_name, canister_id).await; + install_res?; + clear_res?; + } + + Ok(()) +} + +async fn clear_chunk_store( + agent: &Agent, + proxy: Option, + canister_name: &str, + canister_id: Principal, +) -> Result<(), InstallCanisterError> { + mgmt_call::<_, ()>( + agent, + proxy, + "clear_chunk_store", + canister_id, + (ClearChunkStoreArgs { canister_id },), + 0, + ) + .await + .context(ClearChunkStoreSnafu { + canister: canister_name, + }) +} + +/// Guard an upgrade/reinstall of a Running canister by stopping it first and +/// restarting it afterwards (whether or not the install succeeded). +#[allow(clippy::too_many_arguments)] +async fn stop_and_start_if_needed( + agent: &Agent, + proxy: Option, + canister_name: &str, + canister_id: Principal, + mode: CanisterInstallMode, + status: CanisterStatusType, + install: F, +) -> Result<(), InstallCanisterError> +where + F: Future>, +{ + let should_guard = matches!( + mode, + CanisterInstallMode::Upgrade(_) | CanisterInstallMode::Reinstall + ) && matches!(status, CanisterStatusType::Running); + + if should_guard { + mgmt_call::<_, ()>( + agent, + proxy, + "stop_canister", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(StopCanisterSnafu { + canister: canister_name, + })?; + } + + let install_result = install.await; + + if !should_guard { + return install_result; + } + + let start_result = mgmt_call::<_, ()>( + agent, + proxy, + "start_canister", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(StartCanisterSnafu { + canister: canister_name, + }); + + // Restart whether or not the install succeeded. If both failed, the install + // error is the more likely root cause. + match (install_result, start_result) { + (Err(install_err), _) => Err(install_err), + (Ok(()), Err(start_err)) => Err(start_err), + (Ok(()), Ok(())) => Ok(()), + } +} + +/// Start a canister (idempotent; a no-op if already Running). +pub async fn start_canister( + agent: &Agent, + proxy: Option, + canister_name: &str, + canister_id: Principal, +) -> Result<(), InstallCanisterError> { + mgmt_call::<_, ()>( + agent, + proxy, + "start_canister", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(StartCanisterSnafu { + canister: canister_name, + }) +} + +// --------------------------------------------------------------------------- +// Environment variables + sync +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum SyncCanisterError { + #[snafu(display("failed to apply environment variables to canister '{canister}'"))] + ApplyEnvVars { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to run sync step for canister '{canister}'"))] + RunStep { + canister: String, + source: SyncStepError, + }, +} + +#[derive(Debug, Snafu)] +pub enum SyncStepError { + #[snafu(display("failed to resolve plugin wasm"))] + ResolveWasm { + source: crate::canister::recipe::ResolveError, + }, + + #[snafu(display("failed to get identity principal: {err}"))] + GetIdentityPrincipal { err: String }, + + #[snafu(display("plugin run failed"))] + RunPlugin { + source: icp_sync_plugin::RunPluginError, + }, + + #[snafu(transparent)] + Script { + source: crate::sync_exec::ScriptRunError, + }, +} + +/// Compute the environment variables a canister should run with: its manifest +/// `settings` variables merged with the generated `PUBLIC_CANISTER_ID:` +/// variables, resolved against `canister_ids`. +/// +/// Each canister is wired only to the ids it declares in `bindings`, resolved to +/// the ids that exist in this environment; unresolved bindings are skipped. +pub fn binding_env_vars( + canister: &Canister, + canister_ids: &BTreeMap, +) -> Vec { + let mut env_vars = canister + .settings + .environment_variables + .clone() + .unwrap_or_default(); + + for (env_name, referenced_key) in &canister.bindings { + if let Some(principal) = canister_ids.get(referenced_key) { + env_vars.insert( + format!("PUBLIC_CANISTER_ID:{env_name}"), + principal.to_text(), + ); + } + } + + env_vars + .into_iter() + .map(|(name, value)| EnvironmentVariable { name, value }) + .collect() +} + +/// Apply the canister's environment variables (see [`binding_env_vars`]) via +/// `update_settings`. +/// +/// This is the piece that standalone `icp sync` previously skipped: the binding +/// ids must be (re)written whenever a canister is synced, not only on deploy. +pub async fn apply_binding_env_vars( + canister: &Canister, + canister_id: Principal, + canister_ids: &BTreeMap, + agent: &Agent, + proxy: Option, +) -> Result<(), SyncCanisterError> { + let environment_variables = binding_env_vars(canister, canister_ids); + + mgmt_call::<_, ()>( + agent, + proxy, + "update_settings", + canister_id, + (UpdateSettingsArgs { + canister_id, + settings: CanisterSettings { + environment_variables: Some(environment_variables), + ..Default::default() + }, + sender_canister_version: None, + },), + 0, + ) + .await + .context(ApplyEnvVarsSnafu { + canister: canister.name.clone(), + }) +} + +/// Resolve and run one plugin sync step in the sandboxed wasmtime engine. The +/// wasm source is resolved (fetched/verified/cached) through `resolver`. +async fn run_plugin_step( + adapter: &crate::manifest::adapter::plugin::Adapter, + ctx: &SyncStepContext, + agent: &Agent, + resolver: &dyn RemoteResourceResolve, + stdio: Option>, +) -> Result, SyncStepError> { + let wasm_path = resolver + .resolve_wasm( + &adapter.source, + &ctx.canister_path, + adapter.sha256.as_deref(), + stdio.clone(), + ) + .await + .context(ResolveWasmSnafu)?; + + let identity_principal = agent + .get_principal() + .map_err(|err| SyncStepError::GetIdentityPrincipal { err })?; + let dirs = adapter.dirs.clone().unwrap_or_default(); + let files = adapter.files.clone().unwrap_or_default(); + let agent = agent.clone(); + let base_dir = ctx.canister_path.clone(); + let cid = ctx.canister_id; + let proxy = ctx.proxy; + let environment = ctx.environment.clone(); + + // Blocking wasmtime call — signal Tokio that this thread will block. + tokio::task::block_in_place(|| { + icp_sync_plugin::run_plugin( + wasm_path, + base_dir, + dirs, + files, + cid, + agent, + proxy, + identity_principal, + environment, + stdio, + ) + }) + .context(RunPluginSnafu) +} + +/// Run a canister's configured sync steps, collecting any retained stderr lines. +/// Plugin steps run in the sandboxed wasmtime engine (their wasm resolved via +/// `resolver`); script steps run through `script_runner`. Does not apply +/// environment variables (see [`apply_binding_env_vars`] / [`sync_canister`]). +pub async fn run_sync_steps( + canister: &Canister, + ctx: &SyncStepContext, + agent: &Agent, + resolver: &dyn RemoteResourceResolve, + script_runner: &dyn ScriptRunner, + mut progress: Option<&mut dyn StepProgress>, +) -> Result, SyncCanisterError> { + let total = canister.sync.steps.len(); + let mut lines = Vec::new(); + for (i, step) in canister.sync.steps.iter().enumerate() { + let header = format!("\nSyncing: {step} {} of {total}", i + 1); + let stdio = progress.as_deref_mut().and_then(|p| p.begin_step(header)); + let step_lines = match step { + SyncStep::Plugin(adapter) => { + run_plugin_step(adapter, ctx, agent, resolver, stdio).await + } + SyncStep::Script(adapter) => script_runner + .run_script(ScriptInvocation::new(adapter, ctx), stdio) + .await + .map_err(|source| SyncStepError::Script { source }), + }; + if let Some(p) = progress.as_deref_mut() { + p.end_step().await; + } + lines.extend(step_lines.context(RunStepSnafu { + canister: canister.name.clone(), + })?); + } + Ok(lines) +} + +/// Sync a single canister: (re)apply its binding environment variables, then run +/// its sync steps. Applying env vars here is what makes standalone `icp sync` +/// include the generated `PUBLIC_CANISTER_ID:*` variables. +#[allow(clippy::too_many_arguments)] +pub async fn sync_canister( + canister: &Canister, + canister_id: Principal, + ctx: &SyncStepContext, + agent: &Agent, + resolver: &dyn RemoteResourceResolve, + script_runner: &dyn ScriptRunner, + progress: Option<&mut dyn StepProgress>, +) -> Result, SyncCanisterError> { + apply_binding_env_vars(canister, canister_id, &ctx.canister_ids, agent, ctx.proxy).await?; + run_sync_steps(canister, ctx, agent, resolver, script_runner, progress).await +} + +// --------------------------------------------------------------------------- +// Deploy +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum DeployCanisterError { + #[snafu(display("could not find canister '{canister}' in environment '{environment}'"))] + UnknownCanister { + canister: String, + environment: String, + }, + + #[snafu(display("could not find an id for canister '{canister}'; create it first"))] + LookupId { + canister: String, + source: crate::ids::IdStoreError, + }, + + #[snafu(display("failed to encode init args for canister '{canister}'"))] + InitArgs { + canister: String, + source: crate::InitArgsToBytesError, + }, + + #[snafu(transparent)] + Install { source: InstallCanisterError }, + + #[snafu(transparent)] + Sync { source: SyncCanisterError }, +} + +/// Deploy (install then sync) a single already-built canister in `environment`. +/// +/// Assumes the canister already exists (its id is read from `ids`); creating +/// canisters is the caller's responsibility. +#[allow(clippy::too_many_arguments)] +pub async fn deploy_canister( + project: &Project, + canister_name: &str, + environment: &str, + artifact_path: &Path, + mode: InstallMode, + proxy: Option, + agent: &Agent, + ids: &dyn IdStore, + resolver: &dyn RemoteResourceResolve, + script_runner: &dyn ScriptRunner, + progress: Option<&mut dyn StepProgress>, +) -> Result, DeployCanisterError> { + let env = project + .environments + .get(environment) + .context(UnknownCanisterSnafu { + canister: canister_name, + environment, + })?; + let is_cache = matches!(env.network.configuration, Configuration::Managed { .. }); + let network = env.network.name.clone(); + + let (canister_path, canister) = + env.canisters + .get(canister_name) + .context(UnknownCanisterSnafu { + canister: canister_name, + environment, + })?; + + let canister_id = ids + .lookup(is_cache, environment, canister_name) + .context(LookupIdSnafu { + canister: canister_name, + })?; + let canister_ids = ids + .lookup_by_environment(is_cache, environment) + .unwrap_or_default(); + + let init_args = canister + .init_args + .as_ref() + .map(|ia| ia.to_bytes()) + .transpose() + .context(InitArgsSnafu { + canister: canister_name, + })?; + + let wasm = crate::fs::read(artifact_path).context(ReadArtifactSnafu { + canister: canister_name, + })?; + + // Environment variables first, then install, then sync steps. + apply_binding_env_vars(canister, canister_id, &canister_ids, agent, proxy).await?; + install_canister( + canister_name, + canister_id, + &wasm, + mode, + init_args.as_deref(), + None, + agent, + proxy, + ) + .await?; + // Asset sync requires a Running canister; install_code is status-preserving. + start_canister(agent, proxy, canister_name, canister_id).await?; + + let ctx = SyncStepContext { + canister_path: canister_path.clone(), + canister_id, + environment: environment.to_owned(), + network, + canister_ids, + proxy, + }; + let lines = run_sync_steps(canister, &ctx, agent, resolver, script_runner, progress).await?; + Ok(lines) +} + +#[derive(Debug, Snafu)] +#[snafu(display("failed to deploy canister(s): {}", names.join(", ")))] +pub struct DeployError { + pub names: Vec, + pub failures: Vec<(String, DeployCanisterError)>, +} + +/// Deploy the `selected` already-built canisters in `environment`, each through +/// [`deploy_canister`]. `artifact_paths` maps canister name → its built wasm +/// path. Per-canister failures are aggregated. This is a batch entry point with +/// no progress reporting; the CLI drives its own per-canister fan-out instead. +#[allow(clippy::too_many_arguments)] +pub async fn deploy( + project: &Project, + selected: &[String], + environment: &str, + mode: InstallMode, + proxy: Option, + artifact_paths: &BTreeMap, + agent: &Agent, + ids: &dyn IdStore, + resolver: &dyn RemoteResourceResolve, + script_runner: &dyn ScriptRunner, +) -> Result<(), DeployError> { + let mut failures = Vec::new(); + for name in selected { + let Some(artifact_path) = artifact_paths.get(name) else { + failures.push(( + name.clone(), + DeployCanisterError::UnknownCanister { + canister: name.clone(), + environment: environment.to_owned(), + }, + )); + continue; + }; + if let Err(e) = deploy_canister( + project, + name, + environment, + artifact_path, + mode, + proxy, + agent, + ids, + resolver, + script_runner, + None, + ) + .await + { + failures.push((name.clone(), e)); + } + } + + if failures.is_empty() { + Ok(()) + } else { + let names = failures.iter().map(|(n, _)| n.clone()).collect(); + Err(DeployError { names, failures }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::canister::Settings; + use crate::manifest::canister::{BuildSteps, SyncSteps}; + use std::collections::HashMap; + + fn canister(env_vars: &[(&str, &str)], bindings: &[(&str, &str)]) -> Canister { + Canister { + name: "backend".to_owned(), + settings: Settings { + environment_variables: Some( + env_vars + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect::>(), + ), + ..Default::default() + }, + build: BuildSteps { steps: vec![] }, + sync: SyncSteps { steps: vec![] }, + init_args: None, + registry_recipe: None, + bindings: bindings + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(), + friendly_names: vec![], + } + } + + fn vars(canister: &Canister, ids: &[(&str, Principal)]) -> HashMap { + let ids: BTreeMap = + ids.iter().map(|(k, v)| ((*k).to_owned(), *v)).collect(); + binding_env_vars(canister, &ids) + .into_iter() + .map(|v| (v.name, v.value)) + .collect() + } + + /// Manifest env vars and resolved `PUBLIC_CANISTER_ID:` ids are both + /// present in the computed environment. + #[test] + fn merges_manifest_vars_and_resolved_bindings() { + let cid = Principal::anonymous(); + let c = canister(&[("FOO", "bar")], &[("frontend", "frontend")]); + let v = vars(&c, &[("frontend", cid)]); + + assert_eq!(v.get("FOO").map(String::as_str), Some("bar")); + assert_eq!( + v.get("PUBLIC_CANISTER_ID:frontend").map(String::as_str), + Some(cid.to_text().as_str()) + ); + } + + /// A binding whose referenced canister has no id in this environment is + /// skipped rather than emitted with an empty value. + #[test] + fn unresolved_binding_is_skipped() { + let c = canister(&[], &[("frontend", "frontend")]); + let v = vars(&c, &[]); + assert!(v.is_empty(), "expected no vars, got {v:?}"); + } +} diff --git a/crates/icp/src/fs/json.rs b/crates/icp-deploy-canister/src/fs/json.rs similarity index 100% rename from crates/icp/src/fs/json.rs rename to crates/icp-deploy-canister/src/fs/json.rs diff --git a/crates/icp-deploy-canister/src/fs/mod.rs b/crates/icp-deploy-canister/src/fs/mod.rs new file mode 100644 index 000000000..87f65c071 --- /dev/null +++ b/crates/icp-deploy-canister/src/fs/mod.rs @@ -0,0 +1,74 @@ +use snafu::prelude::*; +use std::io::{self, ErrorKind}; + +use crate::prelude::*; + +pub mod json; +pub mod yaml; + +#[derive(Debug, Snafu)] +#[snafu(display("Filesystem operation failed at {path}"))] +pub struct IoError { + source: io::Error, + path: PathBuf, +} + +#[derive(Debug, Snafu)] +#[snafu(display("Failed to rename `{from}` to `{to}`"))] +pub struct RenameError { + source: io::Error, + from: PathBuf, + to: PathBuf, +} + +#[derive(Debug, Snafu)] +#[snafu(display("Failed to copy `{from}` to `{to}`"))] +pub struct CopyError { + source: io::Error, + from: PathBuf, + to: PathBuf, +} + +impl IoError { + pub fn kind(&self) -> ErrorKind { + self.source.kind() + } +} + +pub fn create_dir_all(path: &Path) -> Result<(), IoError> { + std::fs::create_dir_all(path).context(IoSnafu { path }) +} + +pub fn read(path: &Path) -> Result, IoError> { + std::fs::read(path).context(IoSnafu { path }) +} + +pub fn read_to_string(path: &Path) -> Result { + std::fs::read_to_string(path).context(IoSnafu { path }) +} + +pub fn remove_dir_all(path: &Path) -> Result<(), IoError> { + std::fs::remove_dir_all(path).context(IoSnafu { path }) +} + +pub fn remove_file(path: &Path) -> Result<(), IoError> { + std::fs::remove_file(path).context(IoSnafu { path }) +} + +pub fn copy(from: &Path, to: &Path) -> Result<(), CopyError> { + std::fs::copy(from, to) + .map(|_| ()) + .context(CopySnafu { from, to }) +} + +pub fn rename(from: &Path, to: &Path) -> Result<(), RenameError> { + std::fs::rename(from, to).context(RenameSnafu { from, to }) +} + +pub fn write(path: &Path, contents: &[u8]) -> Result<(), IoError> { + std::fs::write(path, contents).context(IoSnafu { path }) +} + +pub fn write_string(path: &Path, contents: &str) -> Result<(), IoError> { + std::fs::write(path, contents.as_bytes()).context(IoSnafu { path }) +} diff --git a/crates/icp/src/fs/yaml.rs b/crates/icp-deploy-canister/src/fs/yaml.rs similarity index 100% rename from crates/icp/src/fs/yaml.rs rename to crates/icp-deploy-canister/src/fs/yaml.rs diff --git a/crates/icp-deploy-canister/src/ids.rs b/crates/icp-deploy-canister/src/ids.rs new file mode 100644 index 000000000..962a11bfa --- /dev/null +++ b/crates/icp-deploy-canister/src/ids.rs @@ -0,0 +1,85 @@ +//! Canister-id store: per-environment `name → principal` mappings. + +use std::{collections::BTreeMap, sync::Mutex}; + +use candid::Principal; +use snafu::{OptionExt, Snafu}; + +/// Mapping of canister names to their principals within an environment. +pub type IdMapping = BTreeMap; + +#[derive(Debug, Snafu)] +pub enum IdStoreError { + #[snafu(display("could not find id for canister '{canister_name}' in environment '{env}'"))] + NotFound { env: String, canister_name: String }, + + #[snafu(display("failed to access canister id store for environment '{env}': {message}"))] + Access { env: String, message: String }, +} + +/// Read/write access to canister-id mappings. +/// +/// The `is_cache` flag lets an implementation that keeps two stores — a +/// managed-network cache and a connected-network data store — pick the right +/// one. `register` mutates through `&self`, so the store is interior-mutable. +pub trait IdStore: Send + Sync { + fn lookup( + &self, + is_cache: bool, + env: &str, + canister_name: &str, + ) -> Result; + + fn lookup_by_environment(&self, is_cache: bool, env: &str) -> Result; + + fn register( + &self, + is_cache: bool, + env: &str, + canister_name: &str, + canister_id: Principal, + ) -> Result<(), IdStoreError>; +} + +#[derive(Debug, Default)] +pub struct InMemoryIdStore(pub Mutex>); + +impl IdStore for InMemoryIdStore { + fn lookup( + &self, + _is_cache: bool, + env: &str, + canister_name: &str, + ) -> Result { + let mapping = self.lookup_by_environment(_is_cache, env)?; + mapping + .get(canister_name) + .cloned() + .context(NotFoundSnafu { env, canister_name }) + } + + fn lookup_by_environment(&self, _is_cache: bool, env: &str) -> Result { + self.0 + .lock() + .unwrap() + .get(env) + .cloned() + .context(AccessSnafu { + env, + message: "environment not found", + }) + } + + fn register( + &self, + _is_cache: bool, + env: &str, + canister_name: &str, + canister_id: Principal, + ) -> Result<(), IdStoreError> { + let mut store = self.0.lock().unwrap(); + let mapping = store.entry(env.to_string()).or_default(); + mapping.insert(canister_name.to_string(), canister_id); + Ok(()) + } +} diff --git a/crates/icp-deploy-canister/src/lib.rs b/crates/icp-deploy-canister/src/lib.rs new file mode 100644 index 000000000..fe63dd90b --- /dev/null +++ b/crates/icp-deploy-canister/src/lib.rs @@ -0,0 +1,311 @@ +//! The project model plus canister installation, syncing, and deploy +//! orchestration over an `ic-agent` `Agent`. +//! +//! Subprocess script execution and remote-resource resolution (recipe templates +//! and plugin wasms, which use the host's package cache) are provided by the +//! host through the [`sync_exec::ScriptRunner`] and +//! [`canister::recipe::RemoteResourceResolve`] traits. + +use std::collections::{BTreeMap, HashMap}; + +use indexmap::IndexMap; +use serde::Serialize; +use snafu::prelude::*; + +use candid_parser::parse_idl_args; + +use crate::{ + canister::Settings, + manifest::{ + ArgsFormat, + canister::{BuildSteps, SyncSteps}, + }, + network::Configuration, + prelude::*, +}; + +pub mod canister; +pub mod deploy; +pub mod fs; +pub mod ids; +pub mod manifest; +pub mod network; +pub mod parsers; +pub mod prelude; +pub mod project; +pub mod sync_exec; + +pub use deploy::{ + DeployCanisterError, DeployError, InstallCanisterError, InstallMode, SyncCanisterError, + SyncStepError, UpdateOrProxyError, apply_binding_env_vars, binding_env_vars, deploy, + deploy_canister, install_canister, install_canister_resolved, resolve_install_mode_and_status, + run_sync_steps, start_canister, sync_canister, +}; +pub use ids::{IdMapping, IdStore, IdStoreError}; +pub use project::{consolidate_manifest, load_project, verify_sandbox}; +pub use sync_exec::{ + ScriptInvocation, ScriptRunError, ScriptRunner, StepProgress, SyncStepContext, system_env_vars, +}; + +/// Resolved initialization arguments, with any file references already loaded. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub enum InitArgs { + /// Text content (inline or loaded from file). Format is always known. + Text { content: String, format: ArgsFormat }, + /// Raw binary bytes (from a file with `format: bin`). Used directly. + Binary(Vec), +} + +#[derive(Debug, Snafu)] +pub enum InitArgsToBytesError { + #[snafu(display("failed to decode hex init args"))] + HexDecode { source: hex::FromHexError }, + + #[snafu(display("failed to parse Candid init args"))] + CandidParse { source: candid_parser::Error }, + + #[snafu(display("failed to encode Candid init args to bytes"))] + CandidEncode { source: candid::Error }, +} + +impl InitArgs { + /// Resolve to raw bytes according to the format. + pub fn to_bytes(&self) -> Result, InitArgsToBytesError> { + match self { + InitArgs::Binary(bytes) => Ok(bytes.clone()), + InitArgs::Text { content, format } => match format { + ArgsFormat::Hex => hex::decode(content.trim()).context(HexDecodeSnafu), + ArgsFormat::Candid => { + let args = parse_idl_args(content.trim()).context(CandidParseSnafu)?; + args.to_bytes().context(CandidEncodeSnafu) + } + ArgsFormat::Bin => { + unreachable!("binary format cannot appear in InitArgs::Text") + } + }, + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Canister { + pub name: String, + + /// Canister settings, such as memory constaints, etc. + pub settings: Settings, + + /// The build configuration specifying how to compile the canister's source + /// code into a WebAssembly module, including the adapter to use. + pub build: BuildSteps, + + /// The configuration specifying how to sync the canister + pub sync: SyncSteps, + + /// Initialization arguments passed to the canister during installation. + /// Resolved from the manifest — file contents are already loaded. + pub init_args: Option, + + /// If the canister was defined via a recipe reference, this holds the + /// original recipe specifier string (e.g. `@dfinity/motoko@v4.0.0`). + /// `None` when the canister uses explicit build/sync instructions. + pub registry_recipe: Option, + + /// Canister-discovery wiring. Maps the name this canister reads in a + /// `PUBLIC_CANISTER_ID:` environment variable to the store key of the + /// referenced canister. Computed during consolidation so each canister sees + /// the view its owning project expects: its own project's canisters under + /// their local names, plus any declared dependencies under their aliases + /// (`:`). For a project with no dependencies this maps every + /// canister's local name to itself, reproducing the flat "every canister sees + /// every sibling" behavior. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub bindings: BTreeMap, + + /// Subdomain prefixes for the canister's friendly URLs, most-specific label + /// first, e.g. `["backend"]` for an own canister or `["backend.openemail"]` + /// for a dependency canister (dot-nested by alias chain). A de-duplicated + /// shared dependency canister carries one entry per alias chain that reaches + /// it. Consumed only at deploy time to build `custom-domains.txt` entries and + /// the printed URLs; a runtime display aid that is always recomputed during + /// consolidation, so it is never serialized. + #[serde(skip)] + pub friendly_names: Vec, +} + +#[derive(Debug, Snafu)] +pub enum BundleModulePathError { + #[snafu(display( + "canister '{canister}' does not have a single build step (found {count}); a bundled \ + canister must be built by exactly one pre-built step" + ))] + NotSingleBuildStep { canister: String, count: usize }, + + #[snafu(display( + "canister '{canister}' is not built by a pre-built step; a bundled canister's module \ + must come from a `pre-built` build step" + ))] + NotPrebuilt { canister: String }, + + #[snafu(display("canister '{canister}' is built from a remote URL, not a local module path"))] + NotLocal { canister: String }, +} + +/// Extract the local wasm module path a bundled canister is built from. +/// +/// A bundle's canisters are each built by a single `pre-built` step pointing at +/// the module on disk; this returns that path, erroring if the build is not that +/// single-prebuilt-local-path shape. +pub fn bundle_get_canister_module_path( + canister: &Canister, +) -> Result<&Path, BundleModulePathError> { + let steps = &canister.build.steps; + let [step] = steps.as_slice() else { + return NotSingleBuildStepSnafu { + canister: canister.name.clone(), + count: steps.len(), + } + .fail(); + }; + let manifest::BuildStep::Prebuilt(adapter) = step else { + return NotPrebuiltSnafu { + canister: canister.name.clone(), + } + .fail(); + }; + match &adapter.source { + manifest::prebuilt::SourceField::Local(local) => Ok(&local.path), + manifest::prebuilt::SourceField::Remote(_) => NotLocalSnafu { + canister: canister.name.clone(), + } + .fail(), + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Network { + pub name: String, + pub configuration: Configuration, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Environment { + pub name: String, + pub network: Network, + pub canisters: IndexMap, +} + +impl Environment { + pub fn get_canister_names(&self) -> Vec { + self.canisters.keys().cloned().collect() + } + + pub fn contains_canister(&self, canister_name: &str) -> bool { + self.canisters.contains_key(canister_name) + } + + pub fn get_canister_info(&self, canister: &str) -> Result<(PathBuf, Canister), String> { + self.canisters + .get(canister) + .ok_or_else(|| { + format!( + "canister '{}' not declared in environment '{}'", + canister, self.name + ) + }) + .cloned() + } +} + +/// Consolidated project definition +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Project { + pub dir: PathBuf, + pub canisters: IndexMap, + pub networks: HashMap, + pub environments: HashMap, + + /// Environments the workspace defines that some vendored member does *not* + /// declare, keyed by environment name → the missing members' store-key + /// prefixes. Enforced when the environment is selected (strict rule). + /// Empty for standalone projects and workspaces whose members are complete. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub member_missing_envs: HashMap>, +} + +impl Project { + pub fn get_canister(&self, canister_name: &str) -> Option<&(PathBuf, Canister)> { + self.canisters.get(canister_name) + } +} + +#[cfg(test)] +mod bundle_tests { + use super::*; + use crate::canister::Settings; + use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; + use crate::manifest::adapter::{prebuilt, script}; + use crate::manifest::canister::{BuildStep, SyncSteps}; + + fn canister_with_build(steps: Vec) -> Canister { + Canister { + name: "backend".to_owned(), + settings: Settings::default(), + build: BuildSteps { steps }, + sync: SyncSteps { steps: vec![] }, + init_args: None, + registry_recipe: None, + bindings: BTreeMap::new(), + friendly_names: vec![], + } + } + + fn prebuilt_local(path: &str) -> BuildStep { + BuildStep::Prebuilt(prebuilt::Adapter { + source: prebuilt::SourceField::Local(LocalSource { path: path.into() }), + sha256: None, + }) + } + + #[test] + fn extracts_the_single_prebuilt_local_path() { + let c = canister_with_build(vec![prebuilt_local("out/backend.wasm")]); + assert_eq!( + bundle_get_canister_module_path(&c).unwrap(), + Path::new("out/backend.wasm") + ); + } + + #[test] + fn rejects_zero_or_multiple_build_steps() { + let two = canister_with_build(vec![prebuilt_local("a.wasm"), prebuilt_local("b.wasm")]); + assert!(matches!( + bundle_get_canister_module_path(&two), + Err(BundleModulePathError::NotSingleBuildStep { count: 2, .. }) + )); + } + + #[test] + fn rejects_a_non_prebuilt_step() { + let c = canister_with_build(vec![BuildStep::Script(script::Adapter { + command: script::CommandField::Command("make".to_owned()), + })]); + assert!(matches!( + bundle_get_canister_module_path(&c), + Err(BundleModulePathError::NotPrebuilt { .. }) + )); + } + + #[test] + fn rejects_a_remote_prebuilt_source() { + let c = canister_with_build(vec![BuildStep::Prebuilt(prebuilt::Adapter { + source: prebuilt::SourceField::Remote(RemoteSource { + url: "https://example.com/backend.wasm".to_owned(), + }), + sha256: Some("abc".to_owned()), + })]); + assert!(matches!( + bundle_get_canister_module_path(&c), + Err(BundleModulePathError::NotLocal { .. }) + )); + } +} diff --git a/crates/icp/src/manifest/adapter/mod.rs b/crates/icp-deploy-canister/src/manifest/adapter/mod.rs similarity index 100% rename from crates/icp/src/manifest/adapter/mod.rs rename to crates/icp-deploy-canister/src/manifest/adapter/mod.rs diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp-deploy-canister/src/manifest/adapter/plugin.rs similarity index 100% rename from crates/icp/src/manifest/adapter/plugin.rs rename to crates/icp-deploy-canister/src/manifest/adapter/plugin.rs diff --git a/crates/icp/src/manifest/adapter/prebuilt.rs b/crates/icp-deploy-canister/src/manifest/adapter/prebuilt.rs similarity index 100% rename from crates/icp/src/manifest/adapter/prebuilt.rs rename to crates/icp-deploy-canister/src/manifest/adapter/prebuilt.rs diff --git a/crates/icp/src/manifest/adapter/script.rs b/crates/icp-deploy-canister/src/manifest/adapter/script.rs similarity index 100% rename from crates/icp/src/manifest/adapter/script.rs rename to crates/icp-deploy-canister/src/manifest/adapter/script.rs diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp-deploy-canister/src/manifest/canister.rs similarity index 100% rename from crates/icp/src/manifest/canister.rs rename to crates/icp-deploy-canister/src/manifest/canister.rs diff --git a/crates/icp/src/manifest/dependency.rs b/crates/icp-deploy-canister/src/manifest/dependency.rs similarity index 100% rename from crates/icp/src/manifest/dependency.rs rename to crates/icp-deploy-canister/src/manifest/dependency.rs diff --git a/crates/icp/src/manifest/environment.rs b/crates/icp-deploy-canister/src/manifest/environment.rs similarity index 100% rename from crates/icp/src/manifest/environment.rs rename to crates/icp-deploy-canister/src/manifest/environment.rs diff --git a/crates/icp-deploy-canister/src/manifest/mod.rs b/crates/icp-deploy-canister/src/manifest/mod.rs new file mode 100644 index 000000000..3ad57482c --- /dev/null +++ b/crates/icp-deploy-canister/src/manifest/mod.rs @@ -0,0 +1,132 @@ +use std::marker::PhantomData; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use snafu::prelude::*; + +use crate::fs; +use crate::prelude::*; + +pub mod adapter; +pub mod canister; +pub mod dependency; +pub mod environment; +pub mod network; +pub mod project; +pub mod recipe; +pub mod serde_helpers; + +pub use { + adapter::plugin, + adapter::prebuilt, + canister::{ + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, + SyncStep, SyncSteps, + }, + dependency::DependencyManifest, + environment::EnvironmentManifest, + network::{ManagedMode, Mode, NetworkManifest}, + project::ProjectManifest, +}; + +pub const PROJECT_MANIFEST: &str = "icp.yaml"; +pub const CANISTER_MANIFEST: &str = "canister.yaml"; + +#[derive(Debug, Snafu)] +pub enum LoadManifestError { + #[snafu(transparent)] + Read { source: fs::IoError }, + + #[snafu(display("failed to parse manifest at '{path}'"))] + Parse { + source: serde_yaml::Error, + path: PathBuf, + }, +} + +/// Load and parse a YAML manifest of type `T` from disk. +pub fn load_manifest(path: &Path) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let content = fs::read(path)?; + let m = serde_yaml::from_slice::(&content).context(ParseSnafu { + path: path.to_path_buf(), + })?; + Ok(m) +} + +// A manifest item that can either be a path to another manifest file or the manifest itself. +// +// The valid path specifications are: +// - CanisterManifest: path or glob pattern to the directory containing "canister.yaml" +// - NetworkManifest: path to network manifest +// - EnvironmentManifest: path to environment manifest +#[derive(Clone, Debug, PartialEq, JsonSchema)] +#[serde(untagged)] +pub enum Item { + /// Path to a manifest + Path(String), + + /// The manifest + Manifest(T), +} + +/// Items in path form serialize back to a bare path string, *not* to the contents of the +/// referenced file. Callers that need a self-contained YAML output (e.g. `icp project bundle`) +/// must convert any `Item::Path` to `Item::Manifest` themselves by loading the referenced +/// manifest first. +impl Serialize for Item { + fn serialize(&self, serializer: S) -> Result { + match self { + Item::Path(p) => p.serialize(serializer), + Item::Manifest(m) => m.serialize(serializer), + } + } +} + +impl<'de, T> Deserialize<'de> for Item +where + T: Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::{MapAccess, Visitor, value::MapAccessDeserializer}; + use std::fmt; + + struct ItemVisitor(PhantomData); + + impl<'de, T: Deserialize<'de>> Visitor<'de> for ItemVisitor { + type Value = Item; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string path or a manifest object") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(Item::Path(v.to_owned())) + } + + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + Ok(Item::Path(v)) + } + + fn visit_map(self, map: M) -> Result + where + M: MapAccess<'de>, + { + T::deserialize(MapAccessDeserializer::new(map)).map(Item::Manifest) + } + } + + deserializer.deserialize_any(ItemVisitor(PhantomData)) + } +} diff --git a/crates/icp/src/manifest/network.rs b/crates/icp-deploy-canister/src/manifest/network.rs similarity index 100% rename from crates/icp/src/manifest/network.rs rename to crates/icp-deploy-canister/src/manifest/network.rs diff --git a/crates/icp/src/manifest/project.rs b/crates/icp-deploy-canister/src/manifest/project.rs similarity index 100% rename from crates/icp/src/manifest/project.rs rename to crates/icp-deploy-canister/src/manifest/project.rs diff --git a/crates/icp/src/manifest/recipe.rs b/crates/icp-deploy-canister/src/manifest/recipe.rs similarity index 100% rename from crates/icp/src/manifest/recipe.rs rename to crates/icp-deploy-canister/src/manifest/recipe.rs diff --git a/crates/icp/src/manifest/serde_helpers.rs b/crates/icp-deploy-canister/src/manifest/serde_helpers.rs similarity index 100% rename from crates/icp/src/manifest/serde_helpers.rs rename to crates/icp-deploy-canister/src/manifest/serde_helpers.rs diff --git a/crates/icp-deploy-canister/src/network/mod.rs b/crates/icp-deploy-canister/src/network/mod.rs new file mode 100644 index 000000000..4f21d3e7f --- /dev/null +++ b/crates/icp-deploy-canister/src/network/mod.rs @@ -0,0 +1,368 @@ +//! Network *configuration* model (the manifest-derived view of a network). +//! +//! Runtime concerns — launching/stopping managed networks, network descriptors, +//! agent access — live in the host `icp` crate. + +use schemars::JsonSchema; +use serde::{Deserialize, Deserializer, Serialize}; +use strum::EnumString; +use url::Url; + +pub use crate::manifest::network::RootKeySpec; +use crate::manifest::network::{ + Connected as ManifestConnected, Endpoints, Gateway as ManifestGateway, Mode, +}; + +pub const DEFAULT_LOCAL_NETWORK_BIND: &str = "127.0.0.1"; +pub const DEFAULT_LOCAL_NETWORK_PORT: u16 = 8000; + +#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] +pub enum Port { + Fixed(u16), + Random, +} + +impl Default for Port { + fn default() -> Self { + Port::Fixed(8000) + } +} + +impl<'de> Deserialize<'de> for Port { + fn deserialize>(d: D) -> Result { + Ok(match u16::deserialize(d)? { + 0 => Port::Random, + p => Port::Fixed(p), + }) + } +} + +fn default_bind() -> String { + "127.0.0.1".to_string() +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct Gateway { + #[serde(default = "default_bind")] + pub bind: String, + + #[serde(default)] + pub port: Port, + + #[serde(default)] + pub domains: Vec, +} + +impl Default for Gateway { + fn default() -> Self { + Self { + bind: default_bind(), + port: Default::default(), + domains: Default::default(), + } + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct Managed { + #[serde(flatten)] + pub mode: ManagedMode, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +#[serde(untagged)] +pub enum ManagedMode { + Image(Box), + Launcher(Box), +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct ManagedLauncherConfig { + pub gateway: Gateway, + pub artificial_delay_ms: Option, + pub ii: bool, + pub nns: bool, + pub subnets: Option>, + pub bitcoind_addr: Option>, + pub dogecoind_addr: Option>, + pub version: Option, +} + +#[derive( + Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize, EnumString, strum::Display, +)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab-case")] +pub enum SubnetKind { + Application, + System, + VerifiedApplication, + Bitcoin, + Fiduciary, + Nns, + Sns, +} + +impl Default for ManagedMode { + fn default() -> Self { + Self::default_for_port(DEFAULT_LOCAL_NETWORK_PORT) + } +} + +impl ManagedMode { + pub fn default_for_port(port: u16) -> Self { + ManagedMode::Launcher(Box::new(ManagedLauncherConfig { + gateway: Gateway { + bind: default_bind(), + port: if port == 0 { + Port::Random + } else { + Port::Fixed(port) + }, + domains: vec![], + }, + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + version: None, + })) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct ManagedImageConfig { + pub image: String, + pub port_mapping: Vec, + pub rm_on_exit: bool, + pub args: Vec, + pub entrypoint: Option>, + pub environment: Vec, + pub volumes: Vec, + pub platform: Option, + pub user: Option, + pub shm_size: Option, + pub status_dir: String, + pub mounts: Vec, + pub extra_hosts: Vec, +} + +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct Connected { + /// The URL this network's API can be reached at. + pub api_url: Url, + + /// The URL this network's HTTP gateway can be reached at. + pub http_gateway_url: Option, + + /// How to obtain the root key used to verify responses from this network. + pub root_key: RootKeySpec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +#[serde(tag = "mode", rename_all = "lowercase")] +pub enum Configuration { + // Note: we must use struct variants to be able to flatten + // and make schemars generate the proper schema + /// A managed network is one which can be controlled and manipulated. + Managed { + #[serde(flatten)] + managed: Managed, + }, + + /// A connected network is one which can be interacted with + /// but cannot be controlled or manipulated. + Connected { + #[serde(flatten)] + connected: Connected, + }, +} + +impl Default for Configuration { + fn default() -> Self { + Configuration::Managed { + managed: Managed::default(), + } + } +} + +impl From for Gateway { + fn from(value: ManifestGateway) -> Self { + let ManifestGateway { + bind, + domains, + port, + } = value; + let bind = bind.unwrap_or("127.0.0.1".to_string()); + let port = match port { + Some(0) => Port::Random, + Some(p) => Port::Fixed(p), + None => Port::default(), + }; + let mut domains = domains.unwrap_or_default(); + if bind == "127.0.0.1" || bind == "0.0.0.0" || bind == "::1" || bind == "::" { + domains.insert(0, "localhost".to_string()); + } + Gateway { + bind, + port, + domains, + } + } +} + +impl From for Connected { + fn from(value: ManifestConnected) -> Self { + let root_key = value.root_key; + match value.endpoints { + Endpoints::Implicit { url } => Connected { + api_url: url.clone(), + http_gateway_url: Some(url), + root_key, + }, + Endpoints::Explicit { + api_url, + http_gateway_url, + } => Connected { + api_url, + http_gateway_url, + root_key, + }, + } + } +} + +impl From for Configuration { + fn from(value: Mode) -> Self { + match value { + Mode::Managed(managed) => match *managed.mode { + crate::manifest::network::ManagedMode::Launcher { + gateway, + artificial_delay_ms, + ii, + nns, + subnets, + bitcoind_addr, + dogecoind_addr, + version, + } => { + let gateway: Gateway = match gateway { + Some(g) => g.into(), + None => Gateway::default(), + }; + let version = match version { + Some(v) => { + if v.starts_with('v') { + Some(v) + } else { + Some(format!("v{v}")) + } + } + None => None, + }; + Configuration::Managed { + managed: Managed { + mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { + gateway, + artificial_delay_ms, + ii: ii.unwrap_or(false), + nns: nns.unwrap_or(false), + subnets, + bitcoind_addr, + dogecoind_addr, + version, + })), + }, + } + } + crate::manifest::network::ManagedMode::Image { + image, + port_mapping, + rm_on_exit, + args, + entrypoint, + environment, + volumes, + platform, + user, + shm_size, + status_dir, + mounts: mount, + extra_hosts, + } => Configuration::Managed { + managed: Managed { + mode: ManagedMode::Image(Box::new(ManagedImageConfig { + image, + port_mapping, + rm_on_exit: rm_on_exit.unwrap_or(false), + args: args.unwrap_or_default(), + entrypoint, + environment: environment.unwrap_or_default(), + volumes: volumes.unwrap_or_default(), + platform, + user, + shm_size, + status_dir: status_dir.unwrap_or_else(|| "/app/status".to_string()), + mounts: mount.unwrap_or_default(), + extra_hosts: extra_hosts.unwrap_or_default(), + })), + }, + }, + }, + Mode::Connected(connected) => Configuration::Connected { + connected: connected.into(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::network::{ + Gateway as ManifestGateway, Managed as ManifestManaged, ManagedMode as ManifestManagedMode, + Mode, + }; + + #[test] + fn from_mode_launcher_with_bitcoind_addr() { + let mode = Mode::Managed(ManifestManaged { + mode: Box::new(ManifestManagedMode::Launcher { + gateway: Some(ManifestGateway { + bind: None, + port: Some(8000), + domains: None, + }), + artificial_delay_ms: None, + ii: None, + nns: None, + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: None, + version: None, + }), + }); + + let config: Configuration = mode.into(); + match config { + Configuration::Managed { + managed: + Managed { + mode: ManagedMode::Launcher(launcher_config), + }, + } => { + assert_eq!( + launcher_config.bitcoind_addr, + Some(vec!["127.0.0.1:18444".to_string()]) + ); + assert_eq!(launcher_config.dogecoind_addr, None); + assert!(!launcher_config.ii); + assert!(!launcher_config.nns); + } + _ => panic!("expected ManagedMode::Launcher"), + } + } +} diff --git a/crates/icp-deploy-canister/src/parsers.rs b/crates/icp-deploy-canister/src/parsers.rs new file mode 100644 index 000000000..b0a74730f --- /dev/null +++ b/crates/icp-deploy-canister/src/parsers.rs @@ -0,0 +1,643 @@ +//! Parsing of token, cycle, memory, and duration amounts with support for suffixes and underscores. + +use bigdecimal::{BigDecimal, Signed}; +use num_bigint::BigUint; +use num_integer::Integer; +use num_traits::{ToPrimitive, Zero}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::str::FromStr; + +/// Parse a token amount with support for suffixes (k, m, b, t) and underscores. +/// +/// Examples: +/// - `1` -> 1 +/// - `1_000` -> 1000 +/// - `1k` or `1K` -> 1000 +/// - `1t` or `1T` -> 1000000000000 +/// - `0.5` -> 0.5 +/// - `0.5k` -> 500 +pub fn parse_token_amount(input: &str) -> Result { + let input = input.trim(); + + if input.is_empty() { + return Err("Token amount cannot be empty".to_string()); + } + + let (number_part, multiplier) = if let Some(last_char) = input.chars().last() { + match last_char { + 'k' | 'K' => (&input[..input.len() - 1], 1_000u128), + 'm' | 'M' => (&input[..input.len() - 1], 1_000_000u128), + 'b' | 'B' => (&input[..input.len() - 1], 1_000_000_000u128), + 't' | 'T' => (&input[..input.len() - 1], 1_000_000_000_000u128), + _ => (input, 1u128), + } + } else { + (input, 1u128) + }; + + let cleaned = number_part.replace('_', ""); + let base = + BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid token amount: '{}'", input))?; + + if base.is_negative() { + return Err(format!("Token amount cannot be negative: '{}'", input)); + } + + let multiplier_decimal = BigDecimal::from(multiplier); + Ok(base * multiplier_decimal) +} + +/// Convert a token amount to the smallest unit by multiplying by 10^token_decimals. +/// E.g. 1.5 with 8 decimals -> 150000000. Fails if the result would be fractional. +pub fn to_token_unit_amount( + token_amount: BigDecimal, + token_decimals: u8, +) -> Result { + use num_bigint::BigInt; + use num_traits::pow::Pow; + + let (mantissa, exponent) = token_amount.into_bigint_and_exponent(); + let scale_adjustment = token_decimals as i64 - exponent; + let ten = BigInt::from(10); + + let result = if scale_adjustment >= 0 { + let multiplier = ten.pow(scale_adjustment as u32); + mantissa * multiplier + } else { + let divisor = ten.pow((-scale_adjustment) as u32); + let (quotient, remainder) = mantissa.div_rem(&divisor); + if !remainder.is_zero() { + return Err(format!( + "Token amount cannot be represented with {} decimals (would result in fractional units)", + token_decimals + )); + } + quotient + }; + + result + .try_into() + .map_err(|_| "Token amount cannot be negative".to_string()) +} + +fn parse_cycles_str(s: &str) -> Result { + let token_amount = parse_token_amount(s)?; + let unit_amount = to_token_unit_amount(token_amount, 0)?; + unit_amount + .to_u128() + .ok_or_else(|| format!("Cycles amount too large: '{}'", s)) +} + +/// An amount of cycles. +/// +/// Deserializes from a number or a string with suffixes (k, m, b, t) and optional underscore separators. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] +#[schemars(untagged)] +pub enum CyclesAmount { + Number(u64), // yaml only supports up to u64 + Str(String), +} + +impl CyclesAmount { + pub fn get(&self) -> u128 { + match self { + CyclesAmount::Number(n) => *n as u128, + CyclesAmount::Str(s) => parse_cycles_str(s) + .unwrap_or_else(|e| panic!("invalid cycles amount '{}': {}", s, e)), + } + } +} + +impl<'de> Deserialize<'de> for CyclesAmount { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + // Identical enum to CyclesAmount. Needed to avoid a circular dependency. + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Number(u64), + Str(String), + } + let v = Raw::deserialize(d).map_err(|_| { + serde::de::Error::custom("cycles amount must be a number or a string with optional suffix (k, m, b, t), e.g. 1000 or \"4t\"") + })?; + let c = match v { + Raw::Number(n) => CyclesAmount::Number(n), + Raw::Str(ref s) => { + parse_cycles_str(s).map_err(serde::de::Error::custom)?; // validate the string is a valid cycles amount + CyclesAmount::Str(s.clone()) + } + }; + Ok(c) + } +} + +impl Serialize for CyclesAmount { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + CyclesAmount::Number(n) => serializer.serialize_u64(*n), + CyclesAmount::Str(s) => serializer.serialize_str(s), + } + } +} + +impl FromStr for CyclesAmount { + type Err = String; + + fn from_str(s: &str) -> Result { + parse_cycles_str(s)?; // validate the string is a valid cycles amount + Ok(CyclesAmount::Str(s.to_string())) + } +} + +impl fmt::Display for CyclesAmount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl From for u128 { + fn from(c: CyclesAmount) -> Self { + c.get() + } +} + +impl From for CyclesAmount { + fn from(n: u128) -> Self { + if let Ok(n64) = u64::try_from(n) { + CyclesAmount::Number(n64) + } else { + CyclesAmount::Str(n.to_string()) + } + } +} + +const KB: u64 = 1000; +const KIB: u64 = 1024; +const MB: u64 = 1_000_000; +const MIB: u64 = 1024 * 1024; +const GB: u64 = 1_000_000_000; +const GIB: u64 = 1024 * 1024 * 1024; + +fn parse_memory_str(s: &str) -> Result { + let s = s.trim(); + if s.is_empty() { + return Err("Memory amount cannot be empty".to_string()); + } + let lower = s.to_lowercase(); + let (number_part, factor) = if lower.ends_with("gib") { + (&s[..s.len() - 3], GIB) + } else if lower.ends_with("gb") { + (&s[..s.len() - 2], GB) + } else if lower.ends_with("mib") { + (&s[..s.len() - 3], MIB) + } else if lower.ends_with("mb") { + (&s[..s.len() - 2], MB) + } else if lower.ends_with("kib") { + (&s[..s.len() - 3], KIB) + } else if lower.ends_with("kb") { + (&s[..s.len() - 2], KB) + } else { + (s, 1u64) + }; + let cleaned = number_part.trim().replace('_', ""); + let amount = + BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid memory amount: '{}'", s))?; + if amount.is_negative() { + return Err(format!("Memory amount cannot be negative: '{}'", s)); + } + let product = amount * BigDecimal::from(factor); + if !product.is_integer() { + return Err( + "Memory amount must be a whole number of bytes (fractional bytes not allowed)" + .to_string(), + ); + } + product + .to_u64() + .ok_or_else(|| format!("Memory amount too large: '{}'", s)) +} + +/// An amount of memory in bytes. +/// +/// Deserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib), +/// optional decimals, and optional underscore separators. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] +#[schemars(untagged)] +pub enum MemoryAmount { + Number(u64), + Str(String), +} + +impl MemoryAmount { + pub fn get(&self) -> u64 { + match self { + MemoryAmount::Number(n) => *n, + MemoryAmount::Str(s) => parse_memory_str(s) + .unwrap_or_else(|e| panic!("invalid memory amount '{}': {}", s, e)), + } + } +} + +impl<'de> Deserialize<'de> for MemoryAmount { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Number(u64), + Str(String), + } + let v = Raw::deserialize(d).map_err(|_| { + serde::de::Error::custom( + "memory amount must be a number or a string with optional suffix (kb, kib, mb, mib, gb, gib), e.g. 1024 or \"2.5gib\"", + ) + })?; + let m = match v { + Raw::Number(n) => MemoryAmount::Number(n), + Raw::Str(ref s) => { + parse_memory_str(s).map_err(serde::de::Error::custom)?; + MemoryAmount::Str(s.clone()) + } + }; + Ok(m) + } +} + +impl Serialize for MemoryAmount { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + MemoryAmount::Number(n) => serializer.serialize_u64(*n), + MemoryAmount::Str(s) => serializer.serialize_str(s), + } + } +} + +impl FromStr for MemoryAmount { + type Err = String; + + fn from_str(s: &str) -> Result { + parse_memory_str(s)?; + Ok(MemoryAmount::Str(s.to_string())) + } +} + +impl fmt::Display for MemoryAmount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl From for u64 { + fn from(m: MemoryAmount) -> Self { + m.get() + } +} + +impl From for MemoryAmount { + fn from(n: u64) -> Self { + MemoryAmount::Number(n) + } +} + +const SECONDS_PER_MINUTE: u64 = 60; +const SECONDS_PER_HOUR: u64 = 3600; +const SECONDS_PER_DAY: u64 = 86400; +const SECONDS_PER_WEEK: u64 = 604800; + +fn parse_duration_str(s: &str) -> Result { + let s = s.trim(); + if s.is_empty() { + return Err("Duration cannot be empty".to_string()); + } + let lower = s.to_lowercase(); + let (number_part, factor) = if lower.ends_with('w') { + (&s[..s.len() - 1], SECONDS_PER_WEEK) + } else if lower.ends_with('d') { + (&s[..s.len() - 1], SECONDS_PER_DAY) + } else if lower.ends_with('h') { + (&s[..s.len() - 1], SECONDS_PER_HOUR) + } else if lower.ends_with('m') { + (&s[..s.len() - 1], SECONDS_PER_MINUTE) + } else if lower.ends_with('s') { + (&s[..s.len() - 1], 1u64) + } else { + (s, 1u64) + }; + let cleaned = number_part.trim().replace('_', ""); + if cleaned.is_empty() { + return Err(format!("Invalid duration: '{s}'")); + } + let value: u64 = cleaned + .parse() + .map_err(|_| format!("Invalid duration: '{s}'"))?; + value + .checked_mul(factor) + .ok_or_else(|| format!("Duration too large: '{s}'")) +} + +/// A duration in seconds. +/// +/// Deserializes from a number (seconds) or a string with duration suffix (s, m, h, d, w) +/// and optional underscore separators. +/// +/// Suffixes (case-insensitive): +/// - `s` — seconds +/// - `m` — minutes (×60) +/// - `h` — hours (×3600) +/// - `d` — days (×86400) +/// - `w` — weeks (×604800) +/// +/// A bare number without suffix is treated as seconds. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] +#[schemars(untagged)] +pub enum DurationAmount { + Number(u64), + Str(String), +} + +impl DurationAmount { + pub fn get(&self) -> u64 { + match self { + DurationAmount::Number(n) => *n, + DurationAmount::Str(s) => { + parse_duration_str(s).unwrap_or_else(|e| panic!("invalid duration '{}': {}", s, e)) + } + } + } +} + +impl<'de> Deserialize<'de> for DurationAmount { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Number(u64), + Str(String), + } + let v = Raw::deserialize(d).map_err(|_| { + serde::de::Error::custom( + "duration must be a number (seconds) or a string with optional suffix (s, m, h, d, w), e.g. 2592000 or \"30d\"", + ) + })?; + let c = match v { + Raw::Number(n) => DurationAmount::Number(n), + Raw::Str(ref s) => { + parse_duration_str(s).map_err(serde::de::Error::custom)?; + DurationAmount::Str(s.clone()) + } + }; + Ok(c) + } +} + +impl Serialize for DurationAmount { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + DurationAmount::Number(n) => serializer.serialize_u64(*n), + DurationAmount::Str(s) => serializer.serialize_str(s), + } + } +} + +impl FromStr for DurationAmount { + type Err = String; + + fn from_str(s: &str) -> Result { + parse_duration_str(s)?; + Ok(DurationAmount::Str(s.to_string())) + } +} + +impl fmt::Display for DurationAmount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl From for u64 { + fn from(d: DurationAmount) -> Self { + d.get() + } +} + +impl From for DurationAmount { + fn from(n: u64) -> Self { + DurationAmount::Number(n) + } +} + +impl PartialEq for DurationAmount { + fn eq(&self, other: &u64) -> bool { + self.get() == *other + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cycles_amount_from_str_plain() { + assert_eq!("1".parse::().unwrap().get(), 1); + assert_eq!("1000".parse::().unwrap().get(), 1000); + } + + #[test] + fn cycles_amount_from_str_suffixes() { + assert_eq!("1k".parse::().unwrap().get(), 1000); + assert_eq!( + "1t".parse::().unwrap().get(), + 1_000_000_000_000 + ); + assert_eq!( + "4t".parse::().unwrap().get(), + 4_000_000_000_000 + ); + assert_eq!( + "0.5t".parse::().unwrap().get(), + 500_000_000_000 + ); + } + + #[test] + fn cycles_amount_from_str_underscores() { + assert_eq!("1_000".parse::().unwrap().get(), 1000); + } + + #[test] + fn cycles_amount_from_str_fractional_rejected() { + assert!("1.5".parse::().is_err()); + } + + #[test] + fn cycles_amount_deserialize() { + let yaml = "4t"; + let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(c.get(), 4_000_000_000_000); + + let yaml = "5000000000000"; + let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(c.get(), 5_000_000_000_000); + } + + #[test] + fn parse_token_amount_plain_and_suffixes() { + use std::str::FromStr; + assert_eq!( + parse_token_amount("1").unwrap(), + BigDecimal::from_str("1").unwrap() + ); + assert_eq!( + parse_token_amount("1k").unwrap(), + BigDecimal::from_str("1000").unwrap() + ); + assert_eq!( + parse_token_amount("0.5t").unwrap(), + BigDecimal::from_str("500000000000").unwrap() + ); + } + + #[test] + fn memory_amount_from_str_plain() { + assert_eq!("1".parse::().unwrap().get(), 1); + assert_eq!("1024".parse::().unwrap().get(), 1024); + } + + #[test] + fn memory_amount_from_str_suffixes() { + assert_eq!("1kb".parse::().unwrap().get(), 1000); + assert_eq!("1kib".parse::().unwrap().get(), 1024); + assert_eq!("1mb".parse::().unwrap().get(), 1_000_000); + assert_eq!("1mib".parse::().unwrap().get(), 1024 * 1024); + assert_eq!("1gb".parse::().unwrap().get(), 1_000_000_000); + assert_eq!( + "1gib".parse::().unwrap().get(), + 1024 * 1024 * 1024 + ); + assert_eq!( + "2 GiB".parse::().unwrap().get(), + 2 * 1024 * 1024 * 1024 + ); + } + + #[test] + fn memory_amount_from_str_decimals() { + assert_eq!("0.5kib".parse::().unwrap().get(), 512); + assert_eq!("1.5gib".parse::().unwrap().get(), 1610612736); + } + + #[test] + fn memory_amount_fractional_bytes_rejected() { + assert!("1.5".parse::().is_err()); // 1.5 bytes + assert!("0.3kib".parse::().is_err()); // 307.2 bytes + } + + #[test] + fn memory_amount_from_str_underscores() { + assert_eq!("1_024".parse::().unwrap().get(), 1024); + } + + #[test] + fn memory_amount_deserialize() { + let yaml = "2gib"; + let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(m.get(), 2 * 1024 * 1024 * 1024); + + let yaml = "4294967296"; + let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(m.get(), 4294967296); + } + + #[test] + fn duration_amount_from_str_plain() { + assert_eq!("60".parse::().unwrap().get(), 60); + assert_eq!("2592000".parse::().unwrap().get(), 2592000); + } + + #[test] + fn duration_amount_from_str_underscores() { + assert_eq!( + "2_592_000".parse::().unwrap().get(), + 2592000 + ); + } + + #[test] + fn duration_amount_from_str_suffixes() { + assert_eq!("60s".parse::().unwrap().get(), 60); + assert_eq!("90m".parse::().unwrap().get(), 5400); + assert_eq!("24h".parse::().unwrap().get(), 86400); + assert_eq!("30d".parse::().unwrap().get(), 2592000); + assert_eq!("4w".parse::().unwrap().get(), 2419200); + } + + #[test] + fn duration_amount_from_str_case_insensitive() { + assert_eq!("30D".parse::().unwrap().get(), 2592000); + assert_eq!("1W".parse::().unwrap().get(), 604800); + assert_eq!("24H".parse::().unwrap().get(), 86400); + assert_eq!("60S".parse::().unwrap().get(), 60); + assert_eq!("90M".parse::().unwrap().get(), 5400); + } + + #[test] + fn duration_amount_from_str_underscores_with_suffix() { + assert_eq!( + "2_592_000s".parse::().unwrap().get(), + 2592000 + ); + } + + #[test] + fn duration_amount_from_str_errors() { + assert!("abc".parse::().is_err()); + assert!("".parse::().is_err()); + assert!("1x".parse::().is_err()); + assert!("1.5d".parse::().is_err()); + assert!("-1d".parse::().is_err()); + } + + #[test] + fn duration_amount_deserialize() { + let yaml = "30d"; + let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(d.get(), 2592000); + + let yaml = "2592000"; + let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(d.get(), 2592000); + + let yaml = "2_592_000"; + let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(d.get(), 2592000); + } + + #[test] + fn duration_amount_partial_eq_u64() { + let d = DurationAmount::Number(2592000); + assert!(d == 2592000); + assert!(d != 0); + + let d = DurationAmount::Str("30d".to_string()); + assert!(d == 2592000); + } +} diff --git a/crates/icp-deploy-canister/src/prelude.rs b/crates/icp-deploy-canister/src/prelude.rs new file mode 100644 index 000000000..3cf6e17e0 --- /dev/null +++ b/crates/icp-deploy-canister/src/prelude.rs @@ -0,0 +1,13 @@ +pub use camino::{FromPathBufError, Utf8Path as Path, Utf8PathBuf as PathBuf}; + +pub const TRILLION: u128 = 1_000_000_000_000; + +pub const SECOND: u64 = 1; +pub const MINUTE: u64 = 60 * SECOND; + +pub const IC_MAINNET_NETWORK_API_URL: &str = "https://icp-api.io"; +pub const IC_MAINNET_NETWORK_GATEWAY_URL: &str = "https://icp.net"; +/// Name of the implicit IC mainnet network and its implicit environment +pub const IC: &str = "ic"; +/// Name of the implicit local managed network and its implicit environment +pub const LOCAL: &str = "local"; diff --git a/crates/icp-deploy-canister/src/project.rs b/crates/icp-deploy-canister/src/project.rs new file mode 100644 index 000000000..b0819c335 --- /dev/null +++ b/crates/icp-deploy-canister/src/project.rs @@ -0,0 +1,1943 @@ +use std::collections::{BTreeMap, HashMap, HashSet, hash_map::Entry}; + +use indexmap::{IndexMap, map::Entry as IndexEntry}; + +use snafu::prelude::*; + +use crate::{ + Canister, Environment, InitArgs, Network, Project, + canister::{ControllerRef, Settings, recipe}, + fs, + manifest::{ + ArgsFormat, CANISTER_MANIFEST, CanisterManifest, DependencyManifest, EnvironmentManifest, + Item, LoadManifestError, ManifestInitArgs, NetworkManifest, PROJECT_MANIFEST, + ProjectManifest, + canister::{Instructions, SyncSteps}, + environment::CanisterSelection, + load_manifest, + network::RootKeySpec, + recipe::RecipeType, + }, + network::{ + Configuration, Connected, DEFAULT_LOCAL_NETWORK_BIND, DEFAULT_LOCAL_NETWORK_PORT, Gateway, + Managed, ManagedLauncherConfig, ManagedMode, Port, + }, + prelude::*, +}; + +#[derive(Debug, Snafu)] +pub enum EnvironmentError { + #[snafu(display("environment '{environment}' points to invalid network '{network}'"))] + InvalidNetwork { + environment: String, + network: String, + }, + + #[snafu(display("environment '{environment}' points to invalid canister '{canister}'"))] + InvalidCanister { + environment: String, + canister: String, + }, +} + +#[derive(Debug, Snafu)] +pub enum ConsolidateManifestError { + #[snafu(display("failed to perform glob parsing"))] + GlobParse { source: glob::PatternError }, + + #[snafu(display("failed to get glob iter"))] + GlobIter { source: glob::GlobError }, + + #[snafu(display("failed to convert path to UTF-8"))] + Utf8Path { source: FromPathBufError }, + + #[snafu(display("failed to load canister manifest"))] + LoadCanister { source: LoadManifestError }, + + #[snafu(display("failed to load network manifest"))] + LoadNetwork { source: LoadManifestError }, + + #[snafu(display("failed to load environment manifest"))] + LoadEnvironment { source: LoadManifestError }, + + #[snafu(display("failed to load {kind} manifest at: {path}"))] + Failed { kind: String, path: String }, + + #[snafu(display("failed to fetch canister recipe: {recipe_type:?}"))] + Recipe { + #[snafu(source(from(recipe::ResolveError, Box::new)))] + source: Box, + recipe_type: RecipeType, + }, + + #[snafu(display("failed to render canister recipe: {recipe_type:?}"))] + RenderRecipe { + #[snafu(source(from(recipe::RenderRecipeError, Box::new)))] + source: Box, + recipe_type: RecipeType, + }, + + #[snafu(display("project contains two similarly named {kind}s: '{name}'"))] + Duplicate { kind: String, name: String }, + + #[snafu(display("`{name}` is a reserved {kind} name."))] + Reserved { kind: String, name: String }, + + #[snafu(display("could not locate a {kind} manifest at: '{path}'"))] + NotFound { kind: String, path: String }, + + #[snafu(display("failed to read init_args file for canister '{canister}'"))] + ReadInitArgs { + source: fs::IoError, + canister: String, + }, + + #[snafu(display( + "init_args for canister '{canister}' uses format 'bin' with inline content; \ + binary format requires a file path" + ))] + BinFormatInlineContent { canister: String }, + + #[snafu(display( + "canister '{canister}' lists controller '{controller}', but no canister with that \ + name is declared in the project" + ))] + UnknownControllerCanister { + canister: String, + controller: String, + }, + + #[snafu(display( + "canister name '{name}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ + (':' is reserved as the dependency namespace separator)" + ))] + InvalidCanisterName { name: String }, + + #[snafu(display( + "dependency alias '{alias}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ + (':' is reserved as the dependency namespace separator)" + ))] + InvalidDependencyAlias { alias: String }, + + #[snafu(display("project declares two dependencies with the same alias '{alias}'"))] + DuplicateDependencyAlias { alias: String }, + + #[snafu(display( + "dependency alias '{alias}' collides with a canister of the same name in the same project" + ))] + DependencyAliasCollision { alias: String }, + + #[snafu(display("could not find a project manifest for dependency '{alias}' at: '{path}'"))] + DependencyNotFound { alias: String, path: String }, + + #[snafu(display("failed to canonicalize path for dependency '{alias}' at: '{path}'"))] + DependencyCanonicalize { alias: String, path: String }, + + #[snafu(display("failed to load project manifest for dependency '{alias}'"))] + LoadDependencyManifest { + source: LoadManifestError, + alias: String, + }, + + #[snafu(display( + "dependency '{alias}' selects canister '{canister}', which the dependency does not declare" + ))] + UnknownDependencyCanister { alias: String, canister: String }, + + #[snafu(display("dependency cycle detected: {chain}"))] + CircularDependency { chain: String }, + + #[snafu(transparent)] + Environment { source: EnvironmentError }, +} + +/// Resolve a [`ManifestInitArgs`] into a canonical [`InitArgs`] by reading +/// any file references relative to `base_path`. +fn resolve_manifest_init_args( + manifest_init_args: &ManifestInitArgs, + base_path: &Path, + canister: &str, +) -> Result { + match manifest_init_args { + ManifestInitArgs::String(content) => Ok(InitArgs::Text { + content: content.trim().to_owned(), + format: ArgsFormat::Candid, + }), + ManifestInitArgs::Path { path, format } => { + let file_path = base_path.join(path); + match format { + ArgsFormat::Bin => { + let bytes = fs::read(&file_path).context(ReadInitArgsSnafu { canister })?; + Ok(InitArgs::Binary(bytes)) + } + fmt => { + let content = + fs::read_to_string(&file_path).context(ReadInitArgsSnafu { canister })?; + Ok(InitArgs::Text { + content: content.trim().to_owned(), + format: fmt.clone(), + }) + } + } + } + ManifestInitArgs::Value { value, format } => match format { + ArgsFormat::Bin => BinFormatInlineContentSnafu { canister }.fail(), + fmt => Ok(InitArgs::Text { + content: value.trim().to_owned(), + format: fmt.clone(), + }), + }, + } +} + +fn is_glob(s: &str) -> bool { + s.contains('*') || s.contains('?') || s.contains('[') || s.contains('{') +} + +/// Whether `name` is a valid canister name or dependency alias: non-empty and +/// containing only ASCII letters, digits, `_`, or `-`. +/// +/// A single strict rule keeps names safe for every purpose they are reused for — +/// store-key segments, `PUBLIC_CANISTER_ID:` env vars, DNS subdomains, and +/// archive paths — so no per-site sanitizing is needed. In particular `:` is the +/// dependency namespace separator, and `.` / `/` would be ambiguous in +/// subdomains and paths. +fn is_valid_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') +} + +/// Builds the canonical canisters declared directly in one project manifest, +/// resolving glob/path/inline entries, recipes, and init-args relative to +/// `pdir`. Returns `(local name, canister dir, canister)` with empty bindings; +/// callers assign store keys and bindings. Does not check for duplicate names +/// across projects — that is the caller's responsibility (via the global map). +async fn build_manifest_canisters( + pdir: &Path, + manifest_canisters: &[Item], + recipe_resolver: &dyn recipe::RemoteResourceResolve, +) -> Result, ConsolidateManifestError> { + let mut result: Vec<(String, PathBuf, Canister)> = Vec::new(); + + for i in manifest_canisters { + let ms = match i { + Item::Path(pattern) => { + let is_glob_pattern = is_glob(pattern); + let paths = match is_glob_pattern { + // Explicit path + false => vec![pdir.join(pattern)], + + // Glob pattern + true => { + let paths = + glob::glob(pdir.join(pattern).as_str()).context(GlobParseSnafu)?; + + let mut v = vec![]; + for p in paths { + let path = p.context(GlobIterSnafu)?; + let utf8_path = PathBuf::try_from(path).context(Utf8PathSnafu)?; + v.push(utf8_path); + } + v + } + }; + + let paths = if is_glob_pattern { + // For glob patterns, filter out non-directories and non-canister directories + paths + .into_iter() + .filter(|p| p.is_dir()) + .filter(|p| p.join(CANISTER_MANIFEST).exists()) + .collect::>() + } else { + // For explicit paths, validate that they exist and contain canister.yaml + let mut validated_paths = vec![]; + for p in paths { + if !p.join(CANISTER_MANIFEST).is_file() { + return NotFoundSnafu { + kind: "canister".to_string(), + path: pattern.to_string(), + } + .fail(); + } + validated_paths.push(p); + } + validated_paths + }; + + let mut ms = vec![]; + for p in paths { + ms.push(( + p.to_owned(), + load_manifest::(&p.join(CANISTER_MANIFEST)) + .context(LoadCanisterSnafu)?, + )); + } + ms + } + + Item::Manifest(m) => vec![(pdir.to_owned(), m.to_owned())], + }; + + for (cdir, m) in ms { + if !is_valid_name(&m.name) { + return InvalidCanisterNameSnafu { + name: m.name.clone(), + } + .fail(); + } + + let registry_recipe = match &m.instructions { + Instructions::BuildSync { .. } => None, + Instructions::Recipe { recipe } => match &recipe.recipe_type { + RecipeType::Registry { .. } => Some(recipe.recipe_type.to_string()), + _ => None, + }, + }; + + let (build, sync) = match &m.instructions { + // Build/Sync + Instructions::BuildSync { build, sync } => ( + build.to_owned(), + match sync { + Some(sync) => sync.to_owned(), + None => SyncSteps::default(), + }, + ), + + // Recipe: fetch the template through the resolver, then render + // and parse it into concrete steps. + Instructions::Recipe { recipe } => { + let ctx = recipe::RecipeContext { + canister_name: m.name.clone(), + }; + let template = + recipe_resolver + .resolve_recipe(recipe) + .await + .context(RecipeSnafu { + recipe_type: recipe.recipe_type.clone(), + })?; + recipe::render_recipe(&template, recipe, &ctx).context(RenderRecipeSnafu { + recipe_type: recipe.recipe_type.clone(), + })? + } + }; + + let init_args = m + .init_args + .as_ref() + .map(|mia| resolve_manifest_init_args(mia, &cdir, &m.name)) + .transpose()?; + + result.push(( + m.name.clone(), + cdir, + Canister { + name: m.name.clone(), + settings: m.settings.clone(), + build, + sync, + init_args, + registry_recipe, + bindings: BTreeMap::new(), + // Default to the bare local name; overwritten with the + // dot-nested alias form when the canister is imported as a + // dependency (see `import_dependency`). + friendly_names: vec![m.name.clone()], + }, + )); + } + } + + Ok(result) +} + +/// A dependency instance imported into the workspace. Returned by +/// [`import_dependency`] and cached per canonical path so diamond dependencies +/// reuse the same instance. +#[derive(Clone)] +struct ImportedInstance { + /// This instance's own canisters, as `(local name, full store key)` — the + /// set exposable to the parent via `canisters:` selection. + own: Vec<(String, String)>, + /// Every canister in this instance's subtree (its own canisters plus all + /// transitively imported ones), as `(store key, local name, alias chain + /// from this instance down to the canister's owning project)`. Used to + /// register a friendly URL per alias chain when the instance is reached + /// again via de-duplication (a diamond), including for its descendants. + subtree: Vec<(String, String, Vec)>, +} + +/// A member environment's per-canister config, to be folded into the root's +/// same-named environment beneath any root overrides. +#[derive(Default, Clone)] +struct MemberCanisterOverride { + settings: Option, + init_args: Option, +} + +/// Per-environment member overrides: env name → store key → override. +type MemberEnvOverrides = HashMap>; + +/// A member's identity (store-key prefix) and the environment names it defines, +/// used to enforce that a member declares every environment the root targets +/// (strict rule). +struct MemberEnvInfo { + prefix: String, + defined: HashSet, +} + +/// Canonicalize a dependency root (resolving symlinks and `..`) for use as a +/// de-dup / cycle-detection identity. +fn canonicalize_dep(alias: &str, dep_root: &Path) -> Result { + let build_err = || { + DependencyCanonicalizeSnafu { + alias: alias.to_owned(), + path: dep_root.to_string(), + } + .build() + }; + let canon = dunce::canonicalize(dep_root.as_std_path()).map_err(|_| build_err())?; + PathBuf::try_from(canon).map_err(|_| build_err()) +} + +/// Store-key prefix for a dependency instance: its canonical directory relative +/// to the canonical app root, forward-slash separated so keys are stable across +/// platforms and independent of how each edge spells the path. +fn relative_prefix(app_root_canonical: &Path, dep_canonical: &Path) -> String { + let rel = pathdiff::diff_utf8_paths(dep_canonical, app_root_canonical) + .unwrap_or_else(|| dep_canonical.to_owned()); + rel.as_str().replace('\\', "/") +} + +/// Build a dependency canister's friendly-URL subdomain prefix: the canister's +/// local name as the most-specific label, followed by its alias chain reversed +/// (root-most alias last). E.g. local `backend` reached via `[service-a, +/// openemail]` → `backend.openemail.service-a`. Dot-nested so it stays a valid, +/// collision-free multi-label host; see DESIGN §17.2. +fn friendly_name_for(local: &str, alias_chain: &[String]) -> String { + let mut labels = Vec::with_capacity(alias_chain.len() + 1); + labels.push(local.to_string()); + labels.extend(alias_chain.iter().rev().cloned()); + labels.join(".") +} + +/// Rewrite `CanisterName` controller references from a dependency's local +/// canister names to their store keys, so global controller validation and +/// deploy-time id lookup operate uniformly on store keys. +fn translate_controllers(canister: &mut Canister, local_to_key: &BTreeMap) { + translate_settings_controllers(&mut canister.settings, local_to_key); +} + +/// Rewrite `CanisterName` controller references in a `Settings` from a +/// dependency's local canister names to their store keys. +fn translate_settings_controllers( + settings: &mut Settings, + local_to_key: &BTreeMap, +) { + if let Some(controllers) = &mut settings.controllers { + for cref in controllers.iter_mut() { + if let ControllerRef::CanisterName(name) = cref + && let Some(key) = local_to_key.get(name) + { + *name = key.clone(); + } + } + } +} + +/// Compute the `PUBLIC_CANISTER_ID` env-var wiring for canisters in one project +/// scope: its own canisters by local name, plus each dependency's exposed +/// canisters under `:`. +fn compute_bindings( + own: &[(String, String)], + edges: &[(String, Vec<(String, String)>)], +) -> BTreeMap { + let mut bindings = BTreeMap::new(); + for (local, key) in own { + bindings.insert(local.clone(), key.clone()); + } + for (alias, exposed) in edges { + for (dep_local, key) in exposed { + bindings.insert(format!("{alias}:{dep_local}"), key.clone()); + } + } + bindings +} + +/// Select which of a dependency instance's own canisters are exposed to the +/// parent, per the dependency's `canisters` selection. +fn select_exposed( + own: &[(String, String)], + selection: &CanisterSelection, + alias: &str, +) -> Result, ConsolidateManifestError> { + match selection { + CanisterSelection::Everything => Ok(own.to_vec()), + CanisterSelection::None => Ok(vec![]), + CanisterSelection::Named(names) => { + let mut out = Vec::new(); + for name in names { + match own.iter().find(|(local, _)| local == name) { + Some(pair) => out.push(pair.clone()), + None => { + return UnknownDependencyCanisterSnafu { + alias: alias.to_owned(), + canister: name.clone(), + } + .fail(); + } + } + } + Ok(out) + } + } +} + +/// Validate the dependency aliases declared in one project scope: no `:`, no +/// collision with a local canister name, and no duplicate alias. +fn validate_dependency_aliases( + deps: &[DependencyManifest], + own_canister_names: &HashSet, +) -> Result<(), ConsolidateManifestError> { + let mut seen: HashSet<&str> = HashSet::new(); + for d in deps { + if !is_valid_name(&d.name) { + return InvalidDependencyAliasSnafu { + alias: d.name.clone(), + } + .fail(); + } + if own_canister_names.contains(&d.name) { + return DependencyAliasCollisionSnafu { + alias: d.name.clone(), + } + .fail(); + } + if !seen.insert(&d.name) { + return DuplicateDependencyAliasSnafu { + alias: d.name.clone(), + } + .fail(); + } + } + Ok(()) +} + +/// Recursively import a dependency's canisters into `canisters`, keyed by their +/// app-root-relative store keys. De-duplicates instances by canonical path +/// (diamond dependencies deploy once) and detects cycles. Returns the imported +/// instance's prefix and its own canisters. +#[allow(clippy::too_many_arguments)] +async fn import_dependency( + app_root_canonical: &Path, + parent_dir: &Path, + dep: &DependencyManifest, + recipe_resolver: &dyn recipe::RemoteResourceResolve, + canisters: &mut IndexMap, + registry: &mut HashMap, + stack: &mut Vec, + member_env_overrides: &mut MemberEnvOverrides, + members: &mut Vec, + // Alias chain from the workspace root to and including this dependency, + // used to build friendly-URL subdomains (§17.2). + alias_chain: &[String], +) -> Result { + let dep_root = parent_dir.join(&dep.path); + let manifest_path = dep_root.join(PROJECT_MANIFEST); + if !manifest_path.is_file() { + return DependencyNotFoundSnafu { + alias: dep.name.clone(), + path: dep_root.to_string(), + } + .fail(); + } + + let canonical = canonicalize_dep(&dep.name, &dep_root)?; + + // Cycle detection. + if stack.contains(&canonical) { + let mut chain: Vec = stack.iter().map(|p| p.to_string()).collect(); + chain.push(canonical.to_string()); + return CircularDependencySnafu { + chain: chain.join(" -> "), + } + .fail(); + } + + // Diamond de-dup: same resolved directory means the same instance, deployed + // once. It is still reachable via this new alias chain, so register an + // additional friendly URL per chain (§17.3) rather than picking one — for + // the whole subtree (its own canisters *and* its transitive dependencies), + // each named by this chain extended with the canister's alias path below the + // instance. + if let Some(inst) = registry.get(&canonical) { + let inst = inst.clone(); + for (key, local, rel_chain) in &inst.subtree { + let mut chain = alias_chain.to_vec(); + chain.extend(rel_chain.iter().cloned()); + let fname = friendly_name_for(local, &chain); + if let Some((_, canister)) = canisters.get_mut(key) + && !canister.friendly_names.contains(&fname) + { + canister.friendly_names.push(fname); + } + } + return Ok(inst); + } + + stack.push(canonical.clone()); + + let prefix = relative_prefix(app_root_canonical, &canonical); + + let dep_manifest: ProjectManifest = + load_manifest(&manifest_path).context(LoadDependencyManifestSnafu { + alias: dep.name.clone(), + })?; + + // Build the dependency's own canisters and key them under the prefix. All of + // them are imported (deploy-all); the `canisters` exposure subset is applied + // by the caller when wiring env vars. + let built = + build_manifest_canisters(&dep_root, &dep_manifest.canisters, recipe_resolver).await?; + + let mut own: Vec<(String, String)> = Vec::new(); + let mut local_to_key: BTreeMap = BTreeMap::new(); + for (local, cdir, mut canister) in built { + let store_key = format!("{prefix}:{local}"); + canister.name = store_key.clone(); + // Friendly URL from the alias chain, not the path-based store key. + canister.friendly_names = vec![friendly_name_for(&local, alias_chain)]; + own.push((local.clone(), store_key.clone())); + local_to_key.insert(local.clone(), store_key.clone()); + match canisters.entry(store_key.clone()) { + IndexEntry::Occupied(_) => { + return DuplicateSnafu { + kind: "canister".to_string(), + name: store_key, + } + .fail(); + } + IndexEntry::Vacant(e) => { + e.insert((cdir, canister)); + } + } + } + + // Now that every sibling's store key is known, translate the dependency's + // controller references (local sibling name -> store key). + for (_, key) in &own { + if let Some((_, canister)) = canisters.get_mut(key) { + translate_controllers(canister, &local_to_key); + } + } + + // Capture the member's own environments so the parent can honor its + // per-canister settings/init_args for the same-named environment + // (standalone-equivalence). The network binding and canister selection are + // ignored; only overrides on the member's *own* canisters are + // folded in — keys naming its dependencies are left to those dependencies. + let mut defined_envs: HashSet = HashSet::new(); + for env_item in &dep_manifest.environments { + let em: EnvironmentManifest = match env_item { + Item::Manifest(m) => m.clone(), + Item::Path(path) => { + let p = dep_root.join(path); + if !p.is_file() { + return NotFoundSnafu { + kind: "environment".to_string(), + path: p.to_string(), + } + .fail(); + } + load_manifest::(&p).context(LoadEnvironmentSnafu)? + } + }; + defined_envs.insert(em.name.clone()); + if let Some(settings) = &em.settings { + for (local, s) in settings { + if let Some(key) = local_to_key.get(local) { + // Translate the override's own controller references from the + // member's local names to store keys, so name-based controllers + // resolve against the workspace id map just like base settings. + let mut s = s.clone(); + translate_settings_controllers(&mut s, &local_to_key); + member_env_overrides + .entry(em.name.clone()) + .or_default() + .entry(key.clone()) + .or_default() + .settings = Some(s); + } + } + } + if let Some(init_args) = &em.init_args { + for (local, ia) in init_args { + if let Some(key) = local_to_key.get(local) { + member_env_overrides + .entry(em.name.clone()) + .or_default() + .entry(key.clone()) + .or_default() + .init_args = Some(ia.clone()); + } + } + } + } + members.push(MemberEnvInfo { + prefix: prefix.clone(), + defined: defined_envs, + }); + + // Recurse into the dependency's own dependencies. + let own_names: HashSet = own.iter().map(|(l, _)| l.clone()).collect(); + validate_dependency_aliases(&dep_manifest.dependencies, &own_names)?; + + // The instance's subtree, for diamond-hit friendly-URL propagation: its own + // canisters sit at the instance root (empty relative alias chain); each + // nested dependency contributes its subtree prefixed with the nested alias. + let mut subtree: Vec<(String, String, Vec)> = own + .iter() + .map(|(local, key)| (key.clone(), local.clone(), Vec::new())) + .collect(); + + let mut edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for nested in &dep_manifest.dependencies { + let mut nested_chain = alias_chain.to_vec(); + nested_chain.push(nested.name.clone()); + let inst = Box::pin(import_dependency( + app_root_canonical, + &dep_root, + nested, + recipe_resolver, + canisters, + registry, + stack, + member_env_overrides, + members, + &nested_chain, + )) + .await?; + for (key, local, rel) in &inst.subtree { + let mut r = Vec::with_capacity(rel.len() + 1); + r.push(nested.name.clone()); + r.extend(rel.iter().cloned()); + subtree.push((key.clone(), local.clone(), r)); + } + let exposed = select_exposed(&inst.own, &nested.canisters, &nested.name)?; + edges.push((nested.name.clone(), exposed)); + } + + // Assign env-var bindings for this instance's own canisters. + let bindings = compute_bindings(&own, &edges); + for (_, key) in &own { + if let Some((_, canister)) = canisters.get_mut(key) { + canister.bindings = bindings.clone(); + } + } + + stack.pop(); + let instance = ImportedInstance { own, subtree }; + registry.insert(canonical, instance.clone()); + Ok(instance) +} + +/// Build one environment's canister map: select from `canisters`, then apply the +/// member overrides for this environment (standalone-equivalence), then +/// the root's own overrides (highest precedence). Precedence is therefore +/// root-explicit > member-env > canister-base. +fn build_environment_canisters( + canisters: &IndexMap, + env_name: &str, + selection: &CanisterSelection, + member_overrides: Option<&HashMap>, + root_settings: Option<&HashMap>, + root_init_args: Option<&HashMap>, +) -> Result, ConsolidateManifestError> { + let mut cs = match selection { + CanisterSelection::None => IndexMap::new(), + CanisterSelection::Everything => canisters.clone(), + CanisterSelection::Named(names) => { + let mut cs: IndexMap = IndexMap::new(); + for name in names { + let v = canisters.get(name).ok_or( + InvalidCanisterSnafu { + environment: env_name.to_owned(), + canister: name.to_owned(), + } + .build(), + )?; + cs.insert(name.to_owned(), v.to_owned()); + } + cs + } + }; + + // Member overrides first (lower precedence than the root's own overrides). + if let Some(overrides) = member_overrides { + for (key, ov) in overrides { + if let Some((cpath, canister)) = cs.get_mut(key) { + if let Some(s) = &ov.settings { + canister.settings = s.clone(); + } + if let Some(ia) = &ov.init_args { + canister.init_args = Some(resolve_manifest_init_args(ia, cpath, key)?); + } + } + } + } + + // Root overrides last (highest precedence). + if let Some(settings) = root_settings { + for (name, s) in settings { + if let Some((_p, canister)) = cs.get_mut(name) { + canister.settings = s.clone(); + } + } + } + if let Some(init_args) = root_init_args { + for (name, ia) in init_args { + if let Some((cpath, canister)) = cs.get_mut(name) { + canister.init_args = Some(resolve_manifest_init_args(ia, cpath, name)?); + } + } + } + + Ok(cs) +} + +/// Turns the ProjectManifest into a Project struct +/// - Adds the default Networks +/// - Adds the default Environment +/// - Imports any dependency projects' canisters +/// - Validates the manifest to make sure that: +/// - There are no duplicates +/// - All the environments have networks +/// - All the referenced canisters exist +/// - All the recipes have been resolved +pub async fn consolidate_manifest( + pdir: &Path, + recipe_resolver: &dyn recipe::RemoteResourceResolve, + m: &ProjectManifest, +) -> Result { + // Canisters. IndexMap (not HashMap) so the order from the project manifest is preserved + // through to consumers like `icp project bundle`, which needs reproducible output. + let mut canisters: IndexMap = IndexMap::new(); + + // Canonical app root, used to derive stable, order-independent store-key + // prefixes for imported dependency canisters. + let app_root_canonical = + canonicalize_dep("", pdir).unwrap_or_else(|_| pdir.to_owned()); + + // This project's own canisters, keyed by their bare local names. + let app_built = build_manifest_canisters(pdir, &m.canisters, recipe_resolver).await?; + let mut app_own: Vec<(String, String)> = Vec::new(); + for (local, cdir, canister) in app_built { + app_own.push((local.clone(), local.clone())); + match canisters.entry(local.clone()) { + IndexEntry::Occupied(e) => { + return DuplicateSnafu { + kind: "canister".to_string(), + name: e.key().to_owned(), + } + .fail(); + } + IndexEntry::Vacant(e) => { + e.insert((cdir, canister)); + } + } + } + + // Import dependency projects. Each dependency is deployed in full and keyed + // under its app-root-relative path; diamonds (the same directory reached via + // multiple edges) resolve to a single instance. + let mut registry: HashMap = HashMap::new(); + let mut stack: Vec = Vec::new(); + // Member environment config folded into the root's same-named environments, + // and the per-member set of declared environment names for the strict rule. + let mut member_env_overrides: MemberEnvOverrides = HashMap::new(); + let mut members: Vec = Vec::new(); + let app_own_names: HashSet = app_own.iter().map(|(l, _)| l.clone()).collect(); + validate_dependency_aliases(&m.dependencies, &app_own_names)?; + + let mut app_edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for dep in &m.dependencies { + let inst = import_dependency( + &app_root_canonical, + pdir, + dep, + recipe_resolver, + &mut canisters, + &mut registry, + &mut stack, + &mut member_env_overrides, + &mut members, + std::slice::from_ref(&dep.name), + ) + .await?; + let exposed = select_exposed(&inst.own, &dep.canisters, &dep.name)?; + app_edges.push((dep.name.clone(), exposed)); + } + + // Assign env-var bindings for this project's own canisters (own canisters by + // local name plus each dependency's exposed canisters under `:`). + let app_bindings = compute_bindings(&app_own, &app_edges); + for (_, key) in &app_own { + if let Some((_, canister)) = canisters.get_mut(key) { + canister.bindings = app_bindings.clone(); + } + } + + // Friendly URLs need no de-collision pass: the strict name rule (no '.') makes + // own canisters single-label and dependency canisters multi-label (dot-nested + // by alias chain), so their hostnames are disjoint by construction (§17.2). + + // Validate that every canister-name controller reference points to a declared canister. + // Catching typos here turns "perpetual warning" into a clear load-time error. + for (canister_name, (_, canister)) in &canisters { + let Some(crefs) = &canister.settings.controllers else { + continue; + }; + for cref in crefs { + if let Some(ref_name) = cref.canister_name() + && !canisters.contains_key(ref_name) + { + return UnknownControllerCanisterSnafu { + canister: canister_name.to_owned(), + controller: ref_name.to_owned(), + } + .fail(); + } + } + } + + // Networks + let mut networks: HashMap = HashMap::new(); + + // Add IC network first - this is always protected and non-overridable + networks.insert( + IC.to_string(), + Network { + name: IC.to_string(), + configuration: Configuration::Connected { + connected: Connected { + api_url: IC_MAINNET_NETWORK_API_URL.parse().unwrap(), + http_gateway_url: Some(IC_MAINNET_NETWORK_GATEWAY_URL.parse().unwrap()), + root_key: RootKeySpec::Mainnet, + }, + }, + }, + ); + + // Track which network names are protected (only IC network) + let protected_network_names: HashSet = [IC.to_string()].into_iter().collect(); + + // Resolve NetworkManifests and add them (including user-defined "local" if provided) + for i in &m.networks { + let m = match i { + Item::Path(path) => { + let path = pdir.join(path); + if !path.exists() || !path.is_file() { + return NotFoundSnafu { + kind: "network".to_string(), + path: path.to_string(), + } + .fail(); + } + load_manifest::(&path).context(LoadNetworkSnafu)? + } + Item::Manifest(ms) => ms.clone(), + }; + + match networks.entry(m.name.to_owned()) { + // Duplicate + Entry::Occupied(e) => { + // Only error if trying to override a protected network + if protected_network_names.contains(&m.name) { + return ReservedSnafu { + kind: "network".to_string(), + name: m.name.to_string(), + } + .fail(); + } + + // For non-protected duplicates, this is a user error (defining same network twice) + return DuplicateSnafu { + kind: "network".to_string(), + name: e.key().to_owned(), + } + .fail(); + } + + // Ok + Entry::Vacant(e) => { + e.insert(Network { + name: m.name.to_owned(), + configuration: m.configuration.into(), // Convert manifest to config struct + }); + } + } + } + + // After processing user networks, add default "local" if not already defined + // This provides backward compatibility for projects that don't define their own "local" network + if !networks.contains_key(LOCAL) { + networks.insert( + LOCAL.to_string(), + Network { + name: LOCAL.to_string(), + configuration: Configuration::Managed { + managed: Managed { + mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { + gateway: Gateway { + bind: DEFAULT_LOCAL_NETWORK_BIND.to_string(), + port: Port::Fixed(DEFAULT_LOCAL_NETWORK_PORT), + domains: vec![], + }, + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + version: None, + })), + }, + }, + }, + ); + } + + // Environments + let mut environments: HashMap = HashMap::new(); + + for i in &m.environments { + let m = match i { + Item::Path(path) => { + let path = pdir.join(path); + if !path.exists() || !path.is_file() { + return NotFoundSnafu { + kind: "environment".to_string(), + path: path.to_string(), + } + .fail(); + } + load_manifest::(&path).context(LoadEnvironmentSnafu)? + } + Item::Manifest(ms) => ms.clone(), + }; + + match environments.entry(m.name.to_owned()) { + // Duplicate + Entry::Occupied(e) => { + return DuplicateSnafu { + kind: "environment".to_string(), + name: e.key().to_owned(), + } + .fail(); + } + + // Ok + Entry::Vacant(e) => { + e.insert(Environment { + name: m.name.to_owned(), + + // Embed network in environment + network: { + let v = networks.get(&m.network).ok_or( + InvalidNetworkSnafu { + environment: m.name.to_owned(), + network: m.network.to_owned(), + } + .build(), + )?; + + v.to_owned() + }, + + // Embed canisters in environment, folding member overrides + // beneath the root's own settings/init_args overrides. + canisters: build_environment_canisters( + &canisters, + &m.name, + &m.canisters, + member_env_overrides.get(&m.name), + m.settings.as_ref(), + m.init_args.as_ref(), + )?, + }); + } + } + } + + // We're done adding all the user environments + // Now we add the implicit `local` and `ic` environment if the user hasn't overriden it + if let Entry::Vacant(vacant_entry) = environments.entry(LOCAL.to_string()) { + let network = networks + .get(LOCAL) + .ok_or( + InvalidNetworkSnafu { + environment: LOCAL.to_owned(), + network: LOCAL.to_owned(), + } + .build(), + )? + .to_owned(); + vacant_entry.insert(Environment { + name: LOCAL.to_string(), + network, + canisters: build_environment_canisters( + &canisters, + LOCAL, + &CanisterSelection::Everything, + member_env_overrides.get(LOCAL), + None, + None, + )?, + }); + } + if let Entry::Vacant(vacant_entry) = environments.entry(IC.to_string()) { + let network = networks + .get(IC) + .ok_or( + InvalidNetworkSnafu { + environment: IC.to_owned(), + network: IC.to_owned(), + } + .build(), + )? + .to_owned(); + vacant_entry.insert(Environment { + name: IC.to_string(), + network, + canisters: build_environment_canisters( + &canisters, + IC, + &CanisterSelection::Everything, + member_env_overrides.get(IC), + None, + None, + )?, + }); + } + + // Strict rule: every member must declare each environment the root targets. + // `local`/`ic` are implicit for every project, so they never count + // as missing; other environments must be declared explicitly by the member. + // Recorded per-environment and enforced lazily when that environment is + // selected (so a missing `staging` never blocks `deploy -e local`). + let mut member_missing_envs: HashMap> = HashMap::new(); + for env_name in environments.keys() { + if env_name == LOCAL || env_name == IC { + continue; + } + for member in &members { + if !member.defined.contains(env_name) { + member_missing_envs + .entry(env_name.clone()) + .or_default() + .push(member.prefix.clone()); + } + } + } + + Ok(Project { + dir: pdir.into(), + canisters, + networks, + environments, + member_missing_envs, + }) +} + +#[derive(Debug, Snafu)] +pub enum LoadProjectError { + #[snafu(display("failed to load project manifest"))] + ProjectManifest { source: LoadManifestError }, + + #[snafu(transparent)] + Consolidate { source: ConsolidateManifestError }, +} + +/// Load and consolidate the project rooted at `project_dir` (already located by +/// the caller), resolving recipes through `recipe`. +pub async fn load_project( + recipe: &dyn recipe::RemoteResourceResolve, + project_dir: &Path, +) -> Result { + let m: ProjectManifest = + load_manifest(&project_dir.join(PROJECT_MANIFEST)).context(ProjectManifestSnafu)?; + let p = consolidate_manifest(project_dir, recipe, &m).await?; + Ok(p) +} + +#[derive(Debug, Snafu)] +pub enum VerifySandboxError { + #[snafu(display( + "canister '{canister}' uses a script {phase} step, which cannot run in the sandbox; \ + only pre-built builds and plugin syncs are permitted" + ))] + ScriptStep { canister: String, phase: String }, +} + +/// Verify that a fully-resolved project (recipes already resolved into concrete +/// steps) contains no script steps. Script build/sync steps spawn host +/// subprocesses and therefore cannot run inside the sandbox; only pre-built +/// builds and plugin syncs are permitted. +pub fn verify_sandbox(project: &Project) -> Result<(), VerifySandboxError> { + use crate::manifest::canister::{BuildStep, SyncStep}; + + for (name, (_, canister)) in &project.canisters { + if canister + .build + .steps + .iter() + .any(|s| matches!(s, BuildStep::Script(_))) + { + return ScriptStepSnafu { + canister: name.clone(), + phase: "build", + } + .fail(); + } + if canister + .sync + .steps + .iter() + .any(|s| matches!(s, SyncStep::Script(_))) + { + return ScriptStepSnafu { + canister: name.clone(), + phase: "sync", + } + .fail(); + } + } + Ok(()) +} + +#[cfg(test)] +mod dependency_tests { + use super::*; + use crate::canister::recipe::{RemoteResourceResolve, ResolveError}; + use crate::manifest::adapter::prebuilt::SourceField; + use crate::manifest::recipe::Recipe; + use camino_tempfile::Utf8TempDir; + use tokio::sync::mpsc::Sender; + + /// Recipes and plugins are never used in these tests; every canister is pre-built. + struct PanicResolver; + + #[async_trait::async_trait] + impl RemoteResourceResolve for PanicResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + panic!("recipe resolver should not be called in dependency tests"); + } + + async fn resolve_wasm( + &self, + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _stdio: Option>, + ) -> Result { + panic!("wasm resolver should not be called in dependency tests"); + } + } + + fn write(dir: &Path, rel: &str, contents: &str) { + let p = dir.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, contents).unwrap(); + } + + /// A minimal `icp.yaml` body declaring the given pre-built canisters, + /// followed by a raw `dependencies:` block (may be empty). + fn manifest(canisters: &[&str], deps: &str) -> String { + let mut s = String::new(); + if canisters.is_empty() { + s.push_str("canisters: []\n"); + } else { + s.push_str("canisters:\n"); + for c in canisters { + s.push_str(&format!( + " - name: {c}\n build:\n steps:\n - type: pre-built\n path: {c}.wasm\n" + )); + } + } + s.push_str(deps); + s + } + + async fn consolidate(pdir: &Path) -> Result { + let m: ProjectManifest = + load_manifest(&pdir.join(PROJECT_MANIFEST)).expect("failed to parse project manifest"); + consolidate_manifest(pdir, &PanicResolver, &m).await + } + + fn bindings_of<'a>(p: &'a Project, key: &str) -> &'a BTreeMap { + &p.canisters + .get(key) + .unwrap_or_else(|| { + panic!( + "canister '{key}' not found; have {:?}", + p.canisters.keys().collect::>() + ) + }) + .1 + .bindings + } + + fn friendly_names_of<'a>(p: &'a Project, key: &str) -> &'a [String] { + &p.canisters + .get(key) + .unwrap_or_else(|| { + panic!( + "canister '{key}' not found; have {:?}", + p.canisters.keys().collect::>() + ) + }) + .1 + .friendly_names + } + + #[tokio::test] + async fn single_project_bindings_are_self_and_siblings() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Flat behavior preserved: every canister maps every sibling (incl. self) + // to itself. + let expected = BTreeMap::from([ + ("backend".to_string(), "backend".to_string()), + ("frontend".to_string(), "frontend".to_string()), + ]); + assert_eq!(bindings_of(&p, "backend"), &expected); + assert_eq!(bindings_of(&p, "frontend"), &expected); + } + + #[tokio::test] + async fn dependency_import_and_exposure_subset() { + let tmp = Utf8TempDir::new().unwrap(); + // Dependency nested inside the app (mirrors a submodule under the app). + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [backend]\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // The whole dependency is deployed (both canisters imported), keyed by path. + assert!(p.canisters.contains_key("backend")); + assert!(p.canisters.contains_key("openemail:backend")); + assert!(p.canisters.contains_key("openemail:frontend")); + + // App's own canister sees itself and only the *exposed* dependency canister. + assert_eq!( + bindings_of(&p, "backend"), + &BTreeMap::from([ + ("backend".to_string(), "backend".to_string()), + ( + "openemail:backend".to_string(), + "openemail:backend".to_string() + ), + ]) + ); + + // The dependency's own canisters keep their standalone view (bare names). + assert_eq!( + bindings_of(&p, "openemail:backend"), + &BTreeMap::from([ + ("backend".to_string(), "openemail:backend".to_string()), + ("frontend".to_string(), "openemail:frontend".to_string()), + ]) + ); + } + + #[tokio::test] + async fn member_env_config_folds_in_with_root_override_winning() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail defines `staging` with per-canister settings for its own + // canisters. + write( + tmp.path(), + "openemail/icp.yaml", + r#" +canisters: + - name: backend + build: + steps: + - type: pre-built + path: backend.wasm + - name: frontend + build: + steps: + - type: pre-built + path: frontend.wasm +environments: + - name: staging + settings: + backend: + compute_allocation: 5 + frontend: + compute_allocation: 7 +"#, + ); + // The app declares openemail and also defines `staging`, overriding the + // imported backend's settings (the root override must win). + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging + settings: + "openemail:backend": + compute_allocation: 99 +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + + // Root override wins over the member's config. + assert_eq!( + staging + .canisters + .get("openemail:backend") + .unwrap() + .1 + .settings + .compute_allocation, + Some(99), + ); + // No root override → the member's own config applies (standalone-equivalence). + assert_eq!( + staging + .canisters + .get("openemail:frontend") + .unwrap() + .1 + .settings + .compute_allocation, + Some(7), + ); + // Both projects declared staging, so nothing is recorded as missing. + assert!(p.member_missing_envs.is_empty()); + } + + #[tokio::test] + async fn missing_member_environment_is_recorded() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + // openemail does not declare `staging`, so it is recorded as missing. + assert_eq!( + p.member_missing_envs.get("staging"), + Some(&vec!["openemail".to_string()]), + ); + // Implicit environments are never recorded as missing. + assert!(!p.member_missing_envs.contains_key("local")); + assert!(!p.member_missing_envs.contains_key("ic")); + } + + #[tokio::test] + async fn diamond_dedups_to_single_instance() { + let tmp = Utf8TempDir::new().unwrap(); + // umbrella layout: service-a and service-b both depend on ../openemail. + write( + tmp.path(), + "umbrella/openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "umbrella/service-a/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "umbrella/service-b/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // openemail is imported exactly once despite two edges reaching it. + let openemail_keys: Vec<_> = p + .canisters + .keys() + .filter(|k| k.contains("openemail")) + .collect(); + assert_eq!( + openemail_keys, + vec![&"umbrella/openemail:backend".to_string()], + "expected a single shared openemail instance" + ); + + // Both services' code reads `openemail:backend`, resolving to the one instance. + assert_eq!( + bindings_of(&p, "umbrella/service-a:backend").get("openemail:backend"), + Some(&"umbrella/openemail:backend".to_string()) + ); + assert_eq!( + bindings_of(&p, "umbrella/service-b:backend").get("openemail:backend"), + Some(&"umbrella/openemail:backend".to_string()) + ); + + // The single shared instance is reachable at one friendly URL per alias + // chain (§17.3) — the store-key path (`umbrella/`) never appears. + assert_eq!( + friendly_names_of(&p, "umbrella/openemail:backend"), + &["backend.openemail.service-a", "backend.openemail.service-b"] + ); + // Each service's own canister is named by its own alias chain. + assert_eq!( + friendly_names_of(&p, "umbrella/service-a:backend"), + &["backend.service-a"] + ); + assert_eq!( + friendly_names_of(&p, "umbrella/service-b:backend"), + &["backend.service-b"] + ); + } + + #[tokio::test] + async fn diamond_transitive_dependency_gets_url_per_chain() { + let tmp = Utf8TempDir::new().unwrap(); + // The shared openemail itself depends on libfoo, and is reached via both + // service-a and service-b. + write( + tmp.path(), + "umbrella/openemail/libfoo/icp.yaml", + &manifest(&["bar"], ""), + ); + write( + tmp.path(), + "umbrella/openemail/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: libfoo\n path: ./libfoo\n", + ), + ); + write( + tmp.path(), + "umbrella/service-a/icp.yaml", + &manifest( + &["service"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "umbrella/service-b/icp.yaml", + &manifest( + &["service"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // The shared instance's own canister gets one URL per chain... + assert_eq!( + friendly_names_of(&p, "umbrella/openemail:backend"), + &["backend.openemail.service-a", "backend.openemail.service-b"] + ); + // ...and so does its *transitive* dependency (the subtree is revisited on + // the diamond hit, not just the instance's own canisters). + assert_eq!( + friendly_names_of(&p, "umbrella/openemail/libfoo:bar"), + &[ + "bar.libfoo.openemail.service-a", + "bar.libfoo.openemail.service-b" + ] + ); + } + + #[tokio::test] + async fn dot_in_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + // '.' is banned: it would be ambiguous in a dot-nested friendly subdomain + // (an own canister named `frontend.openemail` could collide with dependency + // `openemail`'s `frontend`). The strict name rule rejects it up front. + write( + tmp.path(), + "icp.yaml", + &manifest(&["frontend.openemail"], ""), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn invalid_dependency_alias_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["app"], + "dependencies:\n - name: open.email\n path: ./openemail\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidDependencyAlias { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn member_override_controllers_are_translated_to_store_keys() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail's `staging` override names a controller by its local name. + write( + tmp.path(), + "openemail/icp.yaml", + r#" +canisters: + - name: backend + build: + steps: + - type: pre-built + path: backend.wasm + - name: frontend + build: + steps: + - type: pre-built + path: frontend.wasm +environments: + - name: staging + settings: + backend: + controllers: ["frontend"] +"#, + ); + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + let controllers = staging + .canisters + .get("openemail:backend") + .unwrap() + .1 + .settings + .controllers + .clone() + .expect("controllers set by the member override"); + + // The member-local `frontend` must be translated to its store key, so it + // resolves against the workspace id map at deploy time. + assert_eq!( + controllers, + vec![ControllerRef::CanisterName( + "openemail:frontend".to_string() + )] + ); + } + + #[tokio::test] + async fn friendly_names_are_bare_for_own_and_dotted_for_dependencies() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail (with a transitive dep libfoo) vendored under the app. + write( + tmp.path(), + "openemail/libfoo/icp.yaml", + &manifest(&["bar"], ""), + ); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest( + &["backend", "frontend"], + "dependencies:\n - name: libfoo\n path: ./libfoo\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Own canister: bare name (unchanged from single-project behavior). + assert_eq!(friendly_names_of(&p, "backend"), &["backend"]); + // Direct dependency: dot-nested by alias (no `vendor/` path noise). + assert_eq!( + friendly_names_of(&p, "openemail:backend"), + &["backend.openemail"] + ); + assert_eq!( + friendly_names_of(&p, "openemail:frontend"), + &["frontend.openemail"] + ); + // Transitive dependency: full alias chain, canister-most-specific first. + assert_eq!( + friendly_names_of(&p, "openemail/libfoo:bar"), + &["bar.libfoo.openemail"] + ); + } + + #[tokio::test] + async fn cycle_is_detected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest(&[], "dependencies:\n - name: a\n path: ./a\n"), + ); + write( + tmp.path(), + "a/icp.yaml", + &manifest(&["x"], "dependencies:\n - name: b\n path: ../b\n"), + ); + write( + tmp.path(), + "b/icp.yaml", + &manifest(&["y"], "dependencies:\n - name: a\n path: ../a\n"), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::CircularDependency { .. }), + "expected CircularDependency, got {err:?}" + ); + } + + #[tokio::test] + async fn alias_colliding_with_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["openemail"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::DependencyAliasCollision { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn duplicate_alias_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write(tmp.path(), "one/icp.yaml", &manifest(&["backend"], "")); + write(tmp.path(), "two/icp.yaml", &manifest(&["backend"], "")); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: dup\n path: ./one\n - name: dup\n path: ./two\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::DuplicateDependencyAlias { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn colon_in_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write(tmp.path(), "icp.yaml", &manifest(&["foo:bar"], "")); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn unknown_exposed_canister_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [nope]\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::UnknownDependencyCanister { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn missing_dependency_path_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: openemail\n path: ./does-not-exist\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::DependencyNotFound { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn imported_canisters_appear_in_implicit_environments() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Deploy-all: the implicit `local` environment includes the dependency. + let local = p.environments.get("local").unwrap(); + assert!(local.canisters.contains_key("backend")); + assert!(local.canisters.contains_key("openemail:backend")); + } +} diff --git a/crates/icp-deploy-canister/src/sync_exec.rs b/crates/icp-deploy-canister/src/sync_exec.rs new file mode 100644 index 000000000..3105bdbe5 --- /dev/null +++ b/crates/icp-deploy-canister/src/sync_exec.rs @@ -0,0 +1,140 @@ +//! Sync-step support: the resolved step context, the `ICP_CLI_*` script +//! environment, and the host seams the step loop still needs. +//! +//! Plugin steps run inside the sandboxed wasmtime engine, which [`run_sync_steps`] +//! drives directly. Two things it can't do itself stay behind host seams: +//! subprocess script steps (which may be disallowed in a sandboxed environment) +//! go through [`ScriptRunner`], and step framing / output streaming goes through +//! [`StepProgress`]. +//! +//! [`run_sync_steps`]: crate::deploy::run_sync_steps + +use std::collections::BTreeMap; + +use async_trait::async_trait; +use candid::Principal; +use snafu::Snafu; +use tokio::sync::mpsc::Sender; + +use crate::manifest::adapter::script; +use crate::prelude::*; + +/// Resolved context for executing one canister's sync steps. +#[derive(Clone, Debug)] +pub struct SyncStepContext { + /// Directory the canister was declared in (base for relative plugin paths). + pub canister_path: PathBuf, + /// The canister being synced. + pub canister_id: Principal, + /// Name of the environment being synced (e.g. "local", "production"). + pub environment: String, + /// Name of the network (e.g. "local", "ic"). + pub network: String, + /// IDs of all named canisters in the project for this environment. + pub canister_ids: BTreeMap, + /// Proxy canister to route calls through, if `--proxy` was passed. + pub proxy: Option, +} + +/// A fully-resolved script sync step: the command(s), the working directory, and +/// the complete environment the subprocess runs with (see [`system_env_vars`]). +#[derive(Clone, Debug)] +pub struct ScriptInvocation { + /// Shell command(s) to run in order. + pub commands: Vec, + /// Working directory (the canister directory). + pub cwd: PathBuf, + /// Environment variables the subprocess inherits, in insertion order. + pub env: Vec<(String, String)>, +} + +impl ScriptInvocation { + /// Resolve a script step's adapter against the sync context, assembling the + /// `ICP_CLI_*` system environment variables the command runs with. + pub fn new(adapter: &script::Adapter, ctx: &SyncStepContext) -> Self { + Self { + commands: adapter.command.as_vec(), + cwd: ctx.canister_path.clone(), + env: system_env_vars(ctx), + } + } +} + +/// The `ICP_CLI_*` system environment variables every script sync step runs +/// with: the environment and network names, the target canister id, and one +/// `ICP_CLI_CID_` per known canister in the environment (name uppercased, +/// non-alphanumerics replaced with `_`). +pub fn system_env_vars(ctx: &SyncStepContext) -> Vec<(String, String)> { + let mut envs = vec![ + ("ICP_CLI_ENVIRONMENT".to_owned(), ctx.environment.clone()), + ("ICP_CLI_NETWORK".to_owned(), ctx.network.clone()), + ("ICP_CLI_CID".to_owned(), ctx.canister_id.to_text()), + ]; + for (name, id) in &ctx.canister_ids { + let key = format!( + "ICP_CLI_CID_{}", + name.to_uppercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect::() + ); + envs.push((key, id.to_text())); + } + envs +} + +/// Per-step framing and output streaming, implemented by the host over its +/// progress display. Each step is bracketed by `begin_step`/`end_step`; the +/// `Sender` returned by `begin_step` receives the step's streamed output lines. +#[async_trait] +pub trait StepProgress: Send { + /// Start a step with the given header, returning a sink for its output lines + /// (or `None` to discard output). + fn begin_step(&mut self, header: String) -> Option>; + + /// Finish the current step. + async fn end_step(&mut self); +} + +#[derive(Debug, Snafu)] +#[snafu(display("script sync step failed"))] +pub struct ScriptRunError { + pub source: Box, +} + +/// Host execution of subprocess script sync steps. Kept behind a trait because a +/// sandboxed environment may forbid spawning processes; the wasmtime plugin +/// engine, by contrast, is driven directly by [`run_sync_steps`]. +/// +/// [`run_sync_steps`]: crate::deploy::run_sync_steps +#[async_trait] +pub trait ScriptRunner: Sync + Send { + /// Run a resolved script step, streaming output to `stdio`, and return any + /// stderr lines to retain past the streamed view. + async fn run_script( + &self, + invocation: ScriptInvocation, + stdio: Option>, + ) -> Result, ScriptRunError>; +} + +/// A [`ScriptRunner`] that always fails, for environments that don't support +/// subprocess script sync steps. +pub struct NoScripts; + +#[derive(Debug, Snafu)] +#[snafu(display("script sync steps are not supported in this environment"))] +pub struct NoScriptsError; + +#[async_trait] +impl ScriptRunner for NoScripts { + async fn run_script( + &self, + _invocation: ScriptInvocation, + _stdio: Option>, + ) -> Result, ScriptRunError> { + Err(ScriptRunError { + source: Box::new(NoScriptsError), + }) + } +} diff --git a/crates/icp/Cargo.toml b/crates/icp/Cargo.toml index 004c53e61..847c6ffbc 100644 --- a/crates/icp/Cargo.toml +++ b/crates/icp/Cargo.toml @@ -25,7 +25,6 @@ fd-lock = { workspace = true } flate2 = { workspace = true } futures = { workspace = true } glob = { workspace = true } -handlebars = { workspace = true } hex = { workspace = true } hmac = { workspace = true } hybrid-array = { workspace = true } @@ -36,6 +35,7 @@ ic-ledger-types = { workspace = true } ic-management-canister-types = { workspace = true } ic-utils = { workspace = true } icp-canister-interfaces = { workspace = true } +icp-deploy-canister = { workspace = true } icp-sync-plugin = { workspace = true } icrc-ledger-types = { workspace = true } indexmap = { workspace = true } @@ -75,6 +75,11 @@ uuid = { workspace = true } wslpath2 = { workspace = true } zeroize = { workspace = true } +[features] +# Enables `clap::ValueEnum` derives on CLI-facing enums, including the manifest +# enums now defined in `icp-deploy-canister`. +clap = ["dep:clap", "icp-deploy-canister/clap"] + [target.'cfg(windows)'.dependencies] winreg = { workspace = true } diff --git a/crates/icp/src/canister/mod.rs b/crates/icp/src/canister/mod.rs index e3f7fe7a1..f957ffc7c 100644 --- a/crates/icp/src/canister/mod.rs +++ b/crates/icp/src/canister/mod.rs @@ -1,540 +1,15 @@ -use std::collections::HashMap; +//! Host-side canister facade. +//! +//! The canister *model* (`Settings`, `ControllerRef`, `resolve_controllers`, +//! log-visibility types, the `RemoteResourceResolve` interface) lives in +//! `icp_deploy_canister::canister` and is re-exported here. The build/wasm/recipe +//! *executors* (which spawn processes, fetch over HTTP, and cache) stay here. -use candid::{Nat, Principal}; -use ic_management_canister_types::{CanisterSettings, LogVisibility}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use crate::parsers::{CyclesAmount, DurationAmount, MemoryAmount}; +pub use icp_deploy_canister::canister::{ + ControllerRef, LogVisibilityDef, LogVisibilitySimple, Settings, resolve_controllers, +}; pub mod build; pub mod recipe; -pub mod sync; - -mod script; +pub mod script; pub mod wasm; - -/// Controls who can read canister logs. -/// Supports both string format ("controllers", "public") and object format ({ allowed_viewers: [...] }). -#[derive(Clone, Debug, PartialEq, Serialize)] -#[serde(untagged)] -pub enum LogVisibilityDef { - /// Simple string variants for controllers or public - Simple(LogVisibilitySimple), - /// Object format with allowed_viewers list - AllowedViewers { allowed_viewers: Vec }, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum LogVisibilitySimple { - Controllers, - Public, -} - -impl<'de> Deserialize<'de> for LogVisibilityDef { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - use serde::de::{Error, MapAccess, Visitor}; - use std::fmt; - - struct LogVisibilityVisitor; - - impl<'de> Visitor<'de> for LogVisibilityVisitor { - type Value = LogVisibilityDef; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("'controllers', 'public', or object with 'allowed_viewers'") - } - - fn visit_str(self, value: &str) -> Result { - LogVisibilitySimple::deserialize( - serde::de::value::StrDeserializer::::new(value), - ) - .map(LogVisibilityDef::Simple) - .map_err(|_| { - E::custom(format!( - "unknown log_visibility value: '{}', expected 'controllers' or 'public'", - value - )) - }) - } - - fn visit_map(self, mut map: M) -> Result - where - M: MapAccess<'de>, - { - let mut allowed_viewers: Option> = None; - - while let Some(key) = map.next_key::()? { - match key.as_str() { - "allowed_viewers" => { - if allowed_viewers.is_some() { - return Err(Error::duplicate_field("allowed_viewers")); - } - allowed_viewers = Some(map.next_value()?); - } - _ => { - return Err(Error::unknown_field(&key, &["allowed_viewers"])); - } - } - } - - allowed_viewers - .map(|v| LogVisibilityDef::AllowedViewers { allowed_viewers: v }) - .ok_or_else(|| Error::missing_field("allowed_viewers")) - } - } - - deserializer.deserialize_any(LogVisibilityVisitor) - } -} - -impl JsonSchema for LogVisibilityDef { - fn schema_name() -> std::borrow::Cow<'static, str> { - std::borrow::Cow::Borrowed("LogVisibility") - } - - fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ - "description": "Controls who can read canister logs.", - "oneOf": [ - { - "type": "string", - "enum": ["controllers", "public"], - "description": "Simple log visibility: 'controllers' (only controllers can view) or 'public' (anyone can view)" - }, - { - "type": "object", - "properties": { - "allowed_viewers": { - "type": "array", - "items": { - "type": "string", - "description": "A principal ID that can view logs" - }, - "description": "List of principal IDs that can view canister logs" - } - }, - "required": ["allowed_viewers"], - "additionalProperties": false, - "description": "Specific principals that can view logs" - } - ] - }) - } -} - -impl From for LogVisibility { - fn from(value: LogVisibilityDef) -> Self { - match value { - LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) => { - LogVisibility::Controllers - } - LogVisibilityDef::Simple(LogVisibilitySimple::Public) => LogVisibility::Public, - LogVisibilityDef::AllowedViewers { allowed_viewers } => { - LogVisibility::AllowedViewers(allowed_viewers) - } - } - } -} - -/// A reference to a controller: either an explicit principal or a canister name in this project. -/// -/// During deserialization, principal text format is tried first; strings that don't parse as a -/// principal are treated as canister names. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ControllerRef { - /// An explicitly specified principal (e.g. "2vxsx-fae") - Principal(candid::Principal), - /// A canister name from the same project (e.g. "my_canister") - CanisterName(String), -} - -impl ControllerRef { - /// Resolve to a `Principal` using the provided ID mapping. - /// Returns `None` if this is a `CanisterName` not present in `ids`. - pub fn resolve(&self, ids: &crate::store_id::IdMapping) -> Option { - match self { - ControllerRef::Principal(p) => Some(*p), - ControllerRef::CanisterName(name) => ids.get(name).copied(), - } - } - - /// If this is a `CanisterName`, returns the name; otherwise `None`. - pub fn canister_name(&self) -> Option<&str> { - match self { - ControllerRef::CanisterName(n) => Some(n), - ControllerRef::Principal(_) => None, - } - } -} - -/// Partition a slice of controller references into resolved principals and unresolved canister -/// names, using `ids` for name lookup. -pub fn resolve_controllers( - crefs: &[ControllerRef], - ids: &crate::store_id::IdMapping, -) -> (Vec, Vec) { - let mut resolved = Vec::new(); - let mut unresolved = Vec::new(); - for cref in crefs { - match cref.resolve(ids) { - Some(p) => resolved.push(p), - None => { - if let Some(name) = cref.canister_name() { - unresolved.push(name.to_owned()); - } - } - } - } - (resolved, unresolved) -} - -impl schemars::JsonSchema for ControllerRef { - fn schema_name() -> std::borrow::Cow<'static, str> { - std::borrow::Cow::Borrowed("ControllerRef") - } - - fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ - "type": "string", - "description": "A controller: either a principal text (e.g. '2vxsx-fae') or a canister name in this project (e.g. 'my_canister')" - }) - } -} - -/// Canister settings, such as compute and memory allocation. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct Settings { - /// Controls who can read canister logs. - #[serde(skip_serializing_if = "Option::is_none")] - pub log_visibility: Option, - - /// Compute allocation (0 to 100). Represents guaranteed compute capacity. - #[serde(skip_serializing_if = "Option::is_none")] - pub compute_allocation: Option, - - /// Memory allocation in bytes. If unset, memory is allocated dynamically. - /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_allocation: Option, - - /// Freezing threshold in seconds. Controls how long a canister can be inactive before being frozen. - /// Supports duration suffixes in YAML: s, m, h, d, w (e.g. "30d" or "4w"). - #[serde(skip_serializing_if = "Option::is_none")] - pub freezing_threshold: Option, - - /// Upper limit on cycles reserved for future resource payments. - /// Memory allocations that would push the reserved balance above this limit will fail. - /// Supports suffixes in YAML: k, m, b, t (e.g. "4t" or "4.3t"). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reserved_cycles_limit: Option, - - /// Wasm memory limit in bytes. Sets an upper bound for Wasm heap growth. - /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). - #[serde(skip_serializing_if = "Option::is_none")] - pub wasm_memory_limit: Option, - - /// Wasm memory threshold in bytes. Triggers a callback when exceeded. - /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). - #[serde(skip_serializing_if = "Option::is_none")] - pub wasm_memory_threshold: Option, - - /// Log memory limit in bytes (max 2 MiB). Oldest logs are purged when usage exceeds this value. - /// Supports suffixes in YAML: kb, kib, mb, mib (e.g. "2mib" or "256kib"). Canister default is 4096 bytes. - #[serde(skip_serializing_if = "Option::is_none")] - pub log_memory_limit: Option, - - /// Environment variables for the canister as key-value pairs. - /// These variables are accessible within the canister and can be used to configure - /// behavior without hardcoding values in the WASM module. - #[serde(skip_serializing_if = "Option::is_none")] - pub environment_variables: Option>, - - /// Controllers for this canister. Each entry is either a principal text - /// (e.g. "2vxsx-fae") or the name of another canister in this project. - /// Named canisters that do not yet exist will be set as controllers once created. - #[serde(default)] - pub controllers: Option>, -} - -impl From for CanisterSettings { - fn from(settings: Settings) -> Self { - CanisterSettings { - freezing_threshold: settings.freezing_threshold.map(|d| Nat::from(d.get())), - controllers: None, - reserved_cycles_limit: settings.reserved_cycles_limit.map(|c| Nat::from(c.get())), - log_visibility: settings.log_visibility.map(Into::into), - memory_allocation: settings.memory_allocation.map(|m| Nat::from(m.get())), - compute_allocation: settings.compute_allocation.map(Nat::from), - ..Default::default() - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn log_visibility_deserialize_controllers() { - let yaml = "controllers"; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) - ); - } - - #[test] - fn log_visibility_deserialize_public() { - let yaml = "public"; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - LogVisibilityDef::Simple(LogVisibilitySimple::Public) - ); - } - - #[test] - fn log_visibility_deserialize_allowed_viewers() { - let yaml = r#" -allowed_viewers: - - "aaaaa-aa" - - "2vxsx-fae" -"#; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - match result { - LogVisibilityDef::AllowedViewers { allowed_viewers } => { - assert_eq!(allowed_viewers.len(), 2); - assert_eq!( - allowed_viewers[0], - Principal::from_text("aaaaa-aa").unwrap() - ); - assert_eq!( - allowed_viewers[1], - Principal::from_text("2vxsx-fae").unwrap() - ); - } - _ => panic!("Expected AllowedViewers variant"), - } - } - - #[test] - fn log_visibility_deserialize_allowed_viewers_empty() { - let yaml = "allowed_viewers: []"; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - match result { - LogVisibilityDef::AllowedViewers { allowed_viewers } => { - assert!(allowed_viewers.is_empty()); - } - _ => panic!("Expected AllowedViewers variant"), - } - } - - #[test] - fn log_visibility_deserialize_invalid_string() { - let yaml = "invalid"; - let result: Result = serde_yaml::from_str(yaml); - assert!(result.is_err()); - let err = result.unwrap_err().to_string(); - assert!(err.contains("unknown log_visibility value")); - } - - #[test] - fn log_visibility_deserialize_invalid_field() { - let yaml = "unknown_field: []"; - let result: Result = serde_yaml::from_str(yaml); - assert!(result.is_err()); - let err = result.unwrap_err().to_string(); - assert!(err.contains("unknown field")); - } - - #[test] - fn log_visibility_serialize_controllers() { - let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); - let yaml = serde_yaml::to_string(&log_vis).unwrap(); - assert_eq!(yaml.trim(), "controllers"); - } - - #[test] - fn log_visibility_serialize_public() { - let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Public); - let yaml = serde_yaml::to_string(&log_vis).unwrap(); - assert_eq!(yaml.trim(), "public"); - } - - #[test] - fn log_visibility_serialize_allowed_viewers() { - let log_vis = LogVisibilityDef::AllowedViewers { - allowed_viewers: vec![ - Principal::from_text("aaaaa-aa").unwrap(), - Principal::from_text("2vxsx-fae").unwrap(), - ], - }; - let yaml = serde_yaml::to_string(&log_vis).unwrap(); - assert!(yaml.contains("allowed_viewers")); - assert!(yaml.contains("aaaaa-aa")); - assert!(yaml.contains("2vxsx-fae")); - } - - #[test] - fn settings_reserved_cycles_limit_parses_suffix() { - let yaml = "reserved_cycles_limit: 4.3t"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.reserved_cycles_limit.as_ref().map(|c| c.get()), - Some(4_300_000_000_000) - ); - } - - #[test] - fn settings_reserved_cycles_limit_parses_number() { - let yaml = "reserved_cycles_limit: 5000000000000"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.reserved_cycles_limit.as_ref().map(|c| c.get()), - Some(5_000_000_000_000) - ); - } - - #[test] - fn settings_memory_allocation_parses_suffix() { - let yaml = "memory_allocation: 4gib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.memory_allocation.as_ref().map(|m| m.get()), - Some(4 * 1024 * 1024 * 1024) - ); - } - - #[test] - fn settings_memory_allocation_parses_number() { - let yaml = "memory_allocation: 4294967296"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.memory_allocation.as_ref().map(|m| m.get()), - Some(4294967296) - ); - } - - #[test] - fn settings_wasm_memory_limit_parses_suffix() { - let yaml = "wasm_memory_limit: 1.5gib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.wasm_memory_limit.as_ref().map(|m| m.get()), - Some(1610612736) - ); - } - - #[test] - fn settings_log_memory_limit_parses_suffix() { - let yaml = "log_memory_limit: 256kib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.log_memory_limit.as_ref().map(|m| m.get()), - Some(256 * 1024) - ); - } - - #[test] - fn settings_log_memory_limit_parses_mib() { - let yaml = "log_memory_limit: 2mib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.log_memory_limit.as_ref().map(|m| m.get()), - Some(2 * 1024 * 1024) - ); - } - - #[test] - fn controller_ref_deserializes_principal() { - let yaml = "\"2vxsx-fae\""; - let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - ControllerRef::Principal(Principal::from_text("2vxsx-fae").unwrap()) - ); - } - - #[test] - fn controller_ref_deserializes_canister_name() { - let yaml = "\"my_canister\""; - let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - ControllerRef::CanisterName("my_canister".to_owned()) - ); - } - - #[test] - fn controller_ref_resolve_principal() { - let p = Principal::from_text("aaaaa-aa").unwrap(); - let cref = ControllerRef::Principal(p); - let ids = crate::store_id::IdMapping::new(); - assert_eq!(cref.resolve(&ids), Some(p)); - } - - #[test] - fn controller_ref_resolve_canister_name_present() { - let p = Principal::from_text("aaaaa-aa").unwrap(); - let cref = ControllerRef::CanisterName("backend".to_owned()); - let mut ids = crate::store_id::IdMapping::new(); - ids.insert("backend".to_owned(), p); - assert_eq!(cref.resolve(&ids), Some(p)); - } - - #[test] - fn controller_ref_resolve_canister_name_absent() { - let cref = ControllerRef::CanisterName("backend".to_owned()); - let ids = crate::store_id::IdMapping::new(); - assert_eq!(cref.resolve(&ids), None); - } - - #[test] - fn settings_controllers_parses_mixed() { - let yaml = r#" -controllers: - - "aaaaa-aa" - - "my_other_canister" -"#; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - let controllers = settings.controllers.unwrap(); - assert_eq!(controllers.len(), 2); - assert_eq!( - controllers[0], - ControllerRef::Principal(Principal::from_text("aaaaa-aa").unwrap()) - ); - assert_eq!( - controllers[1], - ControllerRef::CanisterName("my_other_canister".to_owned()) - ); - } - - #[test] - fn log_visibility_conversion_to_ic_type() { - let controllers = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); - let ic_controllers: LogVisibility = controllers.into(); - assert!(matches!(ic_controllers, LogVisibility::Controllers)); - - let public = LogVisibilityDef::Simple(LogVisibilitySimple::Public); - let ic_public: LogVisibility = public.into(); - assert!(matches!(ic_public, LogVisibility::Public)); - - let viewers = LogVisibilityDef::AllowedViewers { - allowed_viewers: vec![Principal::from_text("aaaaa-aa").unwrap()], - }; - let ic_viewers: LogVisibility = viewers.into(); - match ic_viewers { - LogVisibility::AllowedViewers(v) => { - assert_eq!(v.len(), 1); - } - _ => panic!("Expected AllowedViewers"), - } - } -} diff --git a/crates/icp/src/canister/recipe/handlebars.rs b/crates/icp/src/canister/recipe/handlebars.rs deleted file mode 100644 index 11c7c59ba..000000000 --- a/crates/icp/src/canister/recipe/handlebars.rs +++ /dev/null @@ -1,559 +0,0 @@ -use std::{str::FromStr, string::FromUtf8Error}; - -use async_trait::async_trait; -use handlebars::{Context, Helper, HelperDef, HelperResult, Output}; -use indoc::formatdoc; -use reqwest::{Method, Request, Url}; -use serde::Deserialize; -use sha2::{Digest, Sha256}; -use snafu::prelude::*; -use tracing::debug; -use url::ParseError; - -use crate::{ - fs::read, - manifest::{ - canister::{BuildSteps, SyncSteps}, - recipe::{Recipe, RecipeType}, - }, - package::{ - PackageCache, cache_registry_recipe, cache_uri_recipe, read_cached_registry_recipe, - read_cached_uri_recipe, - }, - prelude::*, -}; - -use super::{Resolve, ResolveError}; - -pub struct Handlebars { - /// Http client for fetching remote recipe templates - pub http_client: reqwest::Client, - /// Package cache for caching downloaded recipe templates - pub pkg_cache: PackageCache, -} - -pub enum TemplateSource { - LocalPath(PathBuf), - RemoteUrl(String), - - /// Template originating in a remote registry, e.g `@dfinity/rust@v1.0.2` - Registry(String, String, String), -} - -#[derive(Debug, Snafu)] -pub enum HandlebarsError { - #[snafu(display("failed to read local recipe template file"))] - ReadFile { source: crate::fs::IoError }, - - #[snafu(display("failed to decode UTF-8 string"))] - DecodeUtf8 { source: FromUtf8Error }, - - #[snafu(display("failed to parse user-provided url"))] - UrlParse { source: ParseError }, - - #[snafu(display("failed to execute http request"))] - HttpRequest { source: reqwest::Error }, - - #[snafu(display("request to '{url}' returned '{status}' status-code"))] - HttpStatus { url: String, status: u16 }, - - #[snafu(display("the recipe template for recipe type '{recipe}' failed to be rendered"))] - Render { - source: handlebars::RenderError, - recipe: String, - template: String, - }, - - #[snafu(display( - "sha256 checksum mismatch for recipe template: expected {expected}, actual {actual}" - ))] - ChecksumMismatch { expected: String, actual: String }, - - #[snafu(display("failed to read cached recipe template"))] - ReadCache { - source: crate::package::RecipeCacheError, - }, - - #[snafu(display("failed to cache recipe template"))] - CacheRecipe { - source: crate::package::RecipeCacheError, - }, - - #[snafu(display("failed to acquire lock on package cache"))] - LockCache { source: crate::fs::lock::LockError }, -} - -impl Handlebars { - async fn resolve_impl( - &self, - recipe: &Recipe, - recipe_context: &super::RecipeContext, - ) -> Result<(BuildSteps, SyncSteps), HandlebarsError> { - // Determine the template source - let tmpl_source = match &recipe.recipe_type { - RecipeType::File(path) => TemplateSource::LocalPath(Path::new(&path).into()), - RecipeType::Url(url) => TemplateSource::RemoteUrl(url.to_owned()), - RecipeType::Registry { - name, - recipe, - version, - } => TemplateSource::Registry(name.to_owned(), recipe.to_owned(), version.to_owned()), - }; - - // Retrieve the template, using cache for remote/registry sources - let (tmpl, should_cache) = match &tmpl_source { - TemplateSource::LocalPath(path) => { - let bytes = read(path).context(ReadFileSnafu)?; - (parse_bytes_to_string(bytes)?, false) - } - - TemplateSource::RemoteUrl(u) => { - // Check cache - let maybe_cached = self - .pkg_cache - .with_read(async |r| { - read_cached_uri_recipe(r, u, recipe.sha256.as_deref()) - .context(ReadCacheSnafu) - }) - .await - .context(LockCacheSnafu)?; - if let Some(cached) = maybe_cached? { - debug!("Using cached recipe template for {u}"); - (parse_bytes_to_string(cached)?, false) - } else { - // Download the template - let tmpl = self.fetch_remote_bytes(u).await?; - (parse_bytes_to_string(tmpl)?, true) - } - } - - // TMP(or.ricon): Temporarily hardcode a dfinity registry - TemplateSource::Registry(registry, recipe_name, version) => { - if registry != "dfinity" { - panic!("only the dfinity registry is currently supported"); - } - - let package = format!("@{registry}/{recipe_name}"); - let release_tag = format!("{recipe_name}-{version}"); - - // Check cache - let maybe_cached = self - .pkg_cache - .with_read(async |r| { - read_cached_registry_recipe(r, &package, version).context(ReadCacheSnafu) - }) - .await - .context(LockCacheSnafu)?; - if let Some(cached) = maybe_cached? { - debug!("Using cached recipe template for {package}@{version}"); - (parse_bytes_to_string(cached)?, false) - } else { - // Download the template - let url = format!( - "https://github.com/dfinity/icp-cli-recipes/releases/download/{release_tag}/recipe.hbs" - ); - let bytes = self.fetch_remote_bytes(&url).await?; - - (parse_bytes_to_string(bytes)?, true) - } - } - }; - - let hash = if let Some(sha256) = &recipe.sha256 { - verify_checksum(tmpl.as_bytes(), sha256)? - } else { - Sha256::digest(tmpl.as_bytes()).into() - }; - - // Load the template via handlebars - let mut reg = handlebars::Handlebars::new(); - - // Disable HTML escaping since the output is YAML, not HTML - reg.register_escape_fn(handlebars::no_escape); - - // Register helpers - reg.register_helper("replace", Box::new(ReplaceHelper)); - - // Reject unset template variables - reg.set_strict_mode(true); - - debug!( - "{}", - formatdoc! {r#" - Loaded template: - ------ - {tmpl} - ------ - "#} - ); - - // Build render context: user-provided configuration plus injected _.* variables. - // The _ key is reserved and always overrides any user-supplied value. - let mut render_context = recipe.configuration.clone(); - render_context.insert("_".to_string(), recipe_context.to_yaml()); - - // Render the template to YAML - let out = reg - .render_template(&tmpl, &render_context) - .context(RenderSnafu { - recipe: recipe.recipe_type.clone(), - template: tmpl.to_owned(), - })?; - - // Read the rendered YAML canister manifest - // Recipes can only render build/sync - #[derive(Deserialize)] - struct BuildSyncHelper { - build: BuildSteps, - #[serde(default)] - sync: SyncSteps, - } - - let insts = serde_yaml::from_str::(&out); - let (build, sync) = match insts { - Ok(helper) => (helper.build, helper.sync), - Err(e) => panic!( - "{}", - formatdoc! {r#" - Unable to render recipe {} template into valid yaml: {e} - - Rendered content: - ------ - {out} - ------ - "#, recipe.recipe_type} - ), - }; - - // The template is verified good - now cache it if it was remote - if should_cache { - match tmpl_source { - TemplateSource::LocalPath(_) => unreachable!("local files are never cached"), - TemplateSource::RemoteUrl(u) => { - self.pkg_cache - .with_write(async |w| { - cache_uri_recipe(w, &u, &hex::encode(hash), tmpl.as_bytes()) - .context(CacheRecipeSnafu)?; - Ok(()) - }) - .await - .context(LockCacheSnafu)??; - } - TemplateSource::Registry(registry, recipe_name, version) => { - let package = format!("@{registry}/{recipe_name}"); - self.pkg_cache - .with_write(async |w| { - cache_registry_recipe( - w, - &package, - &version, - &hex::encode(hash), - tmpl.as_bytes(), - ) - .context(CacheRecipeSnafu) - }) - .await - .context(LockCacheSnafu)??; - } - } - } - Ok((build, sync)) - } - - /// Fetch raw bytes from a remote URL. - async fn fetch_remote_bytes(&self, url: &str) -> Result, HandlebarsError> { - let u = Url::from_str(url).context(UrlParseSnafu)?; - debug!("Requesting template from: {u}"); - - let resp = self - .http_client - .execute(Request::new(Method::GET, u.clone())) - .await - .context(HttpRequestSnafu)?; - - if !resp.status().is_success() { - return HttpStatusSnafu { - url: u.to_string(), - status: resp.status().as_u16(), - } - .fail(); - } - - Ok(resp.bytes().await.context(HttpRequestSnafu)?.to_vec()) - } -} - -#[async_trait] -impl Resolve for Handlebars { - async fn resolve( - &self, - recipe: &Recipe, - recipe_context: &super::RecipeContext, - ) -> Result<(BuildSteps, SyncSteps), ResolveError> { - self.resolve_impl(recipe, recipe_context) - .await - .context(super::HandlebarsSnafu) - } -} - -/// Handlebars helper for string replacement operations -/// Usage: {{ replace "from" "to" value }} -#[derive(Clone, Copy)] -struct ReplaceHelper; - -impl HelperDef for ReplaceHelper { - fn call<'reg: 'rc, 'rc>( - &self, - h: &Helper, - _: &'reg handlebars::Handlebars<'reg>, - _: &Context, - _: &mut handlebars::RenderContext<'reg, 'rc>, - out: &mut dyn Output, - ) -> HelperResult { - let (from, to) = ( - h.param(0).unwrap().render(), // from - h.param(1).unwrap().render(), // to - ); - - let v = h.param(2).unwrap().render(); - out.write(&v.replace(&from, &to))?; - - Ok(()) - } -} - -/// Helper function to verify sha256 checksum of recipe template bytes -fn verify_checksum(bytes: &[u8], expected: &str) -> Result<[u8; 32], HandlebarsError> { - let actual_hash = { - let mut h = Sha256::new(); - h.update(bytes); - h.finalize() - }; - let actual = hex::encode(actual_hash); - if actual != expected { - return ChecksumMismatchSnafu { - expected: expected.to_string(), - actual, - } - .fail(); - } - Ok(actual_hash.into()) -} - -/// Helper function to parse bytes into a UTF-8 string -fn parse_bytes_to_string(bytes: Vec) -> Result { - String::from_utf8(bytes).context(DecodeUtf8Snafu) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::canister::recipe::RecipeContext; - use crate::manifest::recipe::{Recipe, RecipeType}; - use std::collections::HashMap; - - fn recipe_context(canister_name: &str) -> RecipeContext { - RecipeContext { - canister_name: canister_name.to_string(), - } - } - - #[tokio::test] - async fn template_values_are_not_html_escaped() { - // Create a recipe template that interpolates a value containing - // characters that Handlebars would normally HTML-escape (", =, &, <, >). - // Use double-stache {{ }} which is what real recipe templates use. - let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); - let tmpl_path = tmp.path().join("recipe.hbs"); - std::fs::write( - &tmpl_path, - indoc::indoc! {r#" - build: - steps: - - type: script - command: "{{ command }}" - "#}, - ) - .unwrap(); - - let cache_dir = tmp.path().join("pkg"); - let pkg_cache = PackageCache::new(cache_dir).unwrap(); - let hbs = Handlebars { - http_client: reqwest::Client::new(), - pkg_cache, - }; - - let mut configuration = HashMap::new(); - configuration.insert( - "command".to_string(), - serde_yaml::Value::String("SITE=https://example.com&foo=bar npm run build".to_string()), - ); - - let recipe = Recipe { - recipe_type: RecipeType::File(tmpl_path.to_string()), - configuration, - sha256: None, - }; - - let (build, _sync) = hbs - .resolve_impl(&recipe, &recipe_context("my-canister")) - .await - .unwrap(); - let cmd = build.steps[0].clone(); - - match cmd { - crate::manifest::canister::BuildStep::Script(adapter) => { - let commands = adapter.command.as_vec(); - assert_eq!( - commands[0], "SITE=https://example.com&foo=bar npm run build", - "Template values must not be HTML-escaped (= and & must be preserved)" - ); - } - other => panic!("Expected Script build step, got: {other:?}"), - } - } - - #[tokio::test] - async fn canister_name_is_injected() { - let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); - let tmpl_path = tmp.path().join("recipe.hbs"); - std::fs::write( - &tmpl_path, - indoc::indoc! {r#" - build: - steps: - - type: script - command: "build {{_.canister.name}}" - "#}, - ) - .unwrap(); - - let pkg_cache = PackageCache::new(tmp.path().join("pkg")).unwrap(); - let hbs = Handlebars { - http_client: reqwest::Client::new(), - pkg_cache, - }; - - let recipe = Recipe { - recipe_type: RecipeType::File(tmpl_path.to_string()), - configuration: HashMap::new(), - sha256: None, - }; - - let (build, _sync) = hbs - .resolve_impl(&recipe, &recipe_context("my-canister")) - .await - .unwrap(); - - match build.steps[0].clone() { - crate::manifest::canister::BuildStep::Script(adapter) => { - assert_eq!(adapter.command.as_vec()[0], "build my-canister"); - } - other => panic!("Expected Script build step, got: {other:?}"), - } - } - - #[tokio::test] - async fn canister_name_works_with_replace_helper() { - let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); - let tmpl_path = tmp.path().join("recipe.hbs"); - std::fs::write( - &tmpl_path, - indoc::indoc! {r#" - build: - steps: - - type: script - command: "cp {{ replace "-" "_" _.canister.name }}.wasm out.wasm" - "#}, - ) - .unwrap(); - - let pkg_cache = PackageCache::new(tmp.path().join("pkg")).unwrap(); - let hbs = Handlebars { - http_client: reqwest::Client::new(), - pkg_cache, - }; - - let recipe = Recipe { - recipe_type: RecipeType::File(tmpl_path.to_string()), - configuration: HashMap::new(), - sha256: None, - }; - - let (build, _sync) = hbs - .resolve_impl(&recipe, &recipe_context("my-canister")) - .await - .unwrap(); - - match build.steps[0].clone() { - crate::manifest::canister::BuildStep::Script(adapter) => { - assert_eq!(adapter.command.as_vec()[0], "cp my_canister.wasm out.wasm"); - } - other => panic!("Expected Script build step, got: {other:?}"), - } - } - - #[tokio::test] - async fn reserved_namespace_cannot_be_overridden_by_user_config() { - let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); - let tmpl_path = tmp.path().join("recipe.hbs"); - std::fs::write( - &tmpl_path, - indoc::indoc! {r#" - build: - steps: - - type: script - command: "build {{_.canister.name}}" - "#}, - ) - .unwrap(); - - let pkg_cache = PackageCache::new(tmp.path().join("pkg")).unwrap(); - let hbs = Handlebars { - http_client: reqwest::Client::new(), - pkg_cache, - }; - - let mut configuration = HashMap::new(); - configuration.insert( - "_".to_string(), - serde_yaml::Value::Mapping({ - let mut m = serde_yaml::Mapping::new(); - m.insert( - serde_yaml::Value::String("canister".to_string()), - serde_yaml::Value::Mapping({ - let mut inner = serde_yaml::Mapping::new(); - inner.insert( - serde_yaml::Value::String("name".to_string()), - serde_yaml::Value::String("user-override".to_string()), - ); - inner - }), - ); - m - }), - ); - - let recipe = Recipe { - recipe_type: RecipeType::File(tmpl_path.to_string()), - configuration, - sha256: None, - }; - - let (build, _sync) = hbs - .resolve_impl(&recipe, &recipe_context("real-name")) - .await - .unwrap(); - - match build.steps[0].clone() { - crate::manifest::canister::BuildStep::Script(adapter) => { - assert_eq!( - adapter.command.as_vec()[0], - "build real-name", - "_ namespace must not be overridable by user configuration" - ); - } - other => panic!("Expected Script build step, got: {other:?}"), - } - } -} diff --git a/crates/icp/src/canister/recipe/mod.rs b/crates/icp/src/canister/recipe/mod.rs index 9c43424ef..bb08a16b1 100644 --- a/crates/icp/src/canister/recipe/mod.rs +++ b/crates/icp/src/canister/recipe/mod.rs @@ -1,55 +1,12 @@ -use async_trait::async_trait; -use snafu::prelude::*; - -use crate::manifest::{ - canister::{BuildSteps, SyncSteps}, - recipe::Recipe, +//! Host-side recipe facade. +//! +//! The `RemoteResourceResolve` interface, recipe rendering, and context/error +//! types live in `icp_deploy_canister::canister::recipe`; the concrete resolver +//! (which fetches templates and plugin wasms over HTTP and caches them) stays +//! here. + +pub use icp_deploy_canister::canister::recipe::{ + RecipeContext, RemoteResourceResolve, ResolveError, }; -pub mod handlebars; - -/// Context passed to a recipe resolver, describing the canister being built. -/// -/// Serializes to the shape injected into recipe templates under the `_` namespace: -/// -/// ```yaml -/// canister: -/// name: -/// ``` -pub struct RecipeContext { - pub canister_name: String, -} - -impl RecipeContext { - /// Builds the YAML value injected into recipe templates under the `_` namespace. - /// Constructing the mapping directly is infallible, unlike `serde` serialization. - pub fn to_yaml(&self) -> serde_yaml::Value { - use serde_yaml::{Mapping, Value}; - - let mut canister = Mapping::new(); - canister.insert("name".into(), Value::String(self.canister_name.clone())); - - let mut root = Mapping::new(); - root.insert("canister".into(), Value::Mapping(canister)); - - Value::Mapping(root) - } -} - -/// A recipe resolver takes a recipe that is specified in a canister manifest -/// and resolves it into a set of build/sync steps -#[async_trait] -pub trait Resolve: Sync + Send { - #[allow(clippy::result_large_err)] - async fn resolve( - &self, - recipe: &Recipe, - recipe_context: &RecipeContext, - ) -> Result<(BuildSteps, SyncSteps), ResolveError>; -} - -#[derive(Debug, Snafu)] -pub enum ResolveError { - #[snafu(display("failed to resolve handlebars template"))] - Handlebars { source: handlebars::HandlebarsError }, -} +pub mod resolver; diff --git a/crates/icp/src/canister/recipe/resolver.rs b/crates/icp/src/canister/recipe/resolver.rs new file mode 100644 index 000000000..71b1a9d1b --- /dev/null +++ b/crates/icp/src/canister/recipe/resolver.rs @@ -0,0 +1,318 @@ +use std::{str::FromStr, string::FromUtf8Error}; + +use async_trait::async_trait; +use reqwest::{Method, Request, Url}; +use sha2::{Digest, Sha256}; +use snafu::prelude::*; +use tokio::sync::mpsc::Sender; +use tracing::debug; +use url::ParseError; + +use crate::{ + fs::read, + manifest::recipe::{Recipe, RecipeType}, + package::{ + PackageCache, cache_registry_recipe, cache_uri_recipe, read_cached_registry_recipe, + read_cached_uri_recipe, + }, + prelude::*, +}; + +use super::{RemoteResourceResolve, ResolveError}; +use crate::manifest::adapter::prebuilt::SourceField; + +/// Fetches recipe templates and plugin wasms over HTTP, caching downloads in the +/// package cache. Template *rendering* is the library's job +/// ([`icp_deploy_canister::canister::recipe::render_recipe`]); this only produces +/// the raw template text. +pub struct ResourceResolver { + /// Http client for fetching remote recipe templates + pub http_client: reqwest::Client, + /// Package cache for caching downloaded recipe templates + pub pkg_cache: PackageCache, +} + +enum TemplateSource { + LocalPath(PathBuf), + RemoteUrl(String), + + /// Template originating in a remote registry, e.g `@dfinity/rust@v1.0.2` + Registry(String, String, String), +} + +#[derive(Debug, Snafu)] +pub enum RecipeFetchError { + #[snafu(display("failed to read local recipe template file"))] + ReadFile { source: crate::fs::IoError }, + + #[snafu(display("failed to decode UTF-8 string"))] + DecodeUtf8 { source: FromUtf8Error }, + + #[snafu(display("failed to parse user-provided url"))] + UrlParse { source: ParseError }, + + #[snafu(display("failed to execute http request"))] + HttpRequest { source: reqwest::Error }, + + #[snafu(display("request to '{url}' returned '{status}' status-code"))] + HttpStatus { url: String, status: u16 }, + + #[snafu(display( + "sha256 checksum mismatch for recipe template: expected {expected}, actual {actual}" + ))] + ChecksumMismatch { expected: String, actual: String }, + + #[snafu(display("failed to read cached recipe template"))] + ReadCache { + source: crate::package::RecipeCacheError, + }, + + #[snafu(display("failed to cache recipe template"))] + CacheRecipe { + source: crate::package::RecipeCacheError, + }, + + #[snafu(display("failed to acquire lock on package cache"))] + LockCache { source: crate::fs::lock::LockError }, +} + +impl ResourceResolver { + /// Fetch a recipe's Handlebars template text: read a local file, or fetch + /// (and cache) a remote URL or registry recipe. Verifies `sha256` when set. + async fn fetch_recipe(&self, recipe: &Recipe) -> Result { + // Determine the template source + let tmpl_source = match &recipe.recipe_type { + RecipeType::File(path) => TemplateSource::LocalPath(Path::new(&path).into()), + RecipeType::Url(url) => TemplateSource::RemoteUrl(url.to_owned()), + RecipeType::Registry { + name, + recipe, + version, + } => TemplateSource::Registry(name.to_owned(), recipe.to_owned(), version.to_owned()), + }; + + // Retrieve the template, using cache for remote/registry sources + let (tmpl, should_cache) = match &tmpl_source { + TemplateSource::LocalPath(path) => { + let bytes = read(path).context(ReadFileSnafu)?; + (parse_bytes_to_string(bytes)?, false) + } + + TemplateSource::RemoteUrl(u) => { + // Check cache + let maybe_cached = self + .pkg_cache + .with_read(async |r| { + read_cached_uri_recipe(r, u, recipe.sha256.as_deref()) + .context(ReadCacheSnafu) + }) + .await + .context(LockCacheSnafu)?; + if let Some(cached) = maybe_cached? { + debug!("Using cached recipe template for {u}"); + (parse_bytes_to_string(cached)?, false) + } else { + // Download the template + let tmpl = self.fetch_remote_bytes(u).await?; + (parse_bytes_to_string(tmpl)?, true) + } + } + + // TMP(or.ricon): Temporarily hardcode a dfinity registry + TemplateSource::Registry(registry, recipe_name, version) => { + if registry != "dfinity" { + panic!("only the dfinity registry is currently supported"); + } + + let package = format!("@{registry}/{recipe_name}"); + let release_tag = format!("{recipe_name}-{version}"); + + // Check cache + let maybe_cached = self + .pkg_cache + .with_read(async |r| { + read_cached_registry_recipe(r, &package, version).context(ReadCacheSnafu) + }) + .await + .context(LockCacheSnafu)?; + if let Some(cached) = maybe_cached? { + debug!("Using cached recipe template for {package}@{version}"); + (parse_bytes_to_string(cached)?, false) + } else { + // Download the template + let url = format!( + "https://github.com/dfinity/icp-cli-recipes/releases/download/{release_tag}/recipe.hbs" + ); + let bytes = self.fetch_remote_bytes(&url).await?; + + (parse_bytes_to_string(bytes)?, true) + } + } + }; + + let hash = if let Some(sha256) = &recipe.sha256 { + verify_checksum(tmpl.as_bytes(), sha256)? + } else { + Sha256::digest(tmpl.as_bytes()).into() + }; + + // Cache the fetched template if it was remote. + if should_cache { + match tmpl_source { + TemplateSource::LocalPath(_) => unreachable!("local files are never cached"), + TemplateSource::RemoteUrl(u) => { + self.pkg_cache + .with_write(async |w| { + cache_uri_recipe(w, &u, &hex::encode(hash), tmpl.as_bytes()) + .context(CacheRecipeSnafu)?; + Ok(()) + }) + .await + .context(LockCacheSnafu)??; + } + TemplateSource::Registry(registry, recipe_name, version) => { + let package = format!("@{registry}/{recipe_name}"); + self.pkg_cache + .with_write(async |w| { + cache_registry_recipe( + w, + &package, + &version, + &hex::encode(hash), + tmpl.as_bytes(), + ) + .context(CacheRecipeSnafu) + }) + .await + .context(LockCacheSnafu)??; + } + } + } + Ok(tmpl) + } + + /// Fetch raw bytes from a remote URL. + async fn fetch_remote_bytes(&self, url: &str) -> Result, RecipeFetchError> { + let u = Url::from_str(url).context(UrlParseSnafu)?; + debug!("Requesting template from: {u}"); + + let resp = self + .http_client + .execute(Request::new(Method::GET, u.clone())) + .await + .context(HttpRequestSnafu)?; + + if !resp.status().is_success() { + return HttpStatusSnafu { + url: u.to_string(), + status: resp.status().as_u16(), + } + .fail(); + } + + Ok(resp.bytes().await.context(HttpRequestSnafu)?.to_vec()) + } +} + +#[async_trait] +impl RemoteResourceResolve for ResourceResolver { + async fn resolve_recipe(&self, recipe: &Recipe) -> Result { + self.fetch_recipe(recipe) + .await + .map_err(|source| ResolveError::Resolve { + source: Box::new(source), + }) + } + + async fn resolve_wasm( + &self, + source: &SourceField, + base_dir: &Path, + sha256: Option<&str>, + stdio: Option>, + ) -> Result { + crate::canister::wasm::resolve(source, base_dir, sha256, stdio.as_ref(), &self.pkg_cache) + .await + .map_err(|source| ResolveError::ResolveWasm { + source: Box::new(source), + }) + } +} + +/// Helper function to verify sha256 checksum of recipe template bytes +fn verify_checksum(bytes: &[u8], expected: &str) -> Result<[u8; 32], RecipeFetchError> { + let actual_hash = { + let mut h = Sha256::new(); + h.update(bytes); + h.finalize() + }; + let actual = hex::encode(actual_hash); + if actual != expected { + return ChecksumMismatchSnafu { + expected: expected.to_string(), + actual, + } + .fail(); + } + Ok(actual_hash.into()) +} + +/// Helper function to parse bytes into a UTF-8 string +fn parse_bytes_to_string(bytes: Vec) -> Result { + String::from_utf8(bytes).context(DecodeUtf8Snafu) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::recipe::{Recipe, RecipeType}; + + /// A local recipe file is read back verbatim (rendering is the library's job). + #[tokio::test] + async fn local_recipe_is_read_verbatim() { + let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); + let tmpl_path = tmp.path().join("recipe.hbs"); + let body = indoc::indoc! {r#" + build: + steps: + - type: script + command: "build {{_.canister.name}}" + "#}; + std::fs::write(&tmpl_path, body).unwrap(); + + let resolver = ResourceResolver { + http_client: reqwest::Client::new(), + pkg_cache: PackageCache::new(tmp.path().join("pkg")).unwrap(), + }; + let recipe = Recipe { + recipe_type: RecipeType::File(tmpl_path.to_string()), + configuration: Default::default(), + sha256: None, + }; + + assert_eq!(resolver.fetch_recipe(&recipe).await.unwrap(), body); + } + + /// A sha256 that does not match the file contents is rejected. + #[tokio::test] + async fn checksum_mismatch_is_rejected() { + let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); + let tmpl_path = tmp.path().join("recipe.hbs"); + std::fs::write(&tmpl_path, "build:\n steps: []\n").unwrap(); + + let resolver = ResourceResolver { + http_client: reqwest::Client::new(), + pkg_cache: PackageCache::new(tmp.path().join("pkg")).unwrap(), + }; + let recipe = Recipe { + recipe_type: RecipeType::File(tmpl_path.to_string()), + configuration: Default::default(), + sha256: Some("00".repeat(32)), + }; + + assert!(matches!( + resolver.fetch_recipe(&recipe).await, + Err(RecipeFetchError::ChecksumMismatch { .. }) + )); + } +} diff --git a/crates/icp/src/canister/script.rs b/crates/icp/src/canister/script.rs index afb3b4d68..dc8b98eb5 100644 --- a/crates/icp/src/canister/script.rs +++ b/crates/icp/src/canister/script.rs @@ -49,18 +49,31 @@ pub enum ScriptError { WslBash, } -pub(super) async fn execute( +pub async fn execute( adapter: &Adapter, cwd: &Path, envs: &[(&str, &str)], stdio: Option>, ) -> Result<(), ScriptError> { // Normalize `command` field based on whether it's a single command or multiple. - let cmds = adapter.command.as_vec(); + execute_commands(&adapter.command.as_vec(), cwd, envs, stdio).await +} +/// Run each command in order under a shell, with `envs` set and `cwd` as the +/// working directory, streaming stdout/stderr lines to `stdio`. +/// +/// The sync path resolves its `commands` (and the `ICP_CLI_*` environment) in +/// `icp-deploy-canister`; the build path calls [`execute`] with its adapter. +pub async fn execute_commands( + cmds: &[String], + cwd: &Path, + envs: &[(&str, &str)], + stdio: Option>, +) -> Result<(), ScriptError> { // Iterate over configured commands for input_cmd in cmds { - let mut cmd = shell_command(&input_cmd, cwd)?; + let input_cmd = input_cmd.as_str(); + let mut cmd = shell_command(input_cmd, cwd)?; // Environment Variables for env in envs { diff --git a/crates/icp/src/canister/sync/mod.rs b/crates/icp/src/canister/sync/mod.rs deleted file mode 100644 index a09b827b9..000000000 --- a/crates/icp/src/canister/sync/mod.rs +++ /dev/null @@ -1,97 +0,0 @@ -use std::collections::BTreeMap; - -use async_trait::async_trait; -use candid::Principal; -use ic_agent::Agent; -use snafu::prelude::*; -use tokio::sync::mpsc::Sender; - -use crate::manifest::canister::SyncStep; -use crate::package::PackageCache; -use crate::prelude::*; - -mod plugin; -mod script; - -pub struct Params { - pub path: PathBuf, - pub cid: Principal, - /// Name of the environment being synced (e.g. "local", "production"). - /// Passed to sync plugin steps via `SyncExecInput`. - pub environment: String, - /// Name of the network (e.g. "local", "ic"). - pub network: String, - /// IDs of all named canisters in the project for this environment. - pub canister_ids: BTreeMap, - /// Proxy canister to route calls through, if `--proxy` was passed. - pub proxy: Option, -} - -#[derive(Debug, Snafu)] -pub enum SynchronizeError { - #[snafu(transparent)] - Script { source: super::script::ScriptError }, - - #[snafu(transparent)] - Plugin { source: plugin::PluginError }, -} - -#[async_trait] -pub trait Synchronize: Sync + Send { - async fn sync( - &self, - step: &SyncStep, - params: &Params, - agent: &Agent, - stdio: Option>, - pkg_cache: &PackageCache, - ) -> Result, SynchronizeError>; -} - -pub struct Syncer; - -#[async_trait] -impl Synchronize for Syncer { - async fn sync( - &self, - step: &SyncStep, - params: &Params, - agent: &Agent, - stdio: Option>, - pkg_cache: &PackageCache, - ) -> Result, SynchronizeError> { - match step { - SyncStep::Script(adapter) => Ok(script::sync(adapter, params, stdio).await?), - SyncStep::Plugin(adapter) => Ok(plugin::sync( - adapter, - params, - agent, - ¶ms.environment, - params.proxy, - stdio, - pkg_cache, - ) - .await?), - } - } -} - -#[cfg(test)] -/// Unimplemented mock implementation of `Synchronize`. -/// All methods panic with `unimplemented!()` when called. -pub struct UnimplementedMockSyncer; - -#[cfg(test)] -#[async_trait] -impl Synchronize for UnimplementedMockSyncer { - async fn sync( - &self, - _step: &SyncStep, - _params: &Params, - _agent: &Agent, - _stdio: Option>, - _pkg_cache: &PackageCache, - ) -> Result, SynchronizeError> { - unimplemented!("UnimplementedMockSyncer::sync") - } -} diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs deleted file mode 100644 index be7820013..000000000 --- a/crates/icp/src/canister/sync/plugin.rs +++ /dev/null @@ -1,78 +0,0 @@ -use camino::Utf8PathBuf; -use candid::Principal; -use ic_agent::Agent; -use icp_sync_plugin::{RunPluginError, run_plugin}; -use snafu::prelude::*; -use tokio::sync::mpsc::Sender; - -use crate::{canister::wasm, manifest::adapter::plugin::Adapter, package::PackageCache}; - -use super::Params; - -#[derive(Debug, Snafu)] -pub enum PluginError { - #[snafu(transparent)] - Wasm { source: wasm::WasmError }, - - #[snafu(display("failed to get identity principal: {err}"))] - GetIdentityPrincipal { err: String }, - - #[snafu(display("failed to run plugin"))] - Run { source: RunPluginError }, -} - -pub(super) async fn sync( - adapter: &Adapter, - params: &Params, - agent: &Agent, - environment: &str, - proxy: Option, - stdio: Option>, - pkg_cache: &PackageCache, -) -> Result, PluginError> { - // 1. Determine the on-disk path for the wasm. run_plugin needs a path, not raw bytes. - // - Local: sha256 is verified if present, then the original path is returned. - // - Remote: downloaded to cache (sha256 required, enforced at parse time) and the - // stable cache path is returned — no temp file needed. - let wasm_path = wasm::resolve( - &adapter.source, - ¶ms.path, - adapter.sha256.as_deref(), - stdio.as_ref(), - pkg_cache, - ) - .await?; - - // 2. Collect inputs as manifest strings. `run_plugin` preopens the `dirs` - // and reads the `files` itself — both anchored at `base_dir`, and both - // subject to the runtime's path-safety checks (no escaping or symlinked - // paths). - let base_dir = Utf8PathBuf::from(params.path.as_str()); - let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); - let files: Vec = adapter.files.clone().unwrap_or_default(); - - // 3. Run the plugin (blocking call — signal Tokio that this thread will block). - let identity_principal = agent - .get_principal() - .map_err(|err| PluginError::GetIdentityPrincipal { err })?; - - let agent_clone = agent.clone(); - let environment_owned = environment.to_owned(); - let stdio_clone = stdio.clone(); - - tokio::task::block_in_place(|| { - run_plugin( - wasm_path, - base_dir, - dirs, - files, - params.cid, - agent_clone, - proxy, - identity_principal, - environment_owned, - stdio_clone, - ) - }) - .context(RunSnafu) -} diff --git a/crates/icp/src/canister/sync/script.rs b/crates/icp/src/canister/sync/script.rs deleted file mode 100644 index 13a4799ce..000000000 --- a/crates/icp/src/canister/sync/script.rs +++ /dev/null @@ -1,34 +0,0 @@ -use tokio::sync::mpsc::Sender; - -use crate::manifest::adapter::script::Adapter; - -use super::Params; - -use super::super::script::{ScriptError, execute}; - -pub(super) async fn sync( - adapter: &Adapter, - params: &Params, - stdio: Option>, -) -> Result, ScriptError> { - let mut envs: Vec<(String, String)> = vec![ - ("ICP_CLI_ENVIRONMENT".to_owned(), params.environment.clone()), - ("ICP_CLI_NETWORK".to_owned(), params.network.clone()), - ("ICP_CLI_CID".to_owned(), params.cid.to_text()), - ]; - for (name, id) in ¶ms.canister_ids { - let key = format!( - "ICP_CLI_CID_{}", - name.to_uppercase() - .chars() - .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) - .collect::() - ); - envs.push((key, id.to_text())); - } - let env_refs: Vec<(&str, &str)> = envs.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect(); - execute(adapter, params.path.as_ref(), &env_refs, stdio).await?; - // Persistent stderr is a sync-plugin feature only; script steps don't - // currently retain any output past the rolling step view. - Ok(vec![]) -} diff --git a/crates/icp/src/context/init.rs b/crates/icp/src/context/init.rs index 8a274415b..722839a67 100644 --- a/crates/icp/src/context/init.rs +++ b/crates/icp/src/context/init.rs @@ -3,8 +3,7 @@ use std::{env::current_dir, sync::Arc}; use snafu::prelude::*; use crate::canister::build::Builder; -use crate::canister::recipe::handlebars::Handlebars; -use crate::canister::sync::Syncer; +use crate::canister::recipe::resolver::ResourceResolver; use crate::context::Context; use crate::directories::{Access as _, Directories}; use crate::prelude::*; @@ -90,7 +89,7 @@ pub fn initialize( let pkg_cache = dirs.package_cache().context(PackageCacheSnafu)?; // Recipes - let recipe = Arc::new(Handlebars { + let recipe = Arc::new(ResourceResolver { http_client, pkg_cache, }); @@ -98,9 +97,6 @@ pub fn initialize( // Canister builder let builder = Arc::new(Builder); - // Canister syncer - let syncer = Arc::new(Syncer); - // Project loader let pload = ProjectLoadImpl { project_root_locate: project_root_locate.clone(), @@ -148,7 +144,6 @@ pub fn initialize( network: netaccess, agent: agent_creator, builder, - syncer, debug, telemetry_data, password_func, diff --git a/crates/icp/src/context/mod.rs b/crates/icp/src/context/mod.rs index e05e7fb41..b68cfc983 100644 --- a/crates/icp/src/context/mod.rs +++ b/crates/icp/src/context/mod.rs @@ -4,7 +4,7 @@ use url::Url; use crate::{ Canister, agent::CreateAgentError, - canister::{build::Build, sync::Synchronize}, + canister::build::Build, directories, identity::IdentitySelection, manifest::network::RootKeySpec, @@ -96,9 +96,6 @@ pub struct Context { /// Canister builder pub builder: Arc, - /// Canister synchronizer - pub syncer: Arc, - /// Whether debug is enabled pub debug: bool, @@ -110,6 +107,20 @@ pub struct Context { } impl Context { + /// Build a resolver for the project's remote resources — recipe templates + /// and plugin wasms — backed by the package cache and an HTTP client. + pub fn resource_resolver( + &self, + ) -> Result, crate::fs::lock::LockError> + { + Ok(Arc::new( + crate::canister::recipe::resolver::ResourceResolver { + http_client: reqwest::Client::new(), + pkg_cache: self.dirs.package_cache()?, + }, + )) + } + /// Gets an identity based on the provided identity selection. // TODO: refactor the whole codebase to use this method instead of directly accessing `ctx.identity.load()` pub async fn get_identity( @@ -610,7 +621,6 @@ impl Context { network: Arc::new(crate::network::MockNetworkAccessor::new()), agent: Arc::new(crate::agent::Creator), builder: Arc::new(crate::canister::build::UnimplementedMockBuilder), - syncer: Arc::new(crate::canister::sync::UnimplementedMockSyncer), debug: false, telemetry_data: Arc::new(crate::telemetry_data::TelemetryData::default()), password_func: Arc::new(|| Err("no password available in mock context".to_string())), diff --git a/crates/icp/src/fs/mod.rs b/crates/icp/src/fs/mod.rs index 6b16cfb2d..4dc9174ad 100644 --- a/crates/icp/src/fs/mod.rs +++ b/crates/icp/src/fs/mod.rs @@ -1,75 +1,9 @@ -use snafu::prelude::*; -use std::io::{self, ErrorKind}; +//! Basic filesystem operations and JSON/YAML loaders with path-carrying errors. +//! +//! The base operations and the `json`/`yaml` loaders are defined in +//! `icp_deploy_canister::fs` and re-exported here; the `lock` submodule +//! (cross-process directory locking) is host-only and lives here. -use crate::prelude::*; +pub use icp_deploy_canister::fs::*; -pub mod json; pub mod lock; -pub mod yaml; - -#[derive(Debug, Snafu)] -#[snafu(display("Filesystem operation failed at {path}"))] -pub struct IoError { - source: io::Error, - path: PathBuf, -} - -#[derive(Debug, Snafu)] -#[snafu(display("Failed to rename `{from}` to `{to}`"))] -pub struct RenameError { - source: io::Error, - from: PathBuf, - to: PathBuf, -} - -#[derive(Debug, Snafu)] -#[snafu(display("Failed to copy `{from}` to `{to}`"))] -pub struct CopyError { - source: io::Error, - from: PathBuf, - to: PathBuf, -} - -impl IoError { - pub fn kind(&self) -> ErrorKind { - self.source.kind() - } -} - -pub fn create_dir_all(path: &Path) -> Result<(), IoError> { - std::fs::create_dir_all(path).context(IoSnafu { path }) -} - -pub fn read(path: &Path) -> Result, IoError> { - std::fs::read(path).context(IoSnafu { path }) -} - -pub fn read_to_string(path: &Path) -> Result { - std::fs::read_to_string(path).context(IoSnafu { path }) -} - -pub fn remove_dir_all(path: &Path) -> Result<(), IoError> { - std::fs::remove_dir_all(path).context(IoSnafu { path }) -} - -pub fn remove_file(path: &Path) -> Result<(), IoError> { - std::fs::remove_file(path).context(IoSnafu { path }) -} - -pub fn copy(from: &Path, to: &Path) -> Result<(), CopyError> { - std::fs::copy(from, to) - .map(|_| ()) - .context(CopySnafu { from, to }) -} - -pub fn rename(from: &Path, to: &Path) -> Result<(), RenameError> { - std::fs::rename(from, to).context(RenameSnafu { from, to }) -} - -pub fn write(path: &Path, contents: &[u8]) -> Result<(), IoError> { - std::fs::write(path, contents).context(IoSnafu { path }) -} - -pub fn write_string(path: &Path, contents: &str) -> Result<(), IoError> { - std::fs::write(path, contents.as_bytes()).context(IoSnafu { path }) -} diff --git a/crates/icp/src/lib.rs b/crates/icp/src/lib.rs index 2ef666d1f..be494287f 100644 --- a/crates/icp/src/lib.rs +++ b/crates/icp/src/lib.rs @@ -1,29 +1,29 @@ -use std::{ - collections::{BTreeMap, HashMap}, - sync::Arc, -}; +use std::sync::Arc; use async_trait::async_trait; -use indexmap::IndexMap; -use serde::Serialize; use snafu::prelude::*; use tokio::sync::Mutex; use tracing::debug; -use candid_parser::parse_idl_args; +pub use icp_deploy_canister::{ + Canister, Environment, InitArgs, InitArgsToBytesError, Network, Project, +}; use crate::{ - canister::{Settings, recipe::Resolve}, + canister::recipe::RemoteResourceResolve, manifest::{ - ArgsFormat, LoadManifestFromPathError, PROJECT_MANIFEST, ProjectRootLocate, - ProjectRootLocateError, - canister::{BuildSteps, SyncSteps}, + LoadManifestFromPathError, PROJECT_MANIFEST, ProjectRootLocate, ProjectRootLocateError, load_manifest_from_path, }, - network::Configuration, prelude::*, }; +// Imports used only by the in-crate test mock builders below. +#[cfg(test)] +use std::collections::{BTreeMap, HashMap}; +#[cfg(test)] +use {crate::canister::Settings, indexmap::IndexMap}; + pub mod agent; pub mod canister; pub mod context; @@ -46,148 +46,6 @@ const ICP_BASE: &str = ".icp"; const CACHE_DIR: &str = "cache"; const DATA_DIR: &str = "data"; -/// Resolved initialization arguments, with any file references already loaded. -#[derive(Clone, Debug, PartialEq, Serialize)] -pub enum InitArgs { - /// Text content (inline or loaded from file). Format is always known. - Text { content: String, format: ArgsFormat }, - /// Raw binary bytes (from a file with `format: bin`). Used directly. - Binary(Vec), -} - -#[derive(Debug, Snafu)] -pub enum InitArgsToBytesError { - #[snafu(display("failed to decode hex init args"))] - HexDecode { source: hex::FromHexError }, - - #[snafu(display("failed to parse Candid init args"))] - CandidParse { source: candid_parser::Error }, - - #[snafu(display("failed to encode Candid init args to bytes"))] - CandidEncode { source: candid::Error }, -} - -impl InitArgs { - /// Resolve to raw bytes according to the format. - pub fn to_bytes(&self) -> Result, InitArgsToBytesError> { - match self { - InitArgs::Binary(bytes) => Ok(bytes.clone()), - InitArgs::Text { content, format } => match format { - ArgsFormat::Hex => hex::decode(content.trim()).context(HexDecodeSnafu), - ArgsFormat::Candid => { - let args = parse_idl_args(content.trim()).context(CandidParseSnafu)?; - args.to_bytes().context(CandidEncodeSnafu) - } - ArgsFormat::Bin => { - unreachable!("binary format cannot appear in InitArgs::Text") - } - }, - } - } -} - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Canister { - pub name: String, - - /// Canister settings, such as memory constaints, etc. - pub settings: Settings, - - /// The build configuration specifying how to compile the canister's source - /// code into a WebAssembly module, including the adapter to use. - pub build: BuildSteps, - - /// The configuration specifying how to sync the canister - pub sync: SyncSteps, - - /// Initialization arguments passed to the canister during installation. - /// Resolved from the manifest — file contents are already loaded. - pub init_args: Option, - - /// If the canister was defined via a recipe reference, this holds the - /// original recipe specifier string (e.g. `@dfinity/motoko@v4.0.0`). - /// `None` when the canister uses explicit build/sync instructions. - pub registry_recipe: Option, - - /// Canister-discovery wiring. Maps the name this canister reads in a - /// `PUBLIC_CANISTER_ID:` environment variable to the store key of the - /// referenced canister. Computed during consolidation so each canister sees - /// the view its owning project expects: its own project's canisters under - /// their local names, plus any declared dependencies under their aliases - /// (`:`). For a project with no dependencies this maps every - /// canister's local name to itself, reproducing the flat "every canister sees - /// every sibling" behavior. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub bindings: BTreeMap, - - /// Subdomain prefixes for the canister's friendly URLs, most-specific label - /// first, e.g. `["backend"]` for an own canister or `["backend.openemail"]` - /// for a dependency canister (dot-nested by alias chain). A de-duplicated - /// shared dependency canister carries one entry per alias chain that reaches - /// it. Consumed only at deploy time to build `custom-domains.txt` entries and - /// the printed URLs; a runtime display aid that is always recomputed during - /// consolidation, so it is never serialized. - #[serde(skip)] - pub friendly_names: Vec, -} - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Network { - pub name: String, - pub configuration: Configuration, -} - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Environment { - pub name: String, - pub network: Network, - pub canisters: IndexMap, -} - -impl Environment { - pub fn get_canister_names(&self) -> Vec { - self.canisters.keys().cloned().collect() - } - - pub fn contains_canister(&self, canister_name: &str) -> bool { - self.canisters.contains_key(canister_name) - } - - pub fn get_canister_info(&self, canister: &str) -> Result<(PathBuf, Canister), String> { - self.canisters - .get(canister) - .ok_or_else(|| { - format!( - "canister '{}' not declared in environment '{}'", - canister, self.name - ) - }) - .cloned() - } -} - -/// Consolidated project definition -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Project { - pub dir: PathBuf, - pub canisters: IndexMap, - pub networks: HashMap, - pub environments: HashMap, - - /// Environments the workspace defines that some vendored member does *not* - /// declare, keyed by environment name → the missing members' store-key - /// prefixes. Enforced when the environment is selected (strict rule). - /// Empty for standalone projects and workspaces whose members are complete. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub member_missing_envs: HashMap>, -} - -impl Project { - pub fn get_canister(&self, canister_name: &str) -> Option<&(PathBuf, Canister)> { - self.canisters.get(canister_name) - } -} - #[derive(Debug, Snafu)] pub enum ProjectLoadError { #[snafu(display("failed to locate project directory"))] @@ -218,7 +76,7 @@ pub trait ProjectLoad: Sync + Send { pub struct ProjectLoadImpl { pub project_root_locate: Arc, - pub recipe: Arc, + pub recipe: Arc, } /// Ensures the "operating on a workspace root above your sub-project" notice is @@ -268,7 +126,7 @@ impl ProjectLoad for ProjectLoadImpl { debug!("Loaded project manifest: {m:#?}"); - // Consolidate manifest into project + // Consolidate manifest into project, reading files from the host filesystem. let p = project::consolidate_manifest(&pdir, self.recipe.as_ref(), &m) .await .context(ProjectSnafu)?; @@ -682,14 +540,13 @@ impl ProjectLoad for NoProjectLoader { #[cfg(test)] mod tests { use super::*; - use crate::canister::recipe::{RecipeContext, Resolve, ResolveError}; + use crate::canister::recipe::{RemoteResourceResolve, ResolveError}; use crate::manifest::{ - ProjectRootLocate, ProjectRootLocateError, - canister::{BuildSteps, SyncSteps}, - recipe::Recipe, + ProjectRootLocate, ProjectRootLocateError, adapter::prebuilt::SourceField, recipe::Recipe, }; use camino_tempfile::Utf8TempDir; use indoc::indoc; + use tokio::sync::mpsc::Sender; struct MockProjectRootLocate { path: PathBuf, @@ -714,28 +571,26 @@ mod tests { struct MockRecipeResolver; #[async_trait] - impl Resolve for MockRecipeResolver { - async fn resolve( - &self, - _recipe: &Recipe, - _context: &RecipeContext, - ) -> Result<(BuildSteps, SyncSteps), ResolveError> { - use crate::manifest::adapter::prebuilt::{ - Adapter as PrebuiltAdapter, LocalSource, SourceField, - }; - use crate::manifest::canister::BuildStep; - - // Create a minimal BuildSteps with a dummy prebuilt step - let build_steps = BuildSteps { - steps: vec![BuildStep::Prebuilt(PrebuiltAdapter { - source: SourceField::Local(LocalSource { - path: "dummy.wasm".into(), - }), - sha256: None, - })], - }; + impl RemoteResourceResolve for MockRecipeResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + // A minimal recipe template rendering to a single prebuilt build step. + Ok(indoc! {r#" + build: + steps: + - type: pre-built + path: dummy.wasm + "#} + .to_owned()) + } - Ok((build_steps, SyncSteps::default())) + async fn resolve_wasm( + &self, + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _stdio: Option>, + ) -> Result { + unimplemented!("MockRecipeResolver::resolve_wasm") } } diff --git a/crates/icp/src/manifest/mod.rs b/crates/icp/src/manifest/mod.rs index 0a811e358..c1019bf76 100644 --- a/crates/icp/src/manifest/mod.rs +++ b/crates/icp/src/manifest/mod.rs @@ -1,113 +1,20 @@ +//! Host-side manifest facade. +//! +//! The manifest *model* (types, `Item`, constants) lives in +//! `icp_deploy_canister::manifest` and is re-exported here. This module keeps +//! the pieces that walk the real filesystem: locating the project/workspace root +//! and the `std::fs`-based manifest loader. + use std::collections::HashSet; -use std::marker::PhantomData; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use snafu::prelude::*; +pub use icp_deploy_canister::manifest::*; + use crate::fs; use crate::prelude::*; -pub(crate) mod adapter; -pub(crate) mod canister; -pub(crate) mod dependency; -pub(crate) mod environment; -pub(crate) mod network; -pub(crate) mod project; -pub(crate) mod recipe; -pub(crate) mod serde_helpers; - -pub use { - adapter::plugin, - adapter::prebuilt, - canister::{ - ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, - SyncStep, SyncSteps, - }, - dependency::DependencyManifest, - environment::EnvironmentManifest, - network::{ManagedMode, Mode, NetworkManifest}, - project::ProjectManifest, -}; - -pub const PROJECT_MANIFEST: &str = "icp.yaml"; -pub const CANISTER_MANIFEST: &str = "canister.yaml"; - -// A manifest item that can either be a path to another manifest file or the manifest itself. -// -// The valid path specifications are: -// - CanisterManifest: path or glob pattern to the directory containing "canister.yaml" -// - NetworkManifest: path to network manifest -// - EnvironmentManifest: path to environment manifest -#[derive(Clone, Debug, PartialEq, JsonSchema)] -#[serde(untagged)] -pub enum Item { - /// Path to a manifest - Path(String), - - /// The manifest - Manifest(T), -} - -/// Items in path form serialize back to a bare path string, *not* to the contents of the -/// referenced file. Callers that need a self-contained YAML output (e.g. `icp project bundle`) -/// must convert any `Item::Path` to `Item::Manifest` themselves by loading the referenced -/// manifest first. -impl Serialize for Item { - fn serialize(&self, serializer: S) -> Result { - match self { - Item::Path(p) => p.serialize(serializer), - Item::Manifest(m) => m.serialize(serializer), - } - } -} - -impl<'de, T> Deserialize<'de> for Item -where - T: Deserialize<'de>, -{ - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - use serde::de::{MapAccess, Visitor, value::MapAccessDeserializer}; - use std::fmt; - - struct ItemVisitor(PhantomData); - - impl<'de, T: Deserialize<'de>> Visitor<'de> for ItemVisitor { - type Value = Item; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a string path or a manifest object") - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - Ok(Item::Path(v.to_owned())) - } - - fn visit_string(self, v: String) -> Result - where - E: serde::de::Error, - { - Ok(Item::Path(v)) - } - - fn visit_map(self, map: M) -> Result - where - M: MapAccess<'de>, - { - T::deserialize(MapAccessDeserializer::new(map)).map(Item::Manifest) - } - } - - deserializer.deserialize_any(ItemVisitor(PhantomData)) - } -} - #[derive(Debug, Snafu)] pub enum ProjectRootLocateError { #[snafu(display("project manifest not found in {path}"))] @@ -290,7 +197,7 @@ pub enum LoadManifestFromPathError { }, } -/// Loads a manifest of type `T` from the specified file path. +/// Loads a manifest of type `T` from the specified file path (host filesystem). pub async fn load_manifest_from_path(path: &Path) -> Result where T: for<'de> Deserialize<'de>, diff --git a/crates/icp/src/network/mod.rs b/crates/icp/src/network/mod.rs index 3d025fb40..e075c49e3 100644 --- a/crates/icp/src/network/mod.rs +++ b/crates/icp/src/network/mod.rs @@ -1,29 +1,28 @@ +//! Host-side network facade. +//! +//! The network *configuration* model lives in `icp_deploy_canister::network` +//! and is re-exported here. Runtime access (launching/stopping managed +//! networks, descriptors, agent bootstrap) stays in this crate. + use std::sync::Arc; use async_trait::async_trait; -use schemars::JsonSchema; -use serde::{Deserialize, Deserializer, Serialize}; use snafu::prelude::*; -pub use crate::manifest::network::RootKeySpec; +pub use icp_deploy_canister::network::*; + pub use access::RootKeySource; pub use directory::{LoadPidError, NetworkDirectory, SavePidError}; pub use managed::run::{RunNetworkError, run_network}; -use strum::EnumString; -use url::Url; use crate::{ CACHE_DIR, ICP_BASE, Network, - manifest::{ - ProjectRootLocate, ProjectRootLocateError, - network::{Connected as ManifestConnected, Endpoints, Gateway as ManifestGateway, Mode}, - }, + manifest::{ProjectRootLocate, ProjectRootLocateError}, network::access::{ GetNetworkAccessError, NetworkAccess, get_connected_network_access, get_managed_network_access, }, prelude::*, - project::DEFAULT_LOCAL_NETWORK_PORT, }; pub mod access; @@ -32,309 +31,6 @@ pub mod custom_domains; pub mod directory; pub mod managed; -#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] -pub enum Port { - Fixed(u16), - Random, -} - -impl Default for Port { - fn default() -> Self { - Port::Fixed(8000) - } -} - -impl<'de> Deserialize<'de> for Port { - fn deserialize>(d: D) -> Result { - Ok(match u16::deserialize(d)? { - 0 => Port::Random, - p => Port::Fixed(p), - }) - } -} - -fn default_bind() -> String { - "127.0.0.1".to_string() -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct Gateway { - #[serde(default = "default_bind")] - pub bind: String, - - #[serde(default)] - pub port: Port, - - #[serde(default)] - pub domains: Vec, -} - -impl Default for Gateway { - fn default() -> Self { - Self { - bind: default_bind(), - port: Default::default(), - domains: Default::default(), - } - } -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct Managed { - #[serde(flatten)] - pub mode: ManagedMode, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -#[serde(untagged)] -pub enum ManagedMode { - Image(Box), - Launcher(Box), -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct ManagedLauncherConfig { - pub gateway: Gateway, - pub artificial_delay_ms: Option, - pub ii: bool, - pub nns: bool, - pub subnets: Option>, - pub bitcoind_addr: Option>, - pub dogecoind_addr: Option>, - pub version: Option, -} - -#[derive( - Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize, EnumString, strum::Display, -)] -#[serde(rename_all = "kebab-case")] -#[strum(serialize_all = "kebab-case")] -pub enum SubnetKind { - Application, - System, - VerifiedApplication, - Bitcoin, - Fiduciary, - Nns, - Sns, -} - -impl Default for ManagedMode { - fn default() -> Self { - Self::default_for_port(DEFAULT_LOCAL_NETWORK_PORT) - } -} - -impl ManagedMode { - pub fn default_for_port(port: u16) -> Self { - ManagedMode::Launcher(Box::new(ManagedLauncherConfig { - gateway: Gateway { - bind: default_bind(), - port: if port == 0 { - Port::Random - } else { - Port::Fixed(port) - }, - domains: vec![], - }, - artificial_delay_ms: None, - ii: false, - nns: false, - subnets: None, - bitcoind_addr: None, - dogecoind_addr: None, - version: None, - })) - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct ManagedImageConfig { - pub image: String, - pub port_mapping: Vec, - pub rm_on_exit: bool, - pub args: Vec, - pub entrypoint: Option>, - pub environment: Vec, - pub volumes: Vec, - pub platform: Option, - pub user: Option, - pub shm_size: Option, - pub status_dir: String, - pub mounts: Vec, - pub extra_hosts: Vec, -} - -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub struct Connected { - /// The URL this network's API can be reached at. - pub api_url: Url, - - /// The URL this network's HTTP gateway can be reached at. - pub http_gateway_url: Option, - - /// How to obtain the root key used to verify responses from this network. - pub root_key: RootKeySpec, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -#[serde(tag = "mode", rename_all = "lowercase")] -pub enum Configuration { - // Note: we must use struct variants to be able to flatten - // and make schemars generate the proper schema - /// A managed network is one which can be controlled and manipulated. - Managed { - #[serde(flatten)] - managed: Managed, - }, - - /// A connected network is one which can be interacted with - /// but cannot be controlled or manipulated. - Connected { - #[serde(flatten)] - connected: Connected, - }, -} - -impl Default for Configuration { - fn default() -> Self { - Configuration::Managed { - managed: Managed::default(), - } - } -} - -impl From for Gateway { - fn from(value: ManifestGateway) -> Self { - let ManifestGateway { - bind, - domains, - port, - } = value; - let bind = bind.unwrap_or("127.0.0.1".to_string()); - let port = match port { - Some(0) => Port::Random, - Some(p) => Port::Fixed(p), - None => Port::default(), - }; - let mut domains = domains.unwrap_or_default(); - if bind == "127.0.0.1" || bind == "0.0.0.0" || bind == "::1" || bind == "::" { - domains.insert(0, "localhost".to_string()); - } - Gateway { - bind, - port, - domains, - } - } -} - -impl From for Connected { - fn from(value: ManifestConnected) -> Self { - let root_key = value.root_key; - match value.endpoints { - Endpoints::Implicit { url } => Connected { - api_url: url.clone(), - http_gateway_url: Some(url), - root_key, - }, - Endpoints::Explicit { - api_url, - http_gateway_url, - } => Connected { - api_url, - http_gateway_url, - root_key, - }, - } - } -} - -impl From for Configuration { - fn from(value: Mode) -> Self { - match value { - Mode::Managed(managed) => match *managed.mode { - crate::manifest::network::ManagedMode::Launcher { - gateway, - artificial_delay_ms, - ii, - nns, - subnets, - bitcoind_addr, - dogecoind_addr, - version, - } => { - let gateway: Gateway = match gateway { - Some(g) => g.into(), - None => Gateway::default(), - }; - let version = match version { - Some(v) => { - if v.starts_with('v') { - Some(v) - } else { - Some(format!("v{v}")) - } - } - None => None, - }; - Configuration::Managed { - managed: Managed { - mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { - gateway, - artificial_delay_ms, - ii: ii.unwrap_or(false), - nns: nns.unwrap_or(false), - subnets, - bitcoind_addr, - dogecoind_addr, - version, - })), - }, - } - } - crate::manifest::network::ManagedMode::Image { - image, - port_mapping, - rm_on_exit, - args, - entrypoint, - environment, - volumes, - platform, - user, - shm_size, - status_dir, - mounts: mount, - extra_hosts, - } => Configuration::Managed { - managed: Managed { - mode: ManagedMode::Image(Box::new(ManagedImageConfig { - image, - port_mapping, - rm_on_exit: rm_on_exit.unwrap_or(false), - args: args.unwrap_or_default(), - entrypoint, - environment: environment.unwrap_or_default(), - volumes: volumes.unwrap_or_default(), - platform, - user, - shm_size, - status_dir: status_dir.unwrap_or_else(|| "/app/status".to_string()), - mounts: mount.unwrap_or_default(), - extra_hosts: extra_hosts.unwrap_or_default(), - })), - }, - }, - }, - Mode::Connected(connected) => Configuration::Connected { - connected: connected.into(), - }, - } - } -} - #[derive(Debug, Snafu)] pub enum AccessError { #[snafu(display("failed to find project root"))] @@ -444,51 +140,3 @@ impl Access for MockNetworkAccessor { }) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::manifest::network::{ - Gateway as ManifestGateway, Managed as ManifestManaged, ManagedMode as ManifestManagedMode, - Mode, - }; - - #[test] - fn from_mode_launcher_with_bitcoind_addr() { - let mode = Mode::Managed(ManifestManaged { - mode: Box::new(ManifestManagedMode::Launcher { - gateway: Some(ManifestGateway { - bind: None, - port: Some(8000), - domains: None, - }), - artificial_delay_ms: None, - ii: None, - nns: None, - subnets: None, - bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), - dogecoind_addr: None, - version: None, - }), - }); - - let config: Configuration = mode.into(); - match config { - Configuration::Managed { - managed: - Managed { - mode: ManagedMode::Launcher(launcher_config), - }, - } => { - assert_eq!( - launcher_config.bitcoind_addr, - Some(vec!["127.0.0.1:18444".to_string()]) - ); - assert_eq!(launcher_config.dogecoind_addr, None); - assert!(!launcher_config.ii); - assert!(!launcher_config.nns); - } - _ => panic!("expected ManagedMode::Launcher"), - } - } -} diff --git a/crates/icp/src/parsers.rs b/crates/icp/src/parsers.rs index b0a74730f..44cdff84f 100644 --- a/crates/icp/src/parsers.rs +++ b/crates/icp/src/parsers.rs @@ -1,643 +1,5 @@ -//! Parsing of token, cycle, memory, and duration amounts with support for suffixes and underscores. +//! Parsing of token, cycle, memory, and duration amounts. +//! +//! Defined in `icp_deploy_canister::parsers` and re-exported here. -use bigdecimal::{BigDecimal, Signed}; -use num_bigint::BigUint; -use num_integer::Integer; -use num_traits::{ToPrimitive, Zero}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::fmt; -use std::str::FromStr; - -/// Parse a token amount with support for suffixes (k, m, b, t) and underscores. -/// -/// Examples: -/// - `1` -> 1 -/// - `1_000` -> 1000 -/// - `1k` or `1K` -> 1000 -/// - `1t` or `1T` -> 1000000000000 -/// - `0.5` -> 0.5 -/// - `0.5k` -> 500 -pub fn parse_token_amount(input: &str) -> Result { - let input = input.trim(); - - if input.is_empty() { - return Err("Token amount cannot be empty".to_string()); - } - - let (number_part, multiplier) = if let Some(last_char) = input.chars().last() { - match last_char { - 'k' | 'K' => (&input[..input.len() - 1], 1_000u128), - 'm' | 'M' => (&input[..input.len() - 1], 1_000_000u128), - 'b' | 'B' => (&input[..input.len() - 1], 1_000_000_000u128), - 't' | 'T' => (&input[..input.len() - 1], 1_000_000_000_000u128), - _ => (input, 1u128), - } - } else { - (input, 1u128) - }; - - let cleaned = number_part.replace('_', ""); - let base = - BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid token amount: '{}'", input))?; - - if base.is_negative() { - return Err(format!("Token amount cannot be negative: '{}'", input)); - } - - let multiplier_decimal = BigDecimal::from(multiplier); - Ok(base * multiplier_decimal) -} - -/// Convert a token amount to the smallest unit by multiplying by 10^token_decimals. -/// E.g. 1.5 with 8 decimals -> 150000000. Fails if the result would be fractional. -pub fn to_token_unit_amount( - token_amount: BigDecimal, - token_decimals: u8, -) -> Result { - use num_bigint::BigInt; - use num_traits::pow::Pow; - - let (mantissa, exponent) = token_amount.into_bigint_and_exponent(); - let scale_adjustment = token_decimals as i64 - exponent; - let ten = BigInt::from(10); - - let result = if scale_adjustment >= 0 { - let multiplier = ten.pow(scale_adjustment as u32); - mantissa * multiplier - } else { - let divisor = ten.pow((-scale_adjustment) as u32); - let (quotient, remainder) = mantissa.div_rem(&divisor); - if !remainder.is_zero() { - return Err(format!( - "Token amount cannot be represented with {} decimals (would result in fractional units)", - token_decimals - )); - } - quotient - }; - - result - .try_into() - .map_err(|_| "Token amount cannot be negative".to_string()) -} - -fn parse_cycles_str(s: &str) -> Result { - let token_amount = parse_token_amount(s)?; - let unit_amount = to_token_unit_amount(token_amount, 0)?; - unit_amount - .to_u128() - .ok_or_else(|| format!("Cycles amount too large: '{}'", s)) -} - -/// An amount of cycles. -/// -/// Deserializes from a number or a string with suffixes (k, m, b, t) and optional underscore separators. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] -#[schemars(untagged)] -pub enum CyclesAmount { - Number(u64), // yaml only supports up to u64 - Str(String), -} - -impl CyclesAmount { - pub fn get(&self) -> u128 { - match self { - CyclesAmount::Number(n) => *n as u128, - CyclesAmount::Str(s) => parse_cycles_str(s) - .unwrap_or_else(|e| panic!("invalid cycles amount '{}': {}", s, e)), - } - } -} - -impl<'de> Deserialize<'de> for CyclesAmount { - fn deserialize(d: D) -> Result - where - D: serde::Deserializer<'de>, - { - // Identical enum to CyclesAmount. Needed to avoid a circular dependency. - #[derive(Deserialize)] - #[serde(untagged)] - enum Raw { - Number(u64), - Str(String), - } - let v = Raw::deserialize(d).map_err(|_| { - serde::de::Error::custom("cycles amount must be a number or a string with optional suffix (k, m, b, t), e.g. 1000 or \"4t\"") - })?; - let c = match v { - Raw::Number(n) => CyclesAmount::Number(n), - Raw::Str(ref s) => { - parse_cycles_str(s).map_err(serde::de::Error::custom)?; // validate the string is a valid cycles amount - CyclesAmount::Str(s.clone()) - } - }; - Ok(c) - } -} - -impl Serialize for CyclesAmount { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - CyclesAmount::Number(n) => serializer.serialize_u64(*n), - CyclesAmount::Str(s) => serializer.serialize_str(s), - } - } -} - -impl FromStr for CyclesAmount { - type Err = String; - - fn from_str(s: &str) -> Result { - parse_cycles_str(s)?; // validate the string is a valid cycles amount - Ok(CyclesAmount::Str(s.to_string())) - } -} - -impl fmt::Display for CyclesAmount { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } -} - -impl From for u128 { - fn from(c: CyclesAmount) -> Self { - c.get() - } -} - -impl From for CyclesAmount { - fn from(n: u128) -> Self { - if let Ok(n64) = u64::try_from(n) { - CyclesAmount::Number(n64) - } else { - CyclesAmount::Str(n.to_string()) - } - } -} - -const KB: u64 = 1000; -const KIB: u64 = 1024; -const MB: u64 = 1_000_000; -const MIB: u64 = 1024 * 1024; -const GB: u64 = 1_000_000_000; -const GIB: u64 = 1024 * 1024 * 1024; - -fn parse_memory_str(s: &str) -> Result { - let s = s.trim(); - if s.is_empty() { - return Err("Memory amount cannot be empty".to_string()); - } - let lower = s.to_lowercase(); - let (number_part, factor) = if lower.ends_with("gib") { - (&s[..s.len() - 3], GIB) - } else if lower.ends_with("gb") { - (&s[..s.len() - 2], GB) - } else if lower.ends_with("mib") { - (&s[..s.len() - 3], MIB) - } else if lower.ends_with("mb") { - (&s[..s.len() - 2], MB) - } else if lower.ends_with("kib") { - (&s[..s.len() - 3], KIB) - } else if lower.ends_with("kb") { - (&s[..s.len() - 2], KB) - } else { - (s, 1u64) - }; - let cleaned = number_part.trim().replace('_', ""); - let amount = - BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid memory amount: '{}'", s))?; - if amount.is_negative() { - return Err(format!("Memory amount cannot be negative: '{}'", s)); - } - let product = amount * BigDecimal::from(factor); - if !product.is_integer() { - return Err( - "Memory amount must be a whole number of bytes (fractional bytes not allowed)" - .to_string(), - ); - } - product - .to_u64() - .ok_or_else(|| format!("Memory amount too large: '{}'", s)) -} - -/// An amount of memory in bytes. -/// -/// Deserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib), -/// optional decimals, and optional underscore separators. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] -#[schemars(untagged)] -pub enum MemoryAmount { - Number(u64), - Str(String), -} - -impl MemoryAmount { - pub fn get(&self) -> u64 { - match self { - MemoryAmount::Number(n) => *n, - MemoryAmount::Str(s) => parse_memory_str(s) - .unwrap_or_else(|e| panic!("invalid memory amount '{}': {}", s, e)), - } - } -} - -impl<'de> Deserialize<'de> for MemoryAmount { - fn deserialize(d: D) -> Result - where - D: serde::Deserializer<'de>, - { - #[derive(Deserialize)] - #[serde(untagged)] - enum Raw { - Number(u64), - Str(String), - } - let v = Raw::deserialize(d).map_err(|_| { - serde::de::Error::custom( - "memory amount must be a number or a string with optional suffix (kb, kib, mb, mib, gb, gib), e.g. 1024 or \"2.5gib\"", - ) - })?; - let m = match v { - Raw::Number(n) => MemoryAmount::Number(n), - Raw::Str(ref s) => { - parse_memory_str(s).map_err(serde::de::Error::custom)?; - MemoryAmount::Str(s.clone()) - } - }; - Ok(m) - } -} - -impl Serialize for MemoryAmount { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - MemoryAmount::Number(n) => serializer.serialize_u64(*n), - MemoryAmount::Str(s) => serializer.serialize_str(s), - } - } -} - -impl FromStr for MemoryAmount { - type Err = String; - - fn from_str(s: &str) -> Result { - parse_memory_str(s)?; - Ok(MemoryAmount::Str(s.to_string())) - } -} - -impl fmt::Display for MemoryAmount { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } -} - -impl From for u64 { - fn from(m: MemoryAmount) -> Self { - m.get() - } -} - -impl From for MemoryAmount { - fn from(n: u64) -> Self { - MemoryAmount::Number(n) - } -} - -const SECONDS_PER_MINUTE: u64 = 60; -const SECONDS_PER_HOUR: u64 = 3600; -const SECONDS_PER_DAY: u64 = 86400; -const SECONDS_PER_WEEK: u64 = 604800; - -fn parse_duration_str(s: &str) -> Result { - let s = s.trim(); - if s.is_empty() { - return Err("Duration cannot be empty".to_string()); - } - let lower = s.to_lowercase(); - let (number_part, factor) = if lower.ends_with('w') { - (&s[..s.len() - 1], SECONDS_PER_WEEK) - } else if lower.ends_with('d') { - (&s[..s.len() - 1], SECONDS_PER_DAY) - } else if lower.ends_with('h') { - (&s[..s.len() - 1], SECONDS_PER_HOUR) - } else if lower.ends_with('m') { - (&s[..s.len() - 1], SECONDS_PER_MINUTE) - } else if lower.ends_with('s') { - (&s[..s.len() - 1], 1u64) - } else { - (s, 1u64) - }; - let cleaned = number_part.trim().replace('_', ""); - if cleaned.is_empty() { - return Err(format!("Invalid duration: '{s}'")); - } - let value: u64 = cleaned - .parse() - .map_err(|_| format!("Invalid duration: '{s}'"))?; - value - .checked_mul(factor) - .ok_or_else(|| format!("Duration too large: '{s}'")) -} - -/// A duration in seconds. -/// -/// Deserializes from a number (seconds) or a string with duration suffix (s, m, h, d, w) -/// and optional underscore separators. -/// -/// Suffixes (case-insensitive): -/// - `s` — seconds -/// - `m` — minutes (×60) -/// - `h` — hours (×3600) -/// - `d` — days (×86400) -/// - `w` — weeks (×604800) -/// -/// A bare number without suffix is treated as seconds. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] -#[schemars(untagged)] -pub enum DurationAmount { - Number(u64), - Str(String), -} - -impl DurationAmount { - pub fn get(&self) -> u64 { - match self { - DurationAmount::Number(n) => *n, - DurationAmount::Str(s) => { - parse_duration_str(s).unwrap_or_else(|e| panic!("invalid duration '{}': {}", s, e)) - } - } - } -} - -impl<'de> Deserialize<'de> for DurationAmount { - fn deserialize(d: D) -> Result - where - D: serde::Deserializer<'de>, - { - #[derive(Deserialize)] - #[serde(untagged)] - enum Raw { - Number(u64), - Str(String), - } - let v = Raw::deserialize(d).map_err(|_| { - serde::de::Error::custom( - "duration must be a number (seconds) or a string with optional suffix (s, m, h, d, w), e.g. 2592000 or \"30d\"", - ) - })?; - let c = match v { - Raw::Number(n) => DurationAmount::Number(n), - Raw::Str(ref s) => { - parse_duration_str(s).map_err(serde::de::Error::custom)?; - DurationAmount::Str(s.clone()) - } - }; - Ok(c) - } -} - -impl Serialize for DurationAmount { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - DurationAmount::Number(n) => serializer.serialize_u64(*n), - DurationAmount::Str(s) => serializer.serialize_str(s), - } - } -} - -impl FromStr for DurationAmount { - type Err = String; - - fn from_str(s: &str) -> Result { - parse_duration_str(s)?; - Ok(DurationAmount::Str(s.to_string())) - } -} - -impl fmt::Display for DurationAmount { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } -} - -impl From for u64 { - fn from(d: DurationAmount) -> Self { - d.get() - } -} - -impl From for DurationAmount { - fn from(n: u64) -> Self { - DurationAmount::Number(n) - } -} - -impl PartialEq for DurationAmount { - fn eq(&self, other: &u64) -> bool { - self.get() == *other - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cycles_amount_from_str_plain() { - assert_eq!("1".parse::().unwrap().get(), 1); - assert_eq!("1000".parse::().unwrap().get(), 1000); - } - - #[test] - fn cycles_amount_from_str_suffixes() { - assert_eq!("1k".parse::().unwrap().get(), 1000); - assert_eq!( - "1t".parse::().unwrap().get(), - 1_000_000_000_000 - ); - assert_eq!( - "4t".parse::().unwrap().get(), - 4_000_000_000_000 - ); - assert_eq!( - "0.5t".parse::().unwrap().get(), - 500_000_000_000 - ); - } - - #[test] - fn cycles_amount_from_str_underscores() { - assert_eq!("1_000".parse::().unwrap().get(), 1000); - } - - #[test] - fn cycles_amount_from_str_fractional_rejected() { - assert!("1.5".parse::().is_err()); - } - - #[test] - fn cycles_amount_deserialize() { - let yaml = "4t"; - let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(c.get(), 4_000_000_000_000); - - let yaml = "5000000000000"; - let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(c.get(), 5_000_000_000_000); - } - - #[test] - fn parse_token_amount_plain_and_suffixes() { - use std::str::FromStr; - assert_eq!( - parse_token_amount("1").unwrap(), - BigDecimal::from_str("1").unwrap() - ); - assert_eq!( - parse_token_amount("1k").unwrap(), - BigDecimal::from_str("1000").unwrap() - ); - assert_eq!( - parse_token_amount("0.5t").unwrap(), - BigDecimal::from_str("500000000000").unwrap() - ); - } - - #[test] - fn memory_amount_from_str_plain() { - assert_eq!("1".parse::().unwrap().get(), 1); - assert_eq!("1024".parse::().unwrap().get(), 1024); - } - - #[test] - fn memory_amount_from_str_suffixes() { - assert_eq!("1kb".parse::().unwrap().get(), 1000); - assert_eq!("1kib".parse::().unwrap().get(), 1024); - assert_eq!("1mb".parse::().unwrap().get(), 1_000_000); - assert_eq!("1mib".parse::().unwrap().get(), 1024 * 1024); - assert_eq!("1gb".parse::().unwrap().get(), 1_000_000_000); - assert_eq!( - "1gib".parse::().unwrap().get(), - 1024 * 1024 * 1024 - ); - assert_eq!( - "2 GiB".parse::().unwrap().get(), - 2 * 1024 * 1024 * 1024 - ); - } - - #[test] - fn memory_amount_from_str_decimals() { - assert_eq!("0.5kib".parse::().unwrap().get(), 512); - assert_eq!("1.5gib".parse::().unwrap().get(), 1610612736); - } - - #[test] - fn memory_amount_fractional_bytes_rejected() { - assert!("1.5".parse::().is_err()); // 1.5 bytes - assert!("0.3kib".parse::().is_err()); // 307.2 bytes - } - - #[test] - fn memory_amount_from_str_underscores() { - assert_eq!("1_024".parse::().unwrap().get(), 1024); - } - - #[test] - fn memory_amount_deserialize() { - let yaml = "2gib"; - let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(m.get(), 2 * 1024 * 1024 * 1024); - - let yaml = "4294967296"; - let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(m.get(), 4294967296); - } - - #[test] - fn duration_amount_from_str_plain() { - assert_eq!("60".parse::().unwrap().get(), 60); - assert_eq!("2592000".parse::().unwrap().get(), 2592000); - } - - #[test] - fn duration_amount_from_str_underscores() { - assert_eq!( - "2_592_000".parse::().unwrap().get(), - 2592000 - ); - } - - #[test] - fn duration_amount_from_str_suffixes() { - assert_eq!("60s".parse::().unwrap().get(), 60); - assert_eq!("90m".parse::().unwrap().get(), 5400); - assert_eq!("24h".parse::().unwrap().get(), 86400); - assert_eq!("30d".parse::().unwrap().get(), 2592000); - assert_eq!("4w".parse::().unwrap().get(), 2419200); - } - - #[test] - fn duration_amount_from_str_case_insensitive() { - assert_eq!("30D".parse::().unwrap().get(), 2592000); - assert_eq!("1W".parse::().unwrap().get(), 604800); - assert_eq!("24H".parse::().unwrap().get(), 86400); - assert_eq!("60S".parse::().unwrap().get(), 60); - assert_eq!("90M".parse::().unwrap().get(), 5400); - } - - #[test] - fn duration_amount_from_str_underscores_with_suffix() { - assert_eq!( - "2_592_000s".parse::().unwrap().get(), - 2592000 - ); - } - - #[test] - fn duration_amount_from_str_errors() { - assert!("abc".parse::().is_err()); - assert!("".parse::().is_err()); - assert!("1x".parse::().is_err()); - assert!("1.5d".parse::().is_err()); - assert!("-1d".parse::().is_err()); - } - - #[test] - fn duration_amount_deserialize() { - let yaml = "30d"; - let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(d.get(), 2592000); - - let yaml = "2592000"; - let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(d.get(), 2592000); - - let yaml = "2_592_000"; - let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(d.get(), 2592000); - } - - #[test] - fn duration_amount_partial_eq_u64() { - let d = DurationAmount::Number(2592000); - assert!(d == 2592000); - assert!(d != 0); - - let d = DurationAmount::Str("30d".to_string()); - assert!(d == 2592000); - } -} +pub use icp_deploy_canister::parsers::*; diff --git a/crates/icp/src/project.rs b/crates/icp/src/project.rs index 75b46c858..333a1c041 100644 --- a/crates/icp/src/project.rs +++ b/crates/icp/src/project.rs @@ -1,747 +1,16 @@ -use std::collections::{BTreeMap, HashMap, HashSet, hash_map::Entry}; - -use indexmap::{IndexMap, map::Entry as IndexEntry}; - -use snafu::prelude::*; - -use crate::{ - Canister, Environment, InitArgs, Network, Project, - canister::{ControllerRef, Settings, recipe}, - fs, - manifest::{ - ArgsFormat, CANISTER_MANIFEST, CanisterManifest, DependencyManifest, EnvironmentManifest, - Item, LoadManifestFromPathError, ManifestInitArgs, NetworkManifest, PROJECT_MANIFEST, - ProjectManifest, ProjectRootLocateError, - canister::{Instructions, SyncSteps}, - environment::CanisterSelection, - load_manifest_from_path, - network::RootKeySpec, - recipe::RecipeType, - }, - network::{ - Configuration, Connected, Gateway, Managed, ManagedLauncherConfig, ManagedMode, Port, - }, - prelude::*, +//! Host-side project facade. +//! +//! Consolidation of manifests into a [`Project`] lives in +//! `icp_deploy_canister::project` and is re-exported here. Member-scoping +//! resolves real filesystem paths against the current working directory, so it +//! stays here. + +pub use icp_deploy_canister::project::{ + ConsolidateManifestError, EnvironmentError, LoadProjectError, VerifySandboxError, + consolidate_manifest, load_project, verify_sandbox, }; -pub const DEFAULT_LOCAL_NETWORK_BIND: &str = "127.0.0.1"; -pub const DEFAULT_LOCAL_NETWORK_PORT: u16 = 8000; - -#[derive(Debug, Snafu)] -pub enum EnvironmentError { - #[snafu(display("environment '{environment}' points to invalid network '{network}'"))] - InvalidNetwork { - environment: String, - network: String, - }, - - #[snafu(display("environment '{environment}' points to invalid canister '{canister}'"))] - InvalidCanister { - environment: String, - canister: String, - }, -} - -#[derive(Debug, Snafu)] -pub enum ConsolidateManifestError { - #[snafu(display("failed to locate project directory"))] - Locate { source: ProjectRootLocateError }, - - #[snafu(display("failed to perform glob parsing"))] - GlobParse { source: glob::PatternError }, - - #[snafu(display("failed to get glob iter"))] - GlobIter { source: glob::GlobError }, - - #[snafu(display("failed to convert path to UTF-8"))] - Utf8Path { source: FromPathBufError }, - - #[snafu(display("failed to load canister manifest"))] - LoadCanister { source: LoadManifestFromPathError }, - - #[snafu(display("failed to load network manifest"))] - LoadNetwork { source: LoadManifestFromPathError }, - - #[snafu(display("failed to load environment manifest"))] - LoadEnvironment { source: LoadManifestFromPathError }, - - #[snafu(display("failed to load {kind} manifest at: {path}"))] - Failed { kind: String, path: String }, - - #[snafu(display("failed to resolve canister recipe: {recipe_type:?}"))] - Recipe { - #[snafu(source(from(recipe::ResolveError, Box::new)))] - source: Box, - recipe_type: RecipeType, - }, - - #[snafu(display("project contains two similarly named {kind}s: '{name}'"))] - Duplicate { kind: String, name: String }, - - #[snafu(display("`{name}` is a reserved {kind} name."))] - Reserved { kind: String, name: String }, - - #[snafu(display("could not locate a {kind} manifest at: '{path}'"))] - NotFound { kind: String, path: String }, - - #[snafu(display("failed to read init_args file for canister '{canister}'"))] - ReadInitArgs { - source: fs::IoError, - canister: String, - }, - - #[snafu(display( - "init_args for canister '{canister}' uses format 'bin' with inline content; \ - binary format requires a file path" - ))] - BinFormatInlineContent { canister: String }, - - #[snafu(display( - "canister '{canister}' lists controller '{controller}', but no canister with that \ - name is declared in the project" - ))] - UnknownControllerCanister { - canister: String, - controller: String, - }, - - #[snafu(display( - "canister name '{name}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ - (':' is reserved as the dependency namespace separator)" - ))] - InvalidCanisterName { name: String }, - - #[snafu(display( - "dependency alias '{alias}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ - (':' is reserved as the dependency namespace separator)" - ))] - InvalidDependencyAlias { alias: String }, - - #[snafu(display("project declares two dependencies with the same alias '{alias}'"))] - DuplicateDependencyAlias { alias: String }, - - #[snafu(display( - "dependency alias '{alias}' collides with a canister of the same name in the same project" - ))] - DependencyAliasCollision { alias: String }, - - #[snafu(display("could not find a project manifest for dependency '{alias}' at: '{path}'"))] - DependencyNotFound { alias: String, path: String }, - - #[snafu(display("failed to canonicalize path for dependency '{alias}' at: '{path}'"))] - DependencyCanonicalize { alias: String, path: String }, - - #[snafu(display("failed to load project manifest for dependency '{alias}'"))] - LoadDependencyManifest { - source: LoadManifestFromPathError, - alias: String, - }, - - #[snafu(display( - "dependency '{alias}' selects canister '{canister}', which the dependency does not declare" - ))] - UnknownDependencyCanister { alias: String, canister: String }, - - #[snafu(display("dependency cycle detected: {chain}"))] - CircularDependency { chain: String }, - - #[snafu(transparent)] - Environment { source: EnvironmentError }, -} - -/// Resolve a [`ManifestInitArgs`] into a canonical [`InitArgs`] by reading -/// any file references relative to `base_path`. -fn resolve_manifest_init_args( - manifest_init_args: &ManifestInitArgs, - base_path: &Path, - canister: &str, -) -> Result { - match manifest_init_args { - ManifestInitArgs::String(content) => Ok(InitArgs::Text { - content: content.trim().to_owned(), - format: ArgsFormat::Candid, - }), - ManifestInitArgs::Path { path, format } => { - let file_path = base_path.join(path); - match format { - ArgsFormat::Bin => { - let bytes = fs::read(&file_path).context(ReadInitArgsSnafu { canister })?; - Ok(InitArgs::Binary(bytes)) - } - fmt => { - let content = - fs::read_to_string(&file_path).context(ReadInitArgsSnafu { canister })?; - Ok(InitArgs::Text { - content: content.trim().to_owned(), - format: fmt.clone(), - }) - } - } - } - ManifestInitArgs::Value { value, format } => match format { - ArgsFormat::Bin => BinFormatInlineContentSnafu { canister }.fail(), - fmt => Ok(InitArgs::Text { - content: value.trim().to_owned(), - format: fmt.clone(), - }), - }, - } -} - -fn is_glob(s: &str) -> bool { - s.contains('*') || s.contains('?') || s.contains('[') || s.contains('{') -} - -/// Whether `name` is a valid canister name or dependency alias: non-empty and -/// containing only ASCII letters, digits, `_`, or `-`. -/// -/// A single strict rule keeps names safe for every purpose they are reused for — -/// store-key segments, `PUBLIC_CANISTER_ID:` env vars, DNS subdomains, and -/// archive paths — so no per-site sanitizing is needed. In particular `:` is the -/// dependency namespace separator, and `.` / `/` would be ambiguous in -/// subdomains and paths. -fn is_valid_name(name: &str) -> bool { - !name.is_empty() - && name - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') -} - -/// Builds the canonical canisters declared directly in one project manifest, -/// resolving glob/path/inline entries, recipes, and init-args relative to -/// `pdir`. Returns `(local name, canister dir, canister)` with empty bindings; -/// callers assign store keys and bindings. Does not check for duplicate names -/// across projects — that is the caller's responsibility (via the global map). -async fn build_manifest_canisters( - pdir: &Path, - manifest_canisters: &[Item], - recipe_resolver: &dyn recipe::Resolve, -) -> Result, ConsolidateManifestError> { - let mut result: Vec<(String, PathBuf, Canister)> = Vec::new(); - - for i in manifest_canisters { - let ms = match i { - Item::Path(pattern) => { - let is_glob_pattern = is_glob(pattern); - let paths = match is_glob_pattern { - // Explicit path - false => vec![pdir.join(pattern)], - - // Glob pattern - true => { - let paths = - glob::glob(pdir.join(pattern).as_str()).context(GlobParseSnafu)?; - - let mut v = vec![]; - for p in paths { - let path = p.context(GlobIterSnafu)?; - let utf8_path = PathBuf::try_from(path).context(Utf8PathSnafu)?; - v.push(utf8_path); - } - v - } - }; - - let paths = if is_glob_pattern { - // For glob patterns, filter out non-directories and non-canister directories - paths - .into_iter() - .filter(|p| p.is_dir()) - .filter(|p| p.join(CANISTER_MANIFEST).exists()) - .collect::>() - } else { - // For explicit paths, validate that they exist and contain canister.yaml - let mut validated_paths = vec![]; - for p in paths { - if !p.join(CANISTER_MANIFEST).is_file() { - return NotFoundSnafu { - kind: "canister".to_string(), - path: pattern.to_string(), - } - .fail(); - } - validated_paths.push(p); - } - validated_paths - }; - - let mut ms = vec![]; - for p in paths { - ms.push(( - p.to_owned(), - load_manifest_from_path::(&p.join(CANISTER_MANIFEST)) - .await - .context(LoadCanisterSnafu)?, - )); - } - ms - } - - Item::Manifest(m) => vec![(pdir.to_owned(), m.to_owned())], - }; - - for (cdir, m) in ms { - if !is_valid_name(&m.name) { - return InvalidCanisterNameSnafu { - name: m.name.clone(), - } - .fail(); - } - - let registry_recipe = match &m.instructions { - Instructions::BuildSync { .. } => None, - Instructions::Recipe { recipe } => match &recipe.recipe_type { - RecipeType::Registry { .. } => Some(recipe.recipe_type.to_string()), - _ => None, - }, - }; - - let (build, sync) = match &m.instructions { - // Build/Sync - Instructions::BuildSync { build, sync } => ( - build.to_owned(), - match sync { - Some(sync) => sync.to_owned(), - None => SyncSteps::default(), - }, - ), - - // Recipe - Instructions::Recipe { recipe } => { - let ctx = recipe::RecipeContext { - canister_name: m.name.clone(), - }; - recipe_resolver - .resolve(recipe, &ctx) - .await - .context(RecipeSnafu { - recipe_type: recipe.recipe_type.clone(), - })? - } - }; - - let init_args = m - .init_args - .as_ref() - .map(|mia| resolve_manifest_init_args(mia, &cdir, &m.name)) - .transpose()?; - - result.push(( - m.name.clone(), - cdir, - Canister { - name: m.name.clone(), - settings: m.settings.clone(), - build, - sync, - init_args, - registry_recipe, - bindings: BTreeMap::new(), - // Default to the bare local name; overwritten with the - // dot-nested alias form when the canister is imported as a - // dependency (see `import_dependency`). - friendly_names: vec![m.name.clone()], - }, - )); - } - } - - Ok(result) -} - -/// A dependency instance imported into the workspace. Returned by -/// [`import_dependency`] and cached per canonical path so diamond dependencies -/// reuse the same instance. -#[derive(Clone)] -struct ImportedInstance { - /// This instance's own canisters, as `(local name, full store key)` — the - /// set exposable to the parent via `canisters:` selection. - own: Vec<(String, String)>, - /// Every canister in this instance's subtree (its own canisters plus all - /// transitively imported ones), as `(store key, local name, alias chain - /// from this instance down to the canister's owning project)`. Used to - /// register a friendly URL per alias chain when the instance is reached - /// again via de-duplication (a diamond), including for its descendants. - subtree: Vec<(String, String, Vec)>, -} - -/// A member environment's per-canister config, to be folded into the root's -/// same-named environment beneath any root overrides. -#[derive(Default, Clone)] -struct MemberCanisterOverride { - settings: Option, - init_args: Option, -} - -/// Per-environment member overrides: env name → store key → override. -type MemberEnvOverrides = HashMap>; - -/// A member's identity (store-key prefix) and the environment names it defines, -/// used to enforce that a member declares every environment the root targets -/// (strict rule). -struct MemberEnvInfo { - prefix: String, - defined: HashSet, -} - -/// Canonicalize a dependency root (resolving symlinks and `..`) for use as a -/// de-dup / cycle-detection identity. -fn canonicalize_dep(alias: &str, dep_root: &Path) -> Result { - let build_err = || { - DependencyCanonicalizeSnafu { - alias: alias.to_owned(), - path: dep_root.to_string(), - } - .build() - }; - let canon = dunce::canonicalize(dep_root.as_std_path()).map_err(|_| build_err())?; - PathBuf::try_from(canon).map_err(|_| build_err()) -} - -/// Store-key prefix for a dependency instance: its canonical directory relative -/// to the canonical app root, forward-slash separated so keys are stable across -/// platforms and independent of how each edge spells the path. -fn relative_prefix(app_root_canonical: &Path, dep_canonical: &Path) -> String { - let rel = pathdiff::diff_utf8_paths(dep_canonical, app_root_canonical) - .unwrap_or_else(|| dep_canonical.to_owned()); - rel.as_str().replace('\\', "/") -} - -/// Build a dependency canister's friendly-URL subdomain prefix: the canister's -/// local name as the most-specific label, followed by its alias chain reversed -/// (root-most alias last). E.g. local `backend` reached via `[service-a, -/// openemail]` → `backend.openemail.service-a`. Dot-nested so it stays a valid, -/// collision-free multi-label host; see DESIGN §17.2. -fn friendly_name_for(local: &str, alias_chain: &[String]) -> String { - let mut labels = Vec::with_capacity(alias_chain.len() + 1); - labels.push(local.to_string()); - labels.extend(alias_chain.iter().rev().cloned()); - labels.join(".") -} - -/// Rewrite `CanisterName` controller references from a dependency's local -/// canister names to their store keys, so global controller validation and -/// deploy-time id lookup operate uniformly on store keys. -fn translate_controllers(canister: &mut Canister, local_to_key: &BTreeMap) { - translate_settings_controllers(&mut canister.settings, local_to_key); -} - -/// Rewrite `CanisterName` controller references in a `Settings` from a -/// dependency's local canister names to their store keys. -fn translate_settings_controllers( - settings: &mut Settings, - local_to_key: &BTreeMap, -) { - if let Some(controllers) = &mut settings.controllers { - for cref in controllers.iter_mut() { - if let ControllerRef::CanisterName(name) = cref - && let Some(key) = local_to_key.get(name) - { - *name = key.clone(); - } - } - } -} - -/// Compute the `PUBLIC_CANISTER_ID` env-var wiring for canisters in one project -/// scope: its own canisters by local name, plus each dependency's exposed -/// canisters under `:`. -fn compute_bindings( - own: &[(String, String)], - edges: &[(String, Vec<(String, String)>)], -) -> BTreeMap { - let mut bindings = BTreeMap::new(); - for (local, key) in own { - bindings.insert(local.clone(), key.clone()); - } - for (alias, exposed) in edges { - for (dep_local, key) in exposed { - bindings.insert(format!("{alias}:{dep_local}"), key.clone()); - } - } - bindings -} - -/// Select which of a dependency instance's own canisters are exposed to the -/// parent, per the dependency's `canisters` selection. -fn select_exposed( - own: &[(String, String)], - selection: &CanisterSelection, - alias: &str, -) -> Result, ConsolidateManifestError> { - match selection { - CanisterSelection::Everything => Ok(own.to_vec()), - CanisterSelection::None => Ok(vec![]), - CanisterSelection::Named(names) => { - let mut out = Vec::new(); - for name in names { - match own.iter().find(|(local, _)| local == name) { - Some(pair) => out.push(pair.clone()), - None => { - return UnknownDependencyCanisterSnafu { - alias: alias.to_owned(), - canister: name.clone(), - } - .fail(); - } - } - } - Ok(out) - } - } -} - -/// Validate the dependency aliases declared in one project scope: no `:`, no -/// collision with a local canister name, and no duplicate alias. -fn validate_dependency_aliases( - deps: &[DependencyManifest], - own_canister_names: &HashSet, -) -> Result<(), ConsolidateManifestError> { - let mut seen: HashSet<&str> = HashSet::new(); - for d in deps { - if !is_valid_name(&d.name) { - return InvalidDependencyAliasSnafu { - alias: d.name.clone(), - } - .fail(); - } - if own_canister_names.contains(&d.name) { - return DependencyAliasCollisionSnafu { - alias: d.name.clone(), - } - .fail(); - } - if !seen.insert(&d.name) { - return DuplicateDependencyAliasSnafu { - alias: d.name.clone(), - } - .fail(); - } - } - Ok(()) -} - -/// Recursively import a dependency's canisters into `canisters`, keyed by their -/// app-root-relative store keys. De-duplicates instances by canonical path -/// (diamond dependencies deploy once) and detects cycles. Returns the imported -/// instance's prefix and its own canisters. -#[allow(clippy::too_many_arguments)] -async fn import_dependency( - app_root_canonical: &Path, - parent_dir: &Path, - dep: &DependencyManifest, - recipe_resolver: &dyn recipe::Resolve, - canisters: &mut IndexMap, - registry: &mut HashMap, - stack: &mut Vec, - member_env_overrides: &mut MemberEnvOverrides, - members: &mut Vec, - // Alias chain from the workspace root to and including this dependency, - // used to build friendly-URL subdomains (§17.2). - alias_chain: &[String], -) -> Result { - let dep_root = parent_dir.join(&dep.path); - let manifest_path = dep_root.join(PROJECT_MANIFEST); - if !manifest_path.is_file() { - return DependencyNotFoundSnafu { - alias: dep.name.clone(), - path: dep_root.to_string(), - } - .fail(); - } - - let canonical = canonicalize_dep(&dep.name, &dep_root)?; - - // Cycle detection. - if stack.contains(&canonical) { - let mut chain: Vec = stack.iter().map(|p| p.to_string()).collect(); - chain.push(canonical.to_string()); - return CircularDependencySnafu { - chain: chain.join(" -> "), - } - .fail(); - } - - // Diamond de-dup: same resolved directory means the same instance, deployed - // once. It is still reachable via this new alias chain, so register an - // additional friendly URL per chain (§17.3) rather than picking one — for - // the whole subtree (its own canisters *and* its transitive dependencies), - // each named by this chain extended with the canister's alias path below the - // instance. - if let Some(inst) = registry.get(&canonical) { - let inst = inst.clone(); - for (key, local, rel_chain) in &inst.subtree { - let mut chain = alias_chain.to_vec(); - chain.extend(rel_chain.iter().cloned()); - let fname = friendly_name_for(local, &chain); - if let Some((_, canister)) = canisters.get_mut(key) - && !canister.friendly_names.contains(&fname) - { - canister.friendly_names.push(fname); - } - } - return Ok(inst); - } - - stack.push(canonical.clone()); - - let prefix = relative_prefix(app_root_canonical, &canonical); - - let dep_manifest: ProjectManifest = - load_manifest_from_path(&manifest_path) - .await - .context(LoadDependencyManifestSnafu { - alias: dep.name.clone(), - })?; - - // Build the dependency's own canisters and key them under the prefix. All of - // them are imported (deploy-all); the `canisters` exposure subset is applied - // by the caller when wiring env vars. - let built = - build_manifest_canisters(&dep_root, &dep_manifest.canisters, recipe_resolver).await?; - - let mut own: Vec<(String, String)> = Vec::new(); - let mut local_to_key: BTreeMap = BTreeMap::new(); - for (local, cdir, mut canister) in built { - let store_key = format!("{prefix}:{local}"); - canister.name = store_key.clone(); - // Friendly URL from the alias chain, not the path-based store key. - canister.friendly_names = vec![friendly_name_for(&local, alias_chain)]; - own.push((local.clone(), store_key.clone())); - local_to_key.insert(local.clone(), store_key.clone()); - match canisters.entry(store_key.clone()) { - IndexEntry::Occupied(_) => { - return DuplicateSnafu { - kind: "canister".to_string(), - name: store_key, - } - .fail(); - } - IndexEntry::Vacant(e) => { - e.insert((cdir, canister)); - } - } - } - - // Now that every sibling's store key is known, translate the dependency's - // controller references (local sibling name -> store key). - for (_, key) in &own { - if let Some((_, canister)) = canisters.get_mut(key) { - translate_controllers(canister, &local_to_key); - } - } - - // Capture the member's own environments so the parent can honor its - // per-canister settings/init_args for the same-named environment - // (standalone-equivalence). The network binding and canister selection are - // ignored; only overrides on the member's *own* canisters are - // folded in — keys naming its dependencies are left to those dependencies. - let mut defined_envs: HashSet = HashSet::new(); - for env_item in &dep_manifest.environments { - let em: EnvironmentManifest = match env_item { - Item::Manifest(m) => m.clone(), - Item::Path(path) => { - let p = dep_root.join(path); - if !p.is_file() { - return NotFoundSnafu { - kind: "environment".to_string(), - path: p.to_string(), - } - .fail(); - } - load_manifest_from_path::(&p) - .await - .context(LoadEnvironmentSnafu)? - } - }; - defined_envs.insert(em.name.clone()); - if let Some(settings) = &em.settings { - for (local, s) in settings { - if let Some(key) = local_to_key.get(local) { - // Translate the override's own controller references from the - // member's local names to store keys, so name-based controllers - // resolve against the workspace id map just like base settings. - let mut s = s.clone(); - translate_settings_controllers(&mut s, &local_to_key); - member_env_overrides - .entry(em.name.clone()) - .or_default() - .entry(key.clone()) - .or_default() - .settings = Some(s); - } - } - } - if let Some(init_args) = &em.init_args { - for (local, ia) in init_args { - if let Some(key) = local_to_key.get(local) { - member_env_overrides - .entry(em.name.clone()) - .or_default() - .entry(key.clone()) - .or_default() - .init_args = Some(ia.clone()); - } - } - } - } - members.push(MemberEnvInfo { - prefix: prefix.clone(), - defined: defined_envs, - }); - - // Recurse into the dependency's own dependencies. - let own_names: HashSet = own.iter().map(|(l, _)| l.clone()).collect(); - validate_dependency_aliases(&dep_manifest.dependencies, &own_names)?; - - // The instance's subtree, for diamond-hit friendly-URL propagation: its own - // canisters sit at the instance root (empty relative alias chain); each - // nested dependency contributes its subtree prefixed with the nested alias. - let mut subtree: Vec<(String, String, Vec)> = own - .iter() - .map(|(local, key)| (key.clone(), local.clone(), Vec::new())) - .collect(); - - let mut edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); - for nested in &dep_manifest.dependencies { - let mut nested_chain = alias_chain.to_vec(); - nested_chain.push(nested.name.clone()); - let inst = Box::pin(import_dependency( - app_root_canonical, - &dep_root, - nested, - recipe_resolver, - canisters, - registry, - stack, - member_env_overrides, - members, - &nested_chain, - )) - .await?; - for (key, local, rel) in &inst.subtree { - let mut r = Vec::with_capacity(rel.len() + 1); - r.push(nested.name.clone()); - r.extend(rel.iter().cloned()); - subtree.push((key.clone(), local.clone(), r)); - } - let exposed = select_exposed(&inst.own, &nested.canisters, &nested.name)?; - edges.push((nested.name.clone(), exposed)); - } - - // Assign env-var bindings for this instance's own canisters. - let bindings = compute_bindings(&own, &edges); - for (_, key) in &own { - if let Some((_, canister)) = canisters.get_mut(key) { - canister.bindings = bindings.clone(); - } - } - - stack.pop(); - let instance = ImportedInstance { own, subtree }; - registry.insert(canonical, instance.clone()); - Ok(instance) -} +use crate::{Environment, prelude::*}; /// Canonicalize into a UTF-8 path, or `None` if it does not exist / is not UTF-8. fn canonicalize_or(dir: &Path) -> Option { @@ -785,437 +54,34 @@ pub fn member_scoped_canisters( Some(names) } -/// Build one environment's canister map: select from `canisters`, then apply the -/// member overrides for this environment (standalone-equivalence), then -/// the root's own overrides (highest precedence). Precedence is therefore -/// root-explicit > member-env > canister-base. -fn build_environment_canisters( - canisters: &IndexMap, - env_name: &str, - selection: &CanisterSelection, - member_overrides: Option<&HashMap>, - root_settings: Option<&HashMap>, - root_init_args: Option<&HashMap>, -) -> Result, ConsolidateManifestError> { - let mut cs = match selection { - CanisterSelection::None => IndexMap::new(), - CanisterSelection::Everything => canisters.clone(), - CanisterSelection::Named(names) => { - let mut cs: IndexMap = IndexMap::new(); - for name in names { - let v = canisters.get(name).ok_or( - InvalidCanisterSnafu { - environment: env_name.to_owned(), - canister: name.to_owned(), - } - .build(), - )?; - cs.insert(name.to_owned(), v.to_owned()); - } - cs - } - }; - - // Member overrides first (lower precedence than the root's own overrides). - if let Some(overrides) = member_overrides { - for (key, ov) in overrides { - if let Some((cpath, canister)) = cs.get_mut(key) { - if let Some(s) = &ov.settings { - canister.settings = s.clone(); - } - if let Some(ia) = &ov.init_args { - canister.init_args = Some(resolve_manifest_init_args(ia, cpath, key)?); - } - } - } - } - - // Root overrides last (highest precedence). - if let Some(settings) = root_settings { - for (name, s) in settings { - if let Some((_p, canister)) = cs.get_mut(name) { - canister.settings = s.clone(); - } - } - } - if let Some(init_args) = root_init_args { - for (name, ia) in init_args { - if let Some((cpath, canister)) = cs.get_mut(name) { - canister.init_args = Some(resolve_manifest_init_args(ia, cpath, name)?); - } - } - } - - Ok(cs) -} - -/// Turns the ProjectManifest into a Project struct -/// - Adds the default Networks -/// - Adds the default Environment -/// - Imports any dependency projects' canisters -/// - Validates the manifest to make sure that: -/// - There are no duplicates -/// - All the environments have networks -/// - All the referenced canisters exist -/// - All the recipes have been resolved -pub async fn consolidate_manifest( - pdir: &Path, - recipe_resolver: &dyn recipe::Resolve, - m: &ProjectManifest, -) -> Result { - // Canisters. IndexMap (not HashMap) so the order from the project manifest is preserved - // through to consumers like `icp project bundle`, which needs reproducible output. - let mut canisters: IndexMap = IndexMap::new(); - - // Canonical app root, used to derive stable, order-independent store-key - // prefixes for imported dependency canisters. - let app_root_canonical = - canonicalize_dep("", pdir).unwrap_or_else(|_| pdir.to_owned()); - - // This project's own canisters, keyed by their bare local names. - let app_built = build_manifest_canisters(pdir, &m.canisters, recipe_resolver).await?; - let mut app_own: Vec<(String, String)> = Vec::new(); - for (local, cdir, canister) in app_built { - app_own.push((local.clone(), local.clone())); - match canisters.entry(local.clone()) { - IndexEntry::Occupied(e) => { - return DuplicateSnafu { - kind: "canister".to_string(), - name: e.key().to_owned(), - } - .fail(); - } - IndexEntry::Vacant(e) => { - e.insert((cdir, canister)); - } - } - } - - // Import dependency projects. Each dependency is deployed in full and keyed - // under its app-root-relative path; diamonds (the same directory reached via - // multiple edges) resolve to a single instance. - let mut registry: HashMap = HashMap::new(); - let mut stack: Vec = Vec::new(); - // Member environment config folded into the root's same-named environments, - // and the per-member set of declared environment names for the strict rule. - let mut member_env_overrides: MemberEnvOverrides = HashMap::new(); - let mut members: Vec = Vec::new(); - let app_own_names: HashSet = app_own.iter().map(|(l, _)| l.clone()).collect(); - validate_dependency_aliases(&m.dependencies, &app_own_names)?; - - let mut app_edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); - for dep in &m.dependencies { - let inst = import_dependency( - &app_root_canonical, - pdir, - dep, - recipe_resolver, - &mut canisters, - &mut registry, - &mut stack, - &mut member_env_overrides, - &mut members, - std::slice::from_ref(&dep.name), - ) - .await?; - let exposed = select_exposed(&inst.own, &dep.canisters, &dep.name)?; - app_edges.push((dep.name.clone(), exposed)); - } - - // Assign env-var bindings for this project's own canisters (own canisters by - // local name plus each dependency's exposed canisters under `:`). - let app_bindings = compute_bindings(&app_own, &app_edges); - for (_, key) in &app_own { - if let Some((_, canister)) = canisters.get_mut(key) { - canister.bindings = app_bindings.clone(); - } - } - - // Friendly URLs need no de-collision pass: the strict name rule (no '.') makes - // own canisters single-label and dependency canisters multi-label (dot-nested - // by alias chain), so their hostnames are disjoint by construction (§17.2). - - // Validate that every canister-name controller reference points to a declared canister. - // Catching typos here turns "perpetual warning" into a clear load-time error. - for (canister_name, (_, canister)) in &canisters { - let Some(crefs) = &canister.settings.controllers else { - continue; - }; - for cref in crefs { - if let Some(ref_name) = cref.canister_name() - && !canisters.contains_key(ref_name) - { - return UnknownControllerCanisterSnafu { - canister: canister_name.to_owned(), - controller: ref_name.to_owned(), - } - .fail(); - } - } - } - - // Networks - let mut networks: HashMap = HashMap::new(); - - // Add IC network first - this is always protected and non-overridable - networks.insert( - IC.to_string(), - Network { - name: IC.to_string(), - configuration: Configuration::Connected { - connected: Connected { - api_url: IC_MAINNET_NETWORK_API_URL.parse().unwrap(), - http_gateway_url: Some(IC_MAINNET_NETWORK_GATEWAY_URL.parse().unwrap()), - root_key: RootKeySpec::Mainnet, - }, - }, - }, - ); - - // Track which network names are protected (only IC network) - let protected_network_names: HashSet = [IC.to_string()].into_iter().collect(); - - // Resolve NetworkManifests and add them (including user-defined "local" if provided) - for i in &m.networks { - let m = match i { - Item::Path(path) => { - let path = pdir.join(path); - if !path.exists() || !path.is_file() { - return NotFoundSnafu { - kind: "network".to_string(), - path: path.to_string(), - } - .fail(); - } - load_manifest_from_path::(&path) - .await - .context(LoadNetworkSnafu)? - } - Item::Manifest(ms) => ms.clone(), - }; - - match networks.entry(m.name.to_owned()) { - // Duplicate - Entry::Occupied(e) => { - // Only error if trying to override a protected network - if protected_network_names.contains(&m.name) { - return ReservedSnafu { - kind: "network".to_string(), - name: m.name.to_string(), - } - .fail(); - } - - // For non-protected duplicates, this is a user error (defining same network twice) - return DuplicateSnafu { - kind: "network".to_string(), - name: e.key().to_owned(), - } - .fail(); - } - - // Ok - Entry::Vacant(e) => { - e.insert(Network { - name: m.name.to_owned(), - configuration: m.configuration.into(), // Convert manifest to config struct - }); - } - } - } - - // After processing user networks, add default "local" if not already defined - // This provides backward compatibility for projects that don't define their own "local" network - if !networks.contains_key(LOCAL) { - networks.insert( - LOCAL.to_string(), - Network { - name: LOCAL.to_string(), - configuration: Configuration::Managed { - managed: Managed { - mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { - gateway: Gateway { - bind: DEFAULT_LOCAL_NETWORK_BIND.to_string(), - port: Port::Fixed(DEFAULT_LOCAL_NETWORK_PORT), - domains: vec![], - }, - artificial_delay_ms: None, - ii: false, - nns: false, - subnets: None, - bitcoind_addr: None, - dogecoind_addr: None, - version: None, - })), - }, - }, - }, - ); - } - - // Environments - let mut environments: HashMap = HashMap::new(); - - for i in &m.environments { - let m = match i { - Item::Path(path) => { - let path = pdir.join(path); - if !path.exists() || !path.is_file() { - return NotFoundSnafu { - kind: "environment".to_string(), - path: path.to_string(), - } - .fail(); - } - load_manifest_from_path::(&path) - .await - .context(LoadEnvironmentSnafu)? - } - Item::Manifest(ms) => ms.clone(), - }; - - match environments.entry(m.name.to_owned()) { - // Duplicate - Entry::Occupied(e) => { - return DuplicateSnafu { - kind: "environment".to_string(), - name: e.key().to_owned(), - } - .fail(); - } - - // Ok - Entry::Vacant(e) => { - e.insert(Environment { - name: m.name.to_owned(), - - // Embed network in environment - network: { - let v = networks.get(&m.network).ok_or( - InvalidNetworkSnafu { - environment: m.name.to_owned(), - network: m.network.to_owned(), - } - .build(), - )?; - - v.to_owned() - }, - - // Embed canisters in environment, folding member overrides - // beneath the root's own settings/init_args overrides. - canisters: build_environment_canisters( - &canisters, - &m.name, - &m.canisters, - member_env_overrides.get(&m.name), - m.settings.as_ref(), - m.init_args.as_ref(), - )?, - }); - } - } - } - - // We're done adding all the user environments - // Now we add the implicit `local` and `ic` environment if the user hasn't overriden it - if let Entry::Vacant(vacant_entry) = environments.entry(LOCAL.to_string()) { - let network = networks - .get(LOCAL) - .ok_or( - InvalidNetworkSnafu { - environment: LOCAL.to_owned(), - network: LOCAL.to_owned(), - } - .build(), - )? - .to_owned(); - vacant_entry.insert(Environment { - name: LOCAL.to_string(), - network, - canisters: build_environment_canisters( - &canisters, - LOCAL, - &CanisterSelection::Everything, - member_env_overrides.get(LOCAL), - None, - None, - )?, - }); - } - if let Entry::Vacant(vacant_entry) = environments.entry(IC.to_string()) { - let network = networks - .get(IC) - .ok_or( - InvalidNetworkSnafu { - environment: IC.to_owned(), - network: IC.to_owned(), - } - .build(), - )? - .to_owned(); - vacant_entry.insert(Environment { - name: IC.to_string(), - network, - canisters: build_environment_canisters( - &canisters, - IC, - &CanisterSelection::Everything, - member_env_overrides.get(IC), - None, - None, - )?, - }); - } - - // Strict rule: every member must declare each environment the root targets. - // `local`/`ic` are implicit for every project, so they never count - // as missing; other environments must be declared explicitly by the member. - // Recorded per-environment and enforced lazily when that environment is - // selected (so a missing `staging` never blocks `deploy -e local`). - let mut member_missing_envs: HashMap> = HashMap::new(); - for env_name in environments.keys() { - if env_name == LOCAL || env_name == IC { - continue; - } - for member in &members { - if !member.defined.contains(env_name) { - member_missing_envs - .entry(env_name.clone()) - .or_default() - .push(member.prefix.clone()); - } - } - } - - Ok(Project { - dir: pdir.into(), - canisters, - networks, - environments, - member_missing_envs, - }) -} - #[cfg(test)] -mod dependency_tests { +mod tests { use super::*; - use crate::canister::recipe::{RecipeContext, Resolve, ResolveError}; - use crate::manifest::canister::{BuildSteps, SyncSteps}; + use crate::canister::recipe::{RemoteResourceResolve, ResolveError}; + use crate::manifest::adapter::prebuilt::SourceField; use crate::manifest::recipe::Recipe; + use crate::manifest::{PROJECT_MANIFEST, ProjectManifest, load_manifest_from_path}; + use crate::prelude::LOCAL; use camino_tempfile::Utf8TempDir; + use tokio::sync::mpsc::Sender; - /// Recipes are never used in these tests; every canister is pre-built. + /// Recipes and plugins are never used in this test; every canister is pre-built. struct PanicResolver; #[async_trait::async_trait] - impl Resolve for PanicResolver { - async fn resolve( + impl RemoteResourceResolve for PanicResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + panic!("recipe resolver should not be called in this test"); + } + + async fn resolve_wasm( &self, - _recipe: &Recipe, - _context: &RecipeContext, - ) -> Result<(BuildSteps, SyncSteps), ResolveError> { - panic!("recipe resolver should not be called in dependency tests"); + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _stdio: Option>, + ) -> Result { + panic!("wasm resolver should not be called in this test"); } } @@ -1225,8 +91,6 @@ mod dependency_tests { std::fs::write(p, contents).unwrap(); } - /// A minimal `icp.yaml` body declaring the given pre-built canisters, - /// followed by a raw `dependencies:` block (may be empty). fn manifest(canisters: &[&str], deps: &str) -> String { let mut s = String::new(); if canisters.is_empty() { @@ -1243,64 +107,9 @@ mod dependency_tests { s } - async fn consolidate(pdir: &Path) -> Result { - let m: ProjectManifest = load_manifest_from_path(&pdir.join(PROJECT_MANIFEST)) - .await - .expect("failed to parse project manifest"); - consolidate_manifest(pdir, &PanicResolver, &m).await - } - - fn bindings_of<'a>(p: &'a Project, key: &str) -> &'a BTreeMap { - &p.canisters - .get(key) - .unwrap_or_else(|| { - panic!( - "canister '{key}' not found; have {:?}", - p.canisters.keys().collect::>() - ) - }) - .1 - .bindings - } - - fn friendly_names_of<'a>(p: &'a Project, key: &str) -> &'a [String] { - &p.canisters - .get(key) - .unwrap_or_else(|| { - panic!( - "canister '{key}' not found; have {:?}", - p.canisters.keys().collect::>() - ) - }) - .1 - .friendly_names - } - #[tokio::test] - async fn single_project_bindings_are_self_and_siblings() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - &manifest(&["backend", "frontend"], ""), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // Flat behavior preserved: every canister maps every sibling (incl. self) - // to itself. - let expected = BTreeMap::from([ - ("backend".to_string(), "backend".to_string()), - ("frontend".to_string(), "frontend".to_string()), - ]); - assert_eq!(bindings_of(&p, "backend"), &expected); - assert_eq!(bindings_of(&p, "frontend"), &expected); - } - - #[tokio::test] - async fn dependency_import_and_exposure_subset() { + async fn member_scope_targets_only_the_members_canisters() { let tmp = Utf8TempDir::new().unwrap(); - // Dependency nested inside the app (mirrors a submodule under the app). write( tmp.path(), "openemail/icp.yaml", @@ -1311,645 +120,35 @@ mod dependency_tests { "icp.yaml", &manifest( &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [backend]\n", + "dependencies:\n - name: openemail\n path: ./openemail\n", ), ); - let p = consolidate(tmp.path()).await.unwrap(); + let m: ProjectManifest = load_manifest_from_path(&tmp.path().join(PROJECT_MANIFEST)) + .await + .unwrap(); + let p = consolidate_manifest(tmp.path(), &PanicResolver, &m) + .await + .unwrap(); + let env = p.environments.get(LOCAL).expect("local environment"); - // The whole dependency is deployed (both canisters imported), keyed by path. - assert!(p.canisters.contains_key("backend")); - assert!(p.canisters.contains_key("openemail:backend")); - assert!(p.canisters.contains_key("openemail:frontend")); + // At the workspace root (member == root): no scoping. + assert_eq!(member_scoped_canisters(&p.dir, Some(&p.dir), env), None); - // App's own canister sees itself and only the *exposed* dependency canister. - assert_eq!( - bindings_of(&p, "backend"), - &BTreeMap::from([ - ("backend".to_string(), "backend".to_string()), - ( - "openemail:backend".to_string(), - "openemail:backend".to_string() - ), - ]) - ); + // Unknown member dir: no scoping. + assert_eq!(member_scoped_canisters(&p.dir, None, env), None); - // The dependency's own canisters keep their standalone view (bare names). + // Inside the member: only the member's own canisters, not the app's. + let member = tmp.path().join("openemail"); + let mut scoped = member_scoped_canisters(&p.dir, Some(&member), env) + .expect("should scope when inside a member"); + scoped.sort(); assert_eq!( - bindings_of(&p, "openemail:backend"), - &BTreeMap::from([ - ("backend".to_string(), "openemail:backend".to_string()), - ("frontend".to_string(), "openemail:frontend".to_string()), - ]) + scoped, + vec![ + "openemail:backend".to_string(), + "openemail:frontend".to_string(), + ] ); } - - #[tokio::test] - async fn member_scope_targets_only_the_members_canisters() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend", "frontend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let env = p.environments.get(LOCAL).expect("local environment"); - - // At the workspace root (member == root): no scoping. - assert_eq!(member_scoped_canisters(&p.dir, Some(&p.dir), env), None); - - // Unknown member dir: no scoping. - assert_eq!(member_scoped_canisters(&p.dir, None, env), None); - - // Inside the member: only the member's own canisters, not the app's. - let member = tmp.path().join("openemail"); - let mut scoped = member_scoped_canisters(&p.dir, Some(&member), env) - .expect("should scope when inside a member"); - scoped.sort(); - assert_eq!( - scoped, - vec![ - "openemail:backend".to_string(), - "openemail:frontend".to_string(), - ] - ); - } - - #[tokio::test] - async fn member_env_config_folds_in_with_root_override_winning() { - let tmp = Utf8TempDir::new().unwrap(); - // openemail defines `staging` with per-canister settings for its own - // canisters. - write( - tmp.path(), - "openemail/icp.yaml", - r#" -canisters: - - name: backend - build: - steps: - - type: pre-built - path: backend.wasm - - name: frontend - build: - steps: - - type: pre-built - path: frontend.wasm -environments: - - name: staging - settings: - backend: - compute_allocation: 5 - frontend: - compute_allocation: 7 -"#, - ); - // The app declares openemail and also defines `staging`, overriding the - // imported backend's settings (the root override must win). - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: app - build: - steps: - - type: pre-built - path: app.wasm -dependencies: - - name: openemail - path: ./openemail -environments: - - name: staging - settings: - "openemail:backend": - compute_allocation: 99 -"#, - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - - // Root override wins over the member's config. - assert_eq!( - staging - .canisters - .get("openemail:backend") - .unwrap() - .1 - .settings - .compute_allocation, - Some(99), - ); - // No root override → the member's own config applies (standalone-equivalence). - assert_eq!( - staging - .canisters - .get("openemail:frontend") - .unwrap() - .1 - .settings - .compute_allocation, - Some(7), - ); - // Both projects declared staging, so nothing is recorded as missing. - assert!(p.member_missing_envs.is_empty()); - } - - #[tokio::test] - async fn missing_member_environment_is_recorded() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: app - build: - steps: - - type: pre-built - path: app.wasm -dependencies: - - name: openemail - path: ./openemail -environments: - - name: staging -"#, - ); - - let p = consolidate(tmp.path()).await.unwrap(); - // openemail does not declare `staging`, so it is recorded as missing. - assert_eq!( - p.member_missing_envs.get("staging"), - Some(&vec!["openemail".to_string()]), - ); - // Implicit environments are never recorded as missing. - assert!(!p.member_missing_envs.contains_key("local")); - assert!(!p.member_missing_envs.contains_key("ic")); - } - - #[tokio::test] - async fn diamond_dedups_to_single_instance() { - let tmp = Utf8TempDir::new().unwrap(); - // umbrella layout: service-a and service-b both depend on ../openemail. - write( - tmp.path(), - "umbrella/openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "umbrella/service-a/icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "umbrella/service-b/icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // openemail is imported exactly once despite two edges reaching it. - let openemail_keys: Vec<_> = p - .canisters - .keys() - .filter(|k| k.contains("openemail")) - .collect(); - assert_eq!( - openemail_keys, - vec![&"umbrella/openemail:backend".to_string()], - "expected a single shared openemail instance" - ); - - // Both services' code reads `openemail:backend`, resolving to the one instance. - assert_eq!( - bindings_of(&p, "umbrella/service-a:backend").get("openemail:backend"), - Some(&"umbrella/openemail:backend".to_string()) - ); - assert_eq!( - bindings_of(&p, "umbrella/service-b:backend").get("openemail:backend"), - Some(&"umbrella/openemail:backend".to_string()) - ); - - // The single shared instance is reachable at one friendly URL per alias - // chain (§17.3) — the store-key path (`umbrella/`) never appears. - assert_eq!( - friendly_names_of(&p, "umbrella/openemail:backend"), - &["backend.openemail.service-a", "backend.openemail.service-b"] - ); - // Each service's own canister is named by its own alias chain. - assert_eq!( - friendly_names_of(&p, "umbrella/service-a:backend"), - &["backend.service-a"] - ); - assert_eq!( - friendly_names_of(&p, "umbrella/service-b:backend"), - &["backend.service-b"] - ); - } - - #[tokio::test] - async fn diamond_transitive_dependency_gets_url_per_chain() { - let tmp = Utf8TempDir::new().unwrap(); - // The shared openemail itself depends on libfoo, and is reached via both - // service-a and service-b. - write( - tmp.path(), - "umbrella/openemail/libfoo/icp.yaml", - &manifest(&["bar"], ""), - ); - write( - tmp.path(), - "umbrella/openemail/icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: libfoo\n path: ./libfoo\n", - ), - ); - write( - tmp.path(), - "umbrella/service-a/icp.yaml", - &manifest( - &["service"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "umbrella/service-b/icp.yaml", - &manifest( - &["service"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // The shared instance's own canister gets one URL per chain... - assert_eq!( - friendly_names_of(&p, "umbrella/openemail:backend"), - &["backend.openemail.service-a", "backend.openemail.service-b"] - ); - // ...and so does its *transitive* dependency (the subtree is revisited on - // the diamond hit, not just the instance's own canisters). - assert_eq!( - friendly_names_of(&p, "umbrella/openemail/libfoo:bar"), - &[ - "bar.libfoo.openemail.service-a", - "bar.libfoo.openemail.service-b" - ] - ); - } - - #[tokio::test] - async fn dot_in_canister_name_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - // '.' is banned: it would be ambiguous in a dot-nested friendly subdomain - // (an own canister named `frontend.openemail` could collide with dependency - // `openemail`'s `frontend`). The strict name rule rejects it up front. - write( - tmp.path(), - "icp.yaml", - &manifest(&["frontend.openemail"], ""), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn invalid_dependency_alias_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["app"], - "dependencies:\n - name: open.email\n path: ./openemail\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::InvalidDependencyAlias { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn member_override_controllers_are_translated_to_store_keys() { - let tmp = Utf8TempDir::new().unwrap(); - // openemail's `staging` override names a controller by its local name. - write( - tmp.path(), - "openemail/icp.yaml", - r#" -canisters: - - name: backend - build: - steps: - - type: pre-built - path: backend.wasm - - name: frontend - build: - steps: - - type: pre-built - path: frontend.wasm -environments: - - name: staging - settings: - backend: - controllers: ["frontend"] -"#, - ); - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: app - build: - steps: - - type: pre-built - path: app.wasm -dependencies: - - name: openemail - path: ./openemail -environments: - - name: staging -"#, - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - let controllers = staging - .canisters - .get("openemail:backend") - .unwrap() - .1 - .settings - .controllers - .clone() - .expect("controllers set by the member override"); - - // The member-local `frontend` must be translated to its store key, so it - // resolves against the workspace id map at deploy time. - assert_eq!( - controllers, - vec![ControllerRef::CanisterName( - "openemail:frontend".to_string() - )] - ); - } - - #[tokio::test] - async fn friendly_names_are_bare_for_own_and_dotted_for_dependencies() { - let tmp = Utf8TempDir::new().unwrap(); - // openemail (with a transitive dep libfoo) vendored under the app. - write( - tmp.path(), - "openemail/libfoo/icp.yaml", - &manifest(&["bar"], ""), - ); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest( - &["backend", "frontend"], - "dependencies:\n - name: libfoo\n path: ./libfoo\n", - ), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // Own canister: bare name (unchanged from single-project behavior). - assert_eq!(friendly_names_of(&p, "backend"), &["backend"]); - // Direct dependency: dot-nested by alias (no `vendor/` path noise). - assert_eq!( - friendly_names_of(&p, "openemail:backend"), - &["backend.openemail"] - ); - assert_eq!( - friendly_names_of(&p, "openemail:frontend"), - &["frontend.openemail"] - ); - // Transitive dependency: full alias chain, canister-most-specific first. - assert_eq!( - friendly_names_of(&p, "openemail/libfoo:bar"), - &["bar.libfoo.openemail"] - ); - } - - #[tokio::test] - async fn cycle_is_detected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - &manifest(&[], "dependencies:\n - name: a\n path: ./a\n"), - ); - write( - tmp.path(), - "a/icp.yaml", - &manifest(&["x"], "dependencies:\n - name: b\n path: ../b\n"), - ); - write( - tmp.path(), - "b/icp.yaml", - &manifest(&["y"], "dependencies:\n - name: a\n path: ../a\n"), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::CircularDependency { .. }), - "expected CircularDependency, got {err:?}" - ); - } - - #[tokio::test] - async fn alias_colliding_with_canister_name_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["openemail"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!( - err, - ConsolidateManifestError::DependencyAliasCollision { .. } - ), - "got {err:?}" - ); - } - - #[tokio::test] - async fn duplicate_alias_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write(tmp.path(), "one/icp.yaml", &manifest(&["backend"], "")); - write(tmp.path(), "two/icp.yaml", &manifest(&["backend"], "")); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: dup\n path: ./one\n - name: dup\n path: ./two\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!( - err, - ConsolidateManifestError::DuplicateDependencyAlias { .. } - ), - "got {err:?}" - ); - } - - #[tokio::test] - async fn colon_in_canister_name_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write(tmp.path(), "icp.yaml", &manifest(&["foo:bar"], "")); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn unknown_exposed_canister_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [nope]\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!( - err, - ConsolidateManifestError::UnknownDependencyCanister { .. } - ), - "got {err:?}" - ); - } - - #[tokio::test] - async fn missing_dependency_path_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: openemail\n path: ./does-not-exist\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::DependencyNotFound { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn imported_canisters_appear_in_implicit_environments() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // Deploy-all: the implicit `local` environment includes the dependency. - let local = p.environments.get("local").unwrap(); - assert!(local.canisters.contains_key("backend")); - assert!(local.canisters.contains_key("openemail:backend")); - } }