From bc2e2bf0b4059ac1fff163d38dd5e93d98152b4f Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 7 Oct 2024 06:52:55 +0000 Subject: [PATCH 01/40] chore: add interface definition --- .../rate_limit_canister/interface.did | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 rs/boundary_node/rate_limit_canister/interface.did diff --git a/rs/boundary_node/rate_limit_canister/interface.did b/rs/boundary_node/rate_limit_canister/interface.did new file mode 100644 index 00000000000..cf651e9755c --- /dev/null +++ b/rs/boundary_node/rate_limit_canister/interface.did @@ -0,0 +1,89 @@ +type Version = nat64; // Represents the config version number +type Timestamp = nat64; // Represents timestamp in nanoseconds since the epoch (1970-01-01) +type RuleId = text; // Unique identifier for each rule + + +// Input structure for defining a rule with mandatory fields within a config +type InputRule = record { + id: RuleId; // Unique identifier for the rule + rule_raw: blob; // Raw rule data (in binary format), expected to be a valid json object + description: text; // Textual description of the rule +}; + +// Output structure for rules +// Optional fields rule_raw and description may remain hidden while the rule is under confidentiality restrictions +type OutputRule = record { + id: RuleId; // Unique identifier for the rule + rule_raw: opt blob; // Raw rule data (in binary format), expected to be a valid json object, none if the rule is currently confidential + description: opt text; // Textual description of the rule, none if the rule is currently confidential + disclosed_at: opt Timestamp; // Timestamp when the rule was disclosed, none if the rule is still confidential + disclosed_in_version: opt Version; // Version in which the rule was disclosed, none if the rule is still confidential +}; + +type OutputConfig = record { + rules: vec OutputRule; +}; + +// Response structure for returning the requested configuration and associated metadata +type OutputConfigResponse = record { + version: Version; // Version of the configuration + active_since: Timestamp; // Time when this configuration became added (became active) + config: OutputConfig; // Contains the list of rules +}; + +// Verbose details of an individual rule +// Optional rule_raw and description fields are for restricted publicly viewing access +type OutputRuleMetadata = record { + id: RuleId; // Unique identifier for the rule + rule_raw: opt blob; // Raw rule data (binary format), expected to be a valid json object, none if the rule is currently confidential + description: opt text; // Textual description of the rule, none if the rule is currently confidential + disclosed_at: opt Timestamp; // Timestamp when the rule was disclosed, none if the rule is still confidential + disclosed_in_version: Version; // Version when the rule was disclosed, none if the rule is still confidential + added_in_version: Version; // Version when the rule was added (became active) + removed_in_version: opt Version; // Version when the rule was deactivated (removed), none if the rule is still active +}; + +type GetRuleByIdResponse = variant { + Ok: OutputRuleMetadata; + Err: text; +} + +type GetConfigResponse = variant { + Ok: OutputConfigResponse; + Err: text; +}; + +type DiscloseRulesResponse = variant { + Ok; + Err: text; +}; + +type OverwriteConfigResponse = variant { + Ok; + Err: text; +}; + +// Configuration containing a list of rules that replaces the current configuration +type InputConfig = record { + rules: vec InputRule; +}; + +// Initialization arguments for the service +type InitArg = record { + registry_polling_period_secs: nat64; // IDs of existing API boundary nodes are polled from the registry with this periodicity +}; + +service : (InitArg) -> { + // Replaces the current rate-limit configuration with a new set of rules + overwrite_config: (InputConfig) -> (OverwriteConfigResponse); + + // Make the viewing of the specified rules publicly accessible + disclose_rules: (vec RuleId) -> (DiscloseRulesResponse); + + // Fetches the rate-limit rule configuration for a specified version + // If no version is provided, the latest configuration is returned + get_config: (opt Version) -> (GetConfigResponse); + + // Fetch the rule with metadata by its ID + get_rule_by_id: (RuleId) -> (GetRuleByIdResponse); +} \ No newline at end of file From 13da21b1d70e84ec4478d7b454c4005f75f09428 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Sun, 20 Oct 2024 13:27:12 +0000 Subject: [PATCH 02/40] chore: rate limit canister --- rs/boundary_node/rate_limits/Cargo.toml | 26 ++ rs/boundary_node/rate_limits/api/Cargo.toml | 14 + rs/boundary_node/rate_limits/api/src/lib.rs | 76 +++++ .../rate_limits/canister/access_control.rs | 52 +++ .../rate_limits/canister/add_config.rs | 303 ++++++++++++++++++ .../rate_limits/canister/canister.rs | 122 +++++++ .../canister/confidentiality_formatting.rs | 102 ++++++ .../rate_limits/canister/disclose.rs | 125 ++++++++ .../rate_limits/canister/fetcher.rs | 156 +++++++++ .../rate_limits/canister/interface.did | 104 ++++++ .../rate_limits/canister/state.rs | 147 +++++++++ .../rate_limits/canister/storage.rs | 186 +++++++++++ .../rate_limits/canister/types.rs | 126 ++++++++ .../rate_limits/canister_client/Cargo.toml | 14 + .../rate_limits/canister_client/src/main.rs | 169 ++++++++++ rs/boundary_node/rate_limits/dfx.json | 21 ++ 16 files changed, 1743 insertions(+) create mode 100644 rs/boundary_node/rate_limits/Cargo.toml create mode 100644 rs/boundary_node/rate_limits/api/Cargo.toml create mode 100644 rs/boundary_node/rate_limits/api/src/lib.rs create mode 100644 rs/boundary_node/rate_limits/canister/access_control.rs create mode 100644 rs/boundary_node/rate_limits/canister/add_config.rs create mode 100644 rs/boundary_node/rate_limits/canister/canister.rs create mode 100644 rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs create mode 100644 rs/boundary_node/rate_limits/canister/disclose.rs create mode 100644 rs/boundary_node/rate_limits/canister/fetcher.rs create mode 100644 rs/boundary_node/rate_limits/canister/interface.did create mode 100644 rs/boundary_node/rate_limits/canister/state.rs create mode 100644 rs/boundary_node/rate_limits/canister/storage.rs create mode 100644 rs/boundary_node/rate_limits/canister/types.rs create mode 100644 rs/boundary_node/rate_limits/canister_client/Cargo.toml create mode 100644 rs/boundary_node/rate_limits/canister_client/src/main.rs create mode 100644 rs/boundary_node/rate_limits/dfx.json diff --git a/rs/boundary_node/rate_limits/Cargo.toml b/rs/boundary_node/rate_limits/Cargo.toml new file mode 100644 index 00000000000..532202a8d2c --- /dev/null +++ b/rs/boundary_node/rate_limits/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "rate_limits" +version.workspace = true +authors.workspace = true +edition.workspace = true +description.workspace = true +documentation.workspace = true + +[dependencies] +anyhow = { workspace = true } +bincode = { workspace = true } +candid = { workspace = true } +hex = { workspace = true } +ic-cdk = { workspace = true } +ic-cdk-macros = { workspace = true } +ic-cdk-timers = { workspace = true } +ic-stable-structures = { workspace = true } +rate-limits-api = { path = "./api" } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } + +[lib] +crate-type = ["cdylib"] +path = "canister/canister.rs" \ No newline at end of file diff --git a/rs/boundary_node/rate_limits/api/Cargo.toml b/rs/boundary_node/rate_limits/api/Cargo.toml new file mode 100644 index 00000000000..adcae36afc6 --- /dev/null +++ b/rs/boundary_node/rate_limits/api/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rate-limits-api" +version.workspace = true +authors.workspace = true +edition.workspace = true +description.workspace = true +documentation.workspace = true + +[dependencies] +candid = {workspace = true} +serde = {workspace = true} + +[lib] +path = "src/lib.rs" \ No newline at end of file diff --git a/rs/boundary_node/rate_limits/api/src/lib.rs b/rs/boundary_node/rate_limits/api/src/lib.rs new file mode 100644 index 00000000000..72173b16947 --- /dev/null +++ b/rs/boundary_node/rate_limits/api/src/lib.rs @@ -0,0 +1,76 @@ +use candid::CandidType; +use candid::Principal; +use serde::{Deserialize, Serialize}; +pub type Version = u64; +pub type Timestamp = u64; +pub type RuleId = String; +pub type IncidentId = String; +pub type SchemaVersion = u64; + +pub type GetConfigResponse = Result; +pub type AddConfigResponse = Result<(), String>; +pub type GetRuleByIdResponse = Result; +pub type DiscloseRulesResponse = Result<(), String>; + +#[derive(CandidType, Deserialize, Debug)] +pub enum DiscloseRulesArg { + RuleIds(Vec), + IncidentIds(Vec), +} + +#[derive(CandidType, Deserialize, Debug)] +pub struct ConfigResponse { + pub version: Version, + pub active_since: Timestamp, + pub config: OutputConfig, +} + +#[derive(CandidType, Deserialize, Debug)] +pub struct OutputConfig { + pub schema_version: SchemaVersion, + pub rules: Vec, +} + +#[derive(CandidType, Deserialize, Debug)] +pub struct InputConfig { + pub schema_version: SchemaVersion, + pub rules: Vec, +} + +#[derive(CandidType, Deserialize, Debug)] +pub struct InputRule { + pub incident_id: IncidentId, + pub rule_raw: Vec, + pub description: String, +} + +#[derive(CandidType, Deserialize, Debug)] +pub struct OutputRule { + pub id: RuleId, + pub rule_raw: Option>, + pub description: Option, + pub disclosed_at: Option, +} + +#[derive(CandidType, Deserialize, Debug)] +pub struct OutputRuleMetadata { + pub id: RuleId, + pub rule_raw: Option>, + pub description: Option, + pub disclosed_at: Option, + pub added_in_version: Version, + pub removed_in_version: Option, +} + +#[derive(CandidType, Deserialize, Debug)] +pub struct InitArg { + pub registry_polling_period_secs: u64, +} + +#[derive(CandidType, Deserialize, Clone, Copy, PartialEq, Eq)] +pub struct GetApiBoundaryNodeIdsRequest {} + +#[derive(CandidType, Serialize, Deserialize, Clone, PartialEq, Debug, Eq)] +pub struct ApiBoundaryNodeIdRecord { + pub id: Option, +} diff --git a/rs/boundary_node/rate_limits/canister/access_control.rs b/rs/boundary_node/rate_limits/canister/access_control.rs new file mode 100644 index 00000000000..358d099d915 --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/access_control.rs @@ -0,0 +1,52 @@ +use candid::Principal; + +use crate::storage::API_BOUNDARY_NODE_PRINCIPALS; + +const FULL_ACCESS_ID: &str = "2vxsx-fae"; +const FULL_READ_ID: &str = "2vxsx-fae"; + +pub trait ResolveAccessLevel { + fn get_access_level(&self) -> AccessLevel; +} + +#[derive(Debug, thiserror::Error)] +pub enum AccessLevelError {} + +#[derive(PartialEq, Eq)] +pub enum AccessLevel { + FullAccess, + FullRead, + RestrictedRead, +} + +#[derive(Clone)] +pub struct AccessLevelResolver { + pub caller_id: Principal, +} + +impl AccessLevelResolver { + pub fn new(caller_id: Principal) -> Self { + Self { caller_id } + } +} + +impl ResolveAccessLevel for AccessLevelResolver { + fn get_access_level(&self) -> AccessLevel { + let full_access_principal = Principal::from_text(FULL_ACCESS_ID).unwrap(); + + API_BOUNDARY_NODE_PRINCIPALS.with(|cell| { + let mut full_read_principals = cell.borrow_mut(); + // TODO: this is just for testing, remove later + let full_read_id = Principal::from_text(FULL_READ_ID).unwrap(); + let _ = full_read_principals.insert(full_read_id); + + if self.caller_id == full_access_principal { + return AccessLevel::FullAccess; + } else if full_read_principals.contains(&self.caller_id) { + return AccessLevel::FullRead; + } + + AccessLevel::RestrictedRead + }) + } +} diff --git a/rs/boundary_node/rate_limits/canister/add_config.rs b/rs/boundary_node/rate_limits/canister/add_config.rs new file mode 100644 index 00000000000..9c06aa2c71a --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/add_config.rs @@ -0,0 +1,303 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; + +use crate::storage::StorableIncidentMetadata; +use ic_cdk::api::time; +use rate_limits_api::IncidentId; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +use crate::{ + access_control::{AccessLevel, ResolveAccessLevel}, + state::Repository, + storage::{StorableConfig, StorableRuleMetadata}, + types::{InputConfig, RuleId, Version}, +}; + +pub const INIT_VERSION: Version = 1; + +pub trait AddsConfig { + fn add_config(&self, config: InputConfig) -> Result<(), AddConfigError>; +} + +#[derive(Debug, Error, Clone)] +pub enum RulePolicyError { + #[error("Rule at index={index} is linked to an already disclosed incident_id={incident_id}")] + LinkingRuleToDisclosedIncident { + index: usize, + incident_id: IncidentId, + }, + #[error("Rule at index={index} is already linked to an incident_id={incident_id}, attempted to relink to incident_id={incident_id_new}")] + LinkingRuleToAnotherIncident { + index: usize, + incident_id: IncidentId, + incident_id_new: IncidentId, + }, + #[error("Rule at index={index} with rule_id={rule_id} was deactivated in version={version}, cannot be resubmitted")] + DeactivatedRuleResubmission { + index: usize, + rule_id: RuleId, + version: Version, + }, +} + +#[derive(Debug, Error, Clone)] +pub enum RuleEncodingError { + #[error("Rule doesn't encode a valid JSON object")] + InvalidJsonEncoding, +} + +#[derive(Debug, Error)] +pub enum AddConfigError { + #[error("Not all rules encode valid JSON objects: {0}")] + RuleEncodingError(#[from] RuleEncodingError), + #[error("Rule violates policy: {0}")] + RulePolicyViolation(#[from] RulePolicyError), + #[error("Unauthorized operation")] + Unauthorized, + #[error("An unexpected error occurred: {0}")] + Unexpected(#[from] anyhow::Error), +} + +pub struct ConfigAdder { + pub repository: R, + pub access_resolver: A, +} + +impl ConfigAdder { + pub fn new(repository: R, access_resolver: A) -> Self { + Self { + repository, + access_resolver, + } + } +} + +// Definitions: +// - Each rule has two IDs - rule_id and incident_id (to which a rule is linked) +// - A rule is uniquely identified by rule_id, which is autogenerated by hashing two fields `rule_raw` + `description` (JSON decoded `rule_raw` is used for hashing) +// - incident_id must be provided for each rule by the caller; multiple rules can be linked to one incident_id +// - The same rule (i.e. `rule_raw` + `description`) can be persisted (resubmitted) from one config version to next ones +// If a rule is not resubmitted to the next config, it is marked as "deactivated" (`removed_in_version` field in `StorableRuleMetadata` is set to Some()) +// - Individual rules or incidents (vector holding multiple rule IDs) can be disclosed +// - Disclosing individual rules (or incidents) for the second time has no effect further effect +// Policies: +// - Deactivated rules can't be resubmitted again (DeactivatedRuleResubmission error) +// - Newly added rules can't be linked to an already disclosed incident (LinkingRuleToDisclosedIncident error) + +impl AddsConfig for ConfigAdder { + fn add_config(&self, config: InputConfig) -> Result<(), AddConfigError> { + // Only privileged users can perform this operation + if self.access_resolver.get_access_level() != AccessLevel::FullAccess { + return Err(AddConfigError::Unauthorized); + } + + let current_version = self + .repository + .get_version() + .unwrap_or(INIT_VERSION.into()) + .0; + let next_version = current_version + 1; + + // All rule IDs in the submitted config + let mut rule_ids = Vec::new(); + // Metadata of the newly submitted rules + let mut new_rules_metadata = Vec::<(RuleId, StorableRuleMetadata)>::new(); + // Hashmap of the newly submitted incident IDs + let mut new_incidents = HashMap::>::new(); + // Hashmap of the already existing incident IDs + let mut existing_incidents = HashMap::>::new(); + + // For each rule: + // - Validate its correct JSON encoding + // - Autogenerate rule ID based on hash(JSON encoding + description) + // - Validate rule submission policies + for (rule_idx, rule) in config.rules.iter().enumerate() { + let rule_id = generate_reproducible_id(&rule.rule_raw, &rule.description)?; + rule_ids.push(rule_id.clone()); + + // Check if submitted rule is new + let existing_rule_metadata = self.repository.get_rule(&rule_id); + // Check if an incident related to a rule is new + let existing_related_incident = self.repository.get_incident(&rule.incident_id); + + if let Some(ref metadata) = existing_rule_metadata { + if let Some(version) = metadata.removed_in_version { + Err(AddConfigError::RulePolicyViolation( + RulePolicyError::DeactivatedRuleResubmission { + index: rule_idx, + rule_id: rule_id.clone(), + version, + }, + ))?; + } + // Check if the rule is relinked to another incident + if metadata.incident_id != rule.incident_id { + Err(AddConfigError::RulePolicyViolation( + RulePolicyError::LinkingRuleToAnotherIncident { + index: rule_idx, + incident_id: metadata.incident_id.clone(), + incident_id_new: rule.incident_id.clone(), + }, + ))?; + } + } else { + let rule_metadata = StorableRuleMetadata { + incident_id: rule.incident_id.clone(), + rule_raw: rule.rule_raw.clone(), + description: rule.description.clone(), + disclosed_at: None, + added_in_version: next_version, + removed_in_version: None, + }; + new_rules_metadata.push((rule_id.clone(), rule_metadata)); + } + + // Rule is linked to an already existing incident + if let Some(incident) = existing_related_incident { + // New rule can't be linked to a disclosed incident + if existing_rule_metadata.is_none() && incident.is_disclosed { + Err(AddConfigError::RulePolicyViolation( + RulePolicyError::LinkingRuleToDisclosedIncident { + index: rule_idx, + incident_id: rule.incident_id.clone(), + }, + ))?; + } + existing_incidents + .entry(rule.incident_id.clone()) + .and_modify(|value| value.push(rule_id.clone())) + .or_insert(vec![rule_id.clone()]); + } else { + // Rule is linked to a new incident + new_incidents + .entry(rule.incident_id.clone()) + .and_modify(|value| value.push(rule_id.clone())) + .or_insert(vec![rule_id.clone()]); + } + } + + // Commit all changes to stable memory. + // Note: if any operation below fails canister state can become inconsistent. + // TODO: maybe it is better to panic to rollback changes + + // Mark deactivated rules + let _ = self.repository.get_config(current_version).map(|config| { + let deactivated_ids = find_difference(&config.rule_ids, &rule_ids); + deactivated_ids.iter().for_each(|rule_id| { + if let Some(mut metadata) = self.repository.get_rule(rule_id) { + metadata.removed_in_version = Some(next_version); + let _ = self.repository.update_rule(rule_id.clone(), metadata); + } + }); + }); + + for (rule_id, metadata) in new_rules_metadata.iter().cloned() { + if !self.repository.add_rule(rule_id.clone(), metadata) { + return Err(AddConfigError::Unexpected(anyhow::anyhow!( + "rule_id={rule_id} already existed, failed to add rule" + ))); + } + } + + for (incident_id, rule_ids) in new_incidents { + let incident_metadata = StorableIncidentMetadata { + is_disclosed: false, + rule_ids, + }; + if !self + .repository + .add_incident(incident_id.clone(), incident_metadata) + { + return Err(AddConfigError::Unexpected(anyhow::anyhow!( + "incident_id={incident_id} already exists, failed to add incident" + ))); + } + } + + for (incident_id, rule_ids) in existing_incidents { + let incident_metadata = StorableIncidentMetadata { + is_disclosed: false, + rule_ids, + }; + if !self + .repository + .update_incident(incident_id.clone(), incident_metadata) + { + return Err(AddConfigError::Unexpected(anyhow::anyhow!( + "incident={incident_id} doesn't exist, failed to update" + ))); + } + } + + let storable_config = StorableConfig { + schema_version: config.schema_version, + active_since: time(), + rule_ids, + }; + + if !self.repository.add_config(next_version, storable_config) { + return Err(AddConfigError::Unexpected(anyhow::anyhow!( + "Config for version {next_version} already exists, failed to add" + ))); + } + + Ok(()) + } +} + +fn generate_reproducible_id( + rule_raw: &[u8], + description: &str, +) -> Result { + let json_value = serde_json::from_slice::(rule_raw) + .map_err(|_| RuleEncodingError::InvalidJsonEncoding)?; + + let canonical_json = canonicalize_json(&json_value)?; + let canonical_json_bytes = + serde_json::to_vec(&canonical_json).map_err(|_| RuleEncodingError::InvalidJsonEncoding)?; + + let mut hasher = Sha256::new(); + + hasher.update(canonical_json_bytes); + hasher.update(description.as_bytes()); + + let result = hasher.finalize(); + + let rule_id = hex::encode(result); + + Ok(rule_id) +} + +fn canonicalize_json(value: &Value) -> Result { + match value { + Value::Object(map) => { + let mut sorted_map = BTreeMap::new(); + for (k, v) in map { + sorted_map.insert(k.clone(), canonicalize_json(v)?); + } + let map: Map = sorted_map.into_iter().collect(); + Ok(Value::Object(map)) + } + Value::Array(arr) => { + let canonicalized_array: Result, _> = + arr.iter().map(canonicalize_json).collect(); + Ok(Value::Array(canonicalized_array?)) + } + _ => Ok(value.clone()), + } +} + +fn find_difference(v1: &[T], v2: &[T]) -> Vec { + let set2: HashSet<_> = v2.iter().collect(); + v1.iter() + .filter(|&item| !set2.contains(item)) + .cloned() + .collect() +} + +impl From for String { + fn from(value: AddConfigError) -> Self { + value.to_string() + } +} diff --git a/rs/boundary_node/rate_limits/canister/canister.rs b/rs/boundary_node/rate_limits/canister/canister.rs new file mode 100644 index 00000000000..d90904f70e0 --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/canister.rs @@ -0,0 +1,122 @@ +use std::{collections::HashSet, time::Duration}; + +use access_control::AccessLevelResolver; +use add_config::{AddsConfig, ConfigAdder}; +use candid::{candid_method, Principal}; +use confidentiality_formatting::ConfidentialityFormatterFactory; +use disclose::{DisclosesRules, RulesDiscloser}; +use fetcher::{ConfigFetcher, EntityFetcher, RuleFetcher}; +use ic_cdk::api::call::call; +use ic_cdk_macros::{init, query, update}; +use rate_limits_api::{ + AddConfigResponse, ApiBoundaryNodeIdRecord, DiscloseRulesArg, DiscloseRulesResponse, + GetApiBoundaryNodeIdsRequest, GetConfigResponse, GetRuleByIdResponse, InitArg, InputConfig, + RuleId, Version, +}; +use state::{init_version_and_config, with_state}; +use storage::API_BOUNDARY_NODE_PRINCIPALS; +mod access_control; +mod add_config; +mod confidentiality_formatting; +mod disclose; +mod fetcher; +mod state; +mod storage; +mod types; + +const REGISTRY_CANISTER_ID: &str = "rwlgt-iiaaa-aaaaa-aaaaa-cai"; +const REGISTRY_CANISTER_METHOD: &str = "get_api_boundary_node_ids"; + +#[init] +#[candid_method(init)] +fn init(init_arg: InitArg) { + // Initialize an empty config with version=1 + init_version_and_config(1); + + let interval = std::time::Duration::from_secs(init_arg.registry_polling_period_secs); + + periodically_fetch_api_boundary_nodes_set(interval); +} + +#[query(name = "get_config")] +#[candid_method(query)] +fn get_config(version: Option) -> GetConfigResponse { + let caller_id = ic_cdk::api::caller(); + let response = with_state(|state| { + let access_resolver = AccessLevelResolver::new(caller_id); + let formatter = + ConfidentialityFormatterFactory::new(access_resolver).create_config_formatter(); + let fetcher = ConfigFetcher::new(state, formatter); + fetcher.fetch(version) + })?; + Ok(response.into()) +} + +#[query(name = "get_rule_by_id")] +#[candid_method(query)] +fn get_rule_by_id(rule_id: RuleId) -> GetRuleByIdResponse { + let caller_id = ic_cdk::api::caller(); + let response = with_state(|state| { + let access_resolver = AccessLevelResolver::new(caller_id); + let formatter = + ConfidentialityFormatterFactory::new(access_resolver).create_rule_formatter(); + let fetcher = RuleFetcher::new(state, formatter); + fetcher.fetch(rule_id) + })?; + Ok(response.into()) +} + +#[update(name = "add_config")] +#[candid_method(update)] +fn add_config(config: InputConfig) -> AddConfigResponse { + let caller_id = ic_cdk::api::caller(); + with_state(|state| { + let access_resolver: AccessLevelResolver = AccessLevelResolver::new(caller_id); + let writer = ConfigAdder::new(state, access_resolver); + writer.add_config(config.into()) + })?; + Ok(()) +} + +#[update(name = "disclose_rules")] +#[candid_method(update)] +fn disclose_rules(args: DiscloseRulesArg) -> DiscloseRulesResponse { + let caller_id = ic_cdk::api::caller(); + with_state(|state| { + let access_resolver: AccessLevelResolver = AccessLevelResolver::new(caller_id); + let discloser = RulesDiscloser::new(state, access_resolver); + discloser.disclose_rules(args.into()) + })?; + Ok(()) +} + +fn periodically_fetch_api_boundary_nodes_set(interval: Duration) { + ic_cdk_timers::set_timer_interval(interval, || { + ic_cdk::spawn(async { + if let Ok(canister_id) = Principal::from_text(REGISTRY_CANISTER_ID) { + match call::<_, (Result, String>,)>( + canister_id, + REGISTRY_CANISTER_METHOD, + (&GetApiBoundaryNodeIdsRequest {},), + ) + .await + { + Ok((Ok(api_bns_count),)) => { + API_BOUNDARY_NODE_PRINCIPALS.with(|cell| { + *cell.borrow_mut() = + HashSet::from_iter(api_bns_count.into_iter().filter_map(|n| n.id)) + }); + } + Ok((Err(err),)) => { + ic_cdk::println!("Error fetching API boundary nodes: {}", err); + } + Err(err) => { + ic_cdk::println!("Error calling registry canister: {:?}", err); + } + } + } else { + ic_cdk::println!("Failed to parse registry_canister_id"); + } + }); + }); +} diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs new file mode 100644 index 00000000000..e84d5ec4234 --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -0,0 +1,102 @@ +use std::marker::PhantomData; + +use crate::{ + access_control::{AccessLevel, ResolveAccessLevel}, + types::{OutputConfig, OutputRuleMetadata}, +}; + +/// Trait for formatting confidential data based on access levels +pub trait ConfidentialityFormatting { + type Input: Clone; + + fn format(&self, value: &Self::Input) -> Result; +} + +#[derive(Debug, thiserror::Error)] +pub enum ConfidentialFormatterError { + #[error("Access denied")] + AccessDenied, +} + +/// A generic confidentiality formatter for various data types +pub struct ConfidentialityFormatter { + access_resolver: A, + phantom: PhantomData, +} + +impl ConfidentialityFormatter { + pub fn new(access_resolver: A) -> Self { + Self { + access_resolver, + phantom: PhantomData, + } + } +} + +impl ConfidentialityFormatting + for ConfidentialityFormatter +{ + type Input = OutputConfig; + + fn format(&self, config: &OutputConfig) -> Result { + let mut config = config.clone(); + match self.access_resolver.get_access_level() { + AccessLevel::RestrictedRead => { + config.rules.iter_mut().for_each(|rule| { + if rule.disclosed_at.is_none() { + rule.description = None; + rule.rule_raw = None; + } + }); + Ok(config) + } + AccessLevel::FullRead => Ok(config), + _ => Err(ConfidentialFormatterError::AccessDenied), + } + } +} + +impl ConfidentialityFormatting + for ConfidentialityFormatter +{ + type Input = OutputRuleMetadata; + + fn format( + &self, + rule: &OutputRuleMetadata, + ) -> Result { + let mut rule = rule.clone(); + match self.access_resolver.get_access_level() { + AccessLevel::RestrictedRead => { + if rule.disclosed_at.is_none() { + rule.description = None; + rule.rule_raw = None; + } + Ok(rule) + } + AccessLevel::FullRead => Ok(rule), + _ => Err(ConfidentialFormatterError::AccessDenied), + } + } +} + +/// Factory for creating confidentiality formatters +pub struct ConfidentialityFormatterFactory { + access_resolver: A, +} + +impl ConfidentialityFormatterFactory { + pub fn new(access_resolver: A) -> Self { + Self { access_resolver } + } + + /// Create a confidentiality formatter for OutputConfig + pub fn create_config_formatter(&self) -> ConfidentialityFormatter { + ConfidentialityFormatter::new(self.access_resolver.clone()) + } + + /// Create a confidentiality formatter for OutputRuleMetadata + pub fn create_rule_formatter(&self) -> ConfidentialityFormatter { + ConfidentialityFormatter::new(self.access_resolver.clone()) + } +} diff --git a/rs/boundary_node/rate_limits/canister/disclose.rs b/rs/boundary_node/rate_limits/canister/disclose.rs new file mode 100644 index 00000000000..ba5bb05627f --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/disclose.rs @@ -0,0 +1,125 @@ +use ic_cdk::api::time; + +use crate::{ + access_control::{AccessLevel, ResolveAccessLevel}, + state::Repository, + types::{DiscloseRulesArg, IncidentId, RuleId, Timestamp}, +}; + +pub trait DisclosesRules { + fn disclose_rules(&self, arg: DiscloseRulesArg) -> Result<(), DiscloseRulesError>; +} + +#[derive(Debug, thiserror::Error)] +pub enum DiscloseRulesError { + #[error("Operation is not permitted")] + Unauthorized, + #[error("Incident with ID={0} not found")] + IncidentIdNotFound(IncidentId), + #[error("Rule with ID={0} not found")] + RuleIdNotFound(RuleId), + #[error(transparent)] + UnexpectedError(#[from] anyhow::Error), +} + +pub struct RulesDiscloser { + pub state: S, + pub access_resolver: A, +} + +impl RulesDiscloser { + pub fn new(state: S, access_resolver: A) -> Self { + Self { + state, + access_resolver, + } + } +} + +impl DisclosesRules for RulesDiscloser { + fn disclose_rules(&self, arg: DiscloseRulesArg) -> Result<(), DiscloseRulesError> { + if self.access_resolver.get_access_level() != AccessLevel::FullAccess { + return Err(DiscloseRulesError::Unauthorized); + } + match arg { + DiscloseRulesArg::RuleIds(rule_ids) => { + disclose_rules(&self.state, time(), &rule_ids)?; + } + DiscloseRulesArg::IncidentIds(incident_ids) => { + disclose_incidents(&self.state, time(), &incident_ids)?; + } + } + + Ok(()) + } +} + +fn disclose_rules( + repository: &impl Repository, + time: Timestamp, + rule_ids: &[RuleId], +) -> Result<(), DiscloseRulesError> { + let mut rules = Vec::with_capacity(rule_ids.len()); + + // Return the first error found while assembling metadata + for rule_id in rule_ids.iter() { + match repository.get_rule(rule_id) { + Some(rule_metadata) => { + rules.push((rule_id.clone(), rule_metadata)); + } + None => { + return Err(DiscloseRulesError::RuleIdNotFound(rule_id.to_string())); + } + } + } + + for (rule_id, mut metadata) in rules { + if metadata.disclosed_at.is_none() { + metadata.disclosed_at = Some(time); + assert!( + repository.update_rule(rule_id, metadata), + "rule id not found" + ); + } + } + + Ok(()) +} + +fn disclose_incidents( + repository: &impl Repository, + time: Timestamp, + incident_ids: &[IncidentId], +) -> Result<(), DiscloseRulesError> { + let mut incidents_metadata = Vec::with_capacity(incident_ids.len()); + + // Return the first error while assembling the metadata + for incident_id in incident_ids.iter() { + match repository.get_incident(incident_id) { + Some(incident_metadata) => { + incidents_metadata.push((incident_id.clone(), incident_metadata)); + } + None => { + return Err(DiscloseRulesError::IncidentIdNotFound( + incident_id.to_string(), + )); + } + } + } + + for (incident_id, mut metadata) in incidents_metadata { + if !metadata.is_disclosed { + disclose_rules(repository, time, &metadata.rule_ids)?; + metadata.is_disclosed = true; + let _ = repository.update_incident(incident_id, metadata); + } + } + + Ok(()) +} + +impl From for String { + fn from(value: DiscloseRulesError) -> Self { + value.to_string() + } +} diff --git a/rs/boundary_node/rate_limits/canister/fetcher.rs b/rs/boundary_node/rate_limits/canister/fetcher.rs new file mode 100644 index 00000000000..e7e2757b1ac --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/fetcher.rs @@ -0,0 +1,156 @@ +use crate::{ + confidentiality_formatting::ConfidentialityFormatting, + state::Repository, + types::{ConfigResponse, OutputConfig, OutputRule, OutputRuleMetadata, RuleId, Version}, +}; + +pub trait EntityFetcher { + type Input; + type Output; + type Error; + + fn fetch(&self, input: Self::Input) -> Result; +} +pub struct ConfigFetcher { + pub repository: R, + pub formatter: F, +} + +pub struct RuleFetcher { + pub repository: R, + pub formatter: F, +} + +impl RuleFetcher { + pub fn new(repository: R, formatter: F) -> Self { + Self { + repository, + formatter, + } + } +} + +impl ConfigFetcher { + pub fn new(repository: R, formatter: F) -> Self { + Self { + repository, + formatter, + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum FetchConfigError { + #[error("Config for version={0} not found")] + NotFound(Version), + #[error("No existing config versions")] + NoExistingVersions, + #[error(transparent)] + Unexpected(#[from] anyhow::Error), +} + +#[derive(Debug, thiserror::Error)] +pub enum FetchRuleError { + #[error("Rule with id={0} not found")] + NotFound(RuleId), + #[error(transparent)] + UnexpectedError(#[from] anyhow::Error), +} + +impl> EntityFetcher + for ConfigFetcher +{ + type Input = Option; + type Output = ConfigResponse; + type Error = FetchConfigError; + + fn fetch(&self, version: Option) -> Result { + let current_version = self + .repository + .get_version() + .ok_or_else(|| FetchConfigError::NoExistingVersions)?; + + let version = version.unwrap_or(current_version.0); + + let stored_config = self + .repository + .get_config(version) + .ok_or_else(|| FetchConfigError::NotFound(version))?; + + let mut rules: Vec = vec![]; + + for rule_id in stored_config.rule_ids.iter() { + let rule = self.repository.get_rule(rule_id).ok_or_else(|| { + FetchConfigError::Unexpected(anyhow::anyhow!("Rule with id = {rule_id} not found")) + })?; + + let output_rule = OutputRule { + id: rule_id.clone(), + rule_raw: Some(rule.rule_raw), + description: Some(rule.description), + disclosed_at: rule.disclosed_at, + }; + + rules.push(output_rule); + } + + let config = OutputConfig { + schema_version: stored_config.schema_version, + rules, + }; + + let formatted_config = self.formatter.format(&config).map_err(|_| { + anyhow::anyhow!("Failed to format config with confidentially constraints") + })?; + + let config = ConfigResponse { + version, + active_since: stored_config.active_since, + config: formatted_config, + }; + + Ok(config) + } +} + +impl> EntityFetcher + for RuleFetcher +{ + type Input = RuleId; + type Output = OutputRuleMetadata; + type Error = FetchRuleError; + + fn fetch(&self, rule_id: RuleId) -> Result { + let stored_metadata = self + .repository + .get_rule(&rule_id) + .ok_or_else(|| FetchRuleError::NotFound(rule_id.clone()))?; + + let rule_metadata = OutputRuleMetadata { + id: rule_id.clone(), + rule_raw: Some(stored_metadata.rule_raw), + description: Some(stored_metadata.description), + disclosed_at: stored_metadata.disclosed_at, + added_in_version: stored_metadata.added_in_version, + removed_in_version: stored_metadata.removed_in_version, + }; + + let formatted_rule = self.formatter.format(&rule_metadata).map_err(|_| { + anyhow::anyhow!("Failed to format rule with confidentially constraints") + })?; + + Ok(formatted_rule) + } +} + +impl From for String { + fn from(value: FetchConfigError) -> Self { + value.to_string() + } +} + +impl From for String { + fn from(value: FetchRuleError) -> Self { + value.to_string() + } +} diff --git a/rs/boundary_node/rate_limits/canister/interface.did b/rs/boundary_node/rate_limits/canister/interface.did new file mode 100644 index 00000000000..2a00a2aa025 --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/interface.did @@ -0,0 +1,104 @@ +type Version = nat64; // Represents the config version number +type Timestamp = nat64; // Represents timestamp in nanoseconds since the epoch (1970-01-01) +type RuleId = text; // Unique identifier for each rule +type SchemaVersion = nat64; // Version of the schema for encoding/decoding the rules +type IncidentId = text; // Unique identifier for each incident + + +// Input structure for defining a rule with mandatory fields within a config +type InputRule = record { + incident_id: IncidentId; // Identifier for the incident, to which the rule is related + rule_raw: blob; // Raw rule data (in binary format), expected to be a valid json object + description: text; // Textual description of the rule +}; + +// Output structure for rules +// Optional fields rule_raw and description may remain hidden while the rule is under confidentiality restrictions +type OutputRule = record { + rule_id: RuleId; // Unique identifier for the rule + rule_raw: opt blob; // Raw rule data (in binary format), expected to be a valid json object, none if the rule is currently confidential + description: opt text; // Textual description of the rule, none if the rule is currently confidential +}; + +type OutputConfig = record { + schema_version: SchemaVersion; // schema version needed to deserialize the rules + rules: vec OutputRule; +}; + +// Response structure for returning the requested configuration and associated metadata +type OutputConfigResponse = record { + version: Version; // Version of the configuration + active_since: Timestamp; // Time when this configuration was added (became active) + config: OutputConfig; // Contains the list of rules +}; + +// Verbose details of an individual rule +// Optional rule_raw and description fields are for restricted publicly viewing access +type OutputRuleMetadata = record { + rule_id: RuleId; // Unique identifier for the rule + incident_id: IncidentId; // Identifier for the incident, to which the rule is related + rule_raw: opt blob; // Raw rule data (binary format), expected to be a valid json object, none if the rule is currently confidential + description: opt text; // Textual description of the rule, none if the rule is currently confidential + disclosed_at: opt Timestamp; // Timestamp when the rule was disclosed, none if the rule is still confidential + added_in_version: Version; // Version when the rule was added (became active) + removed_in_version: opt Version; // Version when the rule was deactivated (removed), none if the rule is still active +}; + +type GetRuleByIdResponse = variant { + Ok: OutputRuleMetadata; + Err: text; +}; + +type GetConfigResponse = variant { + Ok: OutputConfigResponse; + Err: text; +}; + +type AddConfigResponse = variant { + Ok; + Err: text; +}; + +type DiscloseRulesResponse = variant { + Ok; + Err: text; +}; + +type DiscloseRulesArg = variant { + RuleIds: vec RuleId; + IncidentIds: vec IncidentId; +}; + +type GetRulesByIncidentIdResponse = variant { + Ok: vec RuleId; + Err: text; +}; + +// Configuration containing a list of rules that replaces the current configuration +type InputConfig = record { + schema_version: SchemaVersion; // schema version used to serialized the rules + rules: vec InputRule; +}; + +// Initialization arguments for the service +type InitArg = record { + registry_polling_period_secs: nat64; // IDs of existing API boundary nodes are polled from the registry with this periodicity +}; + +service : (InitArg) -> { + // Adds a configuration containing a set of rate-limit rules and increments the current version by one + add_config: (InputConfig) -> (AddConfigResponse); + + // Make the viewing of the specified rules publicly accessible + disclose_rules: (DiscloseRulesArg) -> (DiscloseRulesResponse); + + // Fetches the rate-limit rule configuration for a specified version + // If no version is provided, the latest configuration is returned + get_config: (opt Version) -> (GetConfigResponse) query; + + // Fetch the rule with metadata by its ID + get_rule_by_id: (RuleId) -> (GetRuleByIdResponse) query; + + // Fetch all rules IDs related to an ID of the incident + get_rules_by_incident_id: (IncidentId) -> (GetRulesByIncidentIdResponse) query; +} \ No newline at end of file diff --git a/rs/boundary_node/rate_limits/canister/state.rs b/rs/boundary_node/rate_limits/canister/state.rs new file mode 100644 index 00000000000..737c0e23554 --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/state.rs @@ -0,0 +1,147 @@ +use ic_cdk::api::time; +use rate_limits_api::IncidentId; + +use crate::{ + add_config::INIT_VERSION, + storage::{ + LocalRef, StableMap, StorableConfig, StorableIncidentId, StorableIncidentMetadata, + StorableRuleId, StorableRuleMetadata, StorableVersion, CONFIGS, INCIDENTS, RULES, + }, + types::{RuleId, Version}, +}; + +pub trait Repository { + fn get_version(&self) -> Option; + fn get_config(&self, version: Version) -> Option; + fn get_rule(&self, rule_id: &RuleId) -> Option; + fn get_incident(&self, incident_id: &IncidentId) -> Option; + fn add_config(&self, version: Version, config: StorableConfig) -> bool; + fn add_rule(&self, rule_id: RuleId, rule: StorableRuleMetadata) -> bool; + fn add_incident(&self, incident_id: IncidentId, rule_ids: StorableIncidentMetadata) -> bool; + fn update_rule(&self, rule_id: RuleId, rule: StorableRuleMetadata) -> bool; + fn update_incident(&self, incident_id: IncidentId, rule_ids: StorableIncidentMetadata) -> bool; +} + +pub struct State { + configs: LocalRef>, + rules: LocalRef>, + incidents: LocalRef>, +} + +impl State { + pub fn from_static() -> Self { + Self { + configs: &CONFIGS, + rules: &RULES, + incidents: &INCIDENTS, + } + } +} + +impl Repository for State { + fn get_version(&self) -> Option { + self.configs.with(|cell| { + let configs = cell.borrow(); + configs.last_key_value().map(|(key, _)| Some(key))? + }) + } + + fn get_config(&self, version: Version) -> Option { + self.configs + .with(|cell| cell.borrow().get(&StorableVersion(version))) + } + + fn get_rule(&self, rule_id: &RuleId) -> Option { + self.rules + .with(|cell| cell.borrow().get(&StorableRuleId(rule_id.clone()))) + } + + fn get_incident(&self, incident_id: &IncidentId) -> Option { + self.incidents + .with(|cell| cell.borrow().get(&StorableIncidentId(incident_id.clone()))) + } + + fn add_config(&self, version: Version, config: StorableConfig) -> bool { + self.get_config(version).map_or_else( + || { + self.configs.with(|cell| { + let mut configs = cell.borrow_mut(); + configs.insert(StorableVersion(version), config); + }); + true // Successfully inserted + }, + |_| false, // Already exists, return false + ) + } + + fn add_rule(&self, rule_id: RuleId, rule: StorableRuleMetadata) -> bool { + self.get_rule(&rule_id).map_or_else( + || { + self.rules.with(|cell| { + let mut rules = cell.borrow_mut(); + rules.insert(StorableRuleId(rule_id), rule); + }); + true // Successfully inserted + }, + |_| false, // Already exists, return false + ) + } + + fn add_incident(&self, incident_id: IncidentId, rule_ids: StorableIncidentMetadata) -> bool { + self.get_incident(&incident_id).map_or_else( + || { + self.incidents.with(|cell| { + let mut incidents = cell.borrow_mut(); + incidents.insert(StorableIncidentId(incident_id), rule_ids); + }); + true // Successfully inserted + }, + |_| false, // Already exists, return false + ) + } + + fn update_rule(&self, rule_id: RuleId, rule: StorableRuleMetadata) -> bool { + self.get_rule(&rule_id).map_or_else( + || false, // Rule doesn't exist, return false + |_| { + self.rules.with(|cell| { + let mut rules = cell.borrow_mut(); + rules.insert(StorableRuleId(rule_id), rule); + }); + true // Successfully updated + }, + ) + } + + fn update_incident(&self, incident_id: IncidentId, incident: StorableIncidentMetadata) -> bool { + self.get_incident(&incident_id).map_or_else( + || false, // Incident doesn't exist, return false + |_| { + self.incidents.with(|cell| { + let mut incidents = cell.borrow_mut(); + incidents.insert(StorableIncidentId(incident_id), incident); + }); + true // Successfully updated + }, + ) + } +} + +pub fn init_version_and_config(version: Version) { + with_state(|state| { + let config = StorableConfig { + schema_version: 1, + active_since: time(), + rule_ids: vec![], + }; + assert!( + state.add_config(INIT_VERSION, config), + "Config for version={version} already exists!" + ); + }) +} + +pub fn with_state(f: impl FnOnce(State) -> R) -> R { + let state = State::from_static(); + f(state) +} diff --git a/rs/boundary_node/rate_limits/canister/storage.rs b/rs/boundary_node/rate_limits/canister/storage.rs new file mode 100644 index 00000000000..23318a726f0 --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/storage.rs @@ -0,0 +1,186 @@ +use candid::Principal; +use ic_stable_structures::{ + memory_manager::{MemoryId, MemoryManager, VirtualMemory}, + storable::Bound, + DefaultMemoryImpl, StableBTreeMap, Storable, +}; +use rate_limits_api::SchemaVersion; +use serde::{Deserialize, Serialize}; +use std::{borrow::Cow, cell::RefCell, collections::HashSet, thread::LocalKey}; + +use crate::types::{IncidentId, RuleId, Timestamp, Version}; + +// Type aliases for stable memory +type Memory = VirtualMemory; +pub type LocalRef = &'static LocalKey>; +pub type StableMap = StableBTreeMap; + +// Memory IDs for stable memory management +const MEMORY_ID_CONFIGS: MemoryId = MemoryId::new(0); +const MEMORY_ID_RULES: MemoryId = MemoryId::new(1); +const MEMORY_ID_INCIDENTS: MemoryId = MemoryId::new(2); + +// Storables +#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq)] +pub struct StorableVersion(pub Version); + +#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq)] +pub struct StorableRuleId(pub String); + +#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq)] +pub struct StorableIncidentId(pub String); + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StorableRuleMetadata { + pub incident_id: IncidentId, + pub rule_raw: Vec, + pub description: String, + pub disclosed_at: Option, + pub added_in_version: Version, + pub removed_in_version: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StorableConfig { + pub schema_version: SchemaVersion, + pub active_since: Timestamp, + pub rule_ids: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StorableIncidentMetadata { + pub is_disclosed: bool, + pub rule_ids: Vec, +} + +impl Storable for StorableVersion { + fn to_bytes(&self) -> Cow<[u8]> { + Cow::Owned(bincode::serialize(&self.0).expect("StorableVersion serialization failed")) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + Self(bincode::deserialize(&bytes).expect("StorableVersion deserialization failed")) + } + + const BOUND: Bound = Bound::Bounded { + max_size: std::mem::size_of::() as u32, + is_fixed_size: true, + }; +} + +impl Storable for StorableRuleId { + fn to_bytes(&self) -> Cow<[u8]> { + Cow::Owned(bincode::serialize(&self.0).expect("StorableRuleId serialization failed")) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + Self(bincode::deserialize(&bytes).expect("StorableRuleId deserialization failed")) + } + + const BOUND: Bound = Bound::Bounded { + max_size: 256, + is_fixed_size: false, + }; +} + +impl Storable for StorableIncidentId { + fn to_bytes(&self) -> Cow<[u8]> { + Cow::Owned(bincode::serialize(&self.0).expect("StorableIncidentId serialization failed")) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + Self(bincode::deserialize(&bytes).expect("StorableIncidentId deserialization failed")) + } + + const BOUND: Bound = Bound::Bounded { + max_size: 256, + is_fixed_size: false, + }; +} + +impl Storable for StorableConfig { + fn to_bytes(&self) -> Cow<[u8]> { + Cow::Owned(bincode::serialize(self).expect("StorableConfig serialization failed")) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + bincode::deserialize(&bytes).expect("StorableConfig deserialization failed") + } + + // TODO: adjust these bounds + const BOUND: Bound = Bound::Bounded { + max_size: 1024, + is_fixed_size: false, + }; +} + +impl Storable for StorableIncidentMetadata { + fn to_bytes(&self) -> Cow<[u8]> { + Cow::Owned(bincode::serialize(self).expect("StorableIncidentMetadata serialization failed")) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + bincode::deserialize(&bytes).expect("StorableIncidentMetadata deserialization failed") + } + + // TODO: adjust these bounds + const BOUND: Bound = Bound::Bounded { + max_size: 2048, + is_fixed_size: false, + }; +} + +impl Storable for StorableRuleMetadata { + fn to_bytes(&self) -> Cow<[u8]> { + Cow::Owned(bincode::serialize(self).expect("StorableRuleMetadata serialization failed")) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + bincode::deserialize(&bytes).expect("StorableRuleMetadata deserialization failed") + } + + // TODO: adjust these bounds + const BOUND: Bound = Bound::Bounded { + max_size: 4096, + is_fixed_size: false, + }; +} + +// Declare storage variables +// NOTE: initialization is lazy +thread_local! { + static MEMORY_MANAGER: RefCell> = + RefCell::new(MemoryManager::init(DefaultMemoryImpl::default())); + + pub static CONFIGS: RefCell> = RefCell::new( + StableMap::init( + MEMORY_MANAGER.with(|m| m.borrow().get(MEMORY_ID_CONFIGS)), + ) + ); + + pub static RULES: RefCell> = RefCell::new( + StableMap::init( + MEMORY_MANAGER.with(|m| m.borrow().get(MEMORY_ID_RULES)), + ) + ); + + pub static INCIDENTS: RefCell> = RefCell::new( + StableMap::init( + MEMORY_MANAGER.with(|m| m.borrow().get(MEMORY_ID_INCIDENTS)), + ) + ); + + pub static API_BOUNDARY_NODE_PRINCIPALS: RefCell> = RefCell::new(HashSet::new()); +} + +impl From for StorableVersion { + fn from(version: Version) -> Self { + StorableVersion(version) + } +} + +impl From for StorableRuleId { + fn from(rule_id: RuleId) -> Self { + StorableRuleId(rule_id) + } +} diff --git a/rs/boundary_node/rate_limits/canister/types.rs b/rs/boundary_node/rate_limits/canister/types.rs new file mode 100644 index 00000000000..f2717b6baa2 --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/types.rs @@ -0,0 +1,126 @@ +pub type Version = u64; +pub type Timestamp = u64; +pub type SchemaVersion = u64; +pub type RuleId = String; +pub type IncidentId = String; + +pub enum DiscloseRulesArg { + RuleIds(Vec), + IncidentIds(Vec), +} + +pub struct ConfigResponse { + pub version: Version, + pub active_since: Timestamp, + pub config: OutputConfig, +} + +#[derive(Clone)] +pub struct OutputConfig { + pub schema_version: SchemaVersion, + pub rules: Vec, +} + +pub struct InputConfig { + pub schema_version: SchemaVersion, + pub rules: Vec, +} + +pub struct InputRule { + pub incident_id: IncidentId, + pub rule_raw: Vec, + pub description: String, +} + +#[derive(Clone)] +pub struct OutputRule { + pub id: RuleId, + pub rule_raw: Option>, + pub description: Option, + pub disclosed_at: Option, +} + +impl From for InputRule { + fn from(value: rate_limits_api::InputRule) -> Self { + InputRule { + incident_id: value.incident_id, + description: value.description, + rule_raw: value.rule_raw, + } + } +} + +impl From for rate_limits_api::OutputRule { + fn from(value: OutputRule) -> Self { + rate_limits_api::OutputRule { + description: value.description, + disclosed_at: value.disclosed_at, + id: value.id, + rule_raw: value.rule_raw, + } + } +} + +impl From for InputConfig { + fn from(value: rate_limits_api::InputConfig) -> Self { + InputConfig { + schema_version: value.schema_version, + rules: value.rules.into_iter().map(|r| r.into()).collect(), + } + } +} + +impl From for rate_limits_api::OutputConfig { + fn from(value: OutputConfig) -> Self { + rate_limits_api::OutputConfig { + schema_version: value.schema_version, + rules: value.rules.into_iter().map(|r| r.into()).collect(), + } + } +} + +impl From for rate_limits_api::ConfigResponse { + fn from(value: ConfigResponse) -> Self { + rate_limits_api::ConfigResponse { + version: value.version, + active_since: value.active_since, + config: value.config.into(), + } + } +} + +#[derive(Clone)] +pub struct OutputRuleMetadata { + pub id: RuleId, + pub rule_raw: Option>, + pub description: Option, + pub disclosed_at: Option, + pub added_in_version: Version, + pub removed_in_version: Option, +} + +impl From for rate_limits_api::OutputRuleMetadata { + fn from(value: OutputRuleMetadata) -> Self { + rate_limits_api::OutputRuleMetadata { + id: value.id, + rule_raw: value.rule_raw, + description: value.description, + disclosed_at: value.disclosed_at, + added_in_version: value.added_in_version, + removed_in_version: value.removed_in_version, + } + } +} + +impl From for DiscloseRulesArg { + fn from(value: rate_limits_api::DiscloseRulesArg) -> Self { + match value { + rate_limits_api::DiscloseRulesArg::RuleIds(rule_ids) => { + DiscloseRulesArg::RuleIds(rule_ids) + } + rate_limits_api::DiscloseRulesArg::IncidentIds(incident_ids) => { + DiscloseRulesArg::IncidentIds(incident_ids) + } + } + } +} diff --git a/rs/boundary_node/rate_limits/canister_client/Cargo.toml b/rs/boundary_node/rate_limits/canister_client/Cargo.toml new file mode 100644 index 00000000000..77fd77c5784 --- /dev/null +++ b/rs/boundary_node/rate_limits/canister_client/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "canister_client" +version.workspace = true +authors.workspace = true +edition.workspace = true +description.workspace = true +documentation.workspace = true + +[dependencies] +k256 = { version = "0.13.1", features = ["pem"] } +rate-limits-api = { path = "../api" } +ic-agent = { workspace = true } +tokio = { workspace = true } +candid = { workspace = true } \ No newline at end of file diff --git a/rs/boundary_node/rate_limits/canister_client/src/main.rs b/rs/boundary_node/rate_limits/canister_client/src/main.rs new file mode 100644 index 00000000000..205023dc2ff --- /dev/null +++ b/rs/boundary_node/rate_limits/canister_client/src/main.rs @@ -0,0 +1,169 @@ +use candid::{Decode, Encode, Principal}; +use ic_agent::{ + identity::{AnonymousIdentity, Secp256k1Identity}, + Agent, +}; +use rate_limits_api::{ + AddConfigResponse, DiscloseRulesArg, DiscloseRulesResponse, GetConfigResponse, + GetRuleByIdResponse, InputConfig, InputRule, +}; + +const RATE_LIMIT_CANISTER_ID: &str = "ud6i4-iaaaa-aaaab-qadiq-cai"; +const IC_DOMAIN: &str = "https://ic0.app"; + +use k256::elliptic_curve::SecretKey; + +const TEST_PRIVATE_KEY: &str = "-----BEGIN EC PRIVATE KEY----- +MHQCAQEEIIBzyyJ32Kdjixx+ZJvNeUWsqAzSQZfLsOyXKgxc7aH9oAcGBSuBBAAK +oUQDQgAECWc6ZRn9bBP96RM1G6h8ZAtbryO65dKg6cw0Oij2XbnAlb6zSPhU+4hh +gc2Q0JiGrqKks1AVi+8wzmZ+2PQXXA== +-----END EC PRIVATE KEY-----"; + +#[tokio::main] +async fn main() { + let agent_authorized = Agent::builder() + .with_url(IC_DOMAIN) + .with_identity(AnonymousIdentity {}) + .build() + .expect("failed to build the agent"); + agent_authorized.fetch_root_key().await.unwrap(); + + let mut agent_unauthorized = Agent::builder() + .with_url(IC_DOMAIN) + .build() + .expect("failed to build the agent"); + agent_unauthorized.set_identity(Secp256k1Identity::from_private_key( + SecretKey::from_sec1_pem(TEST_PRIVATE_KEY).unwrap(), + )); + agent_unauthorized.fetch_root_key().await.unwrap(); + + let canister_id = Principal::from_text(RATE_LIMIT_CANISTER_ID).unwrap(); + + // Call 1: overwrite_config by authorized + let args = Encode!(&InputConfig { + schema_version: 1, + rules: vec![ + InputRule { + incident_id: "id1".to_string(), + rule_raw: b"{\"canister_id\": 3}".to_vec(), + description: "canister rate-limit".to_string(), + }, + InputRule { + incident_id: "id1".to_string(), + rule_raw: b"{\"subnet_id\": 2}".to_vec(), + description: "subnet rate-limit".to_string(), + }, + InputRule { + incident_id: "id3".to_string(), + rule_raw: b"{\"subnet_id\": 3}".to_vec(), + description: "another subnet rate-limit".to_string(), + }, + InputRule { + incident_id: "id6".to_string(), + rule_raw: b"{\"subnet_id\": 34}".to_vec(), + description: "another subnet rate-limit".to_string(), + }, + ], + }) + .unwrap(); + + let result = agent_authorized + .update(&canister_id, "add_config") + .with_arg(args) + .call_and_wait() + .await + .unwrap(); + + let decoded = Decode!(&result, AddConfigResponse).unwrap(); + + println!("add_config response: {decoded:#?}"); + + // Call 2: get_config by unauthorized user + let version = 2u64; + let args = Encode!(&Some(version)).unwrap(); + + let response = agent_unauthorized + .update(&canister_id, "get_config") + .with_arg(args) + .call_and_wait() + .await + .expect("update call failed"); + + let decoded = Decode!(&response, GetConfigResponse).expect("failed to decode candid response"); + + println!("get_config response: {decoded:#?}"); + + // Call 3: get_rule_by_id unauthorized + let rule_id = "d2f84ec0331266ff19cf0c889b03794232905d39eaff88504ac47939890c8d38".to_string(); + let args = Encode!(&rule_id).unwrap(); + + let response = agent_unauthorized + .query(&canister_id, "get_rule_by_id") + .with_arg(args) + .call() + .await + .expect("update call failed"); + + let decoded = Decode!(&response, GetRuleByIdResponse).unwrap(); + + println!("get_rule_by_id response: {decoded:#?}"); + + // Call 4: disclose_rules by authorized + let disclose_arg = DiscloseRulesArg::RuleIds(vec![rule_id.clone()]); + let args = Encode!(&disclose_arg).unwrap(); + + let response = agent_authorized + .update(&canister_id, "disclose_rules") + .with_arg(args) + .call_and_wait() + .await + .expect("update call failed"); + + let decoded = Decode!(&response, DiscloseRulesResponse).unwrap(); + + println!("disclose_rules response: {decoded:#?}"); + + // Call 5: get_rule_by_id after disclose() for unauthorized + let args = Encode!(&rule_id).unwrap(); + + let response = agent_unauthorized + .query(&canister_id, "get_rule_by_id") + .with_arg(args) + .call() + .await + .expect("update call failed"); + + let decoded = Decode!(&response, GetRuleByIdResponse).unwrap(); + + println!("get_rule_by_id response: {decoded:#?}"); + + // Call 6: disclose_rules by authorized + let disclose_arg = DiscloseRulesArg::IncidentIds(vec!["id2".to_string(), "id3".to_string()]); + let args = Encode!(&disclose_arg).unwrap(); + + let response = agent_authorized + .update(&canister_id, "disclose_rules") + .with_arg(args) + .call_and_wait() + .await + .expect("update call failed"); + + let decoded = Decode!(&response, DiscloseRulesResponse).unwrap(); + + println!("disclose_rules response: {decoded:#?}"); + + // Call 7: get_config by unauthorized user + let version = 2u64; + let args = Encode!(&Some(version)).unwrap(); + + let response = agent_unauthorized + .update(&canister_id, "get_config") + .with_arg(args) + .call_and_wait() + .await + .expect("update call failed"); + + let decoded = Decode!(&response, GetConfigResponse).expect("failed to decode candid response"); + + println!("get_config response: {decoded:#?}"); +} diff --git a/rs/boundary_node/rate_limits/dfx.json b/rs/boundary_node/rate_limits/dfx.json new file mode 100644 index 00000000000..8c7477ff548 --- /dev/null +++ b/rs/boundary_node/rate_limits/dfx.json @@ -0,0 +1,21 @@ +{ + "version": 1, + "canisters": { + "rate_limit_canister": { + "type": "rust", + "package": "rate_limits", + "candid": "canister/interface.did" + } + }, + "defaults": { + "build": { + "packtool": "" + } + }, + "networks": { + "local": { + "bind": "127.0.0.1:8080", + "type": "ephemeral" + } + } + } \ No newline at end of file From 5127eb53f5bf7bffc52525a7a1a2dee0e370f54f Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Sun, 20 Oct 2024 13:27:33 +0000 Subject: [PATCH 03/40] chore: cargo --- Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index f7ca6f8360b..2c62634297d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,9 @@ members = [ "rs/boundary_node/certificate_issuance/certificate_orchestrator", "rs/boundary_node/discower_bowndary", "rs/boundary_node/ic_boundary", + "rs/boundary_node/rate_limits", + "rs/boundary_node/rate_limits/api", + "rs/boundary_node/rate_limits/canister_client", "rs/boundary_node/systemd_journal_gatewayd_shim", "rs/canister_client", "rs/canister_client/sender", @@ -351,6 +354,7 @@ members = [ "rs/test_utilities/tmpdir", "rs/tests", "rs/tests/boundary_nodes", + "rs/tests/boundary_nodes/custom_domains", "rs/tests/boundary_nodes/integration_test_common", "rs/tests/boundary_nodes/performance_test_common", "rs/tests/boundary_nodes/utils", From ecf03796ee3d8c5c728341d53f7807370f26aff0 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Sun, 20 Oct 2024 13:30:29 +0000 Subject: [PATCH 04/40] chore: fix cargo --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2c62634297d..eeacfe3a871 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -354,7 +354,6 @@ members = [ "rs/test_utilities/tmpdir", "rs/tests", "rs/tests/boundary_nodes", - "rs/tests/boundary_nodes/custom_domains", "rs/tests/boundary_nodes/integration_test_common", "rs/tests/boundary_nodes/performance_test_common", "rs/tests/boundary_nodes/utils", From 0e6d348878ad7f7ee18bbecd03beb7c64bded30d Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Sun, 20 Oct 2024 13:31:53 +0000 Subject: [PATCH 05/40] Automatically updated Cargo*.lock --- Cargo.lock | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f2a8523d912..e3ff8c2601c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1759,6 +1759,17 @@ dependencies = [ "wat", ] +[[package]] +name = "canister_client" +version = "0.9.0" +dependencies = [ + "candid", + "ic-agent", + "k256", + "rate-limits-api", + "tokio", +] + [[package]] name = "canister_http" version = "0.9.0" @@ -17543,6 +17554,33 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60fcc7d6849342eff22c4350c8b9a989ee8ceabc4b481253e8946b9fe83d684" +[[package]] +name = "rate-limits-api" +version = "0.9.0" +dependencies = [ + "candid", + "serde", +] + +[[package]] +name = "rate_limits" +version = "0.9.0" +dependencies = [ + "anyhow", + "bincode", + "candid", + "hex", + "ic-cdk 0.16.0", + "ic-cdk-macros 0.9.0", + "ic-cdk-timers", + "ic-stable-structures", + "rate-limits-api", + "serde", + "serde_json", + "sha2 0.10.8", + "thiserror", +] + [[package]] name = "ratelimit" version = "0.9.1" From 51f7d8ea1e0edbede805be72396d7454aea5515d Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Sun, 20 Oct 2024 18:32:12 +0000 Subject: [PATCH 06/40] chore: interface.did --- .../rate_limit_canister/interface.did | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/rs/boundary_node/rate_limit_canister/interface.did b/rs/boundary_node/rate_limit_canister/interface.did index cf651e9755c..2a00a2aa025 100644 --- a/rs/boundary_node/rate_limit_canister/interface.did +++ b/rs/boundary_node/rate_limit_canister/interface.did @@ -1,44 +1,45 @@ -type Version = nat64; // Represents the config version number -type Timestamp = nat64; // Represents timestamp in nanoseconds since the epoch (1970-01-01) -type RuleId = text; // Unique identifier for each rule +type Version = nat64; // Represents the config version number +type Timestamp = nat64; // Represents timestamp in nanoseconds since the epoch (1970-01-01) +type RuleId = text; // Unique identifier for each rule +type SchemaVersion = nat64; // Version of the schema for encoding/decoding the rules +type IncidentId = text; // Unique identifier for each incident // Input structure for defining a rule with mandatory fields within a config type InputRule = record { - id: RuleId; // Unique identifier for the rule - rule_raw: blob; // Raw rule data (in binary format), expected to be a valid json object - description: text; // Textual description of the rule + incident_id: IncidentId; // Identifier for the incident, to which the rule is related + rule_raw: blob; // Raw rule data (in binary format), expected to be a valid json object + description: text; // Textual description of the rule }; // Output structure for rules // Optional fields rule_raw and description may remain hidden while the rule is under confidentiality restrictions type OutputRule = record { - id: RuleId; // Unique identifier for the rule + rule_id: RuleId; // Unique identifier for the rule rule_raw: opt blob; // Raw rule data (in binary format), expected to be a valid json object, none if the rule is currently confidential description: opt text; // Textual description of the rule, none if the rule is currently confidential - disclosed_at: opt Timestamp; // Timestamp when the rule was disclosed, none if the rule is still confidential - disclosed_in_version: opt Version; // Version in which the rule was disclosed, none if the rule is still confidential }; type OutputConfig = record { + schema_version: SchemaVersion; // schema version needed to deserialize the rules rules: vec OutputRule; }; // Response structure for returning the requested configuration and associated metadata type OutputConfigResponse = record { version: Version; // Version of the configuration - active_since: Timestamp; // Time when this configuration became added (became active) + active_since: Timestamp; // Time when this configuration was added (became active) config: OutputConfig; // Contains the list of rules }; // Verbose details of an individual rule // Optional rule_raw and description fields are for restricted publicly viewing access type OutputRuleMetadata = record { - id: RuleId; // Unique identifier for the rule + rule_id: RuleId; // Unique identifier for the rule + incident_id: IncidentId; // Identifier for the incident, to which the rule is related rule_raw: opt blob; // Raw rule data (binary format), expected to be a valid json object, none if the rule is currently confidential description: opt text; // Textual description of the rule, none if the rule is currently confidential disclosed_at: opt Timestamp; // Timestamp when the rule was disclosed, none if the rule is still confidential - disclosed_in_version: Version; // Version when the rule was disclosed, none if the rule is still confidential added_in_version: Version; // Version when the rule was added (became active) removed_in_version: opt Version; // Version when the rule was deactivated (removed), none if the rule is still active }; @@ -46,26 +47,37 @@ type OutputRuleMetadata = record { type GetRuleByIdResponse = variant { Ok: OutputRuleMetadata; Err: text; -} +}; type GetConfigResponse = variant { Ok: OutputConfigResponse; Err: text; }; -type DiscloseRulesResponse = variant { +type AddConfigResponse = variant { Ok; Err: text; }; -type OverwriteConfigResponse = variant { +type DiscloseRulesResponse = variant { Ok; Err: text; }; +type DiscloseRulesArg = variant { + RuleIds: vec RuleId; + IncidentIds: vec IncidentId; +}; + +type GetRulesByIncidentIdResponse = variant { + Ok: vec RuleId; + Err: text; +}; + // Configuration containing a list of rules that replaces the current configuration type InputConfig = record { - rules: vec InputRule; + schema_version: SchemaVersion; // schema version used to serialized the rules + rules: vec InputRule; }; // Initialization arguments for the service @@ -74,16 +86,19 @@ type InitArg = record { }; service : (InitArg) -> { - // Replaces the current rate-limit configuration with a new set of rules - overwrite_config: (InputConfig) -> (OverwriteConfigResponse); + // Adds a configuration containing a set of rate-limit rules and increments the current version by one + add_config: (InputConfig) -> (AddConfigResponse); // Make the viewing of the specified rules publicly accessible - disclose_rules: (vec RuleId) -> (DiscloseRulesResponse); + disclose_rules: (DiscloseRulesArg) -> (DiscloseRulesResponse); // Fetches the rate-limit rule configuration for a specified version // If no version is provided, the latest configuration is returned - get_config: (opt Version) -> (GetConfigResponse); + get_config: (opt Version) -> (GetConfigResponse) query; // Fetch the rule with metadata by its ID - get_rule_by_id: (RuleId) -> (GetRuleByIdResponse); + get_rule_by_id: (RuleId) -> (GetRuleByIdResponse) query; + + // Fetch all rules IDs related to an ID of the incident + get_rules_by_incident_id: (IncidentId) -> (GetRulesByIncidentIdResponse) query; } \ No newline at end of file From c2515211241988aed7f225e2def7b92f0f85a78d Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Sun, 20 Oct 2024 18:55:26 +0000 Subject: [PATCH 07/40] chore: improve description --- .../rate_limits/canister/add_config.rs | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/add_config.rs b/rs/boundary_node/rate_limits/canister/add_config.rs index 9c06aa2c71a..a9954421985 100644 --- a/rs/boundary_node/rate_limits/canister/add_config.rs +++ b/rs/boundary_node/rate_limits/canister/add_config.rs @@ -74,16 +74,22 @@ impl ConfigAdder { } // Definitions: -// - Each rule has two IDs - rule_id and incident_id (to which a rule is linked) -// - A rule is uniquely identified by rule_id, which is autogenerated by hashing two fields `rule_raw` + `description` (JSON decoded `rule_raw` is used for hashing) -// - incident_id must be provided for each rule by the caller; multiple rules can be linked to one incident_id -// - The same rule (i.e. `rule_raw` + `description`) can be persisted (resubmitted) from one config version to next ones -// If a rule is not resubmitted to the next config, it is marked as "deactivated" (`removed_in_version` field in `StorableRuleMetadata` is set to Some()) -// - Individual rules or incidents (vector holding multiple rule IDs) can be disclosed -// - Disclosing individual rules (or incidents) for the second time has no effect further effect +// - A rate-limit config is an ordered set of rate-limit rules: config = [rule_1, rule_2, ..., rule_N]. +// - Rule order within a config is significant, as rules are applied in the order they appear in the config. +// - Adding a new config requires providing the entire list of rules and increments the version by one. +// - Each rule has two IDs: rule_id (unique identifier) and incident_id (linking the rule to an incident). +// - rule_id is autogenerated by hashing JSON-decoded `rule_raw` + `description`. +// - Each rule must be linked to some incident_id, this ID must be provided for each input rule be the caller; multiple rules can be linked to one incident_id. +// - Rules can persist across config versions if resubmitted. +// - Non-resubmitted rules are marked as "deactivated" (StorableRuleMetadata.removed_in_version = Some()). +// - Individual rules or incidents (set of rules with the same incident_id) can be disclosed. This means that a rule can be viewed by the callers with `RestrictedRead` access level. +// - Disclosing rules or incidents multiple times has no additional effect. + // Policies: -// - Deactivated rules can't be resubmitted again (DeactivatedRuleResubmission error) -// - Newly added rules can't be linked to an already disclosed incident (LinkingRuleToDisclosedIncident error) +// - Existing rules cannot be modified; a new rule must be created for changes +// - Deactivated rules cannot be resubmitted (DeactivatedRuleResubmission error) +// - New rules cannot be linked to an already disclosed incident (LinkingRuleToDisclosedIncident error) + impl AddsConfig for ConfigAdder { fn add_config(&self, config: InputConfig) -> Result<(), AddConfigError> { From 32f32d9e263f99a92e14f86bcbc96b73d387e5c1 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 07:33:47 +0000 Subject: [PATCH 08/40] fix: remove redundant interface --- .../rate_limit_canister/interface.did | 104 ------------------ .../rate_limits/canister/add_config.rs | 1 - 2 files changed, 105 deletions(-) delete mode 100644 rs/boundary_node/rate_limit_canister/interface.did diff --git a/rs/boundary_node/rate_limit_canister/interface.did b/rs/boundary_node/rate_limit_canister/interface.did deleted file mode 100644 index 2a00a2aa025..00000000000 --- a/rs/boundary_node/rate_limit_canister/interface.did +++ /dev/null @@ -1,104 +0,0 @@ -type Version = nat64; // Represents the config version number -type Timestamp = nat64; // Represents timestamp in nanoseconds since the epoch (1970-01-01) -type RuleId = text; // Unique identifier for each rule -type SchemaVersion = nat64; // Version of the schema for encoding/decoding the rules -type IncidentId = text; // Unique identifier for each incident - - -// Input structure for defining a rule with mandatory fields within a config -type InputRule = record { - incident_id: IncidentId; // Identifier for the incident, to which the rule is related - rule_raw: blob; // Raw rule data (in binary format), expected to be a valid json object - description: text; // Textual description of the rule -}; - -// Output structure for rules -// Optional fields rule_raw and description may remain hidden while the rule is under confidentiality restrictions -type OutputRule = record { - rule_id: RuleId; // Unique identifier for the rule - rule_raw: opt blob; // Raw rule data (in binary format), expected to be a valid json object, none if the rule is currently confidential - description: opt text; // Textual description of the rule, none if the rule is currently confidential -}; - -type OutputConfig = record { - schema_version: SchemaVersion; // schema version needed to deserialize the rules - rules: vec OutputRule; -}; - -// Response structure for returning the requested configuration and associated metadata -type OutputConfigResponse = record { - version: Version; // Version of the configuration - active_since: Timestamp; // Time when this configuration was added (became active) - config: OutputConfig; // Contains the list of rules -}; - -// Verbose details of an individual rule -// Optional rule_raw and description fields are for restricted publicly viewing access -type OutputRuleMetadata = record { - rule_id: RuleId; // Unique identifier for the rule - incident_id: IncidentId; // Identifier for the incident, to which the rule is related - rule_raw: opt blob; // Raw rule data (binary format), expected to be a valid json object, none if the rule is currently confidential - description: opt text; // Textual description of the rule, none if the rule is currently confidential - disclosed_at: opt Timestamp; // Timestamp when the rule was disclosed, none if the rule is still confidential - added_in_version: Version; // Version when the rule was added (became active) - removed_in_version: opt Version; // Version when the rule was deactivated (removed), none if the rule is still active -}; - -type GetRuleByIdResponse = variant { - Ok: OutputRuleMetadata; - Err: text; -}; - -type GetConfigResponse = variant { - Ok: OutputConfigResponse; - Err: text; -}; - -type AddConfigResponse = variant { - Ok; - Err: text; -}; - -type DiscloseRulesResponse = variant { - Ok; - Err: text; -}; - -type DiscloseRulesArg = variant { - RuleIds: vec RuleId; - IncidentIds: vec IncidentId; -}; - -type GetRulesByIncidentIdResponse = variant { - Ok: vec RuleId; - Err: text; -}; - -// Configuration containing a list of rules that replaces the current configuration -type InputConfig = record { - schema_version: SchemaVersion; // schema version used to serialized the rules - rules: vec InputRule; -}; - -// Initialization arguments for the service -type InitArg = record { - registry_polling_period_secs: nat64; // IDs of existing API boundary nodes are polled from the registry with this periodicity -}; - -service : (InitArg) -> { - // Adds a configuration containing a set of rate-limit rules and increments the current version by one - add_config: (InputConfig) -> (AddConfigResponse); - - // Make the viewing of the specified rules publicly accessible - disclose_rules: (DiscloseRulesArg) -> (DiscloseRulesResponse); - - // Fetches the rate-limit rule configuration for a specified version - // If no version is provided, the latest configuration is returned - get_config: (opt Version) -> (GetConfigResponse) query; - - // Fetch the rule with metadata by its ID - get_rule_by_id: (RuleId) -> (GetRuleByIdResponse) query; - - // Fetch all rules IDs related to an ID of the incident - get_rules_by_incident_id: (IncidentId) -> (GetRulesByIncidentIdResponse) query; -} \ No newline at end of file diff --git a/rs/boundary_node/rate_limits/canister/add_config.rs b/rs/boundary_node/rate_limits/canister/add_config.rs index a9954421985..5eedc698e76 100644 --- a/rs/boundary_node/rate_limits/canister/add_config.rs +++ b/rs/boundary_node/rate_limits/canister/add_config.rs @@ -90,7 +90,6 @@ impl ConfigAdder { // - Deactivated rules cannot be resubmitted (DeactivatedRuleResubmission error) // - New rules cannot be linked to an already disclosed incident (LinkingRuleToDisclosedIncident error) - impl AddsConfig for ConfigAdder { fn add_config(&self, config: InputConfig) -> Result<(), AddConfigError> { // Only privileged users can perform this operation From ce21765817f45e2ea3239f81c875b9718e0f8f57 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 08:17:42 +0000 Subject: [PATCH 09/40] fix: RestrictedRead formatting --- .../canister/confidentiality_formatting.rs | 36 ++++++++----------- .../rate_limits/canister/fetcher.rs | 4 +-- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index e84d5ec4234..8b134fb35ce 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -40,19 +40,15 @@ impl ConfidentialityFormatting fn format(&self, config: &OutputConfig) -> Result { let mut config = config.clone(); - match self.access_resolver.get_access_level() { - AccessLevel::RestrictedRead => { - config.rules.iter_mut().for_each(|rule| { - if rule.disclosed_at.is_none() { - rule.description = None; - rule.rule_raw = None; - } - }); - Ok(config) - } - AccessLevel::FullRead => Ok(config), - _ => Err(ConfidentialFormatterError::AccessDenied), + if self.access_resolver.get_access_level() == AccessLevel::RestrictedRead { + config.rules.iter_mut().for_each(|rule| { + if rule.disclosed_at.is_none() { + rule.description = None; + rule.rule_raw = None; + } + }); } + Ok(config) } } @@ -66,17 +62,13 @@ impl ConfidentialityFormatting rule: &OutputRuleMetadata, ) -> Result { let mut rule = rule.clone(); - match self.access_resolver.get_access_level() { - AccessLevel::RestrictedRead => { - if rule.disclosed_at.is_none() { - rule.description = None; - rule.rule_raw = None; - } - Ok(rule) - } - AccessLevel::FullRead => Ok(rule), - _ => Err(ConfidentialFormatterError::AccessDenied), + if self.access_resolver.get_access_level() == AccessLevel::RestrictedRead + && rule.disclosed_at.is_none() + { + rule.description = None; + rule.rule_raw = None; } + Ok(rule) } } diff --git a/rs/boundary_node/rate_limits/canister/fetcher.rs b/rs/boundary_node/rate_limits/canister/fetcher.rs index e7e2757b1ac..53079df2caa 100644 --- a/rs/boundary_node/rate_limits/canister/fetcher.rs +++ b/rs/boundary_node/rate_limits/canister/fetcher.rs @@ -99,8 +99,8 @@ impl> EntityFe rules, }; - let formatted_config = self.formatter.format(&config).map_err(|_| { - anyhow::anyhow!("Failed to format config with confidentially constraints") + let formatted_config = self.formatter.format(&config).map_err(|err| { + anyhow::anyhow!("Failed to format config with confidentially constraints: {err}") })?; let config = ConfigResponse { From 851f8d82dcd0557938082bef702e82ee1e1ad18b Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 09:28:45 +0000 Subject: [PATCH 10/40] chore: add incident_id --- rs/boundary_node/rate_limits/api/src/lib.rs | 2 +- rs/boundary_node/rate_limits/canister/access_control.rs | 6 +++--- .../rate_limits/canister/confidentiality_formatting.rs | 2 +- rs/boundary_node/rate_limits/canister/fetcher.rs | 1 + rs/boundary_node/rate_limits/canister/interface.did | 1 + rs/boundary_node/rate_limits/canister/types.rs | 3 ++- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/rs/boundary_node/rate_limits/api/src/lib.rs b/rs/boundary_node/rate_limits/api/src/lib.rs index 72173b16947..4e5a3c3bdd9 100644 --- a/rs/boundary_node/rate_limits/api/src/lib.rs +++ b/rs/boundary_node/rate_limits/api/src/lib.rs @@ -47,9 +47,9 @@ pub struct InputRule { #[derive(CandidType, Deserialize, Debug)] pub struct OutputRule { pub id: RuleId, + pub incident_id: IncidentId, pub rule_raw: Option>, pub description: Option, - pub disclosed_at: Option, } #[derive(CandidType, Deserialize, Debug)] diff --git a/rs/boundary_node/rate_limits/canister/access_control.rs b/rs/boundary_node/rate_limits/canister/access_control.rs index 358d099d915..3fd61d318f7 100644 --- a/rs/boundary_node/rate_limits/canister/access_control.rs +++ b/rs/boundary_node/rate_limits/canister/access_control.rs @@ -2,8 +2,8 @@ use candid::Principal; use crate::storage::API_BOUNDARY_NODE_PRINCIPALS; -const FULL_ACCESS_ID: &str = "2vxsx-fae"; -const FULL_READ_ID: &str = "2vxsx-fae"; +const FULL_ACCESS_ID: &str = ""; +const FULL_READ_TESTING_ID: &str = ""; // TODO: remove this pub trait ResolveAccessLevel { fn get_access_level(&self) -> AccessLevel; @@ -37,7 +37,7 @@ impl ResolveAccessLevel for AccessLevelResolver { API_BOUNDARY_NODE_PRINCIPALS.with(|cell| { let mut full_read_principals = cell.borrow_mut(); // TODO: this is just for testing, remove later - let full_read_id = Principal::from_text(FULL_READ_ID).unwrap(); + let full_read_id = Principal::from_text(FULL_READ_TESTING_ID).unwrap(); let _ = full_read_principals.insert(full_read_id); if self.caller_id == full_access_principal { diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index 8b134fb35ce..8283d0c744b 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -15,7 +15,7 @@ pub trait ConfidentialityFormatting { #[derive(Debug, thiserror::Error)] pub enum ConfidentialFormatterError { #[error("Access denied")] - AccessDenied, + _AccessDenied, } /// A generic confidentiality formatter for various data types diff --git a/rs/boundary_node/rate_limits/canister/fetcher.rs b/rs/boundary_node/rate_limits/canister/fetcher.rs index 53079df2caa..d476243edae 100644 --- a/rs/boundary_node/rate_limits/canister/fetcher.rs +++ b/rs/boundary_node/rate_limits/canister/fetcher.rs @@ -86,6 +86,7 @@ impl> EntityFe let output_rule = OutputRule { id: rule_id.clone(), + incident_id: rule.incident_id, rule_raw: Some(rule.rule_raw), description: Some(rule.description), disclosed_at: rule.disclosed_at, diff --git a/rs/boundary_node/rate_limits/canister/interface.did b/rs/boundary_node/rate_limits/canister/interface.did index 2a00a2aa025..0471faef606 100644 --- a/rs/boundary_node/rate_limits/canister/interface.did +++ b/rs/boundary_node/rate_limits/canister/interface.did @@ -16,6 +16,7 @@ type InputRule = record { // Optional fields rule_raw and description may remain hidden while the rule is under confidentiality restrictions type OutputRule = record { rule_id: RuleId; // Unique identifier for the rule + incident_id: IncidentId; // Identifier for the incident, to which the rule is related rule_raw: opt blob; // Raw rule data (in binary format), expected to be a valid json object, none if the rule is currently confidential description: opt text; // Textual description of the rule, none if the rule is currently confidential }; diff --git a/rs/boundary_node/rate_limits/canister/types.rs b/rs/boundary_node/rate_limits/canister/types.rs index f2717b6baa2..660195240bc 100644 --- a/rs/boundary_node/rate_limits/canister/types.rs +++ b/rs/boundary_node/rate_limits/canister/types.rs @@ -35,6 +35,7 @@ pub struct InputRule { #[derive(Clone)] pub struct OutputRule { pub id: RuleId, + pub incident_id: IncidentId, pub rule_raw: Option>, pub description: Option, pub disclosed_at: Option, @@ -54,8 +55,8 @@ impl From for rate_limits_api::OutputRule { fn from(value: OutputRule) -> Self { rate_limits_api::OutputRule { description: value.description, - disclosed_at: value.disclosed_at, id: value.id, + incident_id: value.incident_id, rule_raw: value.rule_raw, } } From 7ed08073511f05a956a99f166934743ac542338a Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 09:32:58 +0000 Subject: [PATCH 11/40] chore: add client e2e tests --- .../rate_limits/canister_client/src/main.rs | 169 +++++++----------- 1 file changed, 65 insertions(+), 104 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister_client/src/main.rs b/rs/boundary_node/rate_limits/canister_client/src/main.rs index 205023dc2ff..be5cc2bb29f 100644 --- a/rs/boundary_node/rate_limits/canister_client/src/main.rs +++ b/rs/boundary_node/rate_limits/canister_client/src/main.rs @@ -1,73 +1,92 @@ use candid::{Decode, Encode, Principal}; use ic_agent::{ identity::{AnonymousIdentity, Secp256k1Identity}, - Agent, + Agent, Identity, }; use rate_limits_api::{ - AddConfigResponse, DiscloseRulesArg, DiscloseRulesResponse, GetConfigResponse, - GetRuleByIdResponse, InputConfig, InputRule, + AddConfigResponse, DiscloseRulesArg, DiscloseRulesResponse, GetConfigResponse, IncidentId, + InputConfig, InputRule, Version, }; -const RATE_LIMIT_CANISTER_ID: &str = "ud6i4-iaaaa-aaaab-qadiq-cai"; +const RATE_LIMIT_CANISTER_ID: &str = "v3x57-gaaaa-aaaab-qadmq-cai"; const IC_DOMAIN: &str = "https://ic0.app"; use k256::elliptic_curve::SecretKey; -const TEST_PRIVATE_KEY: &str = "-----BEGIN EC PRIVATE KEY----- -MHQCAQEEIIBzyyJ32Kdjixx+ZJvNeUWsqAzSQZfLsOyXKgxc7aH9oAcGBSuBBAAK -oUQDQgAECWc6ZRn9bBP96RM1G6h8ZAtbryO65dKg6cw0Oij2XbnAlb6zSPhU+4hh -gc2Q0JiGrqKks1AVi+8wzmZ+2PQXXA== ------END EC PRIVATE KEY-----"; +const TEST_PRIVATE_KEY: &str = ""; #[tokio::main] async fn main() { - let agent_authorized = Agent::builder() - .with_url(IC_DOMAIN) - .with_identity(AnonymousIdentity {}) - .build() - .expect("failed to build the agent"); - agent_authorized.fetch_root_key().await.unwrap(); + let agent_full_access = create_agent(Secp256k1Identity::from_private_key( + SecretKey::from_sec1_pem(TEST_PRIVATE_KEY).unwrap(), + )) + .await; + + let agent_restricted_read = create_agent(AnonymousIdentity {}).await; + + let canister_id = Principal::from_text(RATE_LIMIT_CANISTER_ID).unwrap(); - let mut agent_unauthorized = Agent::builder() + // Call 1. Add a new config containing some rules (FullAccess level of the caller is required) + add_config_with_four_rules(&agent_full_access, canister_id).await; + + // Call 2. Read config by privileged user (FullAccess or FullRead caller level). Response will expose rules/descriptions in their full form. + let version = 2; + read_config(&agent_full_access, version, canister_id).await; + + // Call 3. Read config by non-privileged user (RestrictedRead). Rules/descriptions are hidden in response. + read_config(&agent_restricted_read, version, canister_id).await; + + // Call 4. Disclose rules linked to one single incident. + let incident_id = "incident_id_1".to_string(); + disclose_incident(&agent_full_access, incident_id, canister_id).await; + + // Call 5. Read config by non-privileged user again. Now rules related to the disclosed incident are shown in the full form. + read_config(&agent_restricted_read, version, canister_id).await; +} + +async fn create_agent(identity: I) -> Agent { + let agent = Agent::builder() .with_url(IC_DOMAIN) + .with_identity(identity) .build() .expect("failed to build the agent"); - agent_unauthorized.set_identity(Secp256k1Identity::from_private_key( - SecretKey::from_sec1_pem(TEST_PRIVATE_KEY).unwrap(), - )); - agent_unauthorized.fetch_root_key().await.unwrap(); - - let canister_id = Principal::from_text(RATE_LIMIT_CANISTER_ID).unwrap(); + agent.fetch_root_key().await.unwrap(); + agent +} - // Call 1: overwrite_config by authorized +async fn add_config_with_four_rules(agent: &Agent, canister_id: Principal) { let args = Encode!(&InputConfig { schema_version: 1, rules: vec![ InputRule { - incident_id: "id1".to_string(), - rule_raw: b"{\"canister_id\": 3}".to_vec(), - description: "canister rate-limit".to_string(), + incident_id: "incident_id_1".to_string(), + rule_raw: b"{\"canister_id\": \"abcd-efgh\",\"limit\": \"10req/s\"}".to_vec(), + description: + "Some vulnerability #1 discovered, temporarily rate-limiting the canister calls" + .to_string(), }, InputRule { - incident_id: "id1".to_string(), - rule_raw: b"{\"subnet_id\": 2}".to_vec(), - description: "subnet rate-limit".to_string(), + incident_id: "incident_id_2".to_string(), + rule_raw: b"{\"subnet_id\": \"kjahd-zcsd\",\"limit\": \"5/s\"}".to_vec(), + description: "Some vulnerability #2 discovered".to_string(), }, InputRule { - incident_id: "id3".to_string(), - rule_raw: b"{\"subnet_id\": 3}".to_vec(), - description: "another subnet rate-limit".to_string(), + incident_id: "incident_id_1".to_string(), + rule_raw: b"{\"canister_id\": \"klmo-pqfs\",\"limit\": \"20req/s\"}".to_vec(), + description: + "Some vulnerability #1 discovered, temporarily rate-limiting the canister calls" + .to_string(), }, InputRule { - incident_id: "id6".to_string(), - rule_raw: b"{\"subnet_id\": 34}".to_vec(), - description: "another subnet rate-limit".to_string(), + incident_id: "incident_id_3".to_string(), + rule_raw: b"{\"canister_id\": \"oiaus-zmnxb\",\"limit\": \"20req/s\"}".to_vec(), + description: "Some vulnerability #3 discovered".to_string(), }, ], }) .unwrap(); - let result = agent_authorized + let result = agent .update(&canister_id, "add_config") .with_arg(args) .call_and_wait() @@ -76,13 +95,13 @@ async fn main() { let decoded = Decode!(&result, AddConfigResponse).unwrap(); - println!("add_config response: {decoded:#?}"); + println!("Response to add_config(): {decoded:#?}"); +} - // Call 2: get_config by unauthorized user - let version = 2u64; +async fn read_config(agent: &Agent, version: Version, canister_id: Principal) { let args = Encode!(&Some(version)).unwrap(); - let response = agent_unauthorized + let response = agent .update(&canister_id, "get_config") .with_arg(args) .call_and_wait() @@ -91,57 +110,14 @@ async fn main() { let decoded = Decode!(&response, GetConfigResponse).expect("failed to decode candid response"); - println!("get_config response: {decoded:#?}"); - - // Call 3: get_rule_by_id unauthorized - let rule_id = "d2f84ec0331266ff19cf0c889b03794232905d39eaff88504ac47939890c8d38".to_string(); - let args = Encode!(&rule_id).unwrap(); - - let response = agent_unauthorized - .query(&canister_id, "get_rule_by_id") - .with_arg(args) - .call() - .await - .expect("update call failed"); - - let decoded = Decode!(&response, GetRuleByIdResponse).unwrap(); - - println!("get_rule_by_id response: {decoded:#?}"); - - // Call 4: disclose_rules by authorized - let disclose_arg = DiscloseRulesArg::RuleIds(vec![rule_id.clone()]); - let args = Encode!(&disclose_arg).unwrap(); - - let response = agent_authorized - .update(&canister_id, "disclose_rules") - .with_arg(args) - .call_and_wait() - .await - .expect("update call failed"); - - let decoded = Decode!(&response, DiscloseRulesResponse).unwrap(); - - println!("disclose_rules response: {decoded:#?}"); - - // Call 5: get_rule_by_id after disclose() for unauthorized - let args = Encode!(&rule_id).unwrap(); - - let response = agent_unauthorized - .query(&canister_id, "get_rule_by_id") - .with_arg(args) - .call() - .await - .expect("update call failed"); - - let decoded = Decode!(&response, GetRuleByIdResponse).unwrap(); - - println!("get_rule_by_id response: {decoded:#?}"); + println!("Response to get_config(): {decoded:#?}"); +} - // Call 6: disclose_rules by authorized - let disclose_arg = DiscloseRulesArg::IncidentIds(vec!["id2".to_string(), "id3".to_string()]); +async fn disclose_incident(agent: &Agent, incident_id: IncidentId, canister_id: Principal) { + let disclose_arg = DiscloseRulesArg::IncidentIds(vec![incident_id]); let args = Encode!(&disclose_arg).unwrap(); - let response = agent_authorized + let response = agent .update(&canister_id, "disclose_rules") .with_arg(args) .call_and_wait() @@ -150,20 +126,5 @@ async fn main() { let decoded = Decode!(&response, DiscloseRulesResponse).unwrap(); - println!("disclose_rules response: {decoded:#?}"); - - // Call 7: get_config by unauthorized user - let version = 2u64; - let args = Encode!(&Some(version)).unwrap(); - - let response = agent_unauthorized - .update(&canister_id, "get_config") - .with_arg(args) - .call_and_wait() - .await - .expect("update call failed"); - - let decoded = Decode!(&response, GetConfigResponse).expect("failed to decode candid response"); - - println!("get_config response: {decoded:#?}"); + println!("Response to disclose_rules(): {decoded:#?}"); } From 5c554f281d6487ae171768935f709e701dff27d7 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 11:33:08 +0000 Subject: [PATCH 12/40] chore: add more calls to the canister client --- .../rate_limits/canister_client/src/main.rs | 95 ++++++++++++++++--- 1 file changed, 82 insertions(+), 13 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister_client/src/main.rs b/rs/boundary_node/rate_limits/canister_client/src/main.rs index be5cc2bb29f..d31480c964a 100644 --- a/rs/boundary_node/rate_limits/canister_client/src/main.rs +++ b/rs/boundary_node/rate_limits/canister_client/src/main.rs @@ -4,11 +4,11 @@ use ic_agent::{ Agent, Identity, }; use rate_limits_api::{ - AddConfigResponse, DiscloseRulesArg, DiscloseRulesResponse, GetConfigResponse, IncidentId, - InputConfig, InputRule, Version, + AddConfigResponse, DiscloseRulesArg, DiscloseRulesResponse, GetConfigResponse, + GetRuleByIdResponse, IncidentId, InputConfig, InputRule, RuleId, Version, }; -const RATE_LIMIT_CANISTER_ID: &str = "v3x57-gaaaa-aaaab-qadmq-cai"; +const RATE_LIMIT_CANISTER_ID: &str = "un4fu-tqaaa-aaaab-qadjq-cai"; const IC_DOMAIN: &str = "https://ic0.app"; use k256::elliptic_curve::SecretKey; @@ -26,22 +26,29 @@ async fn main() { let canister_id = Principal::from_text(RATE_LIMIT_CANISTER_ID).unwrap(); - // Call 1. Add a new config containing some rules (FullAccess level of the caller is required) - add_config_with_four_rules(&agent_full_access, canister_id).await; + println!("Call 1. Add a new config (version = 2) containing some rules (FullAccess level of the caller is required)"); + add_config_1(&agent_full_access, canister_id).await; - // Call 2. Read config by privileged user (FullAccess or FullRead caller level). Response will expose rules/descriptions in their full form. + println!("Call 2. Read config by privileged user (FullAccess or FullRead caller level). Response will expose rules/descriptions in their full form"); let version = 2; read_config(&agent_full_access, version, canister_id).await; - // Call 3. Read config by non-privileged user (RestrictedRead). Rules/descriptions are hidden in response. + println!("Call 3. Read config by non-privileged user (RestrictedRead level). Rules and descriptions are hidden in the response"); read_config(&agent_restricted_read, version, canister_id).await; - // Call 4. Disclose rules linked to one single incident. + println!("Call 4. Disclose rules (two rules in this case) linked to a single incident"); let incident_id = "incident_id_1".to_string(); disclose_incident(&agent_full_access, incident_id, canister_id).await; - // Call 5. Read config by non-privileged user again. Now rules related to the disclosed incident are shown in the full form. + println!("Call 5. Read config by non-privileged user again. Now rules related to the disclosed incident are fully shown"); read_config(&agent_restricted_read, version, canister_id).await; + + println!("Call 6. Add another config (version = 3) with one newly added rule, one remove rule"); + add_config_2(&agent_full_access, canister_id).await; + + println!("Call 7. Inspect the metadata of the removed rule. All metadata fields should be visible, including versions when the rule was added/removed"); + let rule_id = "bc652fa8460f9456edb068ef4b8dd4761ebcf298478d00dac8ba3d4e491bf2ff".to_string(); + read_rule(&agent_restricted_read, rule_id, canister_id).await; } async fn create_agent(identity: I) -> Agent { @@ -54,7 +61,9 @@ async fn create_agent(identity: I) -> Agent { agent } -async fn add_config_with_four_rules(agent: &Agent, canister_id: Principal) { +async fn add_config_1(agent: &Agent, canister_id: Principal) { + // Note two rules (indices = [0, 2]) are linked to the same incident_id_1 + // RuleIds are generated on the server side based on the hash(rule_raw + description) let args = Encode!(&InputConfig { schema_version: 1, rules: vec![ @@ -95,7 +104,52 @@ async fn add_config_with_four_rules(agent: &Agent, canister_id: Principal) { let decoded = Decode!(&result, AddConfigResponse).unwrap(); - println!("Response to add_config(): {decoded:#?}"); + println!("Response to add_config() call: {decoded:#?}"); +} + +async fn add_config_2(agent: &Agent, canister_id: Principal) { + // This config differs from config 1 by one rule at index = 2, see comment below. + let args = Encode!(&InputConfig { + schema_version: 1, + rules: vec![ + InputRule { + incident_id: "incident_id_1".to_string(), + rule_raw: b"{\"canister_id\": \"abcd-efgh\",\"limit\": \"10req/s\"}".to_vec(), + description: + "Some vulnerability #1 discovered, temporarily rate-limiting the canister calls" + .to_string(), + }, + InputRule { + incident_id: "incident_id_2".to_string(), + rule_raw: b"{\"subnet_id\": \"kjahd-zcsd\",\"limit\": \"5/s\"}".to_vec(), + description: "Some vulnerability #2 discovered".to_string(), + }, + // Only this rule is different from config 1. + // It means that the old rule is removed (not mutated) and this new rule is applied instead. + InputRule { + incident_id: "incident_id_4".to_string(), + rule_raw: b"{\"canister_id\": \"aaaa-bbbb\",\"limit\": \"50req/s\"}".to_vec(), + description: "Some vulnerability #4 discovered".to_string(), + }, + InputRule { + incident_id: "incident_id_3".to_string(), + rule_raw: b"{\"canister_id\": \"oiaus-zmnxb\",\"limit\": \"20req/s\"}".to_vec(), + description: "Some vulnerability #3 discovered".to_string(), + }, + ], + }) + .unwrap(); + + let result = agent + .update(&canister_id, "add_config") + .with_arg(args) + .call_and_wait() + .await + .unwrap(); + + let decoded = Decode!(&result, AddConfigResponse).unwrap(); + + println!("Response to add_config() call: {decoded:#?}"); } async fn read_config(agent: &Agent, version: Version, canister_id: Principal) { @@ -110,7 +164,7 @@ async fn read_config(agent: &Agent, version: Version, canister_id: Principal) { let decoded = Decode!(&response, GetConfigResponse).expect("failed to decode candid response"); - println!("Response to get_config(): {decoded:#?}"); + println!("Response to get_config() call: {decoded:#?}"); } async fn disclose_incident(agent: &Agent, incident_id: IncidentId, canister_id: Principal) { @@ -126,5 +180,20 @@ async fn disclose_incident(agent: &Agent, incident_id: IncidentId, canister_id: let decoded = Decode!(&response, DiscloseRulesResponse).unwrap(); - println!("Response to disclose_rules(): {decoded:#?}"); + println!("Response to disclose_rules() call: {decoded:#?}"); +} + +async fn read_rule(agent: &Agent, rule_id: RuleId, canister_id: Principal) { + let args = Encode!(&rule_id).unwrap(); + + let response = agent + .update(&canister_id, "get_rule_by_id") + .with_arg(args) + .call_and_wait() + .await + .expect("update call failed"); + + let decoded = Decode!(&response, GetRuleByIdResponse).unwrap(); + + println!("Response to get_rule_by_id() call: {decoded:#?}"); } From 47efd9b985303ba69021a26cfc1a80de1ce93496 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 12:23:10 +0000 Subject: [PATCH 13/40] fix: names --- rs/boundary_node/rate_limits/canister/canister.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/canister.rs b/rs/boundary_node/rate_limits/canister/canister.rs index d90904f70e0..d9d708099d2 100644 --- a/rs/boundary_node/rate_limits/canister/canister.rs +++ b/rs/boundary_node/rate_limits/canister/canister.rs @@ -35,7 +35,7 @@ fn init(init_arg: InitArg) { let interval = std::time::Duration::from_secs(init_arg.registry_polling_period_secs); - periodically_fetch_api_boundary_nodes_set(interval); + periodically_poll_api_boundary_nodes(interval); } #[query(name = "get_config")] @@ -90,7 +90,7 @@ fn disclose_rules(args: DiscloseRulesArg) -> DiscloseRulesResponse { Ok(()) } -fn periodically_fetch_api_boundary_nodes_set(interval: Duration) { +fn periodically_poll_api_boundary_nodes(interval: Duration) { ic_cdk_timers::set_timer_interval(interval, || { ic_cdk::spawn(async { if let Ok(canister_id) = Principal::from_text(REGISTRY_CANISTER_ID) { @@ -101,10 +101,10 @@ fn periodically_fetch_api_boundary_nodes_set(interval: Duration) { ) .await { - Ok((Ok(api_bns_count),)) => { + Ok((Ok(api_bn_records),)) => { API_BOUNDARY_NODE_PRINCIPALS.with(|cell| { *cell.borrow_mut() = - HashSet::from_iter(api_bns_count.into_iter().filter_map(|n| n.id)) + HashSet::from_iter(api_bn_records.into_iter().filter_map(|n| n.id)) }); } Ok((Err(err),)) => { From d134f4661926a0a79113b817f07478df05f32677 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 12:48:10 +0000 Subject: [PATCH 14/40] chore: add test principal --- rs/boundary_node/rate_limits/canister/access_control.rs | 4 ++-- rs/boundary_node/rate_limits/canister_client/src/main.rs | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/access_control.rs b/rs/boundary_node/rate_limits/canister/access_control.rs index 3fd61d318f7..a8dee62161f 100644 --- a/rs/boundary_node/rate_limits/canister/access_control.rs +++ b/rs/boundary_node/rate_limits/canister/access_control.rs @@ -2,8 +2,8 @@ use candid::Principal; use crate::storage::API_BOUNDARY_NODE_PRINCIPALS; -const FULL_ACCESS_ID: &str = ""; -const FULL_READ_TESTING_ID: &str = ""; // TODO: remove this +const FULL_ACCESS_ID: &str = "imx2d-dctwe-ircfz-emzus-bihdn-aoyzy-lkkdi-vi5vw-npnik-noxiy-mae"; +const FULL_READ_TESTING_ID: &str = "un4fu-tqaaa-aaaab-qadjq-cai"; // TODO: remove this pub trait ResolveAccessLevel { fn get_access_level(&self) -> AccessLevel; diff --git a/rs/boundary_node/rate_limits/canister_client/src/main.rs b/rs/boundary_node/rate_limits/canister_client/src/main.rs index d31480c964a..802a1bbc388 100644 --- a/rs/boundary_node/rate_limits/canister_client/src/main.rs +++ b/rs/boundary_node/rate_limits/canister_client/src/main.rs @@ -8,12 +8,16 @@ use rate_limits_api::{ GetRuleByIdResponse, IncidentId, InputConfig, InputRule, RuleId, Version, }; -const RATE_LIMIT_CANISTER_ID: &str = "un4fu-tqaaa-aaaab-qadjq-cai"; +const RATE_LIMIT_CANISTER_ID: &str = "zkfwe-6yaaa-aaaab-qacca-cai"; const IC_DOMAIN: &str = "https://ic0.app"; use k256::elliptic_curve::SecretKey; -const TEST_PRIVATE_KEY: &str = ""; +const TEST_PRIVATE_KEY: &str = "-----BEGIN EC PRIVATE KEY----- +MHQCAQEEIIBzyyJ32Kdjixx+ZJvNeUWsqAzSQZfLsOyXKgxc7aH9oAcGBSuBBAAK +oUQDQgAECWc6ZRn9bBP96RM1G6h8ZAtbryO65dKg6cw0Oij2XbnAlb6zSPhU+4hh +gc2Q0JiGrqKks1AVi+8wzmZ+2PQXXA== +-----END EC PRIVATE KEY-----"; #[tokio::main] async fn main() { From 2dc3dfce538c8ef6a7ccf4562e2ef3caa30a3ce6 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 13:52:40 +0000 Subject: [PATCH 15/40] chore: add simple unit test --- .../rate_limits/canister/access_control.rs | 2 + .../rate_limits/canister/add_config.rs | 43 ++++++++++++++++--- .../rate_limits/canister/canister.rs | 3 +- .../rate_limits/canister/state.rs | 2 + 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/access_control.rs b/rs/boundary_node/rate_limits/canister/access_control.rs index a8dee62161f..8c0a9c53de0 100644 --- a/rs/boundary_node/rate_limits/canister/access_control.rs +++ b/rs/boundary_node/rate_limits/canister/access_control.rs @@ -1,10 +1,12 @@ use candid::Principal; +use mockall::automock; use crate::storage::API_BOUNDARY_NODE_PRINCIPALS; const FULL_ACCESS_ID: &str = "imx2d-dctwe-ircfz-emzus-bihdn-aoyzy-lkkdi-vi5vw-npnik-noxiy-mae"; const FULL_READ_TESTING_ID: &str = "un4fu-tqaaa-aaaab-qadjq-cai"; // TODO: remove this +#[automock] pub trait ResolveAccessLevel { fn get_access_level(&self) -> AccessLevel; } diff --git a/rs/boundary_node/rate_limits/canister/add_config.rs b/rs/boundary_node/rate_limits/canister/add_config.rs index 5eedc698e76..481769d6a7d 100644 --- a/rs/boundary_node/rate_limits/canister/add_config.rs +++ b/rs/boundary_node/rate_limits/canister/add_config.rs @@ -1,7 +1,6 @@ use std::collections::{BTreeMap, HashMap, HashSet}; -use crate::storage::StorableIncidentMetadata; -use ic_cdk::api::time; +use crate::{storage::StorableIncidentMetadata, types::Timestamp}; use rate_limits_api::IncidentId; use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; @@ -17,7 +16,7 @@ use crate::{ pub const INIT_VERSION: Version = 1; pub trait AddsConfig { - fn add_config(&self, config: InputConfig) -> Result<(), AddConfigError>; + fn add_config(&self, config: InputConfig, time: Timestamp) -> Result<(), AddConfigError>; } #[derive(Debug, Error, Clone)] @@ -91,7 +90,7 @@ impl ConfigAdder { // - New rules cannot be linked to an already disclosed incident (LinkingRuleToDisclosedIncident error) impl AddsConfig for ConfigAdder { - fn add_config(&self, config: InputConfig) -> Result<(), AddConfigError> { + fn add_config(&self, config: InputConfig, time: Timestamp) -> Result<(), AddConfigError> { // Only privileged users can perform this operation if self.access_resolver.get_access_level() != AccessLevel::FullAccess { return Err(AddConfigError::Unauthorized); @@ -237,7 +236,7 @@ impl AddsConfig for ConfigAdder { let storable_config = StorableConfig { schema_version: config.schema_version, - active_since: time(), + active_since: time, rule_ids, }; @@ -306,3 +305,37 @@ impl From for String { value.to_string() } } + +#[cfg(test)] +mod tests { + use crate::access_control::MockResolveAccessLevel; + use crate::state::MockRepository; + + use super::*; + + #[test] + fn test_add_config_success() { + let config = InputConfig { + schema_version: 1, + rules: vec![], + }; + let current_time = 0u64; + + let mut mock_access = MockResolveAccessLevel::new(); + mock_access + .expect_get_access_level() + .returning(|| AccessLevel::FullAccess); + let mut mock_repository = MockRepository::new(); + + mock_repository.expect_get_rule().returning(|_| None); + mock_repository.expect_get_version().returning(|| None); + mock_repository.expect_get_config().returning(|_| None); + mock_repository.expect_add_config().returning(|_, _| true); + + let writer = ConfigAdder::new(mock_repository, mock_access); + + writer + .add_config(config, current_time) + .expect("failed to add a new config"); + } +} diff --git a/rs/boundary_node/rate_limits/canister/canister.rs b/rs/boundary_node/rate_limits/canister/canister.rs index d9d708099d2..a3d59d2bea2 100644 --- a/rs/boundary_node/rate_limits/canister/canister.rs +++ b/rs/boundary_node/rate_limits/canister/canister.rs @@ -70,10 +70,11 @@ fn get_rule_by_id(rule_id: RuleId) -> GetRuleByIdResponse { #[candid_method(update)] fn add_config(config: InputConfig) -> AddConfigResponse { let caller_id = ic_cdk::api::caller(); + let current_time = ic_cdk::api::time(); with_state(|state| { let access_resolver: AccessLevelResolver = AccessLevelResolver::new(caller_id); let writer = ConfigAdder::new(state, access_resolver); - writer.add_config(config.into()) + writer.add_config(config.into(), current_time) })?; Ok(()) } diff --git a/rs/boundary_node/rate_limits/canister/state.rs b/rs/boundary_node/rate_limits/canister/state.rs index 737c0e23554..016f8c48e99 100644 --- a/rs/boundary_node/rate_limits/canister/state.rs +++ b/rs/boundary_node/rate_limits/canister/state.rs @@ -1,4 +1,5 @@ use ic_cdk::api::time; +use mockall::automock; use rate_limits_api::IncidentId; use crate::{ @@ -10,6 +11,7 @@ use crate::{ types::{RuleId, Version}, }; +#[automock] pub trait Repository { fn get_version(&self) -> Option; fn get_config(&self, version: Version) -> Option; From fbd6d94badfac136df8521c890917c68515c0368 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 13:56:26 +0000 Subject: [PATCH 16/40] fix: mockall in cargo --- rs/boundary_node/rate_limits/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/boundary_node/rate_limits/Cargo.toml b/rs/boundary_node/rate_limits/Cargo.toml index 532202a8d2c..937244e9482 100644 --- a/rs/boundary_node/rate_limits/Cargo.toml +++ b/rs/boundary_node/rate_limits/Cargo.toml @@ -11,6 +11,7 @@ anyhow = { workspace = true } bincode = { workspace = true } candid = { workspace = true } hex = { workspace = true } +mockall = { workspace = true } ic-cdk = { workspace = true } ic-cdk-macros = { workspace = true } ic-cdk-timers = { workspace = true } From 3ac629f09387945e6c2aabb8697e09a1ea65f904 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Mon, 21 Oct 2024 13:59:54 +0000 Subject: [PATCH 17/40] Automatically updated Cargo*.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index e3ff8c2601c..3c41ba834c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17574,6 +17574,7 @@ dependencies = [ "ic-cdk-macros 0.9.0", "ic-cdk-timers", "ic-stable-structures", + "mockall 0.13.0", "rate-limits-api", "serde", "serde_json", From a4eab8be7ce2bc472d5b07dfe5123260efe4f10f Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Mon, 21 Oct 2024 14:13:03 +0000 Subject: [PATCH 18/40] chore: simplify canister methods --- rs/boundary_node/rate_limits/canister/canister.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/canister.rs b/rs/boundary_node/rate_limits/canister/canister.rs index a3d59d2bea2..60c216a717c 100644 --- a/rs/boundary_node/rate_limits/canister/canister.rs +++ b/rs/boundary_node/rate_limits/canister/canister.rs @@ -38,7 +38,7 @@ fn init(init_arg: InitArg) { periodically_poll_api_boundary_nodes(interval); } -#[query(name = "get_config")] +#[query] #[candid_method(query)] fn get_config(version: Option) -> GetConfigResponse { let caller_id = ic_cdk::api::caller(); @@ -52,7 +52,7 @@ fn get_config(version: Option) -> GetConfigResponse { Ok(response.into()) } -#[query(name = "get_rule_by_id")] +#[query] #[candid_method(query)] fn get_rule_by_id(rule_id: RuleId) -> GetRuleByIdResponse { let caller_id = ic_cdk::api::caller(); @@ -66,7 +66,7 @@ fn get_rule_by_id(rule_id: RuleId) -> GetRuleByIdResponse { Ok(response.into()) } -#[update(name = "add_config")] +#[update] #[candid_method(update)] fn add_config(config: InputConfig) -> AddConfigResponse { let caller_id = ic_cdk::api::caller(); @@ -79,7 +79,7 @@ fn add_config(config: InputConfig) -> AddConfigResponse { Ok(()) } -#[update(name = "disclose_rules")] +#[update] #[candid_method(update)] fn disclose_rules(args: DiscloseRulesArg) -> DiscloseRulesResponse { let caller_id = ic_cdk::api::caller(); From 3406440e12b5a7c74316b174cffac3780d962a85 Mon Sep 17 00:00:00 2001 From: Nikolay Komarevskiy <90605504+nikolay-komarevskiy@users.noreply.github.com> Date: Mon, 21 Oct 2024 22:49:12 +0200 Subject: [PATCH 19/40] Update rs/boundary_node/rate_limits/canister/add_config.rs Co-authored-by: r-birkner <103420898+r-birkner@users.noreply.github.com> --- rs/boundary_node/rate_limits/canister/add_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/boundary_node/rate_limits/canister/add_config.rs b/rs/boundary_node/rate_limits/canister/add_config.rs index 481769d6a7d..124dc0b86ac 100644 --- a/rs/boundary_node/rate_limits/canister/add_config.rs +++ b/rs/boundary_node/rate_limits/canister/add_config.rs @@ -78,7 +78,7 @@ impl ConfigAdder { // - Adding a new config requires providing the entire list of rules and increments the version by one. // - Each rule has two IDs: rule_id (unique identifier) and incident_id (linking the rule to an incident). // - rule_id is autogenerated by hashing JSON-decoded `rule_raw` + `description`. -// - Each rule must be linked to some incident_id, this ID must be provided for each input rule be the caller; multiple rules can be linked to one incident_id. +// - Each rule must be linked to some incident_id, this ID must be provided for each input rule by the caller; multiple rules can be linked to one incident_id. // - Rules can persist across config versions if resubmitted. // - Non-resubmitted rules are marked as "deactivated" (StorableRuleMetadata.removed_in_version = Some()). // - Individual rules or incidents (set of rules with the same incident_id) can be disclosed. This means that a rule can be viewed by the callers with `RestrictedRead` access level. From 306fc08923061c4c237f8aecdc19138568fdf1b4 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Tue, 22 Oct 2024 12:16:21 +0000 Subject: [PATCH 20/40] chore: add schema for rules --- rs/boundary_node/rate_limits/api/Cargo.toml | 2 + rs/boundary_node/rate_limits/api/src/lib.rs | 67 ++++++++++++++ .../api/src/schema_versions/mod.rs | 1 + .../rate_limits/api/src/schema_versions/v1.rs | 69 +++++++++++++++ .../rate_limits/canister_client/Cargo.toml | 7 +- .../rate_limits/canister_client/src/main.rs | 88 +++++++++++++++---- 6 files changed, 215 insertions(+), 19 deletions(-) create mode 100644 rs/boundary_node/rate_limits/api/src/schema_versions/mod.rs create mode 100644 rs/boundary_node/rate_limits/api/src/schema_versions/v1.rs diff --git a/rs/boundary_node/rate_limits/api/Cargo.toml b/rs/boundary_node/rate_limits/api/Cargo.toml index adcae36afc6..58832c57dbe 100644 --- a/rs/boundary_node/rate_limits/api/Cargo.toml +++ b/rs/boundary_node/rate_limits/api/Cargo.toml @@ -8,7 +8,9 @@ documentation.workspace = true [dependencies] candid = {workspace = true} +regex = { workspace = true } serde = {workspace = true} +serde_json = { workspace = true } [lib] path = "src/lib.rs" \ No newline at end of file diff --git a/rs/boundary_node/rate_limits/api/src/lib.rs b/rs/boundary_node/rate_limits/api/src/lib.rs index 4e5a3c3bdd9..3405e95e23c 100644 --- a/rs/boundary_node/rate_limits/api/src/lib.rs +++ b/rs/boundary_node/rate_limits/api/src/lib.rs @@ -1,6 +1,11 @@ use candid::CandidType; use candid::Principal; +use schema_versions::v1::RateLimitRule; use serde::{Deserialize, Serialize}; + +mod schema_versions; +pub use schema_versions::v1; + pub type Version = u64; pub type Timestamp = u64; pub type RuleId = String; @@ -74,3 +79,65 @@ pub struct GetApiBoundaryNodeIdsRequest {} pub struct ApiBoundaryNodeIdRecord { pub id: Option, } + +const INDENT: &str = " "; +const DOUBLE_INDENT: &str = " "; + +impl std::fmt::Display for ConfigResponse { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "\nConfiguration details:")?; + writeln!(f, "{INDENT}Version: {}", self.version)?; + writeln!(f, "{INDENT}Active Since: {}", self.active_since)?; + writeln!(f, "{INDENT}{}", self.config)?; + Ok(()) + } +} + +impl std::fmt::Display for OutputConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "Schema version: {}", self.schema_version)?; + for (i, rule) in self.rules.iter().enumerate() { + writeln!(f, "{DOUBLE_INDENT}Rule {}:", i + 1)?; + writeln!(f, "{DOUBLE_INDENT}ID: {}", rule.id)?; + writeln!(f, "{DOUBLE_INDENT}Incident ID: {}", rule.incident_id)?; + if let Some(ref description) = rule.description { + writeln!(f, "{DOUBLE_INDENT}Description: {description}")?; + } + if let Some(ref rule_raw) = rule.rule_raw { + let decoded_rule = RateLimitRule::from_bytes_json(rule_raw.as_slice()).unwrap(); + writeln!(f, "{DOUBLE_INDENT}Rate-limit rule:\n{decoded_rule}")?; + } + } + Ok(()) + } +} + +impl std::fmt::Display for OutputRuleMetadata { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "\nOutputRuleMetadata")?; + writeln!(f, "{INDENT}ID: {}", self.id)?; + writeln!( + f, + "{INDENT}Disclosed at: {}", + self.disclosed_at + .map(|v| v.to_string()) + .unwrap_or_else(|| "None".to_string()) + )?; + writeln!(f, "{INDENT}Added in version: {}", self.added_in_version)?; + writeln!( + f, + "{INDENT}Removed in version: {}", + self.removed_in_version + .map(|v| v.to_string()) + .unwrap_or_else(|| "None".to_string()) + )?; + if let Some(ref description) = self.description { + writeln!(f, "{INDENT}Description: {description}")?; + } + if let Some(ref rule_raw) = self.rule_raw { + let decoded_rule = RateLimitRule::from_bytes_json(rule_raw.as_slice()).unwrap(); + writeln!(f, "{INDENT}Rate-limit rule:\n{decoded_rule}")?; + } + Ok(()) + } +} diff --git a/rs/boundary_node/rate_limits/api/src/schema_versions/mod.rs b/rs/boundary_node/rate_limits/api/src/schema_versions/mod.rs new file mode 100644 index 00000000000..a3a6d96c3f5 --- /dev/null +++ b/rs/boundary_node/rate_limits/api/src/schema_versions/mod.rs @@ -0,0 +1 @@ +pub mod v1; diff --git a/rs/boundary_node/rate_limits/api/src/schema_versions/v1.rs b/rs/boundary_node/rate_limits/api/src/schema_versions/v1.rs new file mode 100644 index 00000000000..306cc9a2f5a --- /dev/null +++ b/rs/boundary_node/rate_limits/api/src/schema_versions/v1.rs @@ -0,0 +1,69 @@ +use candid::Principal; +use regex::Regex; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +const DOUBLE_INDENT: &str = " "; + +// Defines the rate-limit rule to be stored in the canister +#[derive(Serialize, Deserialize, Debug)] +pub struct RateLimitRule { + pub canister_id: Option, + pub subnet_id: Option, + #[serde(with = "regex_serde")] + pub methods: Regex, + pub limit: String, +} + +impl std::fmt::Display for RateLimitRule { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "{DOUBLE_INDENT}Canister ID: {}", + format_principal_option(&self.canister_id) + )?; + writeln!( + f, + "{DOUBLE_INDENT}Subnet ID: {}", + format_principal_option(&self.subnet_id) + )?; + writeln!(f, "{DOUBLE_INDENT}Methods: {}", &self.methods)?; + write!(f, "{DOUBLE_INDENT}Limit: {}", &self.limit)?; + Ok(()) + } +} + +fn format_principal_option(principal: &Option) -> String { + match principal { + Some(p) => p.to_string(), + None => "None".to_string(), + } +} + +mod regex_serde { + use super::*; + + pub fn serialize(regex: &Regex, serializer: S) -> Result + where + S: Serializer, + { + regex.as_str().serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Regex::new(&s).map_err(serde::de::Error::custom) + } +} + +impl RateLimitRule { + pub fn to_bytes_json(&self) -> Result, serde_json::Error> { + serde_json::to_vec(self) + } + + pub fn from_bytes_json(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes) + } +} diff --git a/rs/boundary_node/rate_limits/canister_client/Cargo.toml b/rs/boundary_node/rate_limits/canister_client/Cargo.toml index 77fd77c5784..c21b3c304df 100644 --- a/rs/boundary_node/rate_limits/canister_client/Cargo.toml +++ b/rs/boundary_node/rate_limits/canister_client/Cargo.toml @@ -8,7 +8,8 @@ documentation.workspace = true [dependencies] k256 = { version = "0.13.1", features = ["pem"] } -rate-limits-api = { path = "../api" } ic-agent = { workspace = true } -tokio = { workspace = true } -candid = { workspace = true } \ No newline at end of file +candid = { workspace = true } +rate-limits-api = { path = "../api" } +regex = { workspace = true } +tokio = { workspace = true } \ No newline at end of file diff --git a/rs/boundary_node/rate_limits/canister_client/src/main.rs b/rs/boundary_node/rate_limits/canister_client/src/main.rs index 802a1bbc388..52369bc45ed 100644 --- a/rs/boundary_node/rate_limits/canister_client/src/main.rs +++ b/rs/boundary_node/rate_limits/canister_client/src/main.rs @@ -4,11 +4,11 @@ use ic_agent::{ Agent, Identity, }; use rate_limits_api::{ - AddConfigResponse, DiscloseRulesArg, DiscloseRulesResponse, GetConfigResponse, - GetRuleByIdResponse, IncidentId, InputConfig, InputRule, RuleId, Version, + v1::RateLimitRule, AddConfigResponse, DiscloseRulesArg, DiscloseRulesResponse, GetConfigResponse, GetRuleByIdResponse, IncidentId, InputConfig, InputRule, RuleId, Version }; +use regex::Regex; -const RATE_LIMIT_CANISTER_ID: &str = "zkfwe-6yaaa-aaaab-qacca-cai"; +const RATE_LIMIT_CANISTER_ID: &str = "w6dgu-3iaaa-aaaab-qadha-cai"; const IC_DOMAIN: &str = "https://ic0.app"; use k256::elliptic_curve::SecretKey; @@ -51,7 +51,7 @@ async fn main() { add_config_2(&agent_full_access, canister_id).await; println!("Call 7. Inspect the metadata of the removed rule. All metadata fields should be visible, including versions when the rule was added/removed"); - let rule_id = "bc652fa8460f9456edb068ef4b8dd4761ebcf298478d00dac8ba3d4e491bf2ff".to_string(); + let rule_id = "5329ff47530283097e983b18f74eb390324fc2dfd08a37db14760c709b341436".to_string(); read_rule(&agent_restricted_read, rule_id, canister_id).await; } @@ -68,31 +68,59 @@ async fn create_agent(identity: I) -> Agent { async fn add_config_1(agent: &Agent, canister_id: Principal) { // Note two rules (indices = [0, 2]) are linked to the same incident_id_1 // RuleIds are generated on the server side based on the hash(rule_raw + description) + let rule_1 = RateLimitRule { + canister_id: Some(canister_id), + subnet_id: None, + methods: Regex::new(r"^(method_1)$").unwrap(), + limit: "1req/s".to_string(), + }; + + let rule_2 = RateLimitRule { + canister_id: Some(canister_id), + subnet_id: None, + methods: Regex::new(r"^(method_2)$").unwrap(), + limit: "2req/s".to_string(), + }; + + let rule_3 = RateLimitRule { + canister_id: Some(canister_id), + subnet_id: None, + methods: Regex::new(r"^(method_3)$").unwrap(), + limit: "3req/s".to_string(), + }; + + let rule_4 = RateLimitRule { + canister_id: Some(canister_id), + subnet_id: None, + methods: Regex::new(r"^(method_4)$").unwrap(), + limit: "4req/s".to_string(), + }; + let args = Encode!(&InputConfig { schema_version: 1, rules: vec![ InputRule { incident_id: "incident_id_1".to_string(), - rule_raw: b"{\"canister_id\": \"abcd-efgh\",\"limit\": \"10req/s\"}".to_vec(), + rule_raw: rule_1.to_bytes_json().unwrap(), description: "Some vulnerability #1 discovered, temporarily rate-limiting the canister calls" .to_string(), }, InputRule { incident_id: "incident_id_2".to_string(), - rule_raw: b"{\"subnet_id\": \"kjahd-zcsd\",\"limit\": \"5/s\"}".to_vec(), + rule_raw: rule_2.to_bytes_json().unwrap(), description: "Some vulnerability #2 discovered".to_string(), }, InputRule { incident_id: "incident_id_1".to_string(), - rule_raw: b"{\"canister_id\": \"klmo-pqfs\",\"limit\": \"20req/s\"}".to_vec(), + rule_raw: rule_3.to_bytes_json().unwrap(), description: "Some vulnerability #1 discovered, temporarily rate-limiting the canister calls" .to_string(), }, InputRule { incident_id: "incident_id_3".to_string(), - rule_raw: b"{\"canister_id\": \"oiaus-zmnxb\",\"limit\": \"20req/s\"}".to_vec(), + rule_raw: rule_4.to_bytes_json().unwrap(), description: "Some vulnerability #3 discovered".to_string(), }, ], @@ -112,32 +140,60 @@ async fn add_config_1(agent: &Agent, canister_id: Principal) { } async fn add_config_2(agent: &Agent, canister_id: Principal) { - // This config differs from config 1 by one rule at index = 2, see comment below. + // This config differs from config 1 by rule_3 at index = 2, see comment below. + let rule_1 = RateLimitRule { + canister_id: Some(canister_id), + subnet_id: None, + methods: Regex::new(r"^(method_1)$").unwrap(), + limit: "1req/s".to_string(), + }; + + let rule_2 = RateLimitRule { + canister_id: Some(canister_id), + subnet_id: None, + methods: Regex::new(r"^(method_2)$").unwrap(), + limit: "2req/s".to_string(), + }; + + let rule_3 = RateLimitRule { + canister_id: Some(canister_id), + subnet_id: None, + methods: Regex::new(r"^(method_3)$").unwrap(), + limit: "3req/s".to_string(), + }; + + let rule_4 = RateLimitRule { + canister_id: Some(canister_id), + subnet_id: None, + methods: Regex::new(r"^(method_4)$").unwrap(), + limit: "4req/s".to_string(), + }; + let args = Encode!(&InputConfig { schema_version: 1, rules: vec![ InputRule { incident_id: "incident_id_1".to_string(), - rule_raw: b"{\"canister_id\": \"abcd-efgh\",\"limit\": \"10req/s\"}".to_vec(), + rule_raw: rule_1.to_bytes_json().unwrap(), description: "Some vulnerability #1 discovered, temporarily rate-limiting the canister calls" .to_string(), }, InputRule { incident_id: "incident_id_2".to_string(), - rule_raw: b"{\"subnet_id\": \"kjahd-zcsd\",\"limit\": \"5/s\"}".to_vec(), + rule_raw: rule_2.to_bytes_json().unwrap(), description: "Some vulnerability #2 discovered".to_string(), }, // Only this rule is different from config 1. // It means that the old rule is removed (not mutated) and this new rule is applied instead. InputRule { incident_id: "incident_id_4".to_string(), - rule_raw: b"{\"canister_id\": \"aaaa-bbbb\",\"limit\": \"50req/s\"}".to_vec(), + rule_raw: rule_3.to_bytes_json().unwrap(), description: "Some vulnerability #4 discovered".to_string(), }, InputRule { incident_id: "incident_id_3".to_string(), - rule_raw: b"{\"canister_id\": \"oiaus-zmnxb\",\"limit\": \"20req/s\"}".to_vec(), + rule_raw: rule_4.to_bytes_json().unwrap(), description: "Some vulnerability #3 discovered".to_string(), }, ], @@ -168,7 +224,7 @@ async fn read_config(agent: &Agent, version: Version, canister_id: Principal) { let decoded = Decode!(&response, GetConfigResponse).expect("failed to decode candid response"); - println!("Response to get_config() call: {decoded:#?}"); + println!("Response to get_config() call: {}", decoded.unwrap()); } async fn disclose_incident(agent: &Agent, incident_id: IncidentId, canister_id: Principal) { @@ -197,7 +253,7 @@ async fn read_rule(agent: &Agent, rule_id: RuleId, canister_id: Principal) { .await .expect("update call failed"); - let decoded = Decode!(&response, GetRuleByIdResponse).unwrap(); + let decoded = Decode!(&response, GetRuleByIdResponse).unwrap().unwrap(); - println!("Response to get_rule_by_id() call: {decoded:#?}"); + println!("Response to get_rule_by_id() call: {decoded}"); } From 3cacb9f33aafa3443da8c186b82d8d94c3623402 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Tue, 22 Oct 2024 12:33:21 +0000 Subject: [PATCH 21/40] Automatically updated Cargo*.lock --- Cargo.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 3c41ba834c0..9cebdba34c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1767,6 +1767,7 @@ dependencies = [ "ic-agent", "k256", "rate-limits-api", + "regex", "tokio", ] @@ -17559,7 +17560,9 @@ name = "rate-limits-api" version = "0.9.0" dependencies = [ "candid", + "regex", "serde", + "serde_json", ] [[package]] From e02f3280128eab1b20390069cd461098de2dbcd1 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Tue, 22 Oct 2024 13:08:40 +0000 Subject: [PATCH 22/40] fix: clippy --- rs/boundary_node/rate_limits/canister_client/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister_client/src/main.rs b/rs/boundary_node/rate_limits/canister_client/src/main.rs index 52369bc45ed..13393f7c35c 100644 --- a/rs/boundary_node/rate_limits/canister_client/src/main.rs +++ b/rs/boundary_node/rate_limits/canister_client/src/main.rs @@ -8,7 +8,7 @@ use rate_limits_api::{ }; use regex::Regex; -const RATE_LIMIT_CANISTER_ID: &str = "w6dgu-3iaaa-aaaab-qadha-cai"; +const RATE_LIMIT_CANISTER_ID: &str = "yoizw-hyaaa-aaaab-qacea-cai"; const IC_DOMAIN: &str = "https://ic0.app"; use k256::elliptic_curve::SecretKey; @@ -51,7 +51,7 @@ async fn main() { add_config_2(&agent_full_access, canister_id).await; println!("Call 7. Inspect the metadata of the removed rule. All metadata fields should be visible, including versions when the rule was added/removed"); - let rule_id = "5329ff47530283097e983b18f74eb390324fc2dfd08a37db14760c709b341436".to_string(); + let rule_id = "e72de4bf25eeb5be951c78dddd4b3bcdac0890e1d90cb051d05c9afa0ce0fb0b".to_string(); read_rule(&agent_restricted_read, rule_id, canister_id).await; } From 4b99933b595519db1d17d130a37ca746b3299411 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Tue, 22 Oct 2024 18:31:05 +0000 Subject: [PATCH 23/40] chore: use Duration --- rs/boundary_node/rate_limits/canister/canister.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/boundary_node/rate_limits/canister/canister.rs b/rs/boundary_node/rate_limits/canister/canister.rs index 60c216a717c..958cc867238 100644 --- a/rs/boundary_node/rate_limits/canister/canister.rs +++ b/rs/boundary_node/rate_limits/canister/canister.rs @@ -33,7 +33,7 @@ fn init(init_arg: InitArg) { // Initialize an empty config with version=1 init_version_and_config(1); - let interval = std::time::Duration::from_secs(init_arg.registry_polling_period_secs); + let interval = Duration::from_secs(init_arg.registry_polling_period_secs); periodically_poll_api_boundary_nodes(interval); } From 6478c6d773a8e44679421f72b559cf4709dba98f Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Wed, 23 Oct 2024 10:01:43 +0000 Subject: [PATCH 24/40] chore: use only wasm compilation for CI --- rs/boundary_node/rate_limits/Cargo.toml | 2 +- .../rate_limits/canister/canister.rs | 22 ++++++------------- rs/boundary_node/rate_limits/canister/lib.rs | 18 +++++++++++++++ 3 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 rs/boundary_node/rate_limits/canister/lib.rs diff --git a/rs/boundary_node/rate_limits/Cargo.toml b/rs/boundary_node/rate_limits/Cargo.toml index 937244e9482..9c26156cec7 100644 --- a/rs/boundary_node/rate_limits/Cargo.toml +++ b/rs/boundary_node/rate_limits/Cargo.toml @@ -24,4 +24,4 @@ thiserror = { workspace = true } [lib] crate-type = ["cdylib"] -path = "canister/canister.rs" \ No newline at end of file +path = "canister/lib.rs" \ No newline at end of file diff --git a/rs/boundary_node/rate_limits/canister/canister.rs b/rs/boundary_node/rate_limits/canister/canister.rs index 958cc867238..fe63582d978 100644 --- a/rs/boundary_node/rate_limits/canister/canister.rs +++ b/rs/boundary_node/rate_limits/canister/canister.rs @@ -1,11 +1,13 @@ use std::{collections::HashSet, time::Duration}; -use access_control::AccessLevelResolver; -use add_config::{AddsConfig, ConfigAdder}; +use crate::access_control::AccessLevelResolver; +use crate::add_config::{AddsConfig, ConfigAdder}; +use crate::confidentiality_formatting::ConfidentialityFormatterFactory; +use crate::disclose::{DisclosesRules, RulesDiscloser}; +use crate::fetcher::{ConfigFetcher, EntityFetcher, RuleFetcher}; +use crate::state::{init_version_and_config, with_state}; +use crate::storage::API_BOUNDARY_NODE_PRINCIPALS; use candid::{candid_method, Principal}; -use confidentiality_formatting::ConfidentialityFormatterFactory; -use disclose::{DisclosesRules, RulesDiscloser}; -use fetcher::{ConfigFetcher, EntityFetcher, RuleFetcher}; use ic_cdk::api::call::call; use ic_cdk_macros::{init, query, update}; use rate_limits_api::{ @@ -13,16 +15,6 @@ use rate_limits_api::{ GetApiBoundaryNodeIdsRequest, GetConfigResponse, GetRuleByIdResponse, InitArg, InputConfig, RuleId, Version, }; -use state::{init_version_and_config, with_state}; -use storage::API_BOUNDARY_NODE_PRINCIPALS; -mod access_control; -mod add_config; -mod confidentiality_formatting; -mod disclose; -mod fetcher; -mod state; -mod storage; -mod types; const REGISTRY_CANISTER_ID: &str = "rwlgt-iiaaa-aaaaa-aaaaa-cai"; const REGISTRY_CANISTER_METHOD: &str = "get_api_boundary_node_ids"; diff --git a/rs/boundary_node/rate_limits/canister/lib.rs b/rs/boundary_node/rate_limits/canister/lib.rs new file mode 100644 index 00000000000..d6b7fbe832e --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/lib.rs @@ -0,0 +1,18 @@ +#[cfg(target_family = "wasm")] +mod access_control; +#[cfg(target_family = "wasm")] +mod add_config; +#[cfg(target_family = "wasm")] +mod canister; +#[cfg(target_family = "wasm")] +mod confidentiality_formatting; +#[cfg(target_family = "wasm")] +mod disclose; +#[cfg(target_family = "wasm")] +mod fetcher; +#[cfg(target_family = "wasm")] +mod state; +#[cfg(target_family = "wasm")] +mod storage; +#[cfg(target_family = "wasm")] +mod types; From cca546a001b7ab97377e570fcba0532538a79273 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Wed, 23 Oct 2024 10:03:23 +0000 Subject: [PATCH 25/40] fix: format --- rs/boundary_node/rate_limits/canister_client/src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister_client/src/main.rs b/rs/boundary_node/rate_limits/canister_client/src/main.rs index 13393f7c35c..dd00da457a7 100644 --- a/rs/boundary_node/rate_limits/canister_client/src/main.rs +++ b/rs/boundary_node/rate_limits/canister_client/src/main.rs @@ -4,11 +4,12 @@ use ic_agent::{ Agent, Identity, }; use rate_limits_api::{ - v1::RateLimitRule, AddConfigResponse, DiscloseRulesArg, DiscloseRulesResponse, GetConfigResponse, GetRuleByIdResponse, IncidentId, InputConfig, InputRule, RuleId, Version + v1::RateLimitRule, AddConfigResponse, DiscloseRulesArg, DiscloseRulesResponse, + GetConfigResponse, GetRuleByIdResponse, IncidentId, InputConfig, InputRule, RuleId, Version, }; use regex::Regex; -const RATE_LIMIT_CANISTER_ID: &str = "yoizw-hyaaa-aaaab-qacea-cai"; +const RATE_LIMIT_CANISTER_ID: &str = "5oxzc-3qaaa-aaaab-qaczq-cai"; const IC_DOMAIN: &str = "https://ic0.app"; use k256::elliptic_curve::SecretKey; From 039499dbc60bf9fb5f3281a361fe826092e95b51 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Wed, 23 Oct 2024 10:08:16 +0000 Subject: [PATCH 26/40] rename: error --- .../canister/confidentiality_formatting.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index 8283d0c744b..a8796c15dc0 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -9,14 +9,11 @@ use crate::{ pub trait ConfidentialityFormatting { type Input: Clone; - fn format(&self, value: &Self::Input) -> Result; + fn format(&self, value: &Self::Input) -> Result; } #[derive(Debug, thiserror::Error)] -pub enum ConfidentialFormatterError { - #[error("Access denied")] - _AccessDenied, -} +pub enum ConfidentialityFormattingrError {} /// A generic confidentiality formatter for various data types pub struct ConfidentialityFormatter { @@ -38,7 +35,10 @@ impl ConfidentialityFormatting { type Input = OutputConfig; - fn format(&self, config: &OutputConfig) -> Result { + fn format( + &self, + config: &OutputConfig, + ) -> Result { let mut config = config.clone(); if self.access_resolver.get_access_level() == AccessLevel::RestrictedRead { config.rules.iter_mut().for_each(|rule| { @@ -60,7 +60,7 @@ impl ConfidentialityFormatting fn format( &self, rule: &OutputRuleMetadata, - ) -> Result { + ) -> Result { let mut rule = rule.clone(); if self.access_resolver.get_access_level() == AccessLevel::RestrictedRead && rule.disclosed_at.is_none() From b67780f46aeec65a9394fc99ad3b2dcb0c432d2e Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Wed, 23 Oct 2024 10:22:46 +0000 Subject: [PATCH 27/40] remove: test code --- .../rate_limits/canister/access_control.rs | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/access_control.rs b/rs/boundary_node/rate_limits/canister/access_control.rs index 8c0a9c53de0..118d786a1e8 100644 --- a/rs/boundary_node/rate_limits/canister/access_control.rs +++ b/rs/boundary_node/rate_limits/canister/access_control.rs @@ -4,7 +4,6 @@ use mockall::automock; use crate::storage::API_BOUNDARY_NODE_PRINCIPALS; const FULL_ACCESS_ID: &str = "imx2d-dctwe-ircfz-emzus-bihdn-aoyzy-lkkdi-vi5vw-npnik-noxiy-mae"; -const FULL_READ_TESTING_ID: &str = "un4fu-tqaaa-aaaab-qadjq-cai"; // TODO: remove this #[automock] pub trait ResolveAccessLevel { @@ -36,19 +35,19 @@ impl ResolveAccessLevel for AccessLevelResolver { fn get_access_level(&self) -> AccessLevel { let full_access_principal = Principal::from_text(FULL_ACCESS_ID).unwrap(); - API_BOUNDARY_NODE_PRINCIPALS.with(|cell| { - let mut full_read_principals = cell.borrow_mut(); - // TODO: this is just for testing, remove later - let full_read_id = Principal::from_text(FULL_READ_TESTING_ID).unwrap(); - let _ = full_read_principals.insert(full_read_id); + if self.caller_id == full_access_principal { + return AccessLevel::FullAccess; + } - if self.caller_id == full_access_principal { - return AccessLevel::FullAccess; - } else if full_read_principals.contains(&self.caller_id) { - return AccessLevel::FullRead; - } + let has_full_read_access = API_BOUNDARY_NODE_PRINCIPALS.with(|cell| { + let full_read_principals = cell.borrow(); + full_read_principals.contains(&self.caller_id) + }); - AccessLevel::RestrictedRead - }) + if has_full_read_access { + return AccessLevel::FullRead; + } + + AccessLevel::RestrictedRead } } From 0d98d465d4ce6ecf197dfb272d0eeb8f42522887 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Wed, 23 Oct 2024 10:33:42 +0000 Subject: [PATCH 28/40] chore: improve canister client scenario --- .../rate_limits/canister_client/src/main.rs | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister_client/src/main.rs b/rs/boundary_node/rate_limits/canister_client/src/main.rs index dd00da457a7..9907459112c 100644 --- a/rs/boundary_node/rate_limits/canister_client/src/main.rs +++ b/rs/boundary_node/rate_limits/canister_client/src/main.rs @@ -39,21 +39,20 @@ async fn main() { read_config(&agent_full_access, version, canister_id).await; println!("Call 3. Read config by non-privileged user (RestrictedRead level). Rules and descriptions are hidden in the response"); - read_config(&agent_restricted_read, version, canister_id).await; + let rule_ids = read_config(&agent_restricted_read, version, canister_id).await; println!("Call 4. Disclose rules (two rules in this case) linked to a single incident"); let incident_id = "incident_id_1".to_string(); disclose_incident(&agent_full_access, incident_id, canister_id).await; println!("Call 5. Read config by non-privileged user again. Now rules related to the disclosed incident are fully shown"); - read_config(&agent_restricted_read, version, canister_id).await; + let _ = read_config(&agent_restricted_read, version, canister_id).await; println!("Call 6. Add another config (version = 3) with one newly added rule, one remove rule"); add_config_2(&agent_full_access, canister_id).await; println!("Call 7. Inspect the metadata of the removed rule. All metadata fields should be visible, including versions when the rule was added/removed"); - let rule_id = "e72de4bf25eeb5be951c78dddd4b3bcdac0890e1d90cb051d05c9afa0ce0fb0b".to_string(); - read_rule(&agent_restricted_read, rule_id, canister_id).await; + read_rule(&agent_restricted_read, &rule_ids[2], canister_id).await; } async fn create_agent(identity: I) -> Agent { @@ -213,7 +212,7 @@ async fn add_config_2(agent: &Agent, canister_id: Principal) { println!("Response to add_config() call: {decoded:#?}"); } -async fn read_config(agent: &Agent, version: Version, canister_id: Principal) { +async fn read_config(agent: &Agent, version: Version, canister_id: Principal) -> Vec { let args = Encode!(&Some(version)).unwrap(); let response = agent @@ -223,9 +222,18 @@ async fn read_config(agent: &Agent, version: Version, canister_id: Principal) { .await .expect("update call failed"); - let decoded = Decode!(&response, GetConfigResponse).expect("failed to decode candid response"); + let decoded = Decode!(&response, GetConfigResponse) + .expect("failed to decode candid response") + .unwrap(); + + println!("Response to get_config() call: {}", decoded); - println!("Response to get_config() call: {}", decoded.unwrap()); + decoded + .config + .rules + .into_iter() + .map(|rule| rule.id) + .collect() } async fn disclose_incident(agent: &Agent, incident_id: IncidentId, canister_id: Principal) { @@ -244,8 +252,8 @@ async fn disclose_incident(agent: &Agent, incident_id: IncidentId, canister_id: println!("Response to disclose_rules() call: {decoded:#?}"); } -async fn read_rule(agent: &Agent, rule_id: RuleId, canister_id: Principal) { - let args = Encode!(&rule_id).unwrap(); +async fn read_rule(agent: &Agent, rule_id: &RuleId, canister_id: Principal) { + let args = Encode!(rule_id).unwrap(); let response = agent .update(&canister_id, "get_rule_by_id") From 5c2d97239100c70e9899e871e5a1b47ebdb96040 Mon Sep 17 00:00:00 2001 From: Nikolay Komarevskiy <90605504+nikolay-komarevskiy@users.noreply.github.com> Date: Thu, 24 Oct 2024 07:51:40 +0200 Subject: [PATCH 29/40] Update rs/boundary_node/rate_limits/canister/disclose.rs Co-authored-by: r-birkner <103420898+r-birkner@users.noreply.github.com> --- .../rate_limits/canister/disclose.rs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/disclose.rs b/rs/boundary_node/rate_limits/canister/disclose.rs index ba5bb05627f..2827f060b26 100644 --- a/rs/boundary_node/rate_limits/canister/disclose.rs +++ b/rs/boundary_node/rate_limits/canister/disclose.rs @@ -38,19 +38,18 @@ impl RulesDiscloser { impl DisclosesRules for RulesDiscloser { fn disclose_rules(&self, arg: DiscloseRulesArg) -> Result<(), DiscloseRulesError> { - if self.access_resolver.get_access_level() != AccessLevel::FullAccess { - return Err(DiscloseRulesError::Unauthorized); - } - match arg { - DiscloseRulesArg::RuleIds(rule_ids) => { - disclose_rules(&self.state, time(), &rule_ids)?; - } - DiscloseRulesArg::IncidentIds(incident_ids) => { - disclose_incidents(&self.state, time(), &incident_ids)?; + if self.access_resolver.get_access_level() == AccessLevel::FullAccess { + match arg { + DiscloseRulesArg::RuleIds(rule_ids) => { + disclose_rules(&self.state, time(), &rule_ids)?; + } + DiscloseRulesArg::IncidentIds(incident_ids) => { + disclose_incidents(&self.state, time(), &incident_ids)?; + } } + return Ok(()); } - - Ok(()) + Err(DiscloseRulesError::Unauthorized); } } From cc534410486ba31246ba0b2720b42fdd00842193 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Thu, 24 Oct 2024 05:28:34 +0000 Subject: [PATCH 30/40] fix: add field --- rs/boundary_node/rate_limits/api/src/lib.rs | 1 + rs/boundary_node/rate_limits/canister/fetcher.rs | 1 + rs/boundary_node/rate_limits/canister/types.rs | 2 ++ 3 files changed, 4 insertions(+) diff --git a/rs/boundary_node/rate_limits/api/src/lib.rs b/rs/boundary_node/rate_limits/api/src/lib.rs index 3405e95e23c..faa44680eaf 100644 --- a/rs/boundary_node/rate_limits/api/src/lib.rs +++ b/rs/boundary_node/rate_limits/api/src/lib.rs @@ -60,6 +60,7 @@ pub struct OutputRule { #[derive(CandidType, Deserialize, Debug)] pub struct OutputRuleMetadata { pub id: RuleId, + pub incident_id: IncidentId, pub rule_raw: Option>, pub description: Option, pub disclosed_at: Option, diff --git a/rs/boundary_node/rate_limits/canister/fetcher.rs b/rs/boundary_node/rate_limits/canister/fetcher.rs index d476243edae..0725716cd97 100644 --- a/rs/boundary_node/rate_limits/canister/fetcher.rs +++ b/rs/boundary_node/rate_limits/canister/fetcher.rs @@ -129,6 +129,7 @@ impl> En let rule_metadata = OutputRuleMetadata { id: rule_id.clone(), + incident_id: stored_metadata.incident_id, rule_raw: Some(stored_metadata.rule_raw), description: Some(stored_metadata.description), disclosed_at: stored_metadata.disclosed_at, diff --git a/rs/boundary_node/rate_limits/canister/types.rs b/rs/boundary_node/rate_limits/canister/types.rs index 660195240bc..68a0e4a3fc8 100644 --- a/rs/boundary_node/rate_limits/canister/types.rs +++ b/rs/boundary_node/rate_limits/canister/types.rs @@ -93,6 +93,7 @@ impl From for rate_limits_api::ConfigResponse { #[derive(Clone)] pub struct OutputRuleMetadata { pub id: RuleId, + pub incident_id: IncidentId, pub rule_raw: Option>, pub description: Option, pub disclosed_at: Option, @@ -104,6 +105,7 @@ impl From for rate_limits_api::OutputRuleMetadata { fn from(value: OutputRuleMetadata) -> Self { rate_limits_api::OutputRuleMetadata { id: value.id, + incident_id: value.incident_id, rule_raw: value.rule_raw, description: value.description, disclosed_at: value.disclosed_at, From ba8103b96e7040fca8f68a4b5b828330f65c674e Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Thu, 24 Oct 2024 05:45:00 +0000 Subject: [PATCH 31/40] remove: some error structs --- .../rate_limits/canister/access_control.rs | 3 --- .../canister/confidentiality_formatting.rs | 19 +++++-------------- .../rate_limits/canister/fetcher.rs | 8 ++------ 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/access_control.rs b/rs/boundary_node/rate_limits/canister/access_control.rs index 118d786a1e8..d314683b13e 100644 --- a/rs/boundary_node/rate_limits/canister/access_control.rs +++ b/rs/boundary_node/rate_limits/canister/access_control.rs @@ -10,9 +10,6 @@ pub trait ResolveAccessLevel { fn get_access_level(&self) -> AccessLevel; } -#[derive(Debug, thiserror::Error)] -pub enum AccessLevelError {} - #[derive(PartialEq, Eq)] pub enum AccessLevel { FullAccess, diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index a8796c15dc0..56548c044ae 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -9,12 +9,9 @@ use crate::{ pub trait ConfidentialityFormatting { type Input: Clone; - fn format(&self, value: &Self::Input) -> Result; + fn format(&self, value: &Self::Input) -> Self::Input; } -#[derive(Debug, thiserror::Error)] -pub enum ConfidentialityFormattingrError {} - /// A generic confidentiality formatter for various data types pub struct ConfidentialityFormatter { access_resolver: A, @@ -35,10 +32,7 @@ impl ConfidentialityFormatting { type Input = OutputConfig; - fn format( - &self, - config: &OutputConfig, - ) -> Result { + fn format(&self, config: &OutputConfig) -> OutputConfig { let mut config = config.clone(); if self.access_resolver.get_access_level() == AccessLevel::RestrictedRead { config.rules.iter_mut().for_each(|rule| { @@ -48,7 +42,7 @@ impl ConfidentialityFormatting } }); } - Ok(config) + config } } @@ -57,10 +51,7 @@ impl ConfidentialityFormatting { type Input = OutputRuleMetadata; - fn format( - &self, - rule: &OutputRuleMetadata, - ) -> Result { + fn format(&self, rule: &OutputRuleMetadata) -> OutputRuleMetadata { let mut rule = rule.clone(); if self.access_resolver.get_access_level() == AccessLevel::RestrictedRead && rule.disclosed_at.is_none() @@ -68,7 +59,7 @@ impl ConfidentialityFormatting rule.description = None; rule.rule_raw = None; } - Ok(rule) + rule } } diff --git a/rs/boundary_node/rate_limits/canister/fetcher.rs b/rs/boundary_node/rate_limits/canister/fetcher.rs index 0725716cd97..70b6fadf718 100644 --- a/rs/boundary_node/rate_limits/canister/fetcher.rs +++ b/rs/boundary_node/rate_limits/canister/fetcher.rs @@ -100,9 +100,7 @@ impl> EntityFe rules, }; - let formatted_config = self.formatter.format(&config).map_err(|err| { - anyhow::anyhow!("Failed to format config with confidentially constraints: {err}") - })?; + let formatted_config = self.formatter.format(&config); let config = ConfigResponse { version, @@ -137,9 +135,7 @@ impl> En removed_in_version: stored_metadata.removed_in_version, }; - let formatted_rule = self.formatter.format(&rule_metadata).map_err(|_| { - anyhow::anyhow!("Failed to format rule with confidentially constraints") - })?; + let formatted_rule = self.formatter.format(&rule_metadata); Ok(formatted_rule) } From 381074dd12375dc192930d9bec38b666b379d0dd Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Thu, 24 Oct 2024 06:56:25 +0000 Subject: [PATCH 32/40] add: fetcher test --- .../canister/confidentiality_formatting.rs | 4 ++- .../rate_limits/canister/disclose.rs | 2 +- .../rate_limits/canister/fetcher.rs | 36 +++++++++++++++++++ rs/boundary_node/rate_limits/canister/lib.rs | 14 ++++---- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index 56548c044ae..e3e8f84d984 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -1,3 +1,4 @@ +use mockall::automock; use std::marker::PhantomData; use crate::{ @@ -6,7 +7,8 @@ use crate::{ }; /// Trait for formatting confidential data based on access levels -pub trait ConfidentialityFormatting { +#[automock(type Input = OutputConfig;)] +pub trait ConfidentialityFormatting { type Input: Clone; fn format(&self, value: &Self::Input) -> Self::Input; diff --git a/rs/boundary_node/rate_limits/canister/disclose.rs b/rs/boundary_node/rate_limits/canister/disclose.rs index 2827f060b26..75e611b113d 100644 --- a/rs/boundary_node/rate_limits/canister/disclose.rs +++ b/rs/boundary_node/rate_limits/canister/disclose.rs @@ -49,7 +49,7 @@ impl DisclosesRules for RulesDiscloser Result; } + pub struct ConfigFetcher { pub repository: R, pub formatter: F, @@ -152,3 +153,38 @@ impl From for String { value.to_string() } } + +#[cfg(test)] +mod tests { + use crate::confidentiality_formatting::MockConfidentialityFormatting; + use crate::state::MockRepository; + use crate::storage::{StorableConfig, StorableVersion}; + + use super::*; + + #[test] + fn test_get_config_success() { + // Arrange + let mut mock_formatter = MockConfidentialityFormatting::new(); + mock_formatter.expect_format().returning(|_| OutputConfig { + schema_version: 1, + rules: vec![], + }); + + let mut mock_repository = MockRepository::new(); + mock_repository + .expect_get_version() + .returning(|| Some(StorableVersion(1))); + mock_repository.expect_get_config().returning(|_| { + Some(StorableConfig { + schema_version: 1, + active_since: 1, + rule_ids: vec![], + }) + }); + + let fetcher = ConfigFetcher::new(mock_repository, mock_formatter); + // Act + assert + fetcher.fetch(Some(1)).expect("failed to get a config"); + } +} diff --git a/rs/boundary_node/rate_limits/canister/lib.rs b/rs/boundary_node/rate_limits/canister/lib.rs index d6b7fbe832e..1a70662345e 100644 --- a/rs/boundary_node/rate_limits/canister/lib.rs +++ b/rs/boundary_node/rate_limits/canister/lib.rs @@ -1,18 +1,16 @@ -#[cfg(target_family = "wasm")] +#[allow(dead_code)] mod access_control; -#[cfg(target_family = "wasm")] +#[allow(dead_code)] mod add_config; #[cfg(target_family = "wasm")] mod canister; -#[cfg(target_family = "wasm")] +#[allow(dead_code)] mod confidentiality_formatting; -#[cfg(target_family = "wasm")] +#[allow(dead_code)] mod disclose; -#[cfg(target_family = "wasm")] +#[allow(dead_code)] mod fetcher; -#[cfg(target_family = "wasm")] +#[allow(dead_code)] mod state; -#[cfg(target_family = "wasm")] mod storage; -#[cfg(target_family = "wasm")] mod types; From 2c54dce57c4b64d77aa6f7e2a388828157820628 Mon Sep 17 00:00:00 2001 From: Nikolay Komarevskiy <90605504+nikolay-komarevskiy@users.noreply.github.com> Date: Thu, 24 Oct 2024 12:12:43 +0200 Subject: [PATCH 33/40] Update rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs Co-authored-by: r-birkner <103420898+r-birkner@users.noreply.github.com> --- .../canister/confidentiality_formatting.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index e3e8f84d984..d3189587af2 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -36,14 +36,17 @@ impl ConfidentialityFormatting fn format(&self, config: &OutputConfig) -> OutputConfig { let mut config = config.clone(); - if self.access_resolver.get_access_level() == AccessLevel::RestrictedRead { - config.rules.iter_mut().for_each(|rule| { - if rule.disclosed_at.is_none() { - rule.description = None; - rule.rule_raw = None; - } - }); + // return full config if authorized + if self.access_resolver.get_access_level() == AccessLevel::FullAccess || self.access_resolver.get_access_level() == AccessLevel::FullRead { + config } + // return the redacted config otherwise + config.rules.iter_mut().for_each(|rule| { + if rule.disclosed_at.is_none() { + rule.description = None; + rule.rule_raw = None; + } + }); config } } From 4dee53866dd241d648845cb665bd440525febbd8 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Thu, 24 Oct 2024 12:07:04 +0000 Subject: [PATCH 34/40] chore: add /metrics --- rs/boundary_node/rate_limits/Cargo.toml | 4 +- rs/boundary_node/rate_limits/api/Cargo.toml | 1 + .../rate_limits/canister/canister.rs | 12 ++++++ .../rate_limits/canister/interface.did | 18 +++++++++ rs/boundary_node/rate_limits/canister/lib.rs | 2 + .../rate_limits/canister/metrics.rs | 38 +++++++++++++++++++ 6 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 rs/boundary_node/rate_limits/canister/metrics.rs diff --git a/rs/boundary_node/rate_limits/Cargo.toml b/rs/boundary_node/rate_limits/Cargo.toml index 9c26156cec7..375d9971b4e 100644 --- a/rs/boundary_node/rate_limits/Cargo.toml +++ b/rs/boundary_node/rate_limits/Cargo.toml @@ -11,11 +11,13 @@ anyhow = { workspace = true } bincode = { workspace = true } candid = { workspace = true } hex = { workspace = true } -mockall = { workspace = true } +ic-canisters-http-types = { path = "../../rust_canisters/http_types" } ic-cdk = { workspace = true } ic-cdk-macros = { workspace = true } ic-cdk-timers = { workspace = true } +ic-metrics-encoder = "1" ic-stable-structures = { workspace = true } +mockall = { workspace = true } rate-limits-api = { path = "./api" } serde = { workspace = true } serde_json = { workspace = true } diff --git a/rs/boundary_node/rate_limits/api/Cargo.toml b/rs/boundary_node/rate_limits/api/Cargo.toml index 58832c57dbe..0ec1f5eac1b 100644 --- a/rs/boundary_node/rate_limits/api/Cargo.toml +++ b/rs/boundary_node/rate_limits/api/Cargo.toml @@ -10,6 +10,7 @@ documentation.workspace = true candid = {workspace = true} regex = { workspace = true } serde = {workspace = true} +serde_bytes = { workspace = true } serde_json = { workspace = true } [lib] diff --git a/rs/boundary_node/rate_limits/canister/canister.rs b/rs/boundary_node/rate_limits/canister/canister.rs index fe63582d978..e580da9e4b4 100644 --- a/rs/boundary_node/rate_limits/canister/canister.rs +++ b/rs/boundary_node/rate_limits/canister/canister.rs @@ -5,9 +5,11 @@ use crate::add_config::{AddsConfig, ConfigAdder}; use crate::confidentiality_formatting::ConfidentialityFormatterFactory; use crate::disclose::{DisclosesRules, RulesDiscloser}; use crate::fetcher::{ConfigFetcher, EntityFetcher, RuleFetcher}; +use crate::metrics::{encode_metrics, serve_metrics}; use crate::state::{init_version_and_config, with_state}; use crate::storage::API_BOUNDARY_NODE_PRINCIPALS; use candid::{candid_method, Principal}; +use ic_canisters_http_types::{HttpRequest, HttpResponse, HttpResponseBuilder}; use ic_cdk::api::call::call; use ic_cdk_macros::{init, query, update}; use rate_limits_api::{ @@ -83,6 +85,16 @@ fn disclose_rules(args: DiscloseRulesArg) -> DiscloseRulesResponse { Ok(()) } +// TODO: adjust quota +#[query(decoding_quota = 10000)] +#[candid_method(query)] +fn http_request(request: HttpRequest) -> HttpResponse { + match request.path() { + "/metrics" => serve_metrics(ic_cdk::api::time() as i64, encode_metrics), + _ => HttpResponseBuilder::not_found().build(), + } +} + fn periodically_poll_api_boundary_nodes(interval: Duration) { ic_cdk_timers::set_timer_interval(interval, || { ic_cdk::spawn(async { diff --git a/rs/boundary_node/rate_limits/canister/interface.did b/rs/boundary_node/rate_limits/canister/interface.did index 0471faef606..1133a848a24 100644 --- a/rs/boundary_node/rate_limits/canister/interface.did +++ b/rs/boundary_node/rate_limits/canister/interface.did @@ -3,6 +3,7 @@ type Timestamp = nat64; // Represents timestamp in nanoseconds since the type RuleId = text; // Unique identifier for each rule type SchemaVersion = nat64; // Version of the schema for encoding/decoding the rules type IncidentId = text; // Unique identifier for each incident +type HeaderField = record { text; text; }; // Input structure for defining a rule with mandatory fields within a config @@ -81,6 +82,20 @@ type InputConfig = record { rules: vec InputRule; }; + +type HttpRequest = record { + method: text; + url: text; + headers: vec HeaderField; + body: blob; +}; + +type HttpResponse = record { + status_code: nat16; + headers: vec HeaderField; + body: blob; +}; + // Initialization arguments for the service type InitArg = record { registry_polling_period_secs: nat64; // IDs of existing API boundary nodes are polled from the registry with this periodicity @@ -102,4 +117,7 @@ service : (InitArg) -> { // Fetch all rules IDs related to an ID of the incident get_rules_by_incident_id: (IncidentId) -> (GetRulesByIncidentIdResponse) query; + + // Canister metrics (Http Interface) + http_request: (HttpRequest) -> (HttpResponse) query; } \ No newline at end of file diff --git a/rs/boundary_node/rate_limits/canister/lib.rs b/rs/boundary_node/rate_limits/canister/lib.rs index 1a70662345e..9dea924330a 100644 --- a/rs/boundary_node/rate_limits/canister/lib.rs +++ b/rs/boundary_node/rate_limits/canister/lib.rs @@ -11,6 +11,8 @@ mod disclose; #[allow(dead_code)] mod fetcher; #[allow(dead_code)] +mod metrics; +#[allow(dead_code)] mod state; mod storage; mod types; diff --git a/rs/boundary_node/rate_limits/canister/metrics.rs b/rs/boundary_node/rate_limits/canister/metrics.rs new file mode 100644 index 00000000000..f331b32150b --- /dev/null +++ b/rs/boundary_node/rate_limits/canister/metrics.rs @@ -0,0 +1,38 @@ +use crate::storage::API_BOUNDARY_NODE_PRINCIPALS; +use ic_canisters_http_types::{HttpResponse, HttpResponseBuilder}; + +/// Encode the metrics in a format that can be understood by Prometheus +pub fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::io::Result<()> { + // retrieve the count of boundary nodes with full access + let api_bns_count = API_BOUNDARY_NODE_PRINCIPALS.with(|cell| cell.borrow().len()); + + // Encode the gauge for Prometheus + w.encode_gauge( + "rate_limit_canister_api_boundary_nodes_total", + api_bns_count as f64, + "Number of API boundary nodes with full read access permission to rate-limit config", + )?; + Ok(()) +} + +/// Serve the encoded metrics as an HTTP response. +pub fn serve_metrics( + time: i64, + encode_metrics: impl FnOnce(&mut ic_metrics_encoder::MetricsEncoder>) -> std::io::Result<()>, +) -> HttpResponse { + let mut writer = ic_metrics_encoder::MetricsEncoder::new(vec![], time); + + // TODO: Consider implementing metrics versioning + + match encode_metrics(&mut writer) { + Ok(()) => HttpResponseBuilder::ok() + .header("Content-Type", "text/plain") + .with_body_and_content_length(writer.into_inner()) + .build(), + Err(err) => { + // Return an HTTP 500 error with detailed error information + HttpResponseBuilder::server_error(format!("Failed to encode metrics: {:?}", err)) + .build() + } + } +} From 226cfaa8c0879c32f1010fbe4192dbf2b8e2eb4c Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Thu, 24 Oct 2024 12:18:26 +0000 Subject: [PATCH 35/40] Automatically updated Cargo*.lock --- Cargo.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 9cebdba34c9..8c92f8fa678 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17562,6 +17562,7 @@ dependencies = [ "candid", "regex", "serde", + "serde_bytes", "serde_json", ] @@ -17573,9 +17574,11 @@ dependencies = [ "bincode", "candid", "hex", + "ic-canisters-http-types", "ic-cdk 0.16.0", "ic-cdk-macros 0.9.0", "ic-cdk-timers", + "ic-metrics-encoder", "ic-stable-structures", "mockall 0.13.0", "rate-limits-api", From 6e1906c0d9852e78d043e4ff6b0ccb35c19a5f38 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Thu, 24 Oct 2024 12:24:23 +0000 Subject: [PATCH 36/40] fix: clippy --- .../rate_limits/canister/confidentiality_formatting.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index d3189587af2..4aaaef5d77c 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -37,7 +37,9 @@ impl ConfidentialityFormatting fn format(&self, config: &OutputConfig) -> OutputConfig { let mut config = config.clone(); // return full config if authorized - if self.access_resolver.get_access_level() == AccessLevel::FullAccess || self.access_resolver.get_access_level() == AccessLevel::FullRead { + if self.access_resolver.get_access_level() == AccessLevel::FullAccess + || self.access_resolver.get_access_level() == AccessLevel::FullRead + { config } // return the redacted config otherwise From cacb6ee168ae1ac98e2960e0b2152c9b8fc96361 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Thu, 24 Oct 2024 16:03:18 +0000 Subject: [PATCH 37/40] fix: return --- .../rate_limits/canister/confidentiality_formatting.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index 4aaaef5d77c..dd5948c39f3 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -40,7 +40,7 @@ impl ConfidentialityFormatting if self.access_resolver.get_access_level() == AccessLevel::FullAccess || self.access_resolver.get_access_level() == AccessLevel::FullRead { - config + return config; } // return the redacted config otherwise config.rules.iter_mut().for_each(|rule| { From 740206318881ed4a992bf287415002b9a8c9b1fb Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Thu, 24 Oct 2024 16:03:33 +0000 Subject: [PATCH 38/40] add: bazelification --- rs/boundary_node/rate_limits/BUILD.bazel | 37 ++++++++++++++++++++ rs/boundary_node/rate_limits/api/BUILD.bazel | 24 +++++++++++++ rs/boundary_node/rate_limits/canister/lib.rs | 3 ++ 3 files changed, 64 insertions(+) create mode 100644 rs/boundary_node/rate_limits/BUILD.bazel create mode 100644 rs/boundary_node/rate_limits/api/BUILD.bazel diff --git a/rs/boundary_node/rate_limits/BUILD.bazel b/rs/boundary_node/rate_limits/BUILD.bazel new file mode 100644 index 00000000000..5e03033774a --- /dev/null +++ b/rs/boundary_node/rate_limits/BUILD.bazel @@ -0,0 +1,37 @@ +load("//bazel:canisters.bzl", "rust_canister") + +package(default_visibility = ["//visibility:public"]) + +DEPENDENCIES = [ + # Keep sorted. + "//rs/boundary_node/rate_limits/api:rate_limits_api", + "//rs/rust_canisters/http_types", + "@crate_index//:anyhow", + "@crate_index//:bincode", + "@crate_index//:candid", + "@crate_index//:hex", + "@crate_index//:ic-cdk", + "@crate_index//:ic-cdk-timers", + "@crate_index//:ic-metrics-encoder", + "@crate_index//:ic-stable-structures", + "@crate_index//:mockall", + "@crate_index//:serde", + "@crate_index//:serde_json", + "@crate_index//:sha2", + "@crate_index//:thiserror", +] + +MACRO_DEPENDENCIES = [ + # Keep sorted. + "@crate_index//:ic-cdk-macros", +] + +rust_canister( + name = "rate_limit_canister", + srcs = glob(["canister/**/*.rs"]), + crate_name = "rate_limit_canister", + crate_root = "canister/lib.rs", + proc_macro_deps = MACRO_DEPENDENCIES, + service_file = "canister/interface.did", + deps = DEPENDENCIES, +) diff --git a/rs/boundary_node/rate_limits/api/BUILD.bazel b/rs/boundary_node/rate_limits/api/BUILD.bazel new file mode 100644 index 00000000000..6e75d3b5fa0 --- /dev/null +++ b/rs/boundary_node/rate_limits/api/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +DEPENDENCIES = [ + # Keep sorted. + "@crate_index//:candid", + "@crate_index//:regex", + "@crate_index//:serde", + "@crate_index//:serde_json", +] + +MACRO_DEPENDENCIES = [] + +ALIASES = {} + +rust_library( + name = "rate_limits_api", + srcs = glob(["src/**/*.rs"]), + aliases = ALIASES, + crate_name = "rate_limits_api", + proc_macro_deps = MACRO_DEPENDENCIES, + deps = DEPENDENCIES, +) diff --git a/rs/boundary_node/rate_limits/canister/lib.rs b/rs/boundary_node/rate_limits/canister/lib.rs index 9dea924330a..ba9362cbba6 100644 --- a/rs/boundary_node/rate_limits/canister/lib.rs +++ b/rs/boundary_node/rate_limits/canister/lib.rs @@ -16,3 +16,6 @@ mod metrics; mod state; mod storage; mod types; + +#[allow(dead_code)] +fn main() {} From 0f92fcff1b1219ea328ef9223580e0566117be53 Mon Sep 17 00:00:00 2001 From: Nikolay Komarevskiy <90605504+nikolay-komarevskiy@users.noreply.github.com> Date: Thu, 24 Oct 2024 22:16:10 +0200 Subject: [PATCH 39/40] Update rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs Co-authored-by: r-birkner <103420898+r-birkner@users.noreply.github.com> --- .../rate_limits/canister/confidentiality_formatting.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index dd5948c39f3..737bea2e258 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -60,9 +60,12 @@ impl ConfidentialityFormatting fn format(&self, rule: &OutputRuleMetadata) -> OutputRuleMetadata { let mut rule = rule.clone(); - if self.access_resolver.get_access_level() == AccessLevel::RestrictedRead - && rule.disclosed_at.is_none() - { + // return full rule if authorized + if self.access_resolver.get_access_level() == AccessLevel::FullAccess || self.access_resolver.get_access_level() == AccessLevel::FullRead { + rule + } + // return the redacted rule otherwise + if rule.disclosed_at.is_none() { rule.description = None; rule.rule_raw = None; } From 67cb1236c9b574e127f2eb59f1bf896b6c53b5d7 Mon Sep 17 00:00:00 2001 From: IDX GitLab Automation Date: Fri, 25 Oct 2024 08:09:09 +0000 Subject: [PATCH 40/40] fix: formatting --- .../rate_limits/canister/confidentiality_formatting.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs index 737bea2e258..5abcabf520a 100644 --- a/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs +++ b/rs/boundary_node/rate_limits/canister/confidentiality_formatting.rs @@ -61,8 +61,10 @@ impl ConfidentialityFormatting fn format(&self, rule: &OutputRuleMetadata) -> OutputRuleMetadata { let mut rule = rule.clone(); // return full rule if authorized - if self.access_resolver.get_access_level() == AccessLevel::FullAccess || self.access_resolver.get_access_level() == AccessLevel::FullRead { - rule + if self.access_resolver.get_access_level() == AccessLevel::FullAccess + || self.access_resolver.get_access_level() == AccessLevel::FullRead + { + return rule; } // return the redacted rule otherwise if rule.disclosed_at.is_none() {