starfish_config/

parameters.rs

// Copyright (c) Mysten Labs, Inc.
// Modifications Copyright (c) 2024 IOTA Stiftung
// SPDX-License-Identifier: Apache-2.0

use std::{path::PathBuf, time::Duration};

use serde::{Deserialize, Serialize};

/// Operational configurations of a consensus authority.
///
/// All fields should tolerate inconsistencies among authorities, without
/// affecting safety of the protocol. Otherwise, they need to be part of IOTA
/// protocol config or epoch state on-chain.
///
/// NOTE: fields with default values are specified in the serde default
/// functions. Most operators should not need to specify any field, except
/// db_path.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Parameters {
    /// Path to consensus DB for this epoch. Required when initializing
    /// consensus. This is calculated based on user configuration for base
    /// directory.
    #[serde(skip)]
    pub db_path: PathBuf,

    /// Time to wait for parent round leader before sealing a block, from when
    /// parent round has a quorum.
    #[serde(default = "Parameters::default_leader_timeout")]
    pub leader_timeout: Duration,

    /// Minimum delay between own blocks. This avoids generating too many rounds
    /// when latency is low. This is especially necessary for tests running
    /// locally. If setting a non-default value, it should be set low enough
    /// to avoid reducing round rate and increasing latency in realistic and
    /// distributed configurations.
    #[serde(default = "Parameters::default_min_block_delay")]
    pub min_block_delay: Duration,

    /// Maximum forward time drift (how far in future) allowed for received
    /// blocks.
    #[serde(default = "Parameters::default_max_forward_time_drift")]
    pub max_forward_time_drift: Duration,

    /// Number of block headers to fetch per commit sync request.
    #[serde(default = "Parameters::default_max_headers_per_commit_sync_fetch")]
    pub max_headers_per_commit_sync_fetch: usize,

    /// Number of transactions to fetch per commit sync request.
    #[serde(default = "Parameters::default_max_transactions_per_commit_sync_fetch")]
    pub max_transactions_per_commit_sync_fetch: usize,

    /// Number of block headers to fetch per periodic or live sync request
    #[serde(default = "Parameters::default_max_headers_per_regular_sync_fetch")]
    pub max_headers_per_regular_sync_fetch: usize,

    /// Number of transactions to fetch per request.
    #[serde(default = "Parameters::default_max_transactions_per_regular_sync_fetch")]
    pub max_transactions_per_regular_sync_fetch: usize,

    /// Time to wait during node start up until the node has synced the last
    /// proposed block via the network peers. When set to `0` the sync
    /// mechanism is disabled. This property is meant to be used for amnesia
    /// recovery.
    #[serde(default = "Parameters::default_sync_last_known_own_block_timeout")]
    pub sync_last_known_own_block_timeout: Duration,

    /// The number of rounds of blocks to be kept in the Dag state cache per
    /// authority. The larger the number the more the blocks that will be
    /// kept in memory allowing minimising any potential disk access.
    /// Value should be at minimum 50 rounds to ensure node performance, but
    /// being too large can be expensive in memory usage.
    #[serde(default = "Parameters::default_dag_state_cached_rounds")]
    pub dag_state_cached_rounds: u32,

    // Number of authorities commit syncer fetches in parallel.
    // Both commits in a range and blocks referenced by the commits are fetched per authority.
    #[serde(default = "Parameters::default_commit_sync_parallel_fetches")]
    pub commit_sync_parallel_fetches: usize,

    // Number of commits to fetch in a batch, also the maximum number of commits returned per
    // fetch. If this value is set too small, fetching becomes inefficient.
    // If this value is set too large, it can result in load imbalance and stragglers.
    #[serde(default = "Parameters::default_commit_sync_batch_size")]
    pub commit_sync_batch_size: u32,

    // This affects the maximum number of commit batches being fetched, and those fetched but not
    // processed as consensus output, before throttling of outgoing commit fetches starts.
    #[serde(default = "Parameters::default_commit_sync_batches_ahead")]
    pub commit_sync_batches_ahead: usize,

    /// Maximum number of headers to be included in a bundle. Headers exceeding
    /// the max allowed limit will be truncated.
    #[serde(default = "Parameters::default_max_headers_per_bundle")]
    pub max_headers_per_bundle: usize,

    /// Maximum number of transaction shards to be included in a bundle. Shards
    /// exceeding the max allowed limit will be truncated.
    #[serde(default = "Parameters::default_max_shards_per_bundle")]
    pub max_shards_per_bundle: usize,

    /// Tonic network settings.
    #[serde(default = "TonicParameters::default")]
    pub tonic: TonicParameters,

    // Number of commits to fetch in a batch for fast commit syncer, also the maximum number of
    // commits returned per fetch. If this value is set too small, fetching becomes
    // inefficient. If this value is set too large, it can result in load imbalance and
    // stragglers.
    #[serde(default = "Parameters::default_fast_commit_sync_batch_size")]
    pub fast_commit_sync_batch_size: u32,

    // Gap threshold for switching between commit syncers. When the gap between quorum and local
    // commit index is larger than this threshold, FastCommitSyncer fetches. Otherwise,
    // CommitSyncer fetches.
    #[serde(default = "Parameters::default_commit_sync_gap_threshold")]
    pub commit_sync_gap_threshold: u32,

    /// Enable FastCommitSyncer for faster recovery from large commit gaps.
    /// This is a local node configuration that works in conjunction with the
    /// protocol-level consensus_fast_commit_sync feature flag. Both must be
    /// enabled for FastCommitSyncer to run. The protocol flag controls
    /// whether gRPC endpoints are available, while this local flag controls
    /// whether this specific node creates and runs the FastCommitSyncer.
    /// Disabled by default; operators can enable it locally once the protocol
    /// flag is active, or disable it again if bugs are discovered, without
    /// affecting protocol-level endpoint availability.
    #[serde(default = "Parameters::default_enable_fast_commit_syncer")]
    pub enable_fast_commit_syncer: bool,
}

impl Parameters {
    pub(crate) fn default_leader_timeout() -> Duration {
        Duration::from_millis(250)
    }

    pub(crate) fn default_min_block_delay() -> Duration {
        if cfg!(msim) || std::env::var("__TEST_ONLY_CONSENSUS_USE_LONG_MIN_BLOCK_DELAY").is_ok() {
            // Checkpoint building and execution cannot keep up with high commit rate in
            // simtests, leading to long reconfiguration delays. This is because
            // simtest is single threaded, and spending too much time in
            // consensus can lead to starvation elsewhere.
            Duration::from_millis(400)
        } else if cfg!(test) {
            // Avoid excessive CPU, data and logs in tests.
            Duration::from_millis(250)
        } else {
            // For production, use min delay between block being set to 50ms, reducing the
            // block rate to 20 blocks/sec
            Duration::from_millis(50)
        }
    }

    pub(crate) fn default_max_forward_time_drift() -> Duration {
        Duration::from_millis(500)
    }

    // Maximum number of block headers to fetch per commit sync request.
    pub(crate) fn default_max_headers_per_commit_sync_fetch() -> usize {
        if cfg!(msim) {
            // Exercise hitting blocks per fetch limit.
            10
        } else {
            1000
        }
    }

    // Maximum number of transactions to fetch per commit sync request.
    pub(crate) fn default_max_transactions_per_commit_sync_fetch() -> usize {
        if cfg!(msim) {
            // Exercise hitting transactions per fetch limit.
            10
        } else {
            1000
        }
    }

    // Maximum number of block headers to fetch per periodic or live sync request.
    pub(crate) fn default_max_headers_per_regular_sync_fetch() -> usize {
        if cfg!(msim) {
            // Exercise hitting blocks per fetch limit.
            10
        } else {
            // TODO: This might should match the value of block headers in the bundle.
            100
        }
    }

    // Maximum number of transactions to fetch per request.
    pub(crate) fn default_max_transactions_per_regular_sync_fetch() -> usize {
        if cfg!(msim) { 10 } else { 1000 }
    }

    pub(crate) fn default_sync_last_known_own_block_timeout() -> Duration {
        if cfg!(msim) {
            Duration::from_millis(500)
        } else {
            // Here we prioritise liveness over the complete de-risking of block
            // equivocation. 5 seconds in the majority of cases should be good
            // enough for this given a healthy network.
            Duration::from_secs(5)
        }
    }

    pub(crate) fn default_dag_state_cached_rounds() -> u32 {
        if cfg!(msim) {
            // Exercise reading blocks from store.
            5
        } else {
            500
        }
    }

    pub(crate) fn default_commit_sync_parallel_fetches() -> usize {
        8
    }

    pub(crate) fn default_commit_sync_batch_size() -> u32 {
        if cfg!(msim) {
            // Exercise commit sync.
            5
        } else {
            100
        }
    }

    pub(crate) fn default_commit_sync_batches_ahead() -> usize {
        // This is set to be a multiple of default commit_sync_parallel_fetches to allow
        // fetching ahead, while keeping the total number of inflight fetches
        // and unprocessed fetched commits limited.
        32
    }

    pub(crate) fn default_max_headers_per_bundle() -> usize {
        150
    }

    pub(crate) fn default_max_shards_per_bundle() -> usize {
        150
    }

    pub(crate) fn default_fast_commit_sync_batch_size() -> u32 {
        if cfg!(msim) {
            // Exercise fast commit sync.
            5
        } else {
            // With ~10KB per commit and 4MB max message size, 1000 commits (~10MB) requires
            // chunking. The server will chunk commits across multiple response messages.
            1000
        }
    }

    pub(crate) fn default_commit_sync_gap_threshold() -> u32 {
        if cfg!(msim) {
            // Use smaller threshold for testing.
            10
        } else {
            // When gap > 1000, FastCommitSyncer is more efficient.
            // When gap <= 1000, CommitSyncer handles incremental sync.
            1000
        }
    }

    pub(crate) fn default_enable_fast_commit_syncer() -> bool {
        // Disabled by default. Operators can enable it locally once the protocol-level
        // consensus_fast_commit_sync flag is active, or disable it again if bugs are
        // discovered, without waiting for a protocol upgrade.
        false
    }
}

impl Default for Parameters {
    fn default() -> Self {
        Self {
            db_path: PathBuf::default(),
            leader_timeout: Parameters::default_leader_timeout(),
            min_block_delay: Parameters::default_min_block_delay(),
            max_forward_time_drift: Parameters::default_max_forward_time_drift(),
            max_headers_per_commit_sync_fetch:
                Parameters::default_max_headers_per_commit_sync_fetch(),
            max_transactions_per_commit_sync_fetch:
                Parameters::default_max_transactions_per_commit_sync_fetch(),
            max_headers_per_regular_sync_fetch:
                Parameters::default_max_headers_per_regular_sync_fetch(),
            max_transactions_per_regular_sync_fetch:
                Parameters::default_max_transactions_per_regular_sync_fetch(),
            sync_last_known_own_block_timeout:
                Parameters::default_sync_last_known_own_block_timeout(),
            dag_state_cached_rounds: Parameters::default_dag_state_cached_rounds(),
            commit_sync_parallel_fetches: Parameters::default_commit_sync_parallel_fetches(),
            commit_sync_batch_size: Parameters::default_commit_sync_batch_size(),
            commit_sync_batches_ahead: Parameters::default_commit_sync_batches_ahead(),
            max_headers_per_bundle: Parameters::default_max_headers_per_bundle(),
            max_shards_per_bundle: Parameters::default_max_shards_per_bundle(),
            tonic: TonicParameters::default(),
            fast_commit_sync_batch_size: Parameters::default_fast_commit_sync_batch_size(),
            commit_sync_gap_threshold: Parameters::default_commit_sync_gap_threshold(),
            enable_fast_commit_syncer: Parameters::default_enable_fast_commit_syncer(),
        }
    }
}

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct TonicParameters {
    /// Keepalive interval and timeouts for both client and server.
    ///
    /// If unspecified, this will default to 5s.
    #[serde(default = "TonicParameters::default_keepalive_interval")]
    pub keepalive_interval: Duration,

    /// Size of various per-connection buffers.
    ///
    /// If unspecified, this will default to 32MiB.
    #[serde(default = "TonicParameters::default_connection_buffer_size")]
    pub connection_buffer_size: usize,

    /// Messages over this size threshold will increment a counter.
    ///
    /// If unspecified, this will default to 16MiB.
    #[serde(default = "TonicParameters::default_excessive_message_size")]
    pub excessive_message_size: usize,

    /// Hard message size limit for both requests and responses.
    /// This value is higher than strictly necessary, to allow overheads.
    /// Message size targets and soft limits are computed based on this value.
    ///
    /// If unspecified, this will default to 1GiB.
    #[serde(default = "TonicParameters::default_message_size_limit")]
    pub message_size_limit: usize,
}

impl TonicParameters {
    fn default_keepalive_interval() -> Duration {
        Duration::from_secs(5)
    }

    fn default_connection_buffer_size() -> usize {
        32 << 20
    }

    fn default_excessive_message_size() -> usize {
        16 << 20
    }

    fn default_message_size_limit() -> usize {
        64 << 20
    }
}

impl Default for TonicParameters {
    fn default() -> Self {
        Self {
            keepalive_interval: TonicParameters::default_keepalive_interval(),
            connection_buffer_size: TonicParameters::default_connection_buffer_size(),
            excessive_message_size: TonicParameters::default_excessive_message_size(),
            message_size_limit: TonicParameters::default_message_size_limit(),
        }
    }
}