Update to spec v1.0.0-rc.0 and BLSv4 (#1765)

## Issue Addressed

Closes #1504 
Closes #1505
Replaces #1703
Closes #1707

## Proposed Changes

* Update BLST and Milagro to versions compatible with BLSv4 spec
* Update Lighthouse to spec v1.0.0-rc.0, and update EF test vectors
* Use the v1.0.0 constants for `MainnetEthSpec`.
* Rename `InteropEthSpec` -> `V012LegacyEthSpec`
    * Change all constants to suit the mainnet `v0.12.3` specification (i.e., Medalla).
* Deprecate the `--spec` flag for the `lighthouse` binary
    * This value is now obtained from the `config_name` field of the `YamlConfig`.
        * Built in testnet YAML files have been updated.
    * Ignore the `--spec` value, if supplied, log a warning that it will be deprecated
    * `lcli` still has the spec flag, that's fine because it's dev tooling.
* Remove the `E: EthSpec` from `YamlConfig`
    * This means we need to deser the genesis `BeaconState` on-demand, but this is fine.
* Swap the old "minimal", "mainnet" strings over to the new `EthSpecId` enum.
* Always require a `CONFIG_NAME` field in `YamlConfig` (it used to have a default).

## Additional Info

Lots of breaking changes, do not merge! ~~We will likely need a Lighthouse v0.4.0 branch, and possibly a long-term v0.3.0 branch to keep Medalla alive~~.

Co-authored-by: Kirk Baird <baird.k@outlook.com>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

Makefile

@@ -42,7 +42,7 @@ endif
 # optimized CPU functions that may not be available on some systems. This
 # results in a more portable binary with ~20% slower BLS verification.
 build-x86_64:
-	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features modern
 build-x86_64-portable:
 	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features portable
 build-aarch64:

account_manager/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "account_manager"
-version = "0.3.1"
+version = "0.3.2"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
 edition = "2018"
 

account_manager/src/validator/slashing_protection.rs

@@ -5,7 +5,7 @@ use slashing_protection::{
 };
 use std::fs::File;
 use std::path::PathBuf;
-use types::EthSpec;
+use types::{BeaconState, EthSpec};
 
 pub const CMD: &str = "slashing-protection";
 pub const IMPORT_CMD: &str = "import";
@@ -46,18 +46,18 @@ pub fn cli_run<T: EthSpec>(
 ) -> Result<(), String> {
     let slashing_protection_db_path = validator_base_dir.join(SLASHING_PROTECTION_FILENAME);
 
-    let genesis_validators_root = env
+    let testnet_config = env
         .testnet
-        .and_then(|testnet_config| {
-            Some(
-                testnet_config
-                    .genesis_state
-                    .as_ref()?
-                    .genesis_validators_root,
+        .ok_or_else(|| "Unable to get testnet configuration from the environment".to_string())?;
+
+    let genesis_validators_root = testnet_config
+        .beacon_state::<T>()
+        .map(|state: BeaconState<T>| state.genesis_validators_root)
+        .map_err(|e| {
+            format!(
+                "Unable to get genesis state, has genesis occurred? Detail: {:?}",
+                e
             )
-        })
-        .ok_or_else(|| {
-            "Unable to get genesis validators root from testnet config, has genesis occurred?"
         })?;
 
     match matches.subcommand() {

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "0.3.1"
+version = "0.3.2"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
 edition = "2018"
 

beacon_node/client/src/config.rs

@@ -5,9 +5,6 @@ use std::fs;
 use std::path::PathBuf;
 use types::Graffiti;
 
-/// The number initial validators when starting the `Minimal`.
-const TESTNET_SPEC_CONSTANTS: &str = "minimal";
-
 /// Default directory name for the freezer database under the top-level data dir.
 const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db";
 
@@ -46,7 +43,6 @@ pub struct Config {
     /// Path where the freezer database will be located.
     pub freezer_db_path: Option<PathBuf>,
     pub log_file: PathBuf,
-    pub spec_constants: String,
     /// If true, the node will use co-ordinated junk for eth1 values.
     ///
     /// This is the method used for the 2019 client interop in Canada.
@@ -81,7 +77,6 @@ impl Default for Config {
             network: NetworkConfig::default(),
             chain: <_>::default(),
             websocket_server: <_>::default(),
-            spec_constants: TESTNET_SPEC_CONSTANTS.into(),
             dummy_eth1_backend: false,
             sync_eth1_chain: false,
             eth1: <_>::default(),

beacon_node/src/config.rs

@@ -5,8 +5,7 @@ use client::{ClientConfig, ClientGenesis};
 use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR};
 use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized};
 use eth2_testnet_config::Eth2TestnetConfig;
-use slog::{crit, info, warn, Logger};
-use ssz::Encode;
+use slog::{info, warn, Logger};
 use std::cmp;
 use std::fs;
 use std::net::{IpAddr, Ipv4Addr, ToSocketAddrs};
@@ -23,7 +22,6 @@ use types::{ChainSpec, Checkpoint, Epoch, EthSpec, Hash256, GRAFFITI_BYTES_LEN};
 /// response of some remote server.
 pub fn get_config<E: EthSpec>(
     cli_args: &ArgMatches,
-    spec_constants: &str,
     spec: &ChainSpec,
     log: Logger,
 ) -> Result<ClientConfig, String> {
@@ -67,8 +65,6 @@ pub fn get_config<E: EthSpec>(
     log_dir.pop();
     info!(log, "Data directory initialised"; "datadir" => log_dir.into_os_string().into_string().expect("Datadir should be a valid os string"));
 
-    client_config.spec_constants = spec_constants.into();
-
     /*
      * Networking
      */
@@ -222,14 +218,6 @@ pub fn get_config<E: EthSpec>(
             .map_err(|_| "block-cache-size is not a valid integer".to_string())?;
     }
 
-    if spec_constants != client_config.spec_constants {
-        crit!(log, "Specification constants do not match.";
-              "client_config" => client_config.spec_constants,
-              "eth2_config" => spec_constants
-        );
-        return Err("Specification constant mismatch".into());
-    }
-
     /*
      * Zero-ports
      *
@@ -255,7 +243,7 @@ pub fn get_config<E: EthSpec>(
     /*
      * Load the eth2 testnet dir to obtain some additional config values.
      */
-    let eth2_testnet_config: Eth2TestnetConfig<E> = get_eth2_testnet_config(&cli_args)?;
+    let eth2_testnet_config = get_eth2_testnet_config(&cli_args)?;
 
     client_config.eth1.deposit_contract_address =
         format!("{:?}", eth2_testnet_config.deposit_contract_address()?);
@@ -275,12 +263,12 @@ pub fn get_config<E: EthSpec>(
         client_config.network.boot_nodes_enr.append(&mut boot_nodes)
     }
 
-    if let Some(genesis_state) = eth2_testnet_config.genesis_state {
+    if let Some(genesis_state_bytes) = eth2_testnet_config.genesis_state_bytes {
         // Note: re-serializing the genesis state is not so efficient, however it avoids adding
         // trait bounds to the `ClientGenesis` enum. This would have significant flow-on
         // effects.
         client_config.genesis = ClientGenesis::SszBytes {
-            genesis_state_bytes: genesis_state.as_ssz_bytes(),
+            genesis_state_bytes,
         };
     } else {
         client_config.genesis = ClientGenesis::DepositContract;
@@ -545,9 +533,7 @@ pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf {
 
 /// Try to parse the eth2 testnet config from the `testnet`, `testnet-dir` flags in that order.
 /// Returns the default hardcoded testnet if neither flags are set.
-pub fn get_eth2_testnet_config<E: EthSpec>(
-    cli_args: &ArgMatches,
-) -> Result<Eth2TestnetConfig<E>, String> {
+pub fn get_eth2_testnet_config(cli_args: &ArgMatches) -> Result<Eth2TestnetConfig, String> {
     let optional_testnet_config = if cli_args.is_present("testnet") {
         clap_utils::parse_hardcoded_network(cli_args, "testnet")?
     } else if cli_args.is_present("testnet-dir") {

beacon_node/src/lib.rs

@@ -53,12 +53,8 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
         context: RuntimeContext<E>,
         matches: ArgMatches<'static>,
     ) -> Result<Self, String> {
-        let client_config = get_config::<E>(
-            &matches,
-            &context.eth2_config.spec_constants,
-            &context.eth2_config().spec,
-            context.log().clone(),
-        )?;
+        let client_config =
+            get_config::<E>(&matches, &context.eth2_config().spec, context.log().clone())?;
         Self::new(context, client_config).await
     }
 

book/src/cross-compiling.md

@@ -18,17 +18,15 @@ project.
 
 The `Makefile` in the project contains four targets for cross-compiling:
 
-- `build-x86_64`: builds an optimized version for x86_64 processors (suitable
-	for most users).
-- `build-x86_64-portable`: builds a version x86_64 processors which avoids
-	using some modern CPU instructions that might cause an "illegal
-	instruction" error on older CPUs.
-- `build-aarch64`: builds an optimized version for 64bit ARM processors
+- `build-x86_64`: builds an optimized version for x86_64 processors (suitable for most users).
+  Supports Intel Broadwell (2014) and newer, and AMD Ryzen (2017) and newer.
+- `build-x86_64-portable`: builds a version for x86_64 processors which avoids using some modern CPU
+  instructions that are incompatible with older CPUs. Suitable for pre-Broadwell/Ryzen CPUs.
+- `build-aarch64`: builds an optimized version for 64-bit ARM processors
 	(suitable for Raspberry Pi 4).
-- `build-aarch64-portable`: builds a version 64 bit ARM processors which avoids
-	using some modern CPU instructions that might cause an "illegal
-	instruction" error on older CPUs.
-
+- `build-aarch64-portable`: builds a version for 64-bit ARM processors which avoids using some
+  modern CPU instructions. In practice, very few ARM processors lack the instructions necessary to
+  run the faster non-portable build.
 
 ### Example
 

book/src/validator-create.md

@@ -43,9 +43,6 @@ OPTIONS:
         --secrets-dir <SECRETS_DIR>
             The path where the validator keystore passwords will be stored. Defaults to ~/.lighthouse/{testnet}/secrets
 
-    -s, --spec <TITLE>
-            Specifies the default eth2 spec type. [default: mainnet]  [possible values: mainnet, minimal, interop]
-
         --testnet <testnet>
             Name of network lighthouse will connect to [possible values: medalla, altona]
 
@@ -81,4 +78,4 @@ This command will:
 		`--testnet` CLI param.
 - Store a password to the validators voting keypair in `~/.lighthouse/{testnet}/secrets`.
 
-where `testnet` is the name of the testnet passed in the `--testnet` parameter (default is `medalla`).
+where `testnet` is the name of the testnet passed in the `--testnet` parameter (default is `medalla`).

book/src/wallet-create.md

@@ -43,9 +43,6 @@ OPTIONS:
             A path to a file containing the password which will unlock the wallet. If the file does not exist, a random
             password will be generated and saved at that path. To avoid confusion, if the file does not already exist it
             must include a '.pass' suffix.
-    -s, --spec <TITLE>
-            Specifies the default eth2 spec type. [default: mainnet]  [possible values: mainnet, minimal, interop]
-
     -t, --testnet-dir <DIR>
             Path to directory containing eth2_testnet specs. Defaults to a hard-coded Lighthouse testnet. Only effective
             if there is no existing database.

boot_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "boot_node"
-version = "0.3.1"
+version = "0.3.2"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 

boot_node/src/config.rs

@@ -5,7 +5,6 @@ use eth2_libp2p::{
     discovery::{create_enr_builder_from_config, use_or_load_enr},
     load_private_key, CombinedKeyExt, NetworkConfig,
 };
-use eth2_testnet_config::Eth2TestnetConfig;
 use ssz::Encode;
 use std::convert::TryFrom;
 use std::marker::PhantomData;
@@ -30,7 +29,7 @@ impl<T: EthSpec> TryFrom<&ArgMatches<'_>> for BootNodeConfig<T> {
         let data_dir = get_data_dir(matches);
 
         // Try and grab testnet config from input CLI params
-        let eth2_testnet_config: Option<Eth2TestnetConfig<T>> = {
+        let eth2_testnet_config = {
             if matches.is_present("testnet") {
                 Some(get_eth2_testnet_config(&matches)?)
             } else {
@@ -95,7 +94,9 @@ impl<T: EthSpec> TryFrom<&ArgMatches<'_>> for BootNodeConfig<T> {
                 .apply_to_chain_spec::<T>(&T::default_spec())
                 .ok_or_else(|| "The loaded config is not compatible with the current spec")?;
 
-            if let Some(genesis_state) = config.genesis_state.as_ref() {
+            if config.beacon_state_is_known() {
+                let genesis_state = config.beacon_state::<T>()?;
+
                 slog::info!(logger, "Genesis state found"; "root" => genesis_state.canonical_root().to_string());
                 let enr_fork = spec.enr_fork_id(
                     types::Slot::from(0u64),

boot_node/src/lib.rs

@@ -8,12 +8,12 @@ mod config;
 mod server;
 pub use cli::cli_app;
 use config::BootNodeConfig;
-use types::EthSpec;
+use types::{EthSpec, EthSpecId};
 
 const LOG_CHANNEL_SIZE: usize = 2048;
 
 /// Run the bootnode given the CLI configuration.
-pub fn run(matches: &ArgMatches<'_>, debug_level: String) {
+pub fn run(matches: &ArgMatches<'_>, eth_spec_id: EthSpecId, debug_level: String) {
     let debug_level = match debug_level.as_str() {
         "trace" => log::Level::Trace,
         "debug" => log::Level::Debug,
@@ -48,11 +48,10 @@ pub fn run(matches: &ArgMatches<'_>, debug_level: String) {
 
     let log = slog_scope::logger();
     // Run the main function emitting any errors
-    if let Err(e) = match matches.value_of("spec") {
-        Some("minimal") => main::<types::MinimalEthSpec>(matches, log),
-        Some("mainnet") => main::<types::MainnetEthSpec>(matches, log),
-        Some("interop") => main::<types::InteropEthSpec>(matches, log),
-        spec => unreachable!("Unknown spec configuration: {:?}", spec),
+    if let Err(e) = match eth_spec_id {
+        EthSpecId::Minimal => main::<types::MinimalEthSpec>(matches, log),
+        EthSpecId::Mainnet => main::<types::MainnetEthSpec>(matches, log),
+        EthSpecId::V012Legacy => main::<types::V012LegacyEthSpec>(matches, log),
     } {
         slog::crit!(slog_scope::logger(), "{}", e);
     }

common/clap_utils/src/lib.rs

@@ -5,7 +5,6 @@ use eth2_testnet_config::Eth2TestnetConfig;
 use ssz::Decode;
 use std::path::PathBuf;
 use std::str::FromStr;
-use types::EthSpec;
 
 pub const BAD_TESTNET_DIR_MESSAGE: &str = "The hard-coded testnet directory was invalid. \
                                         This happens when Lighthouse is migrating between spec versions \
@@ -14,10 +13,10 @@ pub const BAD_TESTNET_DIR_MESSAGE: &str = "The hard-coded testnet directory was
 
 /// Attempts to load the testnet dir at the path if `name` is in `matches`, returning an error if
 /// the path cannot be found or the testnet dir is invalid.
-pub fn parse_testnet_dir<E: EthSpec>(
+pub fn parse_testnet_dir(
     matches: &ArgMatches,
     name: &'static str,
-) -> Result<Option<Eth2TestnetConfig<E>>, String> {
+) -> Result<Option<Eth2TestnetConfig>, String> {
     let path = parse_required::<PathBuf>(matches, name)?;
     Eth2TestnetConfig::load(path.clone())
         .map_err(|e| format!("Unable to open testnet dir at {:?}: {}", path, e))
@@ -26,10 +25,10 @@ pub fn parse_testnet_dir<E: EthSpec>(
 
 /// Attempts to load a hardcoded network config if `name` is in `matches`, returning an error if
 /// the name is not a valid network name.
-pub fn parse_hardcoded_network<E: EthSpec>(
+pub fn parse_hardcoded_network(
     matches: &ArgMatches,
     name: &str,
-) -> Result<Option<Eth2TestnetConfig<E>>, String> {
+) -> Result<Option<Eth2TestnetConfig>, String> {
     let network_name = parse_required::<String>(matches, name)?;
     Eth2TestnetConfig::constant(network_name.as_str())
 }

common/eth2_config/src/lib.rs

@@ -1,7 +1,7 @@
 use serde_derive::{Deserialize, Serialize};
 use std::env;
 use std::path::PathBuf;
-use types::ChainSpec;
+use types::{ChainSpec, EthSpecId};
 
 // A macro is used to define this constant so it can be used with `include_bytes!`.
 #[macro_export]
@@ -19,14 +19,14 @@ pub const GENESIS_ZIP_FILE_NAME: &str = "genesis.ssz.zip";
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default)]
 pub struct Eth2Config {
-    pub spec_constants: String,
+    pub eth_spec_id: EthSpecId,
     pub spec: ChainSpec,
 }
 
 impl Default for Eth2Config {
     fn default() -> Self {
         Self {
-            spec_constants: "minimal".to_string(),
+            eth_spec_id: EthSpecId::Minimal,
             spec: ChainSpec::minimal(),
         }
     }
@@ -35,22 +35,22 @@ impl Default for Eth2Config {
 impl Eth2Config {
     pub fn mainnet() -> Self {
         Self {
-            spec_constants: "mainnet".to_string(),
+            eth_spec_id: EthSpecId::Mainnet,
             spec: ChainSpec::mainnet(),
         }
     }
 
     pub fn minimal() -> Self {
         Self {
-            spec_constants: "minimal".to_string(),
+            eth_spec_id: EthSpecId::Minimal,
             spec: ChainSpec::minimal(),
         }
     }
 
-    pub fn interop() -> Self {
+    pub fn v012_legacy() -> Self {
         Self {
-            spec_constants: "interop".to_string(),
-            spec: ChainSpec::interop(),
+            eth_spec_id: EthSpecId::V012Legacy,
+            spec: ChainSpec::v012_legacy(),
         }
     }
 }

common/eth2_testnet_config/built_in_testnet_configs/altona/config.yaml

@@ -1,3 +1,4 @@
+CONFIG_NAME: "altona"
 MAX_COMMITTEES_PER_SLOT: 64
 TARGET_COMMITTEE_SIZE: 128
 MAX_VALIDATORS_PER_COMMITTEE: 2048
@@ -56,3 +57,4 @@ DOMAIN_SELECTION_PROOF: 0x05000000
 DOMAIN_AGGREGATE_AND_PROOF: 0x06000000
 DEPOSIT_CHAIN_ID: 5
 DEPOSIT_NETWORK_ID: 5
+PROPORTIONAL_SLASHING_MULTIPLIER: 3