From f15b4eed7d80df7e03b34ea7146663e3c982fdea Mon Sep 17 00:00:00 2001 From: David Enyeart Date: Mon, 29 Nov 2021 12:08:50 -0500 Subject: [PATCH] Add command reference doc for ledgerutil Add command reference doc for ledgerutil. Includes minor edits to the command's user messages. Signed-off-by: David Enyeart --- cmd/ledgerutil/main.go | 16 ++---- docs/source/command_ref.rst | 1 + docs/source/commands/ledgerutil.md | 73 ++++++++++++++++++++++++++ docs/source/peer_ledger_snapshot.md | 6 ++- docs/wrappers/ledgerutil_postscript.md | 24 +++++++++ docs/wrappers/ledgerutil_preamble.md | 21 ++++++++ scripts/help_docs.sh | 7 +++ 7 files changed, 135 insertions(+), 13 deletions(-) create mode 100644 docs/source/commands/ledgerutil.md create mode 100644 docs/wrappers/ledgerutil_postscript.md create mode 100644 docs/wrappers/ledgerutil_preamble.md diff --git a/cmd/ledgerutil/main.go b/cmd/ledgerutil/main.go index 40f0a341a7d..55ab3f57966 100644 --- a/cmd/ledgerutil/main.go +++ b/cmd/ledgerutil/main.go @@ -17,23 +17,21 @@ import ( const ( compareErrorMessage = "Ledger Compare Error: " outputDirDesc = "Snapshot comparison json results output directory. Default is the current directory." - firstDiffsDesc = "Number of differences to record in " + ledgerutil.FirstDiffsByHeight + - ". Generating a file of more than the first 10 differences will result in a large amount " + - "of memory usage and is not recommended. Defaults to 10. If set to 0, will not produce " + + firstDiffsDesc = "Maximum number of differences to record in " + ledgerutil.FirstDiffsByHeight + + ". Requesting a report with many differences may result in a large amount " + + "of memory usage. Defaults to 10. If set to 0, will not produce " + ledgerutil.FirstDiffsByHeight + "." ) var ( app = kingpin.New("ledgerutil", "Ledger Utility Tool") - compare = app.Command("compare", "Compare two ledgers via their snapshots.") + compare = app.Command("compare", "Compare channel snapshots from two different peers.") snapshotPath1 = compare.Arg("snapshotPath1", "First ledger snapshot directory.").Required().String() snapshotPath2 = compare.Arg("snapshotPath2", "Second ledger snapshot directory.").Required().String() outputDir = compare.Flag("outputDir", outputDirDesc).Short('o').String() firstDiffs = compare.Flag("firstDiffs", firstDiffsDesc).Short('f').Default("10").Int() - troubleshoot = app.Command("troubleshoot", "Identify potentially divergent transactions.") - args = os.Args[1:] ) @@ -48,7 +46,6 @@ func main() { // Command logic switch command { - case compare.FullCommand(): // Determine result json file location @@ -72,10 +69,5 @@ func main() { } else { fmt.Printf("Results saved to %s. Total differences found: %d\n", outputDirPath, count) } - - case troubleshoot.FullCommand(): - - fmt.Println("Command TBD") - } } diff --git a/docs/source/command_ref.rst b/docs/source/command_ref.rst index 59a348f64c9..96bbddd2c79 100644 --- a/docs/source/command_ref.rst +++ b/docs/source/command_ref.rst @@ -15,5 +15,6 @@ Commands Reference commands/configtxgen.md commands/configtxlator.md commands/cryptogen.md + commands/ledgerutil.md discovery-cli.md commands/fabric-ca-commands diff --git a/docs/source/commands/ledgerutil.md b/docs/source/commands/ledgerutil.md new file mode 100644 index 00000000000..31550ecb3b7 --- /dev/null +++ b/docs/source/commands/ledgerutil.md @@ -0,0 +1,73 @@ + + +# ledgerutil + +The `ledgerutil compare` command allows administrators to compare channel snapshots from two different peers. +Although channel snapshots from the same height should be identical across peers, if the snapshot +files indicate different state hashes (as seen in `_snapshot_signable_metadata.json` file from each snapshot) +then it indicates that at least one of the peers state databases is not correct relative to the blockchain and +may have gotten corrupted. + +The `ledgerutil compare` utility will output a set of JSON files if the snapshots are not identical to assist with troubleshooting in these situations. +Two output JSON files will include any key/value differences sorted by key (one for public key/value differences and one for private key/value differences), +another JSON file will include any key/value differences (public or private) sorted by block and transaction height so that you can identify the height where a divergence may have first occurred. + +The output files may help an administrator to understand the scope of a state database issue and identify which keys are impacted. +Snapshots from additional peers can be compared to determine which peer has incorrect state. +The block and transaction height of the first difference can be used as a reference point when looking at the peer's logs to understand what may have happened to the state database at that time. + +## Syntax + +The `ledgerutil` command has one subcommand + + * compare + +## ledgerutil compare +``` +usage: ledgerutil compare [] + +Compare channel snapshots from two different peers. + +Flags: + --help Show context-sensitive help (also try --help-long + and --help-man). + -o, --outputDir=OUTPUTDIR Snapshot comparison json results output directory. + Default is the current directory. + -f, --firstDiffs=10 Maximum number of differences to record in + first_diffs_by_height.json. Requesting a report + with many differences may result in a large amount + of memory usage. Defaults to 10. If set to 0, will + not produce first_diffs_by_height.json. + +Args: + First ledger snapshot directory. + Second ledger snapshot directory. +``` + +## Example Usage + +### ledgerutil compare example + +Here is an example of the `ledgerutil compare` command. + + * Compare snapshots from two different peers for mychannel at snapshot height 5. + + ``` + ledgerutil compare -o ./compare_output -f 10 ./peer0.org1.example.com/snapshots/completed/mychannel/5 ./peer1.org1.example.com/snapshots/completed/mychannel/5 + + Both snapshot public state and private state hashes were the same. No results were generated. + + ``` + + The response above indicates that the snapshots are identical. If the snapshots were not identical, the command results will indicate where the comparison output files are written, for example: + + ``` + Successfully compared snapshots. Results saved to compare_output/mychannel_5_comparison. Total differences found: 3 + ``` + + * Note that both snapshot locations must be accessible by the command, for example by mounting volumes from two different peers, or by copying the snapshots to a common location. + +Creative Commons License
This work is licensed under a Creative Commons Attribution 4.0 International License. diff --git a/docs/source/peer_ledger_snapshot.md b/docs/source/peer_ledger_snapshot.md index 9b3dd8353ea..5786101ea4d 100644 --- a/docs/source/peer_ledger_snapshot.md +++ b/docs/source/peer_ledger_snapshot.md @@ -42,7 +42,11 @@ The organization that will use the snapshot to join the channel will then: If an organization that is already joined to the channel wants to join a new peer using a snapshot, it might decide to skip the process of having other organizations take snapshots and evaluate them, though it is a best practice for an organization to periodically take snapshots of all its peers and compare them to ensure the no ledger forks have occurred. In that case, the organization can take a snapshot immediately and then use the snapshot to join the new peer to the channel. -**Note: snapshots can also be used to verify that the public state between peers is identical (in other words, that no ledger fork has occurred), even if no new peer will use the snapshot to join the channel. This can be done by ensuring that the `snapshot_hash` in the file `_snapshot_additional_metadata.json` in the snapshots generated across peers is the same.** +## Using snapshots to verify peer integrity + +Snapshots can be used to verify that the state between peers is identical (in other words, that no ledger fork has occurred), even if no new peer will use the snapshot to join the channel. +This can be done by ensuring that the `snapshot_hash` in the file `_snapshot_additional_metadata.json` in the snapshots generated across peers is the same. +If the hashes are not identical, you can use the [`ledgerutil compare` utility](./commands/ledgerutil.html) to troubleshoot which keys are different across any two snapshots and to understand when a divergence may have occurred. ## Taking a snapshot diff --git a/docs/wrappers/ledgerutil_postscript.md b/docs/wrappers/ledgerutil_postscript.md new file mode 100644 index 00000000000..a58389b604e --- /dev/null +++ b/docs/wrappers/ledgerutil_postscript.md @@ -0,0 +1,24 @@ +## Example Usage + +### ledgerutil compare example + +Here is an example of the `ledgerutil compare` command. + + * Compare snapshots from two different peers for mychannel at snapshot height 5. + + ``` + ledgerutil compare -o ./compare_output -f 10 ./peer0.org1.example.com/snapshots/completed/mychannel/5 ./peer1.org1.example.com/snapshots/completed/mychannel/5 + + Both snapshot public state and private state hashes were the same. No results were generated. + + ``` + + The response above indicates that the snapshots are identical. If the snapshots were not identical, the command results will indicate where the comparison output files are written, for example: + + ``` + Successfully compared snapshots. Results saved to compare_output/mychannel_5_comparison. Total differences found: 3 + ``` + + * Note that both snapshot locations must be accessible by the command, for example by mounting volumes from two different peers, or by copying the snapshots to a common location. + +Creative Commons License
This work is licensed under a Creative Commons Attribution 4.0 International License. diff --git a/docs/wrappers/ledgerutil_preamble.md b/docs/wrappers/ledgerutil_preamble.md new file mode 100644 index 00000000000..3b0210b0a63 --- /dev/null +++ b/docs/wrappers/ledgerutil_preamble.md @@ -0,0 +1,21 @@ +# ledgerutil + +The `ledgerutil compare` command allows administrators to compare channel snapshots from two different peers. +Although channel snapshots from the same height should be identical across peers, if the snapshot +files indicate different state hashes (as seen in `_snapshot_signable_metadata.json` file from each snapshot) +then it indicates that at least one of the peers state databases is not correct relative to the blockchain and +may have gotten corrupted. + +The `ledgerutil compare` utility will output a set of JSON files if the snapshots are not identical to assist with troubleshooting in these situations. +Two output JSON files will include any key/value differences sorted by key (one for public key/value differences and one for private key/value differences), +another JSON file will include any key/value differences (public or private) sorted by block and transaction height so that you can identify the height where a divergence may have first occurred. + +The output files may help an administrator to understand the scope of a state database issue and identify which keys are impacted. +Snapshots from additional peers can be compared to determine which peer has incorrect state. +The block and transaction height of the first difference can be used as a reference point when looking at the peer's logs to understand what may have happened to the state database at that time. + +## Syntax + +The `ledgerutil` command has one subcommand + + * compare diff --git a/scripts/help_docs.sh b/scripts/help_docs.sh index 3aefcce9d7e..e99caa1e715 100755 --- a/scripts/help_docs.sh +++ b/scripts/help_docs.sh @@ -141,4 +141,11 @@ generateOrCheck \ docs/wrappers/osnadmin_channel_postscript.md \ "${commands[@]}" +commands=("ledgerutil compare") +generateOrCheck \ + docs/source/commands/ledgerutil.md \ + docs/wrappers/ledgerutil_preamble.md \ + docs/wrappers/ledgerutil_postscript.md \ + "${commands[@]}" + exit