From 403d888020cc5fcba1e0c5751117b0e327dbd06f Mon Sep 17 00:00:00 2001 From: Devan Date: Wed, 18 Dec 2024 14:25:06 -0600 Subject: [PATCH] feat: Adjust tests to include lower level planning function calls --- tsdb/config.go | 8 ++ tsdb/engine/tsm1/compact.go | 5 +- tsdb/engine/tsm1/compact_test.go | 125 ++++++++++++++++++++++++++++--- 3 files changed, 125 insertions(+), 13 deletions(-) diff --git a/tsdb/config.go b/tsdb/config.go index 0dbb0e4228a..a9252b2cc10 100644 --- a/tsdb/config.go +++ b/tsdb/config.go @@ -84,8 +84,16 @@ const ( // MaxTSMFileSize is the maximum size of TSM files. MaxTSMFileSize = uint32(2048 * 1024 * 1024) // 2GB + ) +// SingleGenerationReason outputs a log message for our single generation compaction +// when checked for full compaction. +// 1048576000 is a magic number for bytes per gigabyte. +func SingleGenerationReason() string { + return fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), AggressiveMaxPointsPerBlock) +} + // Config holds the configuration for the tsbd package. type Config struct { Dir string `toml:"dir"` diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go index 730b2d4ba17..945cc650a40 100644 --- a/tsdb/engine/tsm1/compact.go +++ b/tsdb/engine/tsm1/compact.go @@ -225,9 +225,6 @@ func (c *DefaultPlanner) ParseFileName(path string) (int, int, error) { // FullyCompacted returns true if the shard is fully compacted. func (c *DefaultPlanner) FullyCompacted() (bool, string) { gens := c.findGenerations(false) - // 1048576000 is a magic number for bytes per gigabyte - singleGenReason := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock) - if len(gens) > 1 { return false, "not fully compacted and not idle because of more than one generation" } else if gens.hasTombstones() { @@ -251,7 +248,7 @@ func (c *DefaultPlanner) FullyCompacted() (bool, string) { } if filesUnderMaxTsmSizeCount > 1 && aggressivePointsPerBlockCount < len(gens[0].files) { - return false, singleGenReason + return false, tsdb.SingleGenerationReason() } } return true, "" diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go index 11e38fcab6d..3c283d24dae 100644 --- a/tsdb/engine/tsm1/compact_test.go +++ b/tsdb/engine/tsm1/compact_test.go @@ -2330,6 +2330,17 @@ func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) { expFiles = append(expFiles, file) } + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + + tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) + require.Equal(t, 0, len(tsmP), "compaction group; Plan()") + require.Equal(t, int64(0), pLenP, "compaction group length; Plan()") + tsm, pLen, _ := cp.PlanOptimize() require.Equal(t, 1, len(tsm), "compaction group") require.Equal(t, int64(len(tsm)), pLen, "compaction group length") @@ -2372,6 +2383,17 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) { expFiles = append(expFiles, file) } + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + + tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) + require.Equal(t, 0, len(tsmP), "compaction group; Plan()") + require.Equal(t, int64(0), pLenP, "compaction group length; Plan()") + tsm, pLen, gLen := cp.PlanOptimize() require.Equal(t, 1, len(tsm), "compaction group") require.Equal(t, int64(len(tsm)), pLen, "compaction group length") @@ -2410,6 +2432,14 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing for _, file := range data { expFiles = append(expFiles, file) } + + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) require.Equal(t, 0, len(tsmP), "compaction group; Plan()") require.Equal(t, int64(0), pLenP, "compaction group length; Plan()") @@ -2456,12 +2486,24 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) { cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - // 1048576000 is a magic number for bytes per gigabyte - reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock) - compacted, reason := cp.FullyCompacted() - require.Equal(t, reason, reasonExp, "fullyCompacted reason") + require.Equal(t, reason, tsdb.SingleGenerationReason(), "fullyCompacted reason") require.False(t, compacted, "is fully compacted") + + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + + _, cgLen = cp.Plan(time.Now().Add(-1)) + require.Equal(t, int64(0), cgLen, "compaction group length; Plan()") + + cgroup, cgLen, genLen := cp.PlanOptimize() + require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") + require.Equal(t, int64(0), cgLen, "compaction group length") + require.Equal(t, int64(0), genLen, "generation count") } // This test is added to account for halting state after @@ -2488,6 +2530,21 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T) reasonExp := "" require.Equal(t, reason, reasonExp, "fullyCompacted reason") require.True(t, compacted, "is fully compacted") + + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + + _, cgLen = cp.Plan(time.Now().Add(-1)) + require.Equal(t, int64(0), cgLen, "compaction group length; Plan()") + + cgroup, cgLen, genLen := cp.PlanOptimize() + require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") + require.Equal(t, int64(0), cgLen, "compaction group length") + require.Equal(t, int64(0), genLen, "generation count") } // This test is added to account for a single generation that has a group size @@ -2550,14 +2607,24 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock require.NoError(t, err, "SetBlockCounts") cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - - // 1048576000 is a magic number for bytes per gigabyte - reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock) - compacted, reason := cp.FullyCompacted() - require.Equal(t, reason, reasonExp, "fullyCompacted reason") + require.Equal(t, reason, tsdb.SingleGenerationReason(), "fullyCompacted reason") require.False(t, compacted, "is fully compacted") + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + + _, cgLen = cp.Plan(time.Now().Add(-1)) + require.Equal(t, int64(0), cgLen, "compaction group length; Plan()") + + cgroup, cgLen, genLen := cp.PlanOptimize() + require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") + require.Equal(t, int64(0), cgLen, "compaction group length") + require.Equal(t, int64(0), genLen, "generation count") } // This test is added to account for a single generation that has a group size @@ -2596,6 +2663,16 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks( require.Equal(t, reason, reasonExp, "fullyCompacted reason") require.True(t, compacted, "is fully compacted") + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + + _, cgLen = cp.Plan(time.Now().Add(-1)) + require.Equal(t, int64(0), cgLen, "compaction group length; Plan()") + cgroup, cgLen, genLen := cp.PlanOptimize() require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") require.Equal(t, int64(0), cgLen, "compaction group length") @@ -2639,6 +2716,16 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *t require.Equal(t, reason, reasonExp, "fullyCompacted reason") require.True(t, compacted, "is fully compacted") + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + + _, cgLen = cp.Plan(time.Now().Add(-1)) + require.Equal(t, int64(0), cgLen, "compaction group length; Plan()") + cgroup, cgLen, genLen := cp.PlanOptimize() require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") require.Equal(t, int64(0), cgLen, "compaction group length") @@ -2685,6 +2772,16 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t * require.Equal(t, reason, reasonExp, "fullyCompacted reason") require.True(t, compacted, "is fully compacted") + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + + _, cgLen = cp.Plan(time.Now().Add(-1)) + require.Equal(t, int64(0), cgLen, "compaction group length; Plan()") + cgroup, cgLen, genLen := cp.PlanOptimize() require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") require.Equal(t, int64(0), cgLen, "compaction group length") @@ -2734,6 +2831,16 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks( require.Equal(t, reason, reasonExp, "fullyCompacted reason") require.False(t, compacted, "is fully compacted") + _, cgLen := cp.PlanLevel(1) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)") + + _, cgLen = cp.Plan(time.Now().Add(-1)) + require.Equal(t, int64(0), cgLen, "compaction group length; Plan()") + _, cgLen, genLen := cp.PlanOptimize() require.Equal(t, int64(1), cgLen, "compaction group length") require.Equal(t, int64(1), genLen, "generation count")