Skip to content

Commit

Permalink
feat: Adjust tests to include lower level planning function calls
Browse files Browse the repository at this point in the history
  • Loading branch information
@devanbenz
devanbenz committed Dec 18, 2024
1 parent 54c8e1c commit 403d888
Show file tree
Hide file tree
Showing 3 changed files with 125 additions and 13 deletions.
8 changes: 8 additions & 0 deletions tsdb/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,16 @@ const (

// MaxTSMFileSize is the maximum size of TSM files.
MaxTSMFileSize = uint32(2048 * 1024 * 1024) // 2GB

)

// SingleGenerationReason outputs a log message for our single generation compaction
// when checked for full compaction.
// 1048576000 is a magic number for bytes per gigabyte.
func SingleGenerationReason() string {
return fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), AggressiveMaxPointsPerBlock)
}

// Config holds the configuration for the tsbd package.
type Config struct {
Dir string `toml:"dir"`
Expand Down
5 changes: 1 addition & 4 deletions tsdb/engine/tsm1/compact.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,9 +225,6 @@ func (c *DefaultPlanner) ParseFileName(path string) (int, int, error) {
// FullyCompacted returns true if the shard is fully compacted.
func (c *DefaultPlanner) FullyCompacted() (bool, string) {
gens := c.findGenerations(false)
// 1048576000 is a magic number for bytes per gigabyte
singleGenReason := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)

if len(gens) > 1 {
return false, "not fully compacted and not idle because of more than one generation"
} else if gens.hasTombstones() {
Expand All @@ -251,7 +248,7 @@ func (c *DefaultPlanner) FullyCompacted() (bool, string) {
}

if filesUnderMaxTsmSizeCount > 1 && aggressivePointsPerBlockCount < len(gens[0].files) {
return false, singleGenReason
return false, tsdb.SingleGenerationReason()
}
}
return true, ""
Expand Down
125 changes: 116 additions & 9 deletions tsdb/engine/tsm1/compact_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2330,6 +2330,17 @@ func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) {
expFiles = append(expFiles, file)
}

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")

tsm, pLen, _ := cp.PlanOptimize()
require.Equal(t, 1, len(tsm), "compaction group")
require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
Expand Down Expand Up @@ -2372,6 +2383,17 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) {
expFiles = append(expFiles, file)
}

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")

tsm, pLen, gLen := cp.PlanOptimize()
require.Equal(t, 1, len(tsm), "compaction group")
require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
Expand Down Expand Up @@ -2410,6 +2432,14 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing
for _, file := range data {
expFiles = append(expFiles, file)
}

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")
Expand Down Expand Up @@ -2456,12 +2486,24 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {

cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)

// 1048576000 is a magic number for bytes per gigabyte
reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)

compacted, reason := cp.FullyCompacted()
require.Equal(t, reason, reasonExp, "fullyCompacted reason")
require.Equal(t, reason, tsdb.SingleGenerationReason(), "fullyCompacted reason")
require.False(t, compacted, "is fully compacted")

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

_, cgLen = cp.Plan(time.Now().Add(-1))
require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")

cgroup, cgLen, genLen := cp.PlanOptimize()
require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
require.Equal(t, int64(0), cgLen, "compaction group length")
require.Equal(t, int64(0), genLen, "generation count")
}

// This test is added to account for halting state after
Expand All @@ -2488,6 +2530,21 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T)
reasonExp := ""
require.Equal(t, reason, reasonExp, "fullyCompacted reason")
require.True(t, compacted, "is fully compacted")

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

_, cgLen = cp.Plan(time.Now().Add(-1))
require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")

cgroup, cgLen, genLen := cp.PlanOptimize()
require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
require.Equal(t, int64(0), cgLen, "compaction group length")
require.Equal(t, int64(0), genLen, "generation count")
}

// This test is added to account for a single generation that has a group size
Expand Down Expand Up @@ -2550,14 +2607,24 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
require.NoError(t, err, "SetBlockCounts")

cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)

// 1048576000 is a magic number for bytes per gigabyte
reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)

compacted, reason := cp.FullyCompacted()
require.Equal(t, reason, reasonExp, "fullyCompacted reason")
require.Equal(t, reason, tsdb.SingleGenerationReason(), "fullyCompacted reason")
require.False(t, compacted, "is fully compacted")

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

_, cgLen = cp.Plan(time.Now().Add(-1))
require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")

cgroup, cgLen, genLen := cp.PlanOptimize()
require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
require.Equal(t, int64(0), cgLen, "compaction group length")
require.Equal(t, int64(0), genLen, "generation count")
}

// This test is added to account for a single generation that has a group size
Expand Down Expand Up @@ -2596,6 +2663,16 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(
require.Equal(t, reason, reasonExp, "fullyCompacted reason")
require.True(t, compacted, "is fully compacted")

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

_, cgLen = cp.Plan(time.Now().Add(-1))
require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")

cgroup, cgLen, genLen := cp.PlanOptimize()
require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
require.Equal(t, int64(0), cgLen, "compaction group length")
Expand Down Expand Up @@ -2639,6 +2716,16 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *t
require.Equal(t, reason, reasonExp, "fullyCompacted reason")
require.True(t, compacted, "is fully compacted")

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

_, cgLen = cp.Plan(time.Now().Add(-1))
require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")

cgroup, cgLen, genLen := cp.PlanOptimize()
require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
require.Equal(t, int64(0), cgLen, "compaction group length")
Expand Down Expand Up @@ -2685,6 +2772,16 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *
require.Equal(t, reason, reasonExp, "fullyCompacted reason")
require.True(t, compacted, "is fully compacted")

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

_, cgLen = cp.Plan(time.Now().Add(-1))
require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")

cgroup, cgLen, genLen := cp.PlanOptimize()
require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
require.Equal(t, int64(0), cgLen, "compaction group length")
Expand Down Expand Up @@ -2734,6 +2831,16 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(
require.Equal(t, reason, reasonExp, "fullyCompacted reason")
require.False(t, compacted, "is fully compacted")

_, cgLen := cp.PlanLevel(1)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
_, cgLen = cp.PlanLevel(2)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
_, cgLen = cp.PlanLevel(3)
require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")

_, cgLen = cp.Plan(time.Now().Add(-1))
require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")

_, cgLen, genLen := cp.PlanOptimize()
require.Equal(t, int64(1), cgLen, "compaction group length")
require.Equal(t, int64(1), genLen, "generation count")
Expand Down

0 comments on commit 403d888

Please sign in to comment.