simplify the agents, forums, memory and referrer counters

move DirSize into it's own file

common/counters/agents.go

@@ -4,7 +4,8 @@ import (
 	"database/sql"
 
 	c "github.com/Azareal/Gosora/common"
-	"github.com/Azareal/Gosora/query_gen"
+	qgen "github.com/Azareal/Gosora/query_gen"
+	"github.com/pkg/errors"
 )
 
 var AgentViewCounter *DefaultAgentViewCounter
@@ -19,49 +20,49 @@ func NewDefaultAgentViewCounter(acc *qgen.Accumulator) (*DefaultAgentViewCounter
 	for bucketID, _ := range agentBuckets {
 		agentBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
 	}
-	counter := &DefaultAgentViewCounter{
+	co := &DefaultAgentViewCounter{
 		agentBuckets: agentBuckets,
 		insert:       acc.Insert("viewchunks_agents").Columns("count, createdAt, browser").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
 	}
-	c.AddScheduledFifteenMinuteTask(counter.Tick)
-	//c.AddScheduledSecondTask(counter.Tick)
-	c.AddShutdownTask(counter.Tick)
-	return counter, acc.FirstError()
+	c.AddScheduledFifteenMinuteTask(co.Tick)
+	//c.AddScheduledSecondTask(co.Tick)
+	c.AddShutdownTask(co.Tick)
+	return co, acc.FirstError()
 }
 
-func (counter *DefaultAgentViewCounter) Tick() error {
-	for agentID, agentBucket := range counter.agentBuckets {
+func (co *DefaultAgentViewCounter) Tick() error {
+	for agentID, agentBucket := range co.agentBuckets {
 		var count int
 		agentBucket.RLock()
 		count = agentBucket.counter
 		agentBucket.counter = 0
 		agentBucket.RUnlock()
 
-		err := counter.insertChunk(count, agentID) // TODO: Bulk insert for speed?
+		err := co.insertChunk(count, agentID) // TODO: Bulk insert for speed?
 		if err != nil {
-			return err
+			return errors.Wrap(errors.WithStack(err), "agent counter")
 		}
 	}
 	return nil
 }
 
-func (counter *DefaultAgentViewCounter) insertChunk(count int, agent int) error {
+func (co *DefaultAgentViewCounter) insertChunk(count int, agent int) error {
 	if count == 0 {
 		return nil
 	}
-	var agentName = reverseAgentMapEnum[agent]
-	c.DebugLogf("Inserting a viewchunk with a count of %d for agent %s (%d)", count, agentName, agent)
-	_, err := counter.insert.Exec(count, agentName)
+	agentName := reverseAgentMapEnum[agent]
+	c.DebugLogf("Inserting a vchunk with a count of %d for agent %s (%d)", count, agentName, agent)
+	_, err := co.insert.Exec(count, agentName)
 	return err
 }
 
-func (counter *DefaultAgentViewCounter) Bump(agent int) {
+func (co *DefaultAgentViewCounter) Bump(agent int) {
 	// TODO: Test this check
-	c.DebugDetail("counter.agentBuckets[", agent, "]: ", counter.agentBuckets[agent])
-	if len(counter.agentBuckets) <= agent || agent < 0 {
+	c.DebugDetail("co.agentBuckets[", agent, "]: ", co.agentBuckets[agent])
+	if len(co.agentBuckets) <= agent || agent < 0 {
 		return
 	}
-	counter.agentBuckets[agent].Lock()
-	counter.agentBuckets[agent].counter++
-	counter.agentBuckets[agent].Unlock()
+	co.agentBuckets[agent].Lock()
+	co.agentBuckets[agent].counter++
+	co.agentBuckets[agent].Unlock()
 }

common/counters/forums.go

@@ -6,6 +6,7 @@ import (
 
 	c "github.com/Azareal/Gosora/common"
 	"github.com/Azareal/Gosora/query_gen"
+	"github.com/pkg/errors"
 )
 
 var ForumViewCounter *DefaultForumViewCounter
@@ -23,95 +24,84 @@ type DefaultForumViewCounter struct {
 
 func NewDefaultForumViewCounter() (*DefaultForumViewCounter, error) {
 	acc := qgen.NewAcc()
-	counter := &DefaultForumViewCounter{
+	co := &DefaultForumViewCounter{
 		oddMap:  make(map[int]*RWMutexCounterBucket),
 		evenMap: make(map[int]*RWMutexCounterBucket),
 		insert:  acc.Insert("viewchunks_forums").Columns("count, createdAt, forum").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
 	}
-	c.AddScheduledFifteenMinuteTask(counter.Tick) // There could be a lot of routes, so we don't want to be running this every second
-	//c.AddScheduledSecondTask(counter.Tick)
-	c.AddShutdownTask(counter.Tick)
-	return counter, acc.FirstError()
+	c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second
+	//c.AddScheduledSecondTask(co.Tick)
+	c.AddShutdownTask(co.Tick)
+	return co, acc.FirstError()
 }
 
-func (counter *DefaultForumViewCounter) Tick() error {
-	counter.oddLock.RLock()
-	oddMap := counter.oddMap
-	counter.oddLock.RUnlock()
-	for forumID, forum := range oddMap {
-		var count int
-		forum.RLock()
-		count = forum.counter
-		forum.RUnlock()
-		// TODO: Only delete the bucket when it's zero to avoid hitting popular forums?
-		counter.oddLock.Lock()
-		delete(counter.oddMap, forumID)
-		counter.oddLock.Unlock()
-		err := counter.insertChunk(count, forumID)
-		if err != nil {
-			return err
+func (co *DefaultForumViewCounter) Tick() error {
+	cLoop := func(l *sync.RWMutex, m map[int]*RWMutexCounterBucket) error {
+		l.RLock()
+		for forumID, forum := range m {
+			l.RUnlock()
+			var count int
+			forum.RLock()
+			count = forum.counter
+			forum.RUnlock()
+			// TODO: Only delete the bucket when it's zero to avoid hitting popular forums?
+			l.Lock()
+			delete(m, forumID)
+			l.Unlock()
+			err := co.insertChunk(count, forumID)
+			if err != nil {
+				return errors.Wrap(errors.WithStack(err),"forum counter")
+			}
+			l.RLock()
 		}
+		l.RUnlock()
+		return nil
 	}
-
-	counter.evenLock.RLock()
-	evenMap := counter.evenMap
-	counter.evenLock.RUnlock()
-	for forumID, forum := range evenMap {
-		var count int
-		forum.RLock()
-		count = forum.counter
-		forum.RUnlock()
-		// TODO: Only delete the bucket when it's zero to avoid hitting popular forums?
-		counter.evenLock.Lock()
-		delete(counter.evenMap, forumID)
-		counter.evenLock.Unlock()
-		err := counter.insertChunk(count, forumID)
-		if err != nil {
-			return err
-		}
+	err := cLoop(&co.oddLock,co.oddMap)
+	if err != nil {
+		return err
 	}
-
-	return nil
+	return cLoop(&co.evenLock,co.evenMap)
 }
 
-func (counter *DefaultForumViewCounter) insertChunk(count int, forum int) error {
+func (co *DefaultForumViewCounter) insertChunk(count int, forum int) error {
 	if count == 0 {
 		return nil
 	}
-	c.DebugLogf("Inserting a viewchunk with a count of %d for forum %d", count, forum)
-	_, err := counter.insert.Exec(count, forum)
+	c.DebugLogf("Inserting a vchunk with a count of %d for forum %d", count, forum)
+	_, err := co.insert.Exec(count, forum)
 	return err
 }
 
-func (counter *DefaultForumViewCounter) Bump(forumID int) {
+func (co *DefaultForumViewCounter) Bump(forumID int) {
 	// Is the ID even?
 	if forumID%2 == 0 {
-		counter.evenLock.RLock()
-		forum, ok := counter.evenMap[forumID]
-		counter.evenLock.RUnlock()
+		co.evenLock.RLock()
+		forum, ok := co.evenMap[forumID]
+		co.evenLock.RUnlock()
 		if ok {
 			forum.Lock()
 			forum.counter++
 			forum.Unlock()
 		} else {
-			counter.evenLock.Lock()
-			counter.evenMap[forumID] = &RWMutexCounterBucket{counter: 1}
-			counter.evenLock.Unlock()
+			co.evenLock.Lock()
+			co.evenMap[forumID] = &RWMutexCounterBucket{counter: 1}
+			co.evenLock.Unlock()
 		}
 		return
 	}
 
-	counter.oddLock.RLock()
-	forum, ok := counter.oddMap[forumID]
-	counter.oddLock.RUnlock()
+	co.oddLock.RLock()
+	forum, ok := co.oddMap[forumID]
+	co.oddLock.RUnlock()
 	if ok {
 		forum.Lock()
 		forum.counter++
 		forum.Unlock()
 	} else {
-		counter.oddLock.Lock()
-		counter.oddMap[forumID] = &RWMutexCounterBucket{counter: 1}
-		counter.oddLock.Unlock()
+		co.oddLock.Lock()
+		co.oddMap[forumID] = &RWMutexCounterBucket{counter: 1}
+		co.oddLock.Unlock()
 	}
 }
 

common/counters/langs.go

@@ -1,8 +1,12 @@
 package counters
 
-import "database/sql"
-import c "github.com/Azareal/Gosora/common"
-import "github.com/Azareal/Gosora/query_gen"
+import (
+	"database/sql"
+
+	c "github.com/Azareal/Gosora/common"
+	qgen "github.com/Azareal/Gosora/query_gen"
+	"github.com/pkg/errors"
+)
 
 var LangViewCounter *DefaultLangViewCounter
 
@@ -101,70 +105,70 @@ type DefaultLangViewCounter struct {
 }
 
 func NewDefaultLangViewCounter(acc *qgen.Accumulator) (*DefaultLangViewCounter, error) {
-	var langBuckets = make([]*RWMutexCounterBucket, len(langCodes))
+	langBuckets := make([]*RWMutexCounterBucket, len(langCodes))
 	for bucketID, _ := range langBuckets {
 		langBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
 	}
-	var codesToIndices = make(map[string]int)
+	codesToIndices := make(map[string]int, len(langCodes))
 	for index, code := range langCodes {
 		codesToIndices[code] = index
 	}
 
-	counter := &DefaultLangViewCounter{
+	co := &DefaultLangViewCounter{
 		buckets:        langBuckets,
 		codesToIndices: codesToIndices,
 		insert:         acc.Insert("viewchunks_langs").Columns("count, createdAt, lang").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
 	}
 
-	c.AddScheduledFifteenMinuteTask(counter.Tick)
-	//c.AddScheduledSecondTask(counter.Tick)
-	c.AddShutdownTask(counter.Tick)
-	return counter, acc.FirstError()
+	c.AddScheduledFifteenMinuteTask(co.Tick)
+	//c.AddScheduledSecondTask(co.Tick)
+	c.AddShutdownTask(co.Tick)
+	return co, acc.FirstError()
 }
 
-func (counter *DefaultLangViewCounter) Tick() error {
-	for id, bucket := range counter.buckets {
+func (co *DefaultLangViewCounter) Tick() error {
+	for id, bucket := range co.buckets {
 		var count int
 		bucket.RLock()
 		count = bucket.counter
 		bucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters?
 		bucket.RUnlock()
 
-		err := counter.insertChunk(count, id) // TODO: Bulk insert for speed?
+		err := co.insertChunk(count, id) // TODO: Bulk insert for speed?
 		if err != nil {
-			return err
+			return errors.Wrap(errors.WithStack(err), "langview counter")
 		}
 	}
 	return nil
 }
 
-func (counter *DefaultLangViewCounter) insertChunk(count int, id int) error {
+func (co *DefaultLangViewCounter) insertChunk(count int, id int) error {
 	if count == 0 {
 		return nil
 	}
-	var langCode = langCodes[id]
-	c.DebugLogf("Inserting a viewchunk with a count of %d for lang %s (%d)", count, langCode, id)
-	_, err := counter.insert.Exec(count, langCode)
+	langCode := langCodes[id]
+	c.DebugLogf("Inserting a vchunk with a count of %d for lang %s (%d)", count, langCode, id)
+	_, err := co.insert.Exec(count, langCode)
 	return err
 }
 
-func (counter *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
+func (co *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
 	validCode = true
-	id, ok := counter.codesToIndices[langCode]
+	id, ok := co.codesToIndices[langCode]
 	if !ok {
 		// TODO: Tell the caller that the code's invalid
 		id = 0 // Unknown
 		validCode = false
 	}
 
 	// TODO: Test this check
-	c.DebugDetail("counter.buckets[", id, "]: ", counter.buckets[id])
-	if len(counter.buckets) <= id || id < 0 {
+	c.DebugDetail("co.buckets[", id, "]: ", co.buckets[id])
+	if len(co.buckets) <= id || id < 0 {
 		return validCode
 	}
-	counter.buckets[id].Lock()
-	counter.buckets[id].counter++
-	counter.buckets[id].Unlock()
+	co.buckets[id].Lock()
+	co.buckets[id].counter++
+	co.buckets[id].Unlock()
 
 	return validCode
 }

common/counters/memory.go

@@ -1,33 +1,34 @@
 package counters
 
 import (
-	"time"
-	"sync"
-	"runtime"
 	"database/sql"
+	"runtime"
+	"sync"
+	"time"
 
 	c "github.com/Azareal/Gosora/common"
-	"github.com/Azareal/Gosora/query_gen"
+	qgen "github.com/Azareal/Gosora/query_gen"
+	"github.com/pkg/errors"
 )
 
 var MemoryCounter *DefaultMemoryCounter
 
 type DefaultMemoryCounter struct {
 	insert *sql.Stmt
 
-	totMem uint64
-	totCount uint64
-	stackMem uint64
+	totMem     uint64
+	totCount   uint64
+	stackMem   uint64
 	stackCount uint64
-	heapMem uint64
-	heapCount uint64
+	heapMem    uint64
+	heapCount  uint64
 
 	sync.Mutex
 }
 
 func NewMemoryCounter(acc *qgen.Accumulator) (*DefaultMemoryCounter, error) {
 	co := &DefaultMemoryCounter{
-		insert:        acc.Insert("memchunks").Columns("count, stack, heap, createdAt").Fields("?,?,?,UTC_TIMESTAMP()").Prepare(),
+		insert: acc.Insert("memchunks").Columns("count, stack, heap, createdAt").Fields("?,?,?,UTC_TIMESTAMP()").Prepare(),
 	}
 	c.AddScheduledFifteenMinuteTask(co.Tick)
 	//c.AddScheduledSecondTask(co.Tick)
@@ -81,5 +82,8 @@ func (co *DefaultMemoryCounter) Tick() (err error) {
 
 	c.DebugLogf("Inserting a memchunk with a value of %d - %d - %d", avgMem, avgStack, avgHeap)
 	_, err = co.insert.Exec(avgMem, avgStack, avgHeap)
-	return err
-}
+	if err != nil {
+		return errors.Wrap(errors.WithStack(err), "mem counter")
+	}
+	return nil
+}