Skip to content

Commit

Permalink
single entry cache updates, safe cache maps with locks
Browse files Browse the repository at this point in the history
  • Loading branch information
Cristian Vidmar committed Jun 16, 2021
1 parent c7610d0 commit 252941c
Show file tree
Hide file tree
Showing 2 changed files with 96 additions and 31 deletions.
80 changes: 73 additions & 7 deletions erm/templates/contentful_vo_lib.gotmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,24 +11,29 @@ import (
"io"
"regexp"
"strings"
"sync"
"time"
"github.com/foomo/contentful"
"golang.org/x/sync/errgroup"
)

type cacheEntryMaps struct {
{{ range $index , $contentType := $contentTypes }} {{ $contentType.Sys.ID }} map[string]*Cf{{ firstCap $contentType.Sys.ID }}
{{ $contentType.Sys.ID }}GcLock sync.RWMutex
{{ end }}
}

type ClientMode string

type ContentfulCache struct {
assets assetCacheMap
contentTypes []string
entryMaps cacheEntryMaps
idContentTypeMap map[string]string
parentMap map[string][]EntryReference // Maps each entry ID to its parents' IDs
assets assetCacheMap
assetsGcLock sync.RWMutex
contentTypes []string
entryMaps cacheEntryMaps
idContentTypeMap map[string]string
idContentTypeMapGcLock sync.RWMutex
parentMap map[string][]EntryReference
parentMapGcLock sync.RWMutex
}

type assetCacheMap map[string]*contentful.Asset
Expand Down Expand Up @@ -112,6 +117,9 @@ var (
var spaceContentTypes = []string{ {{ range $index , $contentType := $contentTypes }}ContentType{{ firstCap $contentType.Sys.ID }}, {{ end }} }

func (cc *ContentfulClient) CacheHasContentType(contentTypeID string) bool {
if cc.Cache == nil {
return false
}
for _, cachedContentType := range cc.Cache.contentTypes {
if cachedContentType == contentTypeID {
return true
Expand Down Expand Up @@ -155,7 +163,9 @@ func (cc *ContentfulClient) GetAssetByID(id string) (*contentful.Asset, error) {
return nil, errors.New("GetAssetByID: No client available")
}
if cc.Cache != nil && cc.Cache.assets != nil {
cc.Cache.assetsGcLock.RLock()
asset, okAsset := cc.Cache.assets[id]
cc.Cache.assetsGcLock.RUnlock()
if okAsset {
return asset, nil
}
Expand Down Expand Up @@ -194,7 +204,9 @@ func (cc *ContentfulClient) GetContentTypeOfID(id string) (string, error) {
if cc.Cache != nil {
okVo := false
{{ range $index , $contentType := $contentTypes }}
cc.Cache.entryMaps.{{ $contentType.Sys.ID }}GcLock.RLock()
_, okVo = cc.Cache.entryMaps.{{ $contentType.Sys.ID }}[id]
cc.Cache.entryMaps.{{ $contentType.Sys.ID }}GcLock.RUnlock()
if okVo {
return ContentType{{ firstCap $contentType.Sys.ID }}, nil
}
Expand Down Expand Up @@ -289,6 +301,15 @@ func NewContentfulClient(spaceID string, clientMode ClientMode, clientKey string
if spaceID == "" {
return nil, errors.New("NewContentfulClient: SpaceID cannot be empty")
}
if clientMode != ClientModeCMA && clientMode != ClientModeCPA && clientMode != ClientModeCDA {
return nil, errors.New("NewContentfulClient: clientMode not supported")
}
if optimisticPageSize < 10 {
return nil, errors.New("NewContentfulClient: optimisticPageSize must be 10 or bigger")
}
if logLevel < 0 || logLevel > 3 {
return nil, errors.New("NewContentfulClient: logLevel must be between 0 and 3")
}
if clientKey == "" {
return nil, errors.New("NewContentfulClient: Please provide an API key")
}
Expand Down Expand Up @@ -415,7 +436,7 @@ func (cc *ContentfulClient) UpdateCacheForEntity(ctx context.Context, sysType st
if sysType == sysTypeAsset {
contentType = assetWorkerType
}
if !stringSliceContains(spaceContentTypes, contentType) {
if contentType != assetWorkerType && !stringSliceContains(spaceContentTypes, contentType) {
return fmt.Errorf("UpdateCache: Content Type %q not available in this space", contentType)
}
err := updateCacheForContentTypeAndEntity(ctx, cc, sysType, contentType, entityID)
Expand All @@ -437,11 +458,48 @@ func FieldToObject(jsonField interface{}, targetObject interface{}) error {
return nil
}

func (cc *ContentfulClient) cacheGcAssetByID(ctx context.Context, id string) error {
if cc.Client == nil {
return errors.New("cacheGcAssetByID: No client available")
}
col := cc.Client.Assets.List(cc.SpaceID)
col.Query.Locale("*").Equal("sys.id", id)
_, err := col.Next()
if err != nil {
return err
}
if len(col.Items) == 0 {
return errors.New("cacheGcAssetByID: Not found " + id)
}
item := col.Items[0]
asset := contentful.Asset{}
byt, err := json.Marshal(item)
if err != nil {
return err
}
err = json.Unmarshal(byt, &asset)
if err != nil {
return err
}
for _, loc := range []Locale{SpaceLocaleGerman, SpaceLocaleFrench} {
if _, ok := asset.Fields.File[string(loc)]; ok {
asset.Fields.File[string(loc)].URL = "https:" + asset.Fields.File[string(loc)].URL
}
}
cc.Cache.assetsGcLock.Lock()
cc.Cache.assets[id] = &asset
cc.Cache.assetsGcLock.Unlock()
return nil
}

func (cc *ContentfulClient) deleteAssetFromCache(key string) error {
cc.Cache.assetsGcLock.Lock()
if _, ok := cc.Cache.assets[key]; ok {
delete(cc.Cache.assets, key)
cc.Cache.assetsGcLock.Unlock()
return nil
}
cc.Cache.assetsGcLock.Unlock()
return errors.New("asset not found in cache, could not delete")
}

Expand Down Expand Up @@ -908,14 +966,22 @@ func updateCacheForContentTypeAndEntity(ctx context.Context, cc *ContentfulClien
return ctx.Err()
}
switch contentType {
case assetWorkerType:
err := cc.cacheGcAssetByID(ctx, entityID)
if err != nil {
return err
}
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"contentType": "Asset", "method": "updateCacheForContentTypeAndEntity", "entityID": entityID}, LogInfo, InfoUpdatedEntityCache)
}
{{ range $index , $contentType := $contentTypes }}
case ContentType{{ firstCap $contentType.Sys.ID }}:
err := cc.cache{{ firstCap $contentType.Sys.ID }}ByID(ctx, entityID)
if err != nil {
return err
}
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"contentType": "{{ $contentType.Sys.ID }}", "method": "updateCacheForContentType", "entityID": entityID}, LogInfo, InfoUpdatedEntityCache)
cc.logFn(map[string]interface{}{"contentType": "{{ $contentType.Sys.ID }}", "method": "updateCacheForContentTypeAndEntity", "entityID": entityID}, LogInfo, InfoUpdatedEntityCache)
}
{{ end }}
}
Expand Down
47 changes: 23 additions & 24 deletions erm/templates/contentful_vo_lib_contenttype.gotmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,9 @@ func (cc *ContentfulClient) Get{{ firstCap $contentType.Sys.ID }}ByID(id string)
return nil, errors.New("Get{{ firstCap $contentType.Sys.ID }}ByID: No client available")
}
if cc.Cache != nil {
cc.Cache.entryMaps.{{ $contentType.Sys.ID }}GcLock.RLock()
vo, ok := cc.Cache.entryMaps.{{ $contentType.Sys.ID }}[id]
cc.Cache.entryMaps.{{ $contentType.Sys.ID }}GcLock.RUnlock()
if ok {
return vo, nil
}
Expand Down Expand Up @@ -643,54 +645,51 @@ func (cc *ContentfulClient) cache{{ firstCap $contentType.Sys.ID }}ByID(ctx cont
if err != nil {
return err
}
{{ $contentType.Sys.ID }}EntryMap := cc.Cache.entryMaps.{{ $contentType.Sys.ID }}
idContentTypeMap := cc.Cache.idContentTypeMap
parentMap := cc.Cache.parentMap
cc.Cache.entryMaps.{{ $contentType.Sys.ID }}GcLock.Lock()
cc.Cache.idContentTypeMapGcLock.Lock()
cc.Cache.parentMapGcLock.Lock()
// It was deleted
if len(col.Items) == 0 {
delete({{ $contentType.Sys.ID }}EntryMap, id)
delete(idContentTypeMap, id)
delete(cc.Cache.entryMaps.{{ $contentType.Sys.ID }}, id)
delete(cc.Cache.idContentTypeMap, id)
// delete as child
delete(parentMap, id)
delete(cc.Cache.parentMap, id)
// delete as parent
for childID, child := range parentMap {
for childID, child := range cc.Cache.parentMap {
newParents := []EntryReference{}
for _, parent := range child {
if parent.ID != id {
newParents = append(newParents, parent)
}
}
parentMap[childID] = newParents
cc.Cache.parentMap[childID] = newParents
}
cc.Cache.entryMaps.{{ $contentType.Sys.ID }} = {{ $contentType.Sys.ID }}EntryMap
cc.Cache.idContentTypeMap = idContentTypeMap
cc.Cache.parentMap = parentMap
return nil
}
vos, err := colToCf{{ firstCap $contentType.Sys.ID }}(col, cc)
if err != nil {
return fmt.Errorf("cache{{ firstCap $contentType.Sys.ID }}ByID: Error converting %s to VO: "+err.Error(), id)
}
{{ $contentType.Sys.ID }} := vos[0]
{{ $contentType.Sys.ID }}EntryMap[id] = {{ $contentType.Sys.ID }}
idContentTypeMap[id] = {{ $contentType.Sys.ID }}.Sys.ContentType.Sys.ID
cc.Cache.entryMaps.{{ $contentType.Sys.ID }}[id] = {{ $contentType.Sys.ID }}
cc.Cache.idContentTypeMap[id] = {{ $contentType.Sys.ID }}.Sys.ContentType.Sys.ID
{{ range $fieldIndex, $field := $contentType.Fields }}
{{ if fieldIsMultipleReference $field }}
for _, loc := range cc.locales {
children, okChildren := {{ $contentType.Sys.ID }}.Fields.{{ firstCap $field.ID }}[string(loc)]
if okChildren {
for _, child := range children {
if parentMap[child.Sys.ID] == nil {
parentMap[child.Sys.ID] = []EntryReference{}
if cc.Cache.parentMap[child.Sys.ID] == nil {
cc.Cache.parentMap[child.Sys.ID] = []EntryReference{}
}
found := false
for _, parent := range parentMap[child.Sys.ID] {
for _, parent := range cc.Cache.parentMap[child.Sys.ID] {
if parent.ID == id {
found = true
}
}
if !found {
parentMap[child.Sys.ID] = append(parentMap[child.Sys.ID], EntryReference{ContentType: {{ $contentType.Sys.ID }}.Sys.ContentType.Sys.ID, ID: {{ $contentType.Sys.ID }}.Sys.ID, VO: {{ $contentType.Sys.ID }}})
cc.Cache.parentMap[child.Sys.ID] = append(cc.Cache.parentMap[child.Sys.ID], EntryReference{ContentType: {{ $contentType.Sys.ID }}.Sys.ContentType.Sys.ID, ID: {{ $contentType.Sys.ID }}.Sys.ID, VO: {{ $contentType.Sys.ID }}})
}
}
}
Expand All @@ -700,25 +699,25 @@ func (cc *ContentfulClient) cache{{ firstCap $contentType.Sys.ID }}ByID(ctx cont
for _, loc := range cc.locales {
child, okChild := {{ $contentType.Sys.ID }}.Fields.{{ firstCap $field.ID }}[string(loc)]
if okChild {
if parentMap[child.Sys.ID] == nil {
parentMap[child.Sys.ID] = []EntryReference{}
if cc.Cache.parentMap[child.Sys.ID] == nil {
cc.Cache.parentMap[child.Sys.ID] = []EntryReference{}
}
found := false
for _, parent := range parentMap[child.Sys.ID] {
for _, parent := range cc.Cache.parentMap[child.Sys.ID] {
if parent.ID == id {
found = true
}
}
if !found {
parentMap[child.Sys.ID] = append(parentMap[child.Sys.ID], EntryReference{ContentType: {{ $contentType.Sys.ID }}.Sys.ContentType.Sys.ID, ID: {{ $contentType.Sys.ID }}.Sys.ID, VO: {{ $contentType.Sys.ID }}})
cc.Cache.parentMap[child.Sys.ID] = append(cc.Cache.parentMap[child.Sys.ID], EntryReference{ContentType: {{ $contentType.Sys.ID }}.Sys.ContentType.Sys.ID, ID: {{ $contentType.Sys.ID }}.Sys.ID, VO: {{ $contentType.Sys.ID }}})
}
}
}
{{ end }}
{{ end }}
cc.Cache.entryMaps.{{ $contentType.Sys.ID }} = {{ $contentType.Sys.ID }}EntryMap
cc.Cache.idContentTypeMap = idContentTypeMap
cc.Cache.parentMap = parentMap
cc.Cache.entryMaps.{{ $contentType.Sys.ID }}GcLock.Unlock()
cc.Cache.idContentTypeMapGcLock.Unlock()
cc.Cache.parentMapGcLock.Unlock()
return nil
}

Expand Down

0 comments on commit 252941c

Please sign in to comment.