From 0b74002fcba02fa65d7f0976a81988caeedafa0d Mon Sep 17 00:00:00 2001 From: Ken Eucker Date: Tue, 31 Oct 2023 22:00:19 -0700 Subject: [PATCH] feat(client, imgur): adds caching for calls to imgur.getAlbum to improve performance --- examples/node/index.js | 4 ++-- src/client.ts | 40 ++++++++++++++++++++++++++-------------- src/common/data.ts | 3 ++- src/common/getters.ts | 17 +++++------------ src/common/methods.ts | 20 ++++++++++++++++++++ src/imgur/archiveTag.ts | 9 +++++---- src/imgur/getGame.ts | 40 +++++++++++++++++++++++++++++++++++----- src/imgur/getPlayers.ts | 11 +++++++---- src/imgur/getQueue.ts | 13 +++++++++---- src/imgur/getTags.ts | 20 ++++++++++++-------- src/imgur/helpers.ts | 26 +++++++++++++++++--------- src/sanity/getGame.ts | 26 ++++++++++++++++++++++++-- test/src/client.test.ts | 3 +-- 13 files changed, 165 insertions(+), 67 deletions(-) diff --git a/examples/node/index.js b/examples/node/index.js index 22fdcf9..c035181 100644 --- a/examples/node/index.js +++ b/examples/node/index.js @@ -146,7 +146,7 @@ const get10SettingsAsync = async (pre, client, out = false, opts = {}) => { } const runTests = async (out = false) => { - if (biketagDefaultInstance) { + if (false) { console.log(pretty("Default BikeTag Client Instantiated"), biketagDefaultInstanceOpts) await getGameAsync("BikeTag", biketagDefaultInstance, out) await getTag1Async("BikeTag", biketagDefaultInstance, out) @@ -167,7 +167,7 @@ const runTests = async (out = false) => { await get10PlayersAsync("Imgur", bikeTagImgurInstance, out) } - if (bikeTagSanityInstance) { + if (false) { console.log(pretty("Sanity BikeTag Client Instantiated"), sanityInstanceOpts) // await getTag1Async("Sanity", bikeTagSanityInstance, out) // await get10TagsAsync("Sanity", bikeTagSanityInstance, out) diff --git a/src/client.ts b/src/client.ts index d89510b..d16fff8 100644 --- a/src/client.ts +++ b/src/client.ts @@ -69,6 +69,9 @@ import { EventEmitter } from 'events' import { setup } from 'axios-cache-adapter' import { isEqual } from 'lodash' import { getAuthorizationHeader, getClaims } from './common/auth' +import TinyCache from 'tinycache' + +const apiCache = new TinyCache() // export const USERAGENT = // 'biketag-api (https://github.com/keneucker/biketag-api)' @@ -109,16 +112,18 @@ export class BikeTagClient extends EventEmitter { responseType, }) + const authenticationInterceptor = async (config) => { + config.headers = config.headers ? config.headers : {} + config.headers.authorization = await getAuthorizationHeader(this) + return config + } + this.fetcher = axios.create({ headers, responseType, }) this.fetcher.interceptors.request.use( - async (config: AxiosRequestConfig) => { - config.headers = config.headers ? config.headers : {} - config.headers.authorization = await getAuthorizationHeader(this) - return config - }, + authenticationInterceptor, (e: Error) => Promise.reject(e) ) @@ -139,6 +144,10 @@ export class BikeTagClient extends EventEmitter { headers, responseType, }) + this.cachedFetcher.interceptors.request.use( + authenticationInterceptor, + (e: Error) => Promise.reject(e) + ) } /// **************************** protected Class Methods ******************************** /// @@ -232,7 +241,9 @@ export class BikeTagClient extends EventEmitter { options.tagnumber = options.tagnumbers[0] } else if (options.slug && options.slug !== 'current') { options.tagnumber = BikeTagGetters.getTagnumberFromSlug( - options.slug + options.slug, + undefined, + apiCache ) } } @@ -529,7 +540,7 @@ export class BikeTagClient extends EventEmitter { const clientMethod = api.getGame if (clientMethod) { - return clientMethod(client, options) + return clientMethod(client, options, apiCache) .then((retrievedGameResponse) => { if (retrievedGameResponse.success && retrievedGameResponse.data) { /// Set the most important game data (hash, etc) @@ -585,6 +596,7 @@ export class BikeTagClient extends EventEmitter { } return clientMethod(client, options).catch((e) => { + /// TODO: invalidate cache return Promise.resolve({ status: HttpStatusCode.InternalServerError, data: null, @@ -639,7 +651,7 @@ export class BikeTagClient extends EventEmitter { break } - return clientMethod(client, options).catch((e) => { + return clientMethod(client, options, apiCache).catch((e) => { return { status: HttpStatusCode.InternalServerError, data: null, @@ -742,7 +754,7 @@ export class BikeTagClient extends EventEmitter { // } - return clientMethod(client, options).catch((e) => { + return clientMethod(client, options, apiCache).catch((e) => { return { status: HttpStatusCode.InternalServerError, data: null, @@ -785,7 +797,7 @@ export class BikeTagClient extends EventEmitter { // break // } - return clientMethod(client, options).catch((e) => { + return clientMethod(client, options, apiCache).catch((e) => { return { status: HttpStatusCode.InternalServerError, data: null, @@ -993,7 +1005,7 @@ export class BikeTagClient extends EventEmitter { /// If the client adapter implements a direct way to retrieve a single player if (clientMethod) { - return clientMethod(client, options).catch((e) => { + return clientMethod(client, options, apiCache).catch((e) => { return { status: HttpStatusCode.InternalServerError, data: null, @@ -1039,7 +1051,7 @@ export class BikeTagClient extends EventEmitter { break } - return clientMethod(client, options).catch((e) => { + return clientMethod(client, options, apiCache).catch((e) => { return Promise.resolve({ status: HttpStatusCode.InternalServerError, data: null, @@ -1087,7 +1099,7 @@ export class BikeTagClient extends EventEmitter { /// If the client adapter implements a direct way to retrieve a single ambassador if (clientMethod) { - return clientMethod(client, options).catch((e) => { + return clientMethod(client, options, apiCache).catch((e) => { return { status: HttpStatusCode.InternalServerError, data: null, @@ -1137,7 +1149,7 @@ export class BikeTagClient extends EventEmitter { break } - return clientMethod(client, options).catch((e) => { + return clientMethod(client, options, apiCache).catch((e) => { return Promise.resolve({ status: HttpStatusCode.InternalServerError, data: null, diff --git a/src/common/data.ts b/src/common/data.ts index 79ebccd..4feac64 100644 --- a/src/common/data.ts +++ b/src/common/data.ts @@ -3,13 +3,14 @@ import { Tag, Game, Player, Ambassador, Setting } from './schema' export const cacheKeys = { sanityUrlText: `sanity::`, imageHashText: `hash::`, - albumHash: `imgur::`, + albumHash: `album::`, bikeTagImage: `biketag::`, bikeTagsByUser: `userTags::`, hintText: `hint::`, playerText: `player::`, playerData: `playerData::`, playerIdText: `playerId::`, + gameIdText: `gameId::`, gameSlugText: `slug::`, gameText: `game::`, locationText: `location::`, diff --git a/src/common/getters.ts b/src/common/getters.ts index f4006b5..1cda998 100644 --- a/src/common/getters.ts +++ b/src/common/getters.ts @@ -452,16 +452,13 @@ export const getImgurFoundImageHashFromBikeTagData = ( export const getImgurFoundDescriptionFromBikeTagData = ( tag: Tag, - includeCredit = true, - cache?: typeof TinyCache + includeCredit = true ): string => `#${tag.tagnumber} proof${ tag.foundLocation ? ` found at (${tag.foundLocation})` : '' }${includeCredit ? ` by ${tag.foundPlayer}` : ''}` -export const getImgurFoundTitleFromBikeTagData = ( - tag: Tag, - cache?: typeof TinyCache -): string => + +export const getImgurFoundTitleFromBikeTagData = (tag: Tag): string => `${ !tag.gps || (tag.gps.lat === 0 && tag.gps.long === 0) ? '' @@ -478,10 +475,7 @@ export const getImgurMysteryImageHashFromBikeTagData = ( ): string => { return getImageHashFromText(tag.mysteryImageUrl, cache) } -export const getImgurMysteryTitleFromBikeTagData = ( - tag: Tag, - cache?: typeof TinyCache -): string => +export const getImgurMysteryTitleFromBikeTagData = (tag: Tag): string => `${ !tag.gps || (tag.gps.lat === 0 && tag.gps.long === 0) ? '' @@ -491,8 +485,7 @@ export const getImgurMysteryTitleFromBikeTagData = ( export const getImgurMysteryDescriptionFromBikeTagData = ( tag: Tag, includeCredit = true, - includeHint = true, - cache?: typeof TinyCache + includeHint = true ): string => `#${tag.tagnumber} tag ${ includeHint && tag.hint ? `(hint: ${tag.hint})` : '' diff --git a/src/common/methods.ts b/src/common/methods.ts index b80c8c9..7990ebb 100644 --- a/src/common/methods.ts +++ b/src/common/methods.ts @@ -13,6 +13,7 @@ import FormData from 'form-data' import TinyCache from 'tinycache' import { Tag, Game, Player, Ambassador, Setting } from './schema' import { ApiAvailability } from './enums' +import { cacheKeys } from './data' export const putCacheIfExists = ( key: string, @@ -442,3 +443,22 @@ export const sortSettings = ( return limit !== 0 ? sorted.slice(0, limit) : sorted } + +export const getGameAlbumFromCache = async ( + gameAlbumHash: string, + cache?: typeof TinyCache, + fallback?: any +) => { + const cacheKey = `imgur::${cacheKeys.albumHash}${gameAlbumHash}` + const existsInCache = getCacheIfExists(cacheKey, cache) + if (existsInCache) { + return existsInCache + } + + if (fallback) { + const putIntoCache = await fallback() + putCacheIfExists(cacheKey, putIntoCache, cache) + + return putIntoCache + } +} diff --git a/src/imgur/archiveTag.ts b/src/imgur/archiveTag.ts index 6058d44..14ac64f 100644 --- a/src/imgur/archiveTag.ts +++ b/src/imgur/archiveTag.ts @@ -16,7 +16,7 @@ export async function archiveTag( let data let error let success = true - let mysteryTagDeleteResponse + // let mysteryTagDeleteResponse let archiveFoundImageResponse let deleteFoundImageResponse let existingFoundImageResponse @@ -52,9 +52,10 @@ export async function archiveTag( payload as Tag ) const existingMysteryImageResponse = await client.getImage(mysteryImageHash) - mysteryTagDeleteResponse = existingMysteryImageResponse.success - ? await client.deleteImage(existingMysteryImageResponse.data.deletehash) - : { success: false, data: 'delete of existing mystery image failed' } + if (existingMysteryImageResponse.success) { + await client.deleteImage(existingMysteryImageResponse.data.deletehash) + } + // : { success: false, data: 'delete of existing mystery image failed' } } if (archiveFoundImageResponse?.success && deleteFoundImageResponse?.success) { diff --git a/src/imgur/getGame.ts b/src/imgur/getGame.ts index 73c9129..4332ab7 100644 --- a/src/imgur/getGame.ts +++ b/src/imgur/getGame.ts @@ -1,20 +1,32 @@ import type { ImgurClient } from 'imgur' -import { createGameObject } from '../common/data' +import { createGameObject, cacheKeys } from '../common/data' import { getGamePayload } from '../common/payloads' import { BikeTagApiResponse } from '../common/types' import { Game } from '../common/schema' import { AvailableApis, HttpStatusCode } from '../common/enums' import { getGameDataFromText, getGameSlugFromText } from './helpers' +import { + getCacheIfExists, + getGameAlbumFromCache, + putCacheIfExists, +} from '../common/methods' +import TinyCache from 'tinycache' export async function getGame( client: ImgurClient, - payload: getGamePayload + payload: getGamePayload, + cache?: typeof TinyCache ): Promise> { - let game + const cacheKey = `imgur::${cacheKeys.gameIdText}${ + payload.slug ?? payload.name ?? 'biketag' + }` + let game = getCacheIfExists(cacheKey, cache) let error let success = true - if (client) { + if (game) { + /// First do nothing + } else if (client) { const hashes = payload.hash ? [payload.hash] : [] if (!hashes.length) { let stillSearching = true @@ -42,7 +54,9 @@ export async function getGame( } for (const hash of hashes) { - const albumInfo = await client.getAlbum(hash) + const albumInfo = await getGameAlbumFromCache(hash, cache, () => + client.getAlbum(hash) + ) if (albumInfo.data?.images?.length > 0) { /// TODO: save all game settings into the title of the image (serialized) @@ -58,6 +72,22 @@ export async function getGame( if (games.length) { game = createGameObject(games[0]) + } else if ( + albumInfo.data.title + .toLocaleLowerCase() + .indexOf(`${payload.slug} biketag`) !== -1 || + albumInfo.data.title + .toLocaleLowerCase() + .indexOf(`${payload.slug} bike tag`) !== -1 + ) { + game = createGameObject({ + name: payload.slug, + mainhash: albumInfo.data.id, + }) + } + + if (game) { + putCacheIfExists(cacheKey, game, cache) break } } else if (!albumInfo.success) { diff --git a/src/imgur/getPlayers.ts b/src/imgur/getPlayers.ts index c4cc5ae..e12c555 100644 --- a/src/imgur/getPlayers.ts +++ b/src/imgur/getPlayers.ts @@ -1,23 +1,26 @@ import type { ImgurClient } from 'imgur' import { createPlayerObject, createTagObject } from '../common/data' -import { sortPlayers } from '../common/methods' +import { getGameAlbumFromCache, sortPlayers } from '../common/methods' import { getPlayersPayload } from '../common/payloads' import { BikeTagApiResponse } from '../common/types' import { Player } from '../common/schema' import { AvailableApis, HttpStatusCode } from '../common/enums' import { getPlayerDataFromText } from './helpers' +import TinyCache from 'tinycache' export async function getPlayers( client: ImgurClient, - payload: getPlayersPayload + payload: getPlayersPayload, + cache?: typeof TinyCache ): Promise> { const playersData: Player[] = [] const playerNames: string[] = [] if (client) { const { data: tags } = await this.getTags({ sort: 'relevance' }) - /// TODO: this better be cached because it's being called twice now - const albumInfo = await (client.getAlbum(payload.hash) as any) + const albumInfo = await getGameAlbumFromCache(payload.hash, cache, () => + client.getAlbum(payload.hash) + ) const playerImages = albumInfo.data?.images?.reduce((o, i) => { const player = getPlayerDataFromText(i.description) if (player) { diff --git a/src/imgur/getQueue.ts b/src/imgur/getQueue.ts index 61b0ebe..81bfe39 100644 --- a/src/imgur/getQueue.ts +++ b/src/imgur/getQueue.ts @@ -4,11 +4,13 @@ import { BikeTagApiResponse } from '../common/types' import { Tag } from '../common/schema' import { getGroupedImagesByTagnumber, getGroupedTagsByPlayer } from './helpers' import { AvailableApis, HttpStatusCode } from '../common/enums' -import { sortTags } from '../common/methods' +import { getGameAlbumFromCache, sortTags } from '../common/methods' +import TinyCache from 'tinycache' export async function getQueue( client: ImgurClient, - payload: getQueuePayload + payload: getQueuePayload, + cache?: typeof TinyCache ): Promise> { if (!payload.queuehash) { const game = await this.getGame(payload.game) @@ -16,10 +18,13 @@ export async function getQueue( payload.queuehash = game.data.queuehash } } + const albumInfo = await getGameAlbumFromCache(payload.queuehash, cache, () => + client.getAlbum(payload.queuehash) + ) - const albumInfo = await client.getAlbum(payload.queuehash) - const images = getGroupedImagesByTagnumber(albumInfo?.data?.images) + const images = getGroupedImagesByTagnumber(albumInfo?.data?.images, cache) const queuedTags = getGroupedTagsByPlayer(images, payload) + return { data: sortTags(queuedTags, 'relevance'), success: true, diff --git a/src/imgur/getTags.ts b/src/imgur/getTags.ts index 31d7ed1..045b644 100644 --- a/src/imgur/getTags.ts +++ b/src/imgur/getTags.ts @@ -1,5 +1,5 @@ import type { ImgurClient } from 'imgur' -import { sortTags } from '../common/methods' +import { getGameAlbumFromCache, sortTags } from '../common/methods' import { getTagsPayload } from '../common/payloads' import { BikeTagApiResponse } from '../common/types' import { Tag } from '../common/schema' @@ -9,21 +9,26 @@ import { getGroupedTagsByTagnumber, } from './helpers' import { AvailableApis, HttpStatusCode } from '../common/enums' +import TinyCache from 'tinycache' export async function getTags( client: ImgurClient, - payload: getTagsPayload + payload: getTagsPayload, + cache?: typeof TinyCache ): Promise> { let albumImages: any[] = [] let error let success = true - // + const albumInfo = await getGameAlbumFromCache(payload.hash, cache, () => + client.getAlbum(payload.hash) + ) + if (payload.tagnumbers?.length) { - const albumInfo = await (client.getAlbum(payload.hash) as any) albumImages = albumInfo.data?.images?.filter( (image: any) => - payload.tagnumbers.indexOf(getBikeTagNumberFromImage(image)) !== -1 + payload.tagnumbers.indexOf(getBikeTagNumberFromImage(image, cache)) !== + -1 ) } else if (payload.slugs?.length) { const imagePromises: Promise[] = [] @@ -38,16 +43,15 @@ export async function getTags( albumImages = await Promise.all(imagePromises) } else { - const albumInfo = await client.getAlbum(payload.hash) if (albumInfo.success) { - albumImages = albumInfo.data.images ?? [] + albumImages = albumInfo.data?.images ?? [] } else { success = false error = albumInfo.data } } - const images = getGroupedImagesByTagnumber(albumImages) + const images = getGroupedImagesByTagnumber(albumImages, cache) const tags = getGroupedTagsByTagnumber(images, payload) return { diff --git a/src/imgur/helpers.ts b/src/imgur/helpers.ts index 080904c..cb297fd 100644 --- a/src/imgur/helpers.ts +++ b/src/imgur/helpers.ts @@ -342,8 +342,13 @@ export function getHintFromText( return hint } -export function getBikeTagNumberFromImage(image: ImgurImage): number { - return image.description ? getTagNumbersFromText(image.description)[0] : -1 +export function getBikeTagNumberFromImage( + image: ImgurImage, + cache?: typeof TinyCache +): number { + return image.description + ? getTagNumbersFromText(image.description, undefined, cache)[0] + : -1 } export function isPlayerImage(image: ImgurImage): boolean { @@ -366,11 +371,12 @@ export function isFoundImage(image: ImgurImage): boolean { } export function sortImgurImagesByTagNumber( - images: ImgurImage[] = [] + images: ImgurImage[] = [], + cache?: typeof TinyCache ): ImgurImage[] { return images.sort((image1, image2) => { - const tagNumber1 = getBikeTagNumberFromImage(image1) - const tagNumber2 = getBikeTagNumberFromImage(image2) + const tagNumber1 = getBikeTagNumberFromImage(image1, cache) + const tagNumber2 = getBikeTagNumberFromImage(image2, cache) const tagNumber1IsProof = tagNumber1 < 0 const difference = Math.abs(tagNumber2) - Math.abs(tagNumber1) @@ -507,8 +513,7 @@ export const getBikeTagUsernameFromImgurImage = ( } export const getBikeTagDiscussionLinkFromImgurImage = ( - image: ImgurImage, - cache?: typeof TinyCache + image: ImgurImage ): string | null => { const tagTitle = image.title || '' const tagDiscussionLinkIndex = tagTitle.indexOf('{') @@ -664,11 +669,14 @@ export function getUploadTagImagePayloadFromTagData( } } -export const getGroupedImagesByTagnumber = (ungroupedImages = []) => { +export const getGroupedImagesByTagnumber = ( + ungroupedImages = [], + cache?: typeof TinyCache +) => { const groupedImages: any[] = [] ungroupedImages.forEach((image: any) => { - const tagnumber = getBikeTagNumberFromImage(image) + const tagnumber = getBikeTagNumberFromImage(image, cache) groupedImages[tagnumber] = groupedImages[tagnumber] ? groupedImages[tagnumber] diff --git a/src/sanity/getGame.ts b/src/sanity/getGame.ts index 69b3bb1..1b1a8c7 100644 --- a/src/sanity/getGame.ts +++ b/src/sanity/getGame.ts @@ -7,15 +7,32 @@ import { } from './helpers' import { getGamePayload } from '../common/payloads' import { Game } from '../common/schema' +import TinyCache from 'tinycache' +import { cacheKeys } from '../common/data' +import { putCacheIfExists, getCacheIfExists } from '../common/methods' export async function getGame( client: SanityClient, - payload: getGamePayload + payload: getGamePayload, + cache?: typeof TinyCache ): Promise> { if (!payload) { throw new Error('no payload options') } + const cacheKey = `sanity::${cacheKeys.gameIdText}${ + payload.slug ?? payload.name + }` + const gameExistsInCache = getCacheIfExists(cacheKey, cache) + if (gameExistsInCache) { + return Promise.resolve({ + data: gameExistsInCache, + status: HttpStatusCode.Ok, + success: true, + source: AvailableApis[AvailableApis.sanity], + }) + } + const fields = constructSanityFieldsQuery(payload.fields, DataTypes.game) const slugIsSet = payload.slug?.length const nameIsSet = payload.name?.length @@ -37,9 +54,14 @@ export async function getGame( gameData.push(constructGameFromSanityObject(game, payload.fields)) } + const theGame = isArray ? gameData : gameData[0] + if (theGame) { + putCacheIfExists(cacheKey, theGame, cache) + } + // wrap tag in BikeTagApiResponse const response = { - data: isArray ? gameData : gameData[0], + data: theGame, status: HttpStatusCode.Ok, success: true, source: AvailableApis[AvailableApis.sanity], diff --git a/test/src/client.test.ts b/test/src/client.test.ts index 53cec5a..de8a565 100755 --- a/test/src/client.test.ts +++ b/test/src/client.test.ts @@ -1,7 +1,6 @@ /* eslint-disable @typescript-eslint/no-var-requires */ -import { apiCredentials, assetsPath, testPath, packagePath } from './config' -import { execSync } from 'child_process' +import { apiCredentials, testPath, packagePath } from './config' import * as path from 'path' describe(`Built Module`, () => {