From 352e96f3e6c1ceb5d5bc92159110686b27d180b6 Mon Sep 17 00:00:00 2001 From: ChaosD Date: Thu, 23 Nov 2023 22:56:57 +0800 Subject: [PATCH 01/28] Add concurrent hashtable --- libCacheSim/cache/cacheObj.c | 47 +- libCacheSim/dataStructure/CMakeLists.txt | 3 + .../hashtable/cChainedHashTable.c | 313 ++++++++++ .../hashtable/cChainedHashTable.h | 63 ++ .../hashtable/chainedHashTable.h | 2 +- .../hashtable/chainedHashTableV2.c | 43 +- .../hashtable/chainedHashTableV2.h | 5 +- .../hashtable/chainedHashtable.c | 4 +- .../dataStructure/hashtable/hashtable.h | 25 + .../dataStructure/hashtable/hashtableStruct.h | 3 + libCacheSim/dataStructure/test/CMakeLists.txt | 4 + .../test/test_concurrent_hashtable.c | 537 ++++++++++++++++++ libCacheSim/include/libCacheSim/cacheObj.h | 16 + libCacheSim/utils/include/mymath.h | 14 + libCacheSim/utils/include/mymutex.h | 24 + libCacheSim/utils/mymutex.c | 84 +++ sftp-config.json | 45 ++ 17 files changed, 1211 insertions(+), 21 deletions(-) create mode 100644 libCacheSim/dataStructure/hashtable/cChainedHashTable.c create mode 100644 libCacheSim/dataStructure/hashtable/cChainedHashTable.h create mode 100644 libCacheSim/dataStructure/test/CMakeLists.txt create mode 100644 libCacheSim/dataStructure/test/test_concurrent_hashtable.c create mode 100644 libCacheSim/utils/include/mymutex.h create mode 100644 libCacheSim/utils/mymutex.c create mode 100644 sftp-config.json diff --git a/libCacheSim/cache/cacheObj.c b/libCacheSim/cache/cacheObj.c index 07380b02..56b6f008 100644 --- a/libCacheSim/cache/cacheObj.c +++ b/libCacheSim/cache/cacheObj.c @@ -7,6 +7,30 @@ #include "../include/libCacheSim/macro.h" #include "../include/libCacheSim/request.h" +/** + * [verify the fingerprint of cache_obj] + * @method verify_cache_obj_fingerprint + * @author Chaos + * @date 2023-11-22 + * @param cache_obj [handle of cache item] + * @return [is valid or not] + */ +bool verify_cache_obj_fingerprint(const cache_obj_t *cache_obj) { + return cache_obj->obj_id == cache_obj->fingerprint; +} + +/** + * [set the fingerprint of cache_obj] + * @method set_cache_obj_fingerprint + * @author Chaos + * @date 2023-11-22 + * @param cache_obj [handle of cache item] + */ +void set_cache_obj_fingerprint(cache_obj_t *cache_obj) { + cache_obj->fingerprint = cache_obj->obj_id; +} + + /** * copy the cache_obj to req_dest * @param req_dest @@ -17,7 +41,7 @@ void copy_cache_obj_to_request(request_t *req_dest, req_dest->obj_id = cache_obj->obj_id; req_dest->obj_size = cache_obj->obj_size; req_dest->next_access_vtime = cache_obj->misc.next_access_vtime; - req_dest->valid = true; + req_dest->valid = verify_cache_obj_fingerprint(cache_obj); } /** @@ -34,6 +58,7 @@ void copy_request_to_cache_obj(cache_obj_t *cache_obj, const request_t *req) { cache_obj->exp_time = 0; #endif cache_obj->obj_id = req->obj_id; + set_cache_obj_fingerprint(cache_obj); } /** @@ -48,6 +73,26 @@ cache_obj_t *create_cache_obj_from_request(const request_t *req) { return cache_obj; } +/** + * [create a cache_obj from obj_id] + * @method create_cache_obj_from_obj_id + * @author Chaos + * @date 2023-11-22 + * @param obj_id [Given obj_id] + * @return [cache_obj] + */ +cache_obj_t *create_cache_obj_from_obj_id(const obj_id_t obj_id) { + cache_obj_t *cache_obj = my_malloc(cache_obj_t); + memset(cache_obj, 0, sizeof(cache_obj_t)); + cache_obj->obj_id = obj_id; + cache_obj->obj_size = 0; +#ifdef SUPPORT_TTL + cache_obj->exp_time = 0; +#endif + set_cache_obj_fingerprint(cache_obj); + return cache_obj; +} + /** remove the object from the built-in doubly linked list * * @param head diff --git a/libCacheSim/dataStructure/CMakeLists.txt b/libCacheSim/dataStructure/CMakeLists.txt index f1193e1b..0fc28e42 100644 --- a/libCacheSim/dataStructure/CMakeLists.txt +++ b/libCacheSim/dataStructure/CMakeLists.txt @@ -1,4 +1,6 @@ add_subdirectory(hash) +add_subdirectory(test) + set(source pqueue.c splay.c @@ -7,6 +9,7 @@ set(source hash/murmur3.c hashtable/chainedHashtable.c hashtable/chainedHashTableV2.c + hashtable/cChainedHashTable.c ) add_library (dataStructure ${source}) diff --git a/libCacheSim/dataStructure/hashtable/cChainedHashTable.c b/libCacheSim/dataStructure/hashtable/cChainedHashTable.c new file mode 100644 index 00000000..d77eb8ff --- /dev/null +++ b/libCacheSim/dataStructure/hashtable/cChainedHashTable.c @@ -0,0 +1,313 @@ +// +// TODO +// High level view of the concurrent chained hash table. +// - Rwlock pool: a pool of rwlocks. The size of the pool is 2^(hashpower-7). +// - Hashtable: +// - Each bucket is a linked list of cache_obj_t. The head of the linked list is stored in the ptr_table. The size of the ptr_table is 2^hashpower. +// - Each bucket maps to a rw_lock in the rwlock pool. Mulpitle buckets likely maps to the same rwlock. +// Mapping: rwlock_id = bucket_id & (rw_count-1) +// +// Rwlock pool (count=4) A hashtable +// |-----------------| | ---------------| +// | rw_lock 0 | | bucket 0 | ----> cache_obj_t* ----> cache_obj_t* ----> NULL +// |-----------------| | ---------------| +// | rw_lock 1 | | bucket 1 | ----> cache_obj_t* +// |-----------------| | ---------------| +// | rw_lock 2 | | bucket 2 | ----> NULL +// |-----------------| | ---------------| +// | rw_lock 3 | | bucket 3 | ----> cache_obj_t* ----> cache_obj_t* ----> nULL +// |-----------------| | ---------------| +// | bucket 4 | ----> NULL +// | ---------------| +// | bucket 5 | ----> NULL +// | ---------------| + +#ifdef __cplusplus +extern "C" { +#endif + +#include "cChainedHashTable.h" + +#include +#include +#include +#include +#include +#include + +#include "../../include/libCacheSim/logging.h" +#include "../../include/libCacheSim/macro.h" +#include "../../utils/include/mymath.h" +#include "../hash/hash.h" + +/************************ help func ************************/ + +/* free object, called by other functions when iterating through the hashtable + */ +static inline void foreach_free_obj_locked(cache_obj_t *cache_obj, void *user_data) { + my_free(sizeof(cache_obj_t), cache_obj); +} + +/** + * This function finds an object in the hashtable bucket. + * @method find_in_bucket_locked + * @author Chaos + * @date 2023-11-23 + * @param hashtable [Handle of the hashtable.] + * @param bucket_id [The id of the hashtable bucket.] + * @param obj_id [The id of the object to find.] + * @return [The pointer to the object. If not found, return NULL.] + */ +static inline cache_obj_t* find_in_bucket_locked(const hashtable_t *hashtable, uint64_t bucket_id, obj_id_t obj_id){ + cache_obj_t *cache_obj = hashtable->ptr_table[bucket_id]; + while (cache_obj) { + if (cache_obj->obj_id == obj_id) { + assert(verify_cache_obj_fingerprint(cache_obj)); + return cache_obj; + } + cache_obj = cache_obj->hash_next; + } + // cache_obj is NULL + return cache_obj; +} + +/** + * This function adds an object to the hashtable bucket. + * @method add_to_bucket_locked + * @author Chaos + * @date 2023-11-23 + * @param hashtable [Handle of the hashtable.] + * @param bucket_id [The id of the hashtable bucket.] + * @param cache_obj [The pointer to the object to add.] + * @return [The pointer to the object. If the object is already in the hashtable, return the existing one.] + */ +static inline cache_obj_t *add_to_bucket_locked(hashtable_t *hashtable, uint64_t bucket_id, cache_obj_t *cache_obj) { + // If the object is already in the hashtable, free the inserted object and return the existing one. + cache_obj_t* curr_obj = find_in_bucket_locked(hashtable, bucket_id, cache_obj->obj_id); + if(curr_obj != NULL){ + free_cache_obj(cache_obj); + return curr_obj; + } + // If the object is not in the hashtable, insert it to the head of the bucket. + curr_obj = hashtable->ptr_table[bucket_id]; + cache_obj->hash_next = curr_obj; + hashtable->ptr_table[bucket_id] = cache_obj; + __sync_fetch_and_add(&hashtable->n_obj, 1); + return cache_obj; +} + +/** + * This function deletes an object in the hashtable bucket. + * @method delete_in_bucket_locked + * @author Chaos + * @date 2023-11-23 + * @param hashtable [Handle of the hashtable.] + * @param bucket_id [The id of the hashtable bucket.] + * @param cache_obj [The pointer to the object to delete.] + * @return [Success or not.] + */ +static inline bool delete_in_bucket_locked(hashtable_t *hashtable, uint64_t bucket_id, obj_id_t obj_id) { + cache_obj_t *curr_obj = hashtable->ptr_table[bucket_id]; + cache_obj_t *prev_obj = curr_obj; + + if(curr_obj == NULL) return false; + + if(curr_obj->obj_id == obj_id){ + // the object to remove is the head of the bucket + hashtable->ptr_table[bucket_id] = curr_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(curr_obj); + __sync_fetch_and_sub(&hashtable->n_obj, 1); + return true; + } + // the object to remove is not the head of the bucket + do { + prev_obj = curr_obj; + curr_obj = curr_obj->hash_next; + } while(curr_obj != NULL && curr_obj->obj_id != obj_id); + // the object to remove is in the bucket + if (curr_obj != NULL) { + prev_obj->hash_next = curr_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(curr_obj); + hashtable->n_obj -= 1; + return true; + } + // the object to remove is not in the bucket (also not in the hashtable) + return false; +} + +/************************ hashtable func ************************/ +/** + * [This function is not thread-safe. Other threads muse wait for the return of this function.] + * @method create_concurrent_chained_hashtable + * @author Chaos + * @date 2023-11-21 + * @param hashpower [The power of 2 of the size of the hashtable. Default value is 20.] + * @return [Handle of the created hashtable.] + */ +hashtable_t *create_concurrent_chained_hashtable(const uint16_t hashpower) { + hashtable_t *hashtable = my_malloc(hashtable_t); + memset(hashtable, 0, sizeof(hashtable_t)); + size_t size = sizeof(cache_obj_t *) * hashsize(hashtable->hashpower); + hashtable->ptr_table = my_malloc_n(cache_obj_t *, hashsize(hashpower)); + if (hashtable->ptr_table == NULL) { + ERROR("allcoate hash table %zu entry * %lu B = %ld MiB failed\n", + sizeof(cache_obj_t *), (unsigned long)(hashsize(hashpower)), + (long)(sizeof(cache_obj_t *) * hashsize(hashpower) / 1024 / 1024)); + exit(1); + } + memset(hashtable->ptr_table, 0, size); + +#ifdef USE_HUGEPAGE + madvise(hashtable->table, size, MADV_HUGEPAGE); +#endif + hashtable->external_obj = false; + hashtable->hashpower = hashpower; + hashtable->n_obj = 0; + hashtable->rwlocks_ = init_RWLocks((hashpower>7) ? (hashpower-7) : 0); + return hashtable; +} + +cache_obj_t *concurrent_chained_hashtable_find_obj_id(const hashtable_t *hashtable, + const obj_id_t obj_id) { + uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); + /** Add read lock for query */ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); + pthread_rwlock_rdlock(rwlock_); + + cache_obj_t *cache_obj = find_in_bucket_locked(hashtable, hv, obj_id); + + pthread_rwlock_unlock(rwlock_); + return cache_obj; +} + +cache_obj_t *concurrent_chained_hashtable_find(const hashtable_t *hashtable, + const request_t *req) { + return concurrent_chained_hashtable_find_obj_id(hashtable, req->obj_id); +} + +cache_obj_t *concurrent_chained_hashtable_find_obj(const hashtable_t *hashtable, + const cache_obj_t *obj_to_find) { + return concurrent_chained_hashtable_find_obj_id(hashtable, obj_to_find->obj_id); +} + +/** + * This function inserts an object to the hashtable. + * If the object is not in the hashtable: + * - increase the number of objects in the hashtable + * - return the inserted object. + * Else: + * - return the existing one. + * @Author Chaos + * @Date 2023-11-22 + */ +cache_obj_t *concurrent_chained_hashtable_insert_obj(hashtable_t *hashtable, + cache_obj_t *cache_obj) { + // DEBUG_ASSERT(hashtable->external_obj); + uint64_t hv = get_hash_value_int_64(&cache_obj->obj_id) & hashmask(hashtable->hashpower); + /** Add write lock for insertion */ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); + pthread_rwlock_wrlock(rwlock_); + cache_obj_t *inserted_cache_obj = add_to_bucket_locked(hashtable, hv, cache_obj); + // If successfully inserted, increase the number of objects in the hashtable. + pthread_rwlock_unlock(rwlock_); + return inserted_cache_obj; +} + +cache_obj_t *concurrent_chained_hashtable_insert(hashtable_t *hashtable, + const request_t *req) { + cache_obj_t *new_cache_obj = create_cache_obj_from_request(req); + return concurrent_chained_hashtable_insert_obj(hashtable, new_cache_obj); +} + + +/* you need to free the extra_metadata before deleting from hash table */ +bool concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, + const obj_id_t obj_id) { + uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); + /** Add write lock for removal */ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); + pthread_rwlock_wrlock(rwlock_); + + bool res = delete_in_bucket_locked(hashtable, hv, obj_id); + + pthread_rwlock_unlock(rwlock_); + return res; +} + +bool concurrent_chained_hashtable_try_delete(hashtable_t *hashtable, + cache_obj_t *cache_obj) { + uint64_t hv = get_hash_value_int_64(&cache_obj->obj_id) & hashmask(hashtable->hashpower); + return concurrent_chained_hashtable_delete_obj_id(hashtable, cache_obj->obj_id); +} + +void concurrent_chained_hashtable_delete(hashtable_t *hashtable, + cache_obj_t *cache_obj) { + concurrent_chained_hashtable_try_delete(hashtable, cache_obj); +} + + +cache_obj_t *concurrent_chained_hashtable_rand_obj(const hashtable_t *hashtable) { + uint64_t pos = next_rand() & hashmask(hashtable->hashpower); + /** Add read lock for random query */ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, pos); + pthread_rwlock_rdlock(rwlock_); + + while (hashtable->ptr_table[pos] == NULL){ + pthread_rwlock_unlock(rwlock_); + pos = next_rand() & hashmask(hashtable->hashpower); + /** Add read lock for random query */ + rwlock_ = getRWLock(hashtable->rwlocks_, pos); + pthread_rwlock_rdlock(rwlock_); + } + pthread_rwlock_unlock(rwlock_); + return hashtable->ptr_table[pos]; +} + +void concurrent_chained_hashtable_foreach(hashtable_t *hashtable, + hashtable_iter iter_func, void *user_data) { + cache_obj_t *cur_obj, *next_obj; + for (uint64_t i = 0; i < hashsize(hashtable->hashpower); i++) { + /** Write lock for iter_func*/ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, i); + pthread_rwlock_wrlock(rwlock_); + cur_obj = hashtable->ptr_table[i]; + while (cur_obj != NULL) { + next_obj = cur_obj->hash_next; + iter_func(cur_obj, user_data); + cur_obj = next_obj; + } + pthread_rwlock_unlock(rwlock_); + } +} + +void free_concurrent_chained_hashtable(hashtable_t *hashtable) { + if (!hashtable->external_obj) + concurrent_chained_hashtable_foreach(hashtable, foreach_free_obj_locked, NULL); + my_free(sizeof(cache_obj_t *) * hashsize(hashtable->hashpower), + hashtable->ptr_table); + destory_RWLocks(hashtable->rwlocks_); +} + + +static int count_n_obj_in_bucket_locked(cache_obj_t *curr_obj) { + obj_id_t obj_id_arr[64]; + int chain_len = 0; + while (curr_obj != NULL) { + obj_id_arr[chain_len] = curr_obj->obj_id; + for (int i = 0; i < chain_len; i++) { + if (obj_id_arr[i] == curr_obj->obj_id) { + ERROR("obj_id %lu is duplicated in hashtable\n", curr_obj->obj_id); + abort(); + } + } + + curr_obj = curr_obj->hash_next; + chain_len += 1; + } + return chain_len; +} + + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/dataStructure/hashtable/cChainedHashTable.h b/libCacheSim/dataStructure/hashtable/cChainedHashTable.h new file mode 100644 index 00000000..ae28bfe0 --- /dev/null +++ b/libCacheSim/dataStructure/hashtable/cChainedHashTable.h @@ -0,0 +1,63 @@ +// This file defines a concurrent chained hashtable data structure. It is based on chainedHashTableV2. +// Some Features: +// - Static hashtable size +// - Thread safe +// - No global lock +// +// Created by Chaos on 11/23/2023. +// + +#ifndef libCacheSim_CCHAINEDHASHTABLE_H +#define libCacheSim_CCHAINEDHASHTABLE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include "../../include/libCacheSim/cacheObj.h" +#include "../../include/libCacheSim/request.h" +#include "hashtableStruct.h" + +hashtable_t *create_concurrent_chained_hashtable(const uint16_t hashpower_init); + +void free_concurrent_chained_hashtable(hashtable_t *hashtable); + +cache_obj_t *concurrent_chained_hashtable_find_obj_id(const hashtable_t *hashtable, + const obj_id_t obj_id); + +cache_obj_t *concurrent_chained_hashtable_find(const hashtable_t *hashtable, + const request_t *req); + +cache_obj_t *concurrent_chained_hashtable_find_obj(const hashtable_t *hashtable, + const cache_obj_t *obj_to_evict); + +/* return an empty cache_obj_t */ +cache_obj_t *concurrent_chained_hashtable_insert(hashtable_t *hashtable, + const request_t *req); + +cache_obj_t *concurrent_chained_hashtable_insert_obj(hashtable_t *hashtable, + cache_obj_t *cache_obj); + +bool concurrent_chained_hashtable_try_delete(hashtable_t *hashtable, + cache_obj_t *cache_obj); + +void concurrent_chained_hashtable_delete(hashtable_t *hashtable, + cache_obj_t *cache_obj); + +bool concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, + const obj_id_t obj_id); + +cache_obj_t *concurrent_chained_hashtable_rand_obj(const hashtable_t *hashtable); + +void concurrent_chained_hashtable_foreach(hashtable_t *hashtable, + hashtable_iter iter_func, void *user_data); + + +#ifdef __cplusplus +} +#endif + +#endif // libCacheSim_CCHAINEDHASHTABLE_H diff --git a/libCacheSim/dataStructure/hashtable/chainedHashTable.h b/libCacheSim/dataStructure/hashtable/chainedHashTable.h index bb4a2965..4c7e9b04 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashTable.h +++ b/libCacheSim/dataStructure/hashtable/chainedHashTable.h @@ -67,4 +67,4 @@ void _chained_hashtable_expand(hashtable_t *hashtable); } #endif -#endif // libCacheSim_CHAINEDHASHTABLE_H +#endif // libCacheSim_CHAINEDHASHTABLE_H \ No newline at end of file diff --git a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c index 8a5eb960..7d1214bb 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c +++ b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c @@ -248,26 +248,37 @@ bool chained_hashtable_try_delete_v2(hashtable_t *hashtable, return false; } -void chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, +bool chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, const obj_id_t obj_id) { - hashtable->n_obj -= 1; - uint64_t hv = get_hash_value_int_64(obj_id) & hashmask(hashtable->hashpower); - cache_obj_t *cache_obj = hashtable->ptr_table[hv]; - if (cache_obj != NULL && cache_obj->obj_id == obj_id) { - hashtable->ptr_table[hv] = cache_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(cache_obj); - return; - } - cache_obj = cache_obj->hash_next; - while (cache_obj != NULL && cache_obj->obj_id != obj_id) { - cache_obj = cache_obj->hash_next; + uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); + cache_obj_t *cur_obj = hashtable->ptr_table[hv]; + // the hash bucket is empty + if(cur_obj == NULL) return false; + + // the object to remove is the first object in the hash bucket + if (cur_obj->obj_id == obj_id) { + hashtable->ptr_table[hv] = cur_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(cur_obj); + hashtable->n_obj -= 1; + return true; } - if (cache_obj != NULL) { - cache_obj->hash_next = cache_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(cache_obj); + cache_obj_t *prev_obj; + + do { + prev_obj = cur_obj; + cur_obj = cur_obj->hash_next; + } while(cur_obj != NULL && cur_obj->obj_id != obj_id); + + if (cur_obj != NULL) { + prev_obj->hash_next = cur_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(cur_obj); + hashtable->n_obj -= 1; + return true; } + // the object to remove is not in the hash table + return false; } cache_obj_t *chained_hashtable_rand_obj_v2(const hashtable_t *hashtable) { @@ -384,4 +395,4 @@ static void print_hashbucket_item_distribution(const hashtable_t *hashtable) { #ifdef __cplusplus } -#endif +#endif \ No newline at end of file diff --git a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.h b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.h index ade28eb9..5ec88c7b 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.h +++ b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.h @@ -40,6 +40,9 @@ bool chained_hashtable_try_delete_v2(hashtable_t *hashtable, void chained_hashtable_delete_v2(hashtable_t *hashtable, cache_obj_t *cache_obj); +bool chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, + const obj_id_t obj_id); + cache_obj_t *chained_hashtable_rand_obj_v2(const hashtable_t *hashtable); void chained_hashtable_foreach_v2(hashtable_t *hashtable, @@ -56,4 +59,4 @@ void check_hashtable_integrity2_v2(const hashtable_t *hashtable, } #endif -#endif // libCacheSim_CHAINEDHASHTABLEV2_H +#endif // libCacheSim_CHAINEDHASHTABLEV2_H \ No newline at end of file diff --git a/libCacheSim/dataStructure/hashtable/chainedHashtable.c b/libCacheSim/dataStructure/hashtable/chainedHashtable.c index 1f8cb6fe..38281fcf 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashtable.c +++ b/libCacheSim/dataStructure/hashtable/chainedHashtable.c @@ -196,13 +196,13 @@ cache_obj_t *chained_hashtable_insert(hashtable_t *hashtable, request_t *req) { cache_obj->hash_next = new_cache_obj; cache_obj = new_cache_obj; } - hashtable->n_obj += 1; + __sync_fetch_and_add(&hashtable->n_obj, 1); return cache_obj; } /* you need to free the extra_metadata before deleting from hash table */ void chained_hashtable_delete(hashtable_t *hashtable, cache_obj_t *cache_obj) { - hashtable->n_obj -= 1; + __sync_fetch_and_sub(&hashtable->n_obj, 1); uint64_t hv = get_hash_value_int_64(&cache_obj->obj_id) & hashmask(hashtable->hashpower); cache_obj_t *cache_obj_in_bucket = &hashtable->table[hv]; diff --git a/libCacheSim/dataStructure/hashtable/hashtable.h b/libCacheSim/dataStructure/hashtable/hashtable.h index 7185a551..7f7a9877 100644 --- a/libCacheSim/dataStructure/hashtable/hashtable.h +++ b/libCacheSim/dataStructure/hashtable/hashtable.h @@ -61,6 +61,31 @@ extern "C" { #define hashtable_add_ptr_to_monitoring(hashtable, ptr) #define HASHTABLE_VER 2 +// #elif HASHTABLE_TYPE == CONCURRENT_CHAINED_HASHTABLE +// #include "cChainedHashTable.h" +// #define create_hashtable(hashpower) create_concurrent_chained_hashtable(hashpower) +// #define hashtable_find(hashtable, req) concurrent_chained_hashtable_find(hashtable, req) +// #define hashtable_find_obj_id(hashtable, obj_id) \ +// concurrent_chained_hashtable_find_obj_id(hashtable, obj_id) +// #define hashtable_find_obj(hashtable, cache_obj) \ +// concurrent_chained_hashtable_find_obj(hashtable, cache_obj) +// #define hashtable_insert(hashtable, req) \ +// concurrent_chained_hashtable_insert(hashtable, req) +// #define hashtable_insert_obj(hashtable, cache_obj) \ +// concurrent_chained_hashtable_insert_obj(hashtable, cache_obj) +// #define hashtable_delete(hashtable, cache_obj) \ +// concurrent_chained_hashtable_delete(hashtable, cache_obj) +// #define hashtable_try_delete(hashtable, cache_obj) \ +// concurrent_chained_hashtable_try_delete(hashtable, cache_obj) +// #define hashtable_delete_obj_id(hashtable, obj_id) \ +// concurrent_chained_hashtable_delete_obj_id(hashtable, obj_id) +// #define hashtable_rand_obj(hashtable) concurrent_chained_hashtable_rand_obj(hashtable) +// #define hashtable_foreach(hashtable, iter_func, user_data) \ +// concurrent_chained_hashtable_foreach(hashtable, iter_func, user_data) +// #define free_hashtable(hashtable) free_concurrent_chained_hashtable(hashtable) +// #define hashtable_add_ptr_to_monitoring(hashtable, ptr) +// #define HASHTABLE_VER 3 + #elif HASHTABLE_TYPE == CUCKCOO_HASHTABLE #include "cuckooHashTable.h" #error not implemented diff --git a/libCacheSim/dataStructure/hashtable/hashtableStruct.h b/libCacheSim/dataStructure/hashtable/hashtableStruct.h index 13ad7b1b..1d24c36d 100644 --- a/libCacheSim/dataStructure/hashtable/hashtableStruct.h +++ b/libCacheSim/dataStructure/hashtable/hashtableStruct.h @@ -10,8 +10,10 @@ extern "C" { #endif #include +#include #include "../../include/libCacheSim/cacheObj.h" +#include "../../utils/include/mymutex.h" #define hashsize(n) ((uint64_t)1 << (uint16_t)(n)) #define hashsizeULL(n) ((unsigned long long)1 << (uint16_t)(n)) @@ -40,6 +42,7 @@ typedef struct hashtable { }; void *extra_data; }; + RWLocks_t* rwlocks_; } hashtable_t; #ifdef __cplusplus diff --git a/libCacheSim/dataStructure/test/CMakeLists.txt b/libCacheSim/dataStructure/test/CMakeLists.txt new file mode 100644 index 00000000..1bd287e6 --- /dev/null +++ b/libCacheSim/dataStructure/test/CMakeLists.txt @@ -0,0 +1,4 @@ +add_executable(testCHashtable test_concurrent_hashtable.c) +target_link_libraries(testCHashtable dataStructure utils cachelib pthread) + +add_test(NAME testCHashtable COMMAND testCHashtable) diff --git a/libCacheSim/dataStructure/test/test_concurrent_hashtable.c b/libCacheSim/dataStructure/test/test_concurrent_hashtable.c new file mode 100644 index 00000000..0173219e --- /dev/null +++ b/libCacheSim/dataStructure/test/test_concurrent_hashtable.c @@ -0,0 +1,537 @@ +/* + * Created by ChaosD on 11/22/23. + * This file is used to test the concurrent throughput of hashtable structures. + * Now the test only supports two types of hashtable: chainedHashTableV2, and cChainedHashTable. + * It supports three types of operations: read obj, insert obj, and delete obj. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "../../utils/include/mymath.h" +#include "../../include/libCacheSim/cacheObj.h" +#include "../hashtable/chainedHashTableV2.h" +#include "../hashtable/cChainedHashTable.h" + +/** Power of number of cache objects to insert. */ +size_t g_power = 18; + +/** Number of cache objects to insert. */ +size_t g_numkeys; // 2^18=262144 + +/** Number of threads to spawn for each type of operation. */ +size_t g_thread_num = 1; + +/** Number of seconds to run read test. */ +size_t g_test_len = 10; // 10s + +/** Type of test. */ +size_t g_test_type = 0; // 0: Three tests: Insert, Read, and Delete. Each test is non-overlapping. + // 1: Mixed. Each thread performs insert, read, and delete in turn. + // The three operations are overlapping between different threads. + +/** Type of hashtable. */ +size_t g_ht_type = 0; // 0: cChainedHashTable; 1: chainedHashTableV2; + + +/** Function pointer to find a cache object in the cache instance. */ +typedef cache_obj_t* (*func_hashtable_find_obj_id_t)(const hashtable_t *hashtable, + const obj_id_t obj_id); +/** Function pointer to insert a cache object in the cache instance. */ +typedef cache_obj_t* (*func_hashtable_insert_obj_t)(hashtable_t *hashtable, + cache_obj_t *cache_obj); +/** Function pointer to remove a cache object in the cache instance. */ +typedef bool (*func_hashtable_delete_obj_id_t)(hashtable_t *hashtable, + const obj_id_t obj_id); + +typedef hashtable_t *(*func_create_hashtable_t)(const uint16_t hashpower_init); + +/** Function pointer to free a hashtable instance. */ +typedef void (*func_free_hashtable_t)(hashtable_t *hashtable); + + +struct handle_hashtable_t{ + func_hashtable_find_obj_id_t func_find_obj; + func_hashtable_insert_obj_t func_insert_obj; + func_hashtable_delete_obj_id_t func_delete_obj; + func_create_hashtable_t func_create; + func_free_hashtable_t func_free; +} ht_handle; + +/** + * Struct of parameters for test threads. + * @author Chaos + * @date 2023-11-22 + * @number table [Handle of a cache instance] + * @number r_counter [Number of cache objects read] + * @number i_counter [Number of cache objects inserted] + * @number d_counter [Number of cache objects deleted] + * @number finished [Signal of the end of read] + * @number range [Range of obj ID to read] + * @number thread_id [thread ID number] + * @number func_find [Function pointer to find a cache object] + * @number func_insert [Function pointer to insert a cache object] + * @number func_delete [Function pointer to delete a cache object] + */ +typedef struct thread_para{ + hashtable_t* table; + uint64_t* r_counter; + uint64_t* i_counter; + uint64_t* d_counter; + bool* finished; + uint64_t range; + uint32_t thread_id; +}thread_para_t; + +/** + * [This function reads a cache object from the cache instance. It is used as a thread function.] + * @method func_read + * @author Chaos + * @date 2023-11-22 + */ +void *func_read(void* arg){ + thread_para_t* para = (thread_para_t*) arg; + hashtable_t* table = para->table; + uint64_t* r_counter = para->r_counter; + bool* finished = para->finished; + uint64_t range = para->range; + uint32_t thread_id = para->thread_id; + +// We keep track of our own local counter for reads, to avoid +// over-burdening the shared atomic counter + size_t reads = 0; + size_t success_reads = 0; + size_t fail_reads = 0; + + struct timeval start, end; + // We use thread id as seed to generate a reproducible pseudo random sequence. + // The items in the sequence are operated obj IDs. + obj_id_t cur_obj_id = thread_id; + // The function keeps runing until the flag `finished` == True. + gettimeofday(&start, NULL); + for (uint64_t it = 0;; it++) { + if(it == range){ + it = 0; + cur_obj_id = thread_id; + } + if (*finished) { + __sync_fetch_and_add(r_counter, reads); + gettimeofday(&end, NULL); + double timeuse = (end.tv_sec - start.tv_sec) + + (end.tv_usec - start.tv_usec) / 1000000.0; + INFO("Thread %d read %zu objects in %.1f seconds, throughput is %.2f MQPS. %zu success, %zu fail\n", \ + thread_id, reads, timeuse, reads / timeuse / 1000000, success_reads, fail_reads); + + return NULL; + } + cur_obj_id = get_next_rand(cur_obj_id); + if(ht_handle.func_find_obj(table, cur_obj_id)){ + success_reads++; + } + else{ + fail_reads++; + } + reads++; + } +} + +/** + * [This function inserts a cache object to the cache instance. It is used as a thread function.] + * @method func_insert + * @author Chaos + * @date 2023-11-22 + */ +void *func_insert(void* arg){ + thread_para_t* para = (thread_para_t*) arg; + hashtable_t* table = para->table; + uint64_t* i_counter = para->i_counter; + uint64_t range = para->range; + uint32_t thread_id = para->thread_id; + +// We keep track of our own local counter for inserts, to avoid +// over-burdening the shared atomic counter + size_t inserts = 0; + // We use thread id as seed to generate a reproducible pseudo random sequence. + // The items in the sequence are operated obj IDs. + obj_id_t cur_obj_id = thread_id; + struct timeval start, end; + gettimeofday(&start, NULL); + for (uint64_t it = 0; it < range; it++) { + cur_obj_id = get_next_rand(cur_obj_id); + cache_obj_t* cur_obj = create_cache_obj_from_obj_id(cur_obj_id); + ht_handle.func_insert_obj(table, cur_obj); + inserts++; + } + /** Update the shared atomic counter */ + __sync_fetch_and_add(i_counter, inserts); + gettimeofday(&end, NULL); + double timeuse = (end.tv_sec - start.tv_sec) + + (end.tv_usec - start.tv_usec) / 1000000.0; + INFO("Thread %d inserted %zu objects in %.1f seconds, throughput is %.2f MQPS\n", \ + thread_id, inserts, timeuse, inserts / timeuse / 1000000); +} + +/** + * [This function inserts a cache object to the cache instance. It is used as a thread function.] + * @method func_remove + * @author Chaos + * @date 2023-11-22 + */ +void *func_remove(void* arg){ + thread_para_t* para = (thread_para_t*) arg; + hashtable_t* table = para->table; + uint64_t* d_counter = para->d_counter; + uint64_t range = para->range; + uint32_t thread_id = para->thread_id; + +// We keep track of our own local counter for removals, to avoid +// over-burdening the shared atomic counter + size_t removals = 0; + size_t success_removal = 0; + size_t fail_removal = 0; + + // We use thread id as seed to generate a reproducible pseudo random sequence. + // The items in the sequence are operated obj IDs. + obj_id_t cur_obj_id = thread_id; + struct timeval start, end; + gettimeofday(&start, NULL); + for (uint64_t it = 0; it < range; it++) { + cur_obj_id = get_next_rand(cur_obj_id); + if(ht_handle.func_delete_obj(table, cur_obj_id)){ + success_removal++; + } + else{ + fail_removal++; + } + removals++; + } + // Update the shared atomic counter + __sync_fetch_and_add(d_counter, removals); + gettimeofday(&end, NULL); + double timeuse = (end.tv_sec - start.tv_sec) + + (end.tv_usec - start.tv_usec) / 1000000.0; + INFO("Thread %d removed %zu objects in %.1f seconds, throughput is %.2f MQPS. %zu success, %zu fail\n", \ + thread_id, removals, timeuse, removals / timeuse / 1000000, success_removal, fail_removal); +} + + +/** + * This function is for mixed tests. It is used as a thread function. It works as follows: + * 1. Insert `range` objects to the cache instance. + * 2. Read `range` objects from the cache instance for 10 rounds. + * 3. Delete `range` objects from the cache instance. + * 4. Repeat step 1-3 until the flag `finished` == True. + * The `range` objects are generated by a pseudo random sequence. The seed of the sequence is the thread ID. + * @method func_mixed + * @author Chaos + * @date 2023-11-23 + */ +void *func_mixed(void* arg){ + thread_para_t* para = (thread_para_t*) arg; + hashtable_t* table = para->table; + uint64_t* r_counter = para->r_counter; + uint64_t* i_counter = para->i_counter; + uint64_t* d_counter = para->d_counter; + bool* finished = para->finished; + uint64_t range = para->range; + uint32_t thread_id = para->thread_id; + + +// We keep track of our own local counter for reads, to avoid +// over-burdening the shared atomic counter + size_t reads = 0; + size_t inserts = 0; + size_t removals = 0; + + // We use thread id as seed to generate a reproducible pseudo random sequence. + // The items in the sequence are operated obj IDs. + obj_id_t cur_obj_id = thread_id; + // The loop is broken when the flag `finished` == True. + while(true){ + cur_obj_id = thread_id; + inserts = 0; + removals = 0; + for(uint64_t it = 0; it < range; it++){ + cur_obj_id = get_next_rand(cur_obj_id); + cache_obj_t* cur_obj = create_cache_obj_from_obj_id(cur_obj_id); + ht_handle.func_insert_obj(table, cur_obj); + inserts++; + } + for(uint64_t rounds = 0; rounds < 10; rounds++){ + cur_obj_id = thread_id; + reads = 0; + for (uint64_t it = 0; it < range; it++){ + cur_obj_id = get_next_rand(cur_obj_id); + ht_handle.func_find_obj(table, cur_obj_id); + reads++; + } + __sync_fetch_and_add(r_counter, reads); + } + for(uint64_t it = 0; it < range; it++){ + cur_obj_id = get_next_rand(cur_obj_id); + ht_handle.func_delete_obj(table, cur_obj_id); + removals++; + } + __sync_fetch_and_add(i_counter, inserts); + __sync_fetch_and_add(d_counter, removals); + if(*finished){ + return NULL; + } + } +} + +/** + * [This function parses the command line arguments.] + * @method parse_arg + * @author Chaos + * @date 2023-11-22 + * @param argc [count of input arguments] + * @param argv [input arguments] + * @param description [description of the arguments] + * @param args [optional arguments] + * @param arg_vars [variables to store the argument values] + * @param arg_help [help message for each argument] + * @param arg_num [number of optional arguments] + */ +void parse_arg(int argc, char const **argv, const char *description, + const char *args[], size_t *arg_vars[], const char *arg_help[], + size_t arg_num) { + errno = 0; + for (int i = 0; i < argc; i++) { + for (size_t j = 0; j < arg_num; j++) { + if (strcmp(argv[i], args[j]) == 0) { + if (i == argc - 1) { + ERROR("You must provide a positive integer argument" \ + " after the %s argument\n", args[j]); + } else { + size_t argval = strtoull(argv[i + 1], NULL, 10); + if (errno != 0) { + ERROR("The argument to %s must be a valid size_t\n", args[j]); + } else { + *(arg_vars[j]) = argval; + } + } + } + } + + if (strcmp(argv[i], "--help") == 0) { + printf("%s\n", description); + printf("Arguments:\n"); + for (size_t j = 0; j < arg_num; j++) { + printf("%s\t(default %zu):\t%s\n", args[j], *arg_vars[j], arg_help[j]); + } + exit(0); + } + } +} + +/** + * This function runs a stress test on inserts, finds, and deletes. + * @method StressTest + * @author Chaos + * @date 2023-11-22 + */ +void stress_test() { + // Initialize threads; + pthread_t insert_threads[g_thread_num]; + pthread_t read_threads[g_thread_num]; + pthread_t remove_threads[g_thread_num]; + + int rc; + uint64_t num_inserts = 0; + uint64_t num_removals = 0; + uint64_t num_reads = 0; + bool finished = false; + + // Based on the hashtable type, we assign the function pointers to the function handles. + // If hashtable type is cChainedHashTable. + if(g_ht_type == 0){ + ht_handle.func_find_obj = concurrent_chained_hashtable_find_obj_id; + ht_handle.func_insert_obj = concurrent_chained_hashtable_insert_obj; + ht_handle.func_delete_obj = concurrent_chained_hashtable_delete_obj_id; + ht_handle.func_create = create_concurrent_chained_hashtable; + ht_handle.func_free = free_concurrent_chained_hashtable; + } + // If hashtable type is chainedHashTableV2. + else if(g_ht_type == 1){ + ht_handle.func_find_obj = chained_hashtable_find_obj_id_v2; + ht_handle.func_insert_obj = chained_hashtable_insert_obj_v2; + ht_handle.func_delete_obj = chained_hashtable_delete_obj_id_v2; + ht_handle.func_create = create_chained_hashtable_v2; + ht_handle.func_free = free_chained_hashtable_v2; + } + + // If hashtable type is invalid. + else{ + ERROR("ERROR: Invalid hashtable type\n"); + } + + // Creates a hashtable + hashtable_t *table = ht_handle.func_create(g_power); + + // If the user select the first test. + if(g_test_type == 0){ + // Spawns insert threads + for (size_t i = 0; i < g_thread_num; i++) { + thread_para_t para = { + .table = table, + .i_counter = &num_inserts, + .range = g_numkeys / g_thread_num, + .thread_id = i, + }; + rc = pthread_create(&insert_threads[i], NULL, func_insert, (void *)¶); + if(rc){ + ERROR("ERROR; return code from pthread_create() is %d\n", rc); + } + } + + // Joins insert threads + for (size_t i = 0; i < g_thread_num; i++) { + rc = pthread_join(insert_threads[i], NULL); + if(rc){ + ERROR("ERROR; return code from pthread_join() is %d\n", rc); + } + } + + // Spawns read threads + for(size_t i = 0; i < g_thread_num; i++){ + thread_para_t para = { + .table = table, + .r_counter = &num_reads, + .finished = &finished, + .range = g_numkeys / g_thread_num, + .thread_id = i, + }; + rc = pthread_create(&read_threads[i], NULL, func_read, (void *)¶); + if(rc){ + ERROR("ERROR; return code from pthread_create() is %d\n", rc); + } + } + + // Sleeps before ending the threads + sleep(g_test_len); + finished = true; + // Joins read threads + for (size_t i = 0; i < g_thread_num; i++) { + rc = pthread_join(read_threads[i], NULL); + if(rc){ + ERROR("ERROR; return code from pthread_join() is %d\n", rc); + } + } + + // Spawns remove threads + for(size_t i = 0; i < g_thread_num; i++){ + thread_para_t para = { + .table = table, + .d_counter = &num_removals, + .range = g_numkeys / g_thread_num, + .thread_id = i, + }; + rc = pthread_create(&remove_threads[i], NULL, func_remove, (void *)¶); + if(rc){ + ERROR("ERROR; return code from pthread_create() is %d\n", rc); + } + } + + // Joins delete threads + for (size_t i = 0; i < g_thread_num; i++) { + rc = pthread_join(remove_threads[i], NULL); + if(rc){ + ERROR("ERROR; return code from pthread_join() is %d\n", rc); + } + } + } + // If the user select the second test -- mixed test. + else if(g_test_type == 1){ + uint64_t num_inserts_last_sec = 0; + uint64_t num_removals_last_sec = 0; + uint64_t num_reads_last_sec = 0; + + // Spawns mixed threads + for(size_t i = 0; i < g_thread_num; i++){ + thread_para_t para = { + .table = table, + .r_counter = &num_reads, + .i_counter = &num_inserts, + .d_counter = &num_removals, + .finished = &finished, + .range = g_numkeys / g_thread_num, + .thread_id = i, + }; + rc = pthread_create(&read_threads[i], NULL, func_mixed, (void *)¶); + if(rc){ + ERROR("ERROR; return code from pthread_create() is %d\n", rc); + } + } + printf("----------Throughput MQPS----------\n"); + printf("Seconds\tInsert\tRead\tDelete\tTotal\n"); + + // Prints the throughput every second + for(int i = 0; i < g_test_len; i++){ + sleep(1); + uint64_t num_inserts_this_sec = num_inserts - num_inserts_last_sec; + uint64_t num_reads_this_sec = num_reads - num_reads_last_sec; + uint64_t num_removals_this_sec = num_removals - num_removals_last_sec; + printf("%d\t%.2lf\t%.2lf\t%.2lf\t%.2lf\n", i+1, num_inserts_this_sec/1000000.0, num_reads_this_sec/1000000.0, num_removals_this_sec/1000000.0, (num_inserts_this_sec + num_reads_this_sec + num_removals_this_sec)/1000000.0); + num_inserts_last_sec = num_inserts; + num_reads_last_sec = num_reads; + num_removals_last_sec = num_removals; + } + // Ends the threads. The threads will end when the flag `finished` == True. + finished = true; + // Joins threads + for (size_t i = 0; i < g_thread_num; i++) { + rc = pthread_join(read_threads[i], NULL); + if(rc){ + ERROR("ERROR; return code from pthread_join() is %d\n", rc); + } + } + } + // If the user select an invalid test type. + else{ + ERROR("Invalid test type: %lu. It should be 0 or 1.\n", g_test_type); + } + // Frees the hashtable + ht_handle.func_free(table); + + printf("----------Results----------\n"); + printf("Number of inserts:\t%lu\n", num_inserts); + printf("Number of reads:\t%lu\n", num_reads); + printf("Number of removals:\t%lu\n", num_removals); + printf("Total throughput:\t%.2lf MQPS\n", (num_inserts + num_reads + num_removals)/g_test_len/1000000.0); +} + + +int main(int argc, char const *argv[]) +{ + const char *args[] = {"--power", "--thread-num", "--time", "--test-type", "--ht-type"}; + size_t *arg_vars[] = {&g_power, &g_thread_num, &g_test_len, &g_test_type, &g_ht_type}; + const char *arg_help[] = { + "The number of keys to size the table with, expressed as a power of 2", + "The number of threads to spawn for each type of operation", + "The number of seconds to run the test for lookup", + "The type of test. \n\ + 0: Three tests: Insert, Read, and Delete. Each test is non-overlapping. \n\ + 1: Mixed. Each thread performs insert, read, and delete in turn. The three operations are overlapping between different threads.", + "The type of hashtable. \n\ + 0: cChainedHashTable; \n\ + 1:chainedHashTableV2"}; + parse_arg(argc, argv, "Runs a stress test on concurrent hashtables for inserts, finds, and deletes.", + args, arg_vars, arg_help, sizeof(args) / sizeof(const char *)); + g_numkeys = 1 << g_power; + stress_test(); + return 0; +} + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/libCacheSim/include/libCacheSim/cacheObj.h b/libCacheSim/include/libCacheSim/cacheObj.h index 86a378fd..271c130d 100644 --- a/libCacheSim/include/libCacheSim/cacheObj.h +++ b/libCacheSim/include/libCacheSim/cacheObj.h @@ -139,6 +139,7 @@ struct cache_obj; typedef struct cache_obj { struct cache_obj *hash_next; obj_id_t obj_id; + uint64_t fingerprint; uint32_t obj_size; struct { struct cache_obj *prev; @@ -183,6 +184,14 @@ typedef struct cache_obj { } __attribute__((packed)) cache_obj_t; struct request; + +/** + * verify the fingerprint of current cache_obj + * @param cache_obj + * @param bool [true if the fingerprint is correct] + */ +bool verify_cache_obj_fingerprint(const cache_obj_t *cache_obj); + /** * copy the cache_obj to req_dest * @param req_dest @@ -206,6 +215,13 @@ void copy_request_to_cache_obj(cache_obj_t *cache_obj, */ cache_obj_t *create_cache_obj_from_request(const struct request *req); +/** + * create a empty cache_obj from obj_id + * @param obj_id + * @return + */ +cache_obj_t *create_cache_obj_from_obj_id(const obj_id_t obj_id); + /** * the cache_obj has built-in a doubly list, in the case the list is used as * a singly list (list_prev is not used, next is used) diff --git a/libCacheSim/utils/include/mymath.h b/libCacheSim/utils/include/mymath.h index 6b9137ae..8c6f09f5 100644 --- a/libCacheSim/utils/include/mymath.h +++ b/libCacheSim/utils/include/mymath.h @@ -29,6 +29,20 @@ static inline uint64_t next_rand() { return rand_seed; } + +/** + * generate reproducible pseudo rand sequence, used to test concurrent hashtable. + * @method get_next_rand + * @author Chaos + * @date 2023-11-22 + * @param cur [current random number] + * @return [next random number in the sequence] + */ +static inline uint64_t get_next_rand(uint64_t cur) { + uint64_t next = 6364136223846793005 * cur + 1442695040888963407; + return next; +} + static inline long long next_power_of_2(long long N) { // if N is a power of two simply return it if (!(N & (N - 1))) return N; diff --git a/libCacheSim/utils/include/mymutex.h b/libCacheSim/utils/include/mymutex.h new file mode 100644 index 00000000..2b63e467 --- /dev/null +++ b/libCacheSim/utils/include/mymutex.h @@ -0,0 +1,24 @@ +#pragma once + +// +// Created by Chaos on 11/20/23. +// + +#include +#include + +typedef struct RWLocks +{ + uint64_t locksMask_; + pthread_rwlock_t* locks_; +}RWLocks_t; + + +RWLocks_t* init_RWLocks(uint32_t locksPower); + +void expand_RWLocks(RWLocks_t* rwlocks); + +void destory_RWLocks(RWLocks_t* rwlocks); + +pthread_rwlock_t* getRWLock(RWLocks_t* rwlocks, uint64_t hash); + diff --git a/libCacheSim/utils/mymutex.c b/libCacheSim/utils/mymutex.c new file mode 100644 index 00000000..dc16511f --- /dev/null +++ b/libCacheSim/utils/mymutex.c @@ -0,0 +1,84 @@ +// +// Created by Chaos on 11/20/23. +// + +#ifdef __cplusplus +extern "C" { +#endif + +#include "include/mymutex.h" +#include "../include/libCacheSim/mem.h" +#include +/** + * [init_RWLocks: Initiate a rw_lock pool. The size of the pool is determined by locksPower. + * For example, if the locksPower is 4, the size is 2^4 = 16. ] + * @author Chaos + * @date 2023-11-20 + * @param locksPower [The power of the number of rwlocks.] + * @return [Handler of the created rw_locks. ] + */ +RWLocks_t* init_RWLocks(uint32_t locksPower){ + RWLocks_t* rwlocks = my_malloc(RWLocks_t); + rwlocks->locksMask_ = (1ULL << locksPower) - 1; + rwlocks->locks_ = my_malloc_n(pthread_rwlock_t, 1ULL << locksPower); + + for(uint64_t s = 0; s <= rwlocks->locksMask_; s++){ + pthread_rwlock_init(rwlocks->locks_ + s, NULL); + } + return rwlocks; +} + +/** + * [expand_RWLocks: Grow the rw_locks pool to the next power of 2..] + * @method expand_RWLocks + * @author Chaos + * @date 2023-11-21 + * @param rwlocks [Handler of operated rw_locks. ] + */ +void expand_RWLocks(RWLocks_t* rwlocks){ + uint64_t oldLocksMask = rwlocks->locksMask_; + rwlocks->locksMask_ = (oldLocksMask << 1) | 1; + pthread_rwlock_t* oldLocks = rwlocks->locks_; + rwlocks->locks_ = my_malloc_n(pthread_rwlock_t, rwlocks->locksMask_ + 1); + for(uint64_t s = 0; s <= oldLocksMask; s++){ + pthread_rwlock_init(rwlocks->locks_ + s, NULL); + } + for(uint64_t s = oldLocksMask + 1; s <= rwlocks->locksMask_; s++){ + pthread_rwlock_init(rwlocks->locks_ + s, NULL); + } + my_free(sizeof(pthread_rwlock_t) * (oldLocksMask + 1), oldLocks); +} + +/** + * [destory_RWLocks: Destory the rw_locks.] + * @method destory_RWLocks + * @author Chaos + * @date 2023-11-20 + * @param rwlocks [Handler of the destoryed rw_locks. ] + */ +void destory_RWLocks(RWLocks_t* rwlocks){ + for(uint64_t s = 0; s <= rwlocks->locksMask_; s++){ + pthread_rwlock_destroy(rwlocks->locks_ + s); + } + my_free(sizeof(pthread_rwlock_t) * (rwlocks->locksMask_ + 1), rwlocks->locks_); + my_free(sizeof(RWLocks_t), rwlocks); +} + +/** + * [getRWLock: Get a rw_lock by a random hash number.] + * @method getRWLock + * @author Chaos + * @date 2023-11-20 + * @param rwlocks [Handler] + * @param hash [A random hash number] + * @return [Current rw_lock.] + */ +pthread_rwlock_t* getRWLock(RWLocks_t* rwlocks, uint64_t hash) { + uint64_t index = hash & rwlocks->locksMask_; + return rwlocks->locks_ + index; +} + + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/sftp-config.json b/sftp-config.json new file mode 100644 index 00000000..0fce488b --- /dev/null +++ b/sftp-config.json @@ -0,0 +1,45 @@ +{ + // The tab key will cycle through the settings when first created + // Visit https://codexns.io/products/sftp_for_subime/settings for help + + // sftp, ftp or ftps + "type": "sftp", + + "save_before_upload": true, + "upload_on_save": true, + "sync_down_on_open": false, + "sync_skip_deletes": false, + "sync_same_age": true, + "confirm_downloads": false, + "confirm_sync": true, + "confirm_overwrite_newer": false, + + "host": "localhost", + "user": "chaos", + "password": "hdx12280222", + "port": "1002", + + "remote_path": "/home/chaos/libCacheSim", + "ignore_regexes": [ + "\\.sublime-(project|workspace)", "sftp-config(-alt\\d?)?\\.json", + "sftp-settings\\.json", "/venv/", "\\.svn/", "\\.hg/", "\\.git/", + "\\.bzr", "_darcs", "CVS", "\\.DS_Store", "Thumbs\\.db", "desktop\\.ini" + ], + //"file_permissions": "664", + //"dir_permissions": "775", + + //"extra_list_connections": 0, + + "connect_timeout": 30, + //"keepalive": 120, + //"ftp_passive_mode": true, + //"ftp_obey_passive_host": false, + //"ssh_key_file": "~/.ssh/id_rsa", + //"sftp_flags": ["-F", "/path/to/ssh_config"], + + //"preserve_modification_times": false, + //"remote_time_offset_in_hours": 0, + //"remote_encoding": "utf-8", + //"remote_locale": "C", + //"allow_config_upload": false, +} From 9338024deec16666605c3eb9e252ab2de9b6dd2f Mon Sep 17 00:00:00 2001 From: ChaosD Date: Fri, 24 Nov 2023 12:55:45 +0800 Subject: [PATCH 02/28] add concurrent hashtable implementation and tests --- .../hashtable/cChainedHashTable.c | 56 ++++++++++++++++--- .../test/test_concurrent_hashtable.c | 30 +++++----- 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/libCacheSim/dataStructure/hashtable/cChainedHashTable.c b/libCacheSim/dataStructure/hashtable/cChainedHashTable.c index d77eb8ff..5179ff25 100644 --- a/libCacheSim/dataStructure/hashtable/cChainedHashTable.c +++ b/libCacheSim/dataStructure/hashtable/cChainedHashTable.c @@ -98,7 +98,7 @@ static inline cache_obj_t *add_to_bucket_locked(hashtable_t *hashtable, uint64_t /** * This function deletes an object in the hashtable bucket. - * @method delete_in_bucket_locked + * @method delete_obj_id_in_bucket_locked * @author Chaos * @date 2023-11-23 * @param hashtable [Handle of the hashtable.] @@ -106,7 +106,36 @@ static inline cache_obj_t *add_to_bucket_locked(hashtable_t *hashtable, uint64_t * @param cache_obj [The pointer to the object to delete.] * @return [Success or not.] */ -static inline bool delete_in_bucket_locked(hashtable_t *hashtable, uint64_t bucket_id, obj_id_t obj_id) { +static inline bool delete_in_bucket_locked(hashtable_t *hashtable, uint64_t bucket_id, cache_obj_t *cache_obj) { + cache_obj_t *curr_obj = hashtable->ptr_table[bucket_id]; + // If the object to delete is NULL, return false. + if(curr_obj == NULL || cache_obj == NULL){ + return false; + } + // If the object to delete is the head of the bucket, delete it and return true. + if (curr_obj == cache_obj) { + hashtable->ptr_table[bucket_id] = cache_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(cache_obj); + __sync_fetch_and_sub(&hashtable->n_obj, 1); + return true; + } + // If the object to delete is not the head of the bucket, find it. + while (curr_obj != NULL && curr_obj->hash_next != cache_obj) { + curr_obj = curr_obj->hash_next; + } + // If the object to delete is in the bucket, delete it and return true. + if (curr_obj != NULL) { + curr_obj->hash_next = cache_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(cache_obj); + __sync_fetch_and_sub(&hashtable->n_obj, 1); + return true; + } + // If the object to delete is not in the bucket, return false. + return false; +} + + +static inline bool delete_obj_id_in_bucket_locked(hashtable_t *hashtable, uint64_t bucket_id, obj_id_t obj_id) { cache_obj_t *curr_obj = hashtable->ptr_table[bucket_id]; cache_obj_t *prev_obj = curr_obj; @@ -128,7 +157,7 @@ static inline bool delete_in_bucket_locked(hashtable_t *hashtable, uint64_t buck if (curr_obj != NULL) { prev_obj->hash_next = curr_obj->hash_next; if (!hashtable->external_obj) free_cache_obj(curr_obj); - hashtable->n_obj -= 1; + __sync_fetch_and_sub(&hashtable->n_obj, 1); return true; } // the object to remove is not in the bucket (also not in the hashtable) @@ -220,15 +249,23 @@ cache_obj_t *concurrent_chained_hashtable_insert(hashtable_t *hashtable, } -/* you need to free the extra_metadata before deleting from hash table */ -bool concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, +/** + * This function deletes an object in the hashtable. + * If the object is in the hashtable: + * - decrease the number of objects in the hashtable + * - return true. + * Else: + * - return false. + * @Author Chaos + * @Date 2023-11-22 + */bool concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, const obj_id_t obj_id) { uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); /** Add write lock for removal */ pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); pthread_rwlock_wrlock(rwlock_); - bool res = delete_in_bucket_locked(hashtable, hv, obj_id); + bool res = delete_obj_id_in_bucket_locked(hashtable, hv, obj_id); pthread_rwlock_unlock(rwlock_); return res; @@ -237,7 +274,12 @@ bool concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, bool concurrent_chained_hashtable_try_delete(hashtable_t *hashtable, cache_obj_t *cache_obj) { uint64_t hv = get_hash_value_int_64(&cache_obj->obj_id) & hashmask(hashtable->hashpower); - return concurrent_chained_hashtable_delete_obj_id(hashtable, cache_obj->obj_id); + /** Add write lock for removal */ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); + pthread_rwlock_wrlock(rwlock_); + bool res = delete_in_bucket_locked(hashtable, hv, cache_obj); + pthread_rwlock_unlock(rwlock_); + return res; } void concurrent_chained_hashtable_delete(hashtable_t *hashtable, diff --git a/libCacheSim/dataStructure/test/test_concurrent_hashtable.c b/libCacheSim/dataStructure/test/test_concurrent_hashtable.c index 0173219e..da03555b 100644 --- a/libCacheSim/dataStructure/test/test_concurrent_hashtable.c +++ b/libCacheSim/dataStructure/test/test_concurrent_hashtable.c @@ -23,7 +23,7 @@ extern "C" { #include "../hashtable/cChainedHashTable.h" /** Power of number of cache objects to insert. */ -size_t g_power = 18; +size_t g_power = 19; /** Number of cache objects to insert. */ size_t g_numkeys; // 2^18=262144 @@ -32,7 +32,7 @@ size_t g_numkeys; // 2^18=262144 size_t g_thread_num = 1; /** Number of seconds to run read test. */ -size_t g_test_len = 10; // 10s +size_t g_test_len = 20; // 10s /** Type of test. */ size_t g_test_type = 0; // 0: Three tests: Insert, Read, and Delete. Each test is non-overlapping. @@ -63,8 +63,8 @@ struct handle_hashtable_t{ func_hashtable_find_obj_id_t func_find_obj; func_hashtable_insert_obj_t func_insert_obj; func_hashtable_delete_obj_id_t func_delete_obj; - func_create_hashtable_t func_create; - func_free_hashtable_t func_free; + func_create_hashtable_t func_create_hashtable; + func_free_hashtable_t func_free_hashtable; } ht_handle; /** @@ -128,7 +128,7 @@ void *func_read(void* arg){ gettimeofday(&end, NULL); double timeuse = (end.tv_sec - start.tv_sec) + (end.tv_usec - start.tv_usec) / 1000000.0; - INFO("Thread %d read %zu objects in %.1f seconds, throughput is %.2f MQPS. %zu success, %zu fail\n", \ + INFO("Thread %d read %zu objects in %.1f seconds, throughput is %.2f MOPS. %zu success, %zu fail\n", \ thread_id, reads, timeuse, reads / timeuse / 1000000, success_reads, fail_reads); return NULL; @@ -176,7 +176,7 @@ void *func_insert(void* arg){ gettimeofday(&end, NULL); double timeuse = (end.tv_sec - start.tv_sec) + (end.tv_usec - start.tv_usec) / 1000000.0; - INFO("Thread %d inserted %zu objects in %.1f seconds, throughput is %.2f MQPS\n", \ + INFO("Thread %d inserted %zu objects in %.1f seconds, throughput is %.2f MOPS\n", \ thread_id, inserts, timeuse, inserts / timeuse / 1000000); } @@ -219,7 +219,7 @@ void *func_remove(void* arg){ gettimeofday(&end, NULL); double timeuse = (end.tv_sec - start.tv_sec) + (end.tv_usec - start.tv_usec) / 1000000.0; - INFO("Thread %d removed %zu objects in %.1f seconds, throughput is %.2f MQPS. %zu success, %zu fail\n", \ + INFO("Thread %d removed %zu objects in %.1f seconds, throughput is %.2f MOPS. %zu success, %zu fail\n", \ thread_id, removals, timeuse, removals / timeuse / 1000000, success_removal, fail_removal); } @@ -358,16 +358,16 @@ void stress_test() { ht_handle.func_find_obj = concurrent_chained_hashtable_find_obj_id; ht_handle.func_insert_obj = concurrent_chained_hashtable_insert_obj; ht_handle.func_delete_obj = concurrent_chained_hashtable_delete_obj_id; - ht_handle.func_create = create_concurrent_chained_hashtable; - ht_handle.func_free = free_concurrent_chained_hashtable; + ht_handle.func_create_hashtable = create_concurrent_chained_hashtable; + ht_handle.func_free_hashtable = free_concurrent_chained_hashtable; } // If hashtable type is chainedHashTableV2. else if(g_ht_type == 1){ ht_handle.func_find_obj = chained_hashtable_find_obj_id_v2; ht_handle.func_insert_obj = chained_hashtable_insert_obj_v2; ht_handle.func_delete_obj = chained_hashtable_delete_obj_id_v2; - ht_handle.func_create = create_chained_hashtable_v2; - ht_handle.func_free = free_chained_hashtable_v2; + ht_handle.func_create_hashtable = create_chained_hashtable_v2; + ht_handle.func_free_hashtable = free_chained_hashtable_v2; } // If hashtable type is invalid. @@ -376,7 +376,7 @@ void stress_test() { } // Creates a hashtable - hashtable_t *table = ht_handle.func_create(g_power); + hashtable_t *table = ht_handle.func_create_hashtable(g_power); // If the user select the first test. if(g_test_type == 0){ @@ -472,7 +472,7 @@ void stress_test() { ERROR("ERROR; return code from pthread_create() is %d\n", rc); } } - printf("----------Throughput MQPS----------\n"); + printf("----------Throughput MOPS----------\n"); printf("Seconds\tInsert\tRead\tDelete\tTotal\n"); // Prints the throughput every second @@ -501,13 +501,13 @@ void stress_test() { ERROR("Invalid test type: %lu. It should be 0 or 1.\n", g_test_type); } // Frees the hashtable - ht_handle.func_free(table); + ht_handle.func_free_hashtable(table); printf("----------Results----------\n"); printf("Number of inserts:\t%lu\n", num_inserts); printf("Number of reads:\t%lu\n", num_reads); printf("Number of removals:\t%lu\n", num_removals); - printf("Total throughput:\t%.2lf MQPS\n", (num_inserts + num_reads + num_removals)/g_test_len/1000000.0); + printf("Total throughput:\t%.2lf MOPS\n", (num_inserts + num_reads + num_removals)/g_test_len/1000000.0); } From fe287b84f87db47391387223a3bb41f3a3c5ccac Mon Sep 17 00:00:00 2001 From: ChaosD Date: Mon, 27 Nov 2023 11:40:11 +0800 Subject: [PATCH 03/28] add concurrent hashtable --- libCacheSim/cache/cacheObj.c | 4 +- .../dataStructure/hashtable/hashtable.h | 49 ++++++++++--------- libCacheSim/include/config.h | 2 +- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/libCacheSim/cache/cacheObj.c b/libCacheSim/cache/cacheObj.c index 56b6f008..34fe480e 100644 --- a/libCacheSim/cache/cacheObj.c +++ b/libCacheSim/cache/cacheObj.c @@ -12,7 +12,7 @@ * @method verify_cache_obj_fingerprint * @author Chaos * @date 2023-11-22 - * @param cache_obj [handle of cache item] + * @param cache_obj [pointer of cache item] * @return [is valid or not] */ bool verify_cache_obj_fingerprint(const cache_obj_t *cache_obj) { @@ -24,7 +24,7 @@ bool verify_cache_obj_fingerprint(const cache_obj_t *cache_obj) { * @method set_cache_obj_fingerprint * @author Chaos * @date 2023-11-22 - * @param cache_obj [handle of cache item] + * @param cache_obj [pointer of cache item] */ void set_cache_obj_fingerprint(cache_obj_t *cache_obj) { cache_obj->fingerprint = cache_obj->obj_id; diff --git a/libCacheSim/dataStructure/hashtable/hashtable.h b/libCacheSim/dataStructure/hashtable/hashtable.h index 7f7a9877..98f33059 100644 --- a/libCacheSim/dataStructure/hashtable/hashtable.h +++ b/libCacheSim/dataStructure/hashtable/hashtable.h @@ -6,6 +6,7 @@ #ifdef __cplusplus extern "C" { + #endif #include "../../include/config.h" @@ -61,30 +62,30 @@ extern "C" { #define hashtable_add_ptr_to_monitoring(hashtable, ptr) #define HASHTABLE_VER 2 -// #elif HASHTABLE_TYPE == CONCURRENT_CHAINED_HASHTABLE -// #include "cChainedHashTable.h" -// #define create_hashtable(hashpower) create_concurrent_chained_hashtable(hashpower) -// #define hashtable_find(hashtable, req) concurrent_chained_hashtable_find(hashtable, req) -// #define hashtable_find_obj_id(hashtable, obj_id) \ -// concurrent_chained_hashtable_find_obj_id(hashtable, obj_id) -// #define hashtable_find_obj(hashtable, cache_obj) \ -// concurrent_chained_hashtable_find_obj(hashtable, cache_obj) -// #define hashtable_insert(hashtable, req) \ -// concurrent_chained_hashtable_insert(hashtable, req) -// #define hashtable_insert_obj(hashtable, cache_obj) \ -// concurrent_chained_hashtable_insert_obj(hashtable, cache_obj) -// #define hashtable_delete(hashtable, cache_obj) \ -// concurrent_chained_hashtable_delete(hashtable, cache_obj) -// #define hashtable_try_delete(hashtable, cache_obj) \ -// concurrent_chained_hashtable_try_delete(hashtable, cache_obj) -// #define hashtable_delete_obj_id(hashtable, obj_id) \ -// concurrent_chained_hashtable_delete_obj_id(hashtable, obj_id) -// #define hashtable_rand_obj(hashtable) concurrent_chained_hashtable_rand_obj(hashtable) -// #define hashtable_foreach(hashtable, iter_func, user_data) \ -// concurrent_chained_hashtable_foreach(hashtable, iter_func, user_data) -// #define free_hashtable(hashtable) free_concurrent_chained_hashtable(hashtable) -// #define hashtable_add_ptr_to_monitoring(hashtable, ptr) -// #define HASHTABLE_VER 3 +#elif HASHTABLE_TYPE == CONCURRENT_CHAINED_HASHTABLE +#include "cChainedHashTable.h" +#define create_hashtable(hashpower) create_concurrent_chained_hashtable(hashpower) +#define hashtable_find(hashtable, req) concurrent_chained_hashtable_find(hashtable, req) +#define hashtable_find_obj_id(hashtable, obj_id) \ + concurrent_chained_hashtable_find_obj_id(hashtable, obj_id) +#define hashtable_find_obj(hashtable, cache_obj) \ + concurrent_chained_hashtable_find_obj(hashtable, cache_obj) +#define hashtable_insert(hashtable, req) \ + concurrent_chained_hashtable_insert(hashtable, req) +#define hashtable_insert_obj(hashtable, cache_obj) \ + concurrent_chained_hashtable_insert_obj(hashtable, cache_obj) +#define hashtable_delete(hashtable, cache_obj) \ + concurrent_chained_hashtable_delete(hashtable, cache_obj) +#define hashtable_try_delete(hashtable, cache_obj) \ + concurrent_chained_hashtable_try_delete(hashtable, cache_obj) +#define hashtable_delete_obj_id(hashtable, obj_id) \ + concurrent_chained_hashtable_delete_obj_id(hashtable, obj_id) +#define hashtable_rand_obj(hashtable) concurrent_chained_hashtable_rand_obj(hashtable) +#define hashtable_foreach(hashtable, iter_func, user_data) \ + concurrent_chained_hashtable_foreach(hashtable, iter_func, user_data) +#define free_hashtable(hashtable) free_concurrent_chained_hashtable(hashtable) +#define hashtable_add_ptr_to_monitoring(hashtable, ptr) +#define HASHTABLE_VER 3 #elif HASHTABLE_TYPE == CUCKCOO_HASHTABLE #include "cuckooHashTable.h" diff --git a/libCacheSim/include/config.h b/libCacheSim/include/config.h index a9db61f4..6e89e6f9 100644 --- a/libCacheSim/include/config.h +++ b/libCacheSim/include/config.h @@ -32,7 +32,7 @@ extern "C" { #endif #ifndef HASHTABLE_TYPE -#define HASHTABLE_TYPE CHAINED_HASHTABLEV2 +#define HASHTABLE_TYPE CONCURRENT_CHAINED_HASHTABLE #endif #ifndef HASH_POWER_DEFAULT From ae0de5c4221160e1a02d9729a6539272871863cd Mon Sep 17 00:00:00 2001 From: Chaos Dong <948914729@qq.com> Date: Mon, 27 Nov 2023 16:24:09 +0800 Subject: [PATCH 04/28] Delete sftp-config.json Delete a personal file --- sftp-config.json | 45 --------------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 sftp-config.json diff --git a/sftp-config.json b/sftp-config.json deleted file mode 100644 index 0fce488b..00000000 --- a/sftp-config.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - // The tab key will cycle through the settings when first created - // Visit https://codexns.io/products/sftp_for_subime/settings for help - - // sftp, ftp or ftps - "type": "sftp", - - "save_before_upload": true, - "upload_on_save": true, - "sync_down_on_open": false, - "sync_skip_deletes": false, - "sync_same_age": true, - "confirm_downloads": false, - "confirm_sync": true, - "confirm_overwrite_newer": false, - - "host": "localhost", - "user": "chaos", - "password": "hdx12280222", - "port": "1002", - - "remote_path": "/home/chaos/libCacheSim", - "ignore_regexes": [ - "\\.sublime-(project|workspace)", "sftp-config(-alt\\d?)?\\.json", - "sftp-settings\\.json", "/venv/", "\\.svn/", "\\.hg/", "\\.git/", - "\\.bzr", "_darcs", "CVS", "\\.DS_Store", "Thumbs\\.db", "desktop\\.ini" - ], - //"file_permissions": "664", - //"dir_permissions": "775", - - //"extra_list_connections": 0, - - "connect_timeout": 30, - //"keepalive": 120, - //"ftp_passive_mode": true, - //"ftp_obey_passive_host": false, - //"ssh_key_file": "~/.ssh/id_rsa", - //"sftp_flags": ["-F", "/path/to/ssh_config"], - - //"preserve_modification_times": false, - //"remote_time_offset_in_hours": 0, - //"remote_encoding": "utf-8", - //"remote_locale": "C", - //"allow_config_upload": false, -} From 9b910509031b0a0ca12774cec739c89f1b3b9ea1 Mon Sep 17 00:00:00 2001 From: ChaosD Date: Tue, 28 Nov 2023 13:41:01 +0800 Subject: [PATCH 05/28] Fix function chained_hashtable_delete_obj_id_v2 --- .gitignore | 2 + .../hashtable/chainedHashTableV2.c | 55 ++++++++++++++----- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 4deb53b8..dc00d77f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ example/cacheSimulatorC/cmake-build-debug .vscode/* *.log fig/ +# Chaos +sftp-config.json diff --git a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c index 8a5eb960..5cc736bd 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c +++ b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c @@ -248,26 +248,51 @@ bool chained_hashtable_try_delete_v2(hashtable_t *hashtable, return false; } -void chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, +/** + * This function is used to delete an object from the hash table by object id. + * - if the object is in the hash table + * remove it + * return true. + * - if the object is not in the hash table + * return false. + * + * @method chained_hashtable_delete_obj_id_v2 + * @date 2023-11-28 + * @param hashtable [Handle to the hashtable] + * @param obj_id [The object id to remove] + * @return [true or false] + */ +bool chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, const obj_id_t obj_id) { - hashtable->n_obj -= 1; - uint64_t hv = get_hash_value_int_64(obj_id) & hashmask(hashtable->hashpower); - cache_obj_t *cache_obj = hashtable->ptr_table[hv]; - if (cache_obj != NULL && cache_obj->obj_id == obj_id) { - hashtable->ptr_table[hv] = cache_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(cache_obj); - return; - } + uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); + cache_obj_t *cur_obj = hashtable->ptr_table[hv]; + // the hash bucket is empty + if(cur_obj == NULL) return false; - cache_obj = cache_obj->hash_next; - while (cache_obj != NULL && cache_obj->obj_id != obj_id) { - cache_obj = cache_obj->hash_next; + // the object to remove is the first object in the hash bucket + if (cur_obj->obj_id == obj_id) { + hashtable->ptr_table[hv] = cur_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(cur_obj); + hashtable->n_obj -= 1; + return true; } - if (cache_obj != NULL) { - cache_obj->hash_next = cache_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(cache_obj); + cache_obj_t *prev_obj; + + do { + prev_obj = cur_obj; + cur_obj = cur_obj->hash_next; + } while(cur_obj != NULL && cur_obj->obj_id != obj_id); + + // the object to remove is in the hash bucket + if (cur_obj != NULL) { + prev_obj->hash_next = cur_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(cur_obj); + hashtable->n_obj -= 1; + return true; } + // the object to remove is not in the hash table + return false; } cache_obj_t *chained_hashtable_rand_obj_v2(const hashtable_t *hashtable) { From 64f9eeedd20e2b2cd71f8f4a86ef2bcc2de6bad1 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 28 Nov 2023 20:09:15 -0500 Subject: [PATCH 06/28] update README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e9834377..430e5b90 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # libCacheSim - building and running cache simulations -[![build](https://github.com/1a1a11a/libCacheSimPrv/actions/workflows/build.yml/badge.svg)](https://github.com/1a1a11a/libCacheSimPrv/actions/workflows/build.yml) +[![build](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml/badge.svg)](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml) **The stable development of libCacheSim has moved to [https://github.com/cacheMon/libCacheSim](https://github.com/cacheMon/libCacheSim)**. @@ -65,8 +65,8 @@ Please see [install.md](/doc/install.md) for how to install the dependencies. ### Build libCacheSim cmake recommends **out-of-source build**, so we do it in a new directory: ``` -git clone https://github.com/1a1a11a/libCacheSimPrv -pushd libCachesimPrv; +git clone https://github.com/1a1a11a/libCacheSim +pushd libCachesim; mkdir _build && cd _build; cmake .. && make -j; [sudo] make install; From 715be58a40ddba33ed6bf3ea932f702c4284fad2 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 15 Dec 2023 14:22:01 -0500 Subject: [PATCH 07/28] update README --- README.md | 57 ++++++++++++++++++++++++++++++--------------------- references.md | 22 +++++++++++++++++--- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 430e5b90..a10ba1ff 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,10 @@ [![build](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml/badge.svg)](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml) -**The stable development of libCacheSim has moved to [https://github.com/cacheMon/libCacheSim](https://github.com/cacheMon/libCacheSim)**. - ## News -* **2023 June**: **QD-LP** is available now, see [our paper](https://dl.acm.org/doi/10.1145/3593856.3595887) for details. -* **2023 Oct**: **S3-FIFO** and **Sieve** are generally available! These are simple algorithms that are very effective in reducing cache misses. Try them out in libCacheSim and your production! +* **2023 June**: **QDLP** is available now, see [our paper](https://dl.acm.org/doi/10.1145/3593856.3595887) for details. +* **2023 Oct**: **[S3-FIFO](https://dl.acm.org/doi/10.1145/3600006.3613147)** and **SIEVE** are available! These are very simple algorithms that are very effective in reducing cache misses. Try them out in libCacheSim and your production! --- ## What is libCacheSim @@ -23,8 +21,8 @@ * **State-of-the-art algorithms** - eviction algorithms, admission algorithms, sampling techniques, approximate miss ratio computation, see [here](/doc/quickstart_cachesim.md). * Parallelism out-of-the-box - uses the many CPU cores to speed up trace analysis and cache simulations. * **The ONLY feature-rich trace analyzer** - all types of trace analysis you need, see [here](/doc/quickstart_traceAnalyzer.md). -* **Simple API** - easy to build cache clusters, multi-layer caching, etc, see [here](/doc/API.md). -* **Extensible** - easy to support new trace types or eviction algorithms, see [here](/doc/advanced_lib_extend.md). +* **Simple API** - easy to build cache clusters, multi-layer caching, etc.; see [here](/doc/API.md). +* **Extensible** - easy to support new trace types or eviction algorithms; see [here](/doc/advanced_lib_extend.md). --- ## Supported algorithms @@ -59,7 +57,7 @@ If this does not work, please ### Install dependency libCacheSim uses [cmake](https://cmake.org/) build system and has a few dependencies: [glib](https://developer.gnome.org/glib/), [tcmalloc](https://github.com/google/tcmalloc), [zstd](https://github.com/facebook/zstd). -Please see [install.md](/doc/install.md) for how to install the dependencies. +Please see [install.md](/doc/install.md) for instructions on how to install the dependencies. ### Build libCacheSim @@ -88,16 +86,16 @@ use `./bin/cachesim --help` to get more information. Run the example traces with LRU eviction algorithm and 1GB cache size. ```bash -# Note that no space between the cache size and the unit, unit is not case sensitive +# Note that no space between the cache size and the unit, and the unit is not case-sensitive ./bin/cachesim ../data/trace.vscsi vscsi lru 1gb ``` #### Run multiple cache simulations with different cache sizes ```bash -# Note that no space between the cache sizes +# Note that there is no space between the cache sizes ./bin/cachesim ../data/trace.vscsi vscsi lru 1mb,16mb,256mb,8gb -# besides absolute cache size, you can also use fraction of working set size +# Besides absolute cache size, you can also use a fraction of the working set size ./bin/cachesim ../data/trace.vscsi vscsi lru 0.001,0.01,0.1,0.2 # besides using byte as the unit, you can also treat all objects having the same size, and the size is the number of objects @@ -140,7 +138,7 @@ See [trace analysis](/doc/quickstart_traceAnalyzer.md) for more details. ### Using libCacheSim as a library libCacheSim can be used as a library for building cache simulators. -For example, you can build a cache cluster with consistent hashing, or a multi-layer cache simulator. +For example, you can build a cache cluster with consistent hashing or a multi-layer cache simulator. Here is a simplified example showing the basic APIs. ```c @@ -149,7 +147,7 @@ Here is a simplified example showing the basic APIs. /* open trace, see quickstart_lib.md for opening csv and binary trace */ reader_t *reader = open_trace("../data/trace.vscsi", VSCSI_TRACE, NULL); -/* craete a container for reading from trace */ +/* create a container for reading from trace */ request_t *req = new_request(); /* create a LRU cache */ @@ -175,7 +173,7 @@ free_request(req); cache->cache_free(cache); ``` -save this to `test.c` and compile with +save this to `test.c` and compile it with ``` bash gcc test.c $(pkg-config --cflags --libs libCacheSim glib-2.0) -o test.out ``` @@ -201,14 +199,14 @@ Please join the Google group https://groups.google.com/g/libcachesim and ask que --- ### Contributions We gladly welcome pull requests. -Before making any changes, we recommend opening an issue and discussing your proposed changes. -This will let us give you advice on the proposed changes. If the changes are minor, then feel free to make them without discussion. +Before making any large changes, we recommend opening an issue and discussing your proposed changes. +If the changes are minor, then feel free to make them without discussion. This project adheres to Google's coding style. By participating, you are expected to uphold this code. --- ### Reference ``` -@inproceedings{yang20-workload, +@inproceedings{yang2020-workload, author = {Juncheng Yang and Yao Yue and K. V. Rashmi}, title = {A large scale analysis of hundreds of in-memory cache clusters at Twitter}, booktitle = {14th USENIX Symposium on Operating Systems Design and Implementation (OSDI 20)}, @@ -217,17 +215,32 @@ This project adheres to Google's coding style. By participating, you are expecte pages = {191--208}, url = {https://www.usenix.org/conference/osdi20/presentation/yang}, publisher = {USENIX Association}, - month = nov, } @inproceedings{yang2023-s3fifo, - title={FIFO queues are all you need for cache eviction}, - author={Yang, Juncheng and Zhang, Yazhuo and Qiu, Ziyue and Yue, Yao and Rashmi, K.V.}, - booktitle={Symposium on Operating Systems Principles (SOSP'23)}, + title = {FIFO Queues Are All You Need for Cache Eviction}, + author = {Juncheng Yang and Yazhuo Zhang and Ziyue Qiu and Yao Yue and K.V. Rashmi}, + isbn = {9798400702297}, + publisher = {Association for Computing Machinery}, + booktitle = {Symposium on Operating Systems Principles (SOSP'23)}, + pages = {130–149}, + numpages = {20}, year={2023} } + +@inproceedings{yang2023-qdlp, + author = {Juncheng Yang and Ziyue Qiu and Yazhuo Zhang and Yao Yue and K.V. Rashmi}, + title = {FIFO Can Be Better than LRU: The Power of Lazy Promotion and Quick Demotion}, + year = {2023}, + isbn = {9798400701955}, + publisher = {Association for Computing Machinery}, + doi = {10.1145/3593856.3595887}, + booktitle = {Proceedings of the 19th Workshop on Hot Topics in Operating Systems (HotOS23)}, + pages = {70–79}, + numpages = {10}, +} ``` -If you used libCacheSim in your research, please cite the above papers. And we welcome you to send us a link to your paper and add reference to (references.md)[references.md]. +If you used libCacheSim in your research, please cite the above papers. And we welcome you to send us a link to your paper and add a reference to [references.md](references.md). --- @@ -238,5 +251,3 @@ See [LICENSE](LICENSE) for details. ### Related * [PyMimircache](https://github.com/1a1a11a/PyMimircache): a python based cache trace analysis platform, now deprecated --- - - diff --git a/references.md b/references.md index e1982e25..eb05ea13 100644 --- a/references.md +++ b/references.md @@ -14,12 +14,28 @@ } @inproceedings{yang2023-s3fifo, - title={FIFO queues are all you need for cache eviction}, - author={Yang, Juncheng and Zhang, Yazhuo and Qiu, Ziyue and Yue, Yao and Rashmi, K.V.}, - booktitle={Symposium on Operating Systems Principles (SOSP'23)}, + title = {FIFO Queues Are All You Need for Cache Eviction}, + author = {Juncheng Yang and Yazhuo Zhang and Ziyue Qiu and Yao Yue and K.V. Rashmi}, + isbn = {9798400702297}, + publisher = {Association for Computing Machinery}, + booktitle = {Symposium on Operating Systems Principles (SOSP'23)}, + pages = {130–149}, + numpages = {20}, year={2023} } +@inproceedings{yang2023-qdlp, + author = {Juncheng Yang and Ziyue Qiu and Yazhuo Zhang and Yao Yue and K.V. Rashmi}, + title = {FIFO Can Be Better than LRU: The Power of Lazy Promotion and Quick Demotion}, + year = {2023}, + isbn = {9798400701955}, + publisher = {Association for Computing Machinery}, + doi = {10.1145/3593856.3595887}, + booktitle = {Proceedings of the 19th Workshop on Hot Topics in Operating Systems (HotOS23)}, + pages = {70–79}, + numpages = {10}, +} + @inproceedings{yazhuo2024-sieve, title={FIFO queues are all you need for cache eviction}, author={Yang, Juncheng and Zhang, Yazhuo and Qiu, Ziyue and Yue, Yao and Rashmi, K.V.}, From 26dffad24259e9c9b390b069a613d09431ce1dda Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 17 Dec 2023 15:30:15 -0500 Subject: [PATCH 08/28] remove self-hosted testbed --- .github/workflows/build.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ddc93d9b..155a78d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,17 +44,17 @@ jobs: working-directory: ${{github.workspace}}/build run: ctest -C ${{env.BUILD_TYPE}} - selfhosted: - runs-on: self-hosted - steps: - - uses: actions/checkout@v2 - - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_GLCACHE=on -DENABLE_LRB=on - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Test - working-directory: ${{github.workspace}}/build - run: ctest -C ${{env.BUILD_TYPE}} -j4 + # selfhosted: + # runs-on: self-hosted + # steps: + # - uses: actions/checkout@v2 + # - name: Configure CMake + # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_GLCACHE=on -DENABLE_LRB=on + # - name: Build + # run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + # - name: Test + # working-directory: ${{github.workspace}}/build + # run: ctest -C ${{env.BUILD_TYPE}} -j4 From 6a89a064caaea131772c57cc227e65f7e315d8d5 Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:33:21 -0500 Subject: [PATCH 09/28] add dataset and change names (#43) add TOC --- README.md | 84 ++++++++++++++++-- data/{trace.csv => cloudphyicsIO.csv} | 0 ...al.bin => cloudphyicsIO.oracleGeneral.bin} | Bin data/{trace.txt => cloudphyicsIO.txt} | 0 data/{trace.vscsi => cloudphyicsIO.vscsi} | Bin 5 files changed, 79 insertions(+), 5 deletions(-) rename data/{trace.csv => cloudphyicsIO.csv} (100%) rename data/{trace.oracleGeneral.bin => cloudphyicsIO.oracleGeneral.bin} (100%) rename data/{trace.txt => cloudphyicsIO.txt} (100%) rename data/{trace.vscsi => cloudphyicsIO.vscsi} (100%) diff --git a/README.md b/README.md index a10ba1ff..4d4cf1cd 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,33 @@ [![build](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml/badge.svg)](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml) + +* [What is libCacheSim](#what-is-libcachesim) +* [libCacheSim features ](#libcachesim-features) +* [Supported algorithms](#supported-algorithms) +* [Build and Install libCacheSim](#build-and-install-libcachesim) + + [One-line install](#one-line-install) + + [Install dependency](#install-dependency) + + [Build libCacheSim](#build-libcachesim) +* [Usage](#usage) + + [cachesim (a high-performance cache simulator)](#cachesim-a-high-performance-cache-simulator) + + [Trace analysis](#trace-analysis) + + [Using libCacheSim as a library ](#using-libcachesim-as-a-library) + + [Extending libCacheSim (new algorithms and trace types)](#extending-libcachesim-new-algorithms-and-trace-types) +* [Open source cache traces](#open-source-cache-traces) +* [Questions? ](#questions) +* [Reference](#reference) +* [License](#license) + + + + ## News * **2023 June**: **QDLP** is available now, see [our paper](https://dl.acm.org/doi/10.1145/3593856.3595887) for details. * **2023 Oct**: **[S3-FIFO](https://dl.acm.org/doi/10.1145/3600006.3613147)** and **SIEVE** are available! These are very simple algorithms that are very effective in reducing cache misses. Try them out in libCacheSim and your production! --- + ## What is libCacheSim * a high-performance **cache simulator** for running cache simulations. * a high-performance and versatile trace analyzer for **analyzing different cache traces**. @@ -15,6 +37,7 @@ --- + ## libCacheSim features * **High performance** - over 20M requests/sec for a realistic trace replay. * **High memory efficiency** - predictable and small memory footprint. @@ -25,6 +48,7 @@ * **Extensible** - easy to support new trace types or eviction algorithms; see [here](/doc/advanced_lib_extend.md). --- + ## Supported algorithms cachesim supports the following algorithms: * [FIFO](/libCacheSim/cache/eviction/FIFO.c), [LRU](/libCacheSim/cache/eviction/LRU.c), [Clock](/libCacheSim/cache/eviction/Clock.c), [SLRU](/libCacheSim/cache/eviction/SLRU.c) @@ -45,7 +69,9 @@ cachesim supports the following algorithms: --- + ## Build and Install libCacheSim + ### One-line install We provide some scripts for quick installation of libCacheSim. ```bash @@ -55,11 +81,13 @@ If this does not work, please 1. let us know what system you are using and what error you get 2. read the following sections for self-installation. + ### Install dependency libCacheSim uses [cmake](https://cmake.org/) build system and has a few dependencies: [glib](https://developer.gnome.org/glib/), [tcmalloc](https://github.com/google/tcmalloc), [zstd](https://github.com/facebook/zstd). Please see [install.md](/doc/install.md) for instructions on how to install the dependencies. + ### Build libCacheSim cmake recommends **out-of-source build**, so we do it in a new directory: ``` @@ -72,9 +100,12 @@ popd; ``` --- + ## Usage + ### cachesim (a high-performance cache simulator) After building and installing libCacheSim, `cachesim` should be in the `_build/bin/` directory. + #### basic usage ``` ./bin/cachesim trace_path trace_type eviction_algo cache_size [OPTION...] @@ -82,6 +113,7 @@ After building and installing libCacheSim, `cachesim` should be in the `_build/b use `./bin/cachesim --help` to get more information. + #### Run a single cache simulation Run the example traces with LRU eviction algorithm and 1GB cache size. @@ -90,6 +122,7 @@ Run the example traces with LRU eviction algorithm and 1GB cache size. ./bin/cachesim ../data/trace.vscsi vscsi lru 1gb ``` + #### Run multiple cache simulations with different cache sizes ```bash # Note that there is no space between the cache sizes @@ -111,6 +144,7 @@ Run the example traces with LRU eviction algorithm and 1GB cache size. See [quick start cachesim](/doc/quickstart_cachesim.md) for more usages. + #### Plot miss ratio curve You can plot miss ratios of different algorithms and sizes, and plot the miss ratios over time. @@ -125,6 +159,7 @@ python3 plot_mrc_time.py --tracepath ../data/twitter_cluster52.csv --trace-forma --- + ### Trace analysis libCacheSim also has a trace analyzer that provides a lot of useful information about the trace. And it is very fast, designed to work with billions of requests. @@ -136,6 +171,7 @@ See [trace analysis](/doc/quickstart_traceAnalyzer.md) for more details. --- + ### Using libCacheSim as a library libCacheSim can be used as a library for building cache simulators. For example, you can build a cache cluster with consistent hashing or a multi-layer cache simulator. @@ -183,6 +219,7 @@ See [here](/doc/advanced_lib.md) for more details, and see [example folder](/exa --- + ### Extending libCacheSim (new algorithms and trace types) libCacheSim supports *txt*, *csv*, and *binary* traces. We prefer binary traces because it allows libCacheSim to run faster, and the traces are more compact. @@ -192,19 +229,53 @@ If you need to add a new trace type or a new algorithm, please see [here](/doc/a --- -### Questions? + +## Open source cache traces +In the [repo](/data/), there are sample (one from cloudphysics and one from twitter) traces in different formats (csv, txt, vscsi, and oracleGeneral). Note that the provided traces are **very small** samples and __should not be used for evaluating different algorithms' miss ratios__. The full traces can be found either with the original release or the processed oracleGeneral format. + +Note that the oracleGeneral traces are compressed with [zstd](https://github.com/facebook/zstd) and have the following format: + +``` +struct { + uint32_t timestamp; + uint64_t obj_id; + uint32_t obj_size; + int64_t next_access_vtime; // -1 if no next access +} +``` +The compressed traces can be used with libCacheSim without decompression. And libCacheSim provides a `tracePrint` tool to print the trace in human-readable format. + + +| Dataset | Year | Type | Original release | OracleGeneral format | +|---------------|------|:---------:|:-----------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------:| +| Tencent Photo | 2018 | object | [link](http://iotta.snia.org/traces/parallel?only=27476) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/tencentPhoto/) | +| WikiCDN | 2019 | object | [link](https://wikitech.wikimedia.org/wiki/Analytics/Data_Lake/Traffic/Caching) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/wiki/) | +| Tencent CBS | 2020 | block | [link](http://iotta.snia.org/traces/parallel?only=27917) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/tencentBlock/) | +| Alibaba CBS | 2020 | block | [link](https://github.com/alibaba/block-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/alibabaBlock/) | +| Twitter | 2020 | key-value | [link](https://github.com/twitter/cache-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/twitter/) | +| MetaKV | 2022 | key-value | [link](https://cachelib.org/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval/#list-of-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/metaKV/) | +| MetaCDN | 2023 | object | [link](https://cachelib.org/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval/#list-of-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/metaCDN/) | + +Among the large number of traces, I recommend using the newer traces from Twitter (cluster52), Wiki, and Meta. + + +--- + +## Questions? Please join the Google group https://groups.google.com/g/libcachesim and ask questions. --- -### Contributions + +## Contributions We gladly welcome pull requests. Before making any large changes, we recommend opening an issue and discussing your proposed changes. If the changes are minor, then feel free to make them without discussion. This project adheres to Google's coding style. By participating, you are expected to uphold this code. --- -### Reference + +## Reference ``` @inproceedings{yang2020-workload, author = {Juncheng Yang and Yao Yue and K. V. Rashmi}, @@ -245,9 +316,12 @@ If you used libCacheSim in your research, please cite the above papers. And we w --- -### License + +## License See [LICENSE](LICENSE) for details. -### Related + +## Related * [PyMimircache](https://github.com/1a1a11a/PyMimircache): a python based cache trace analysis platform, now deprecated --- + diff --git a/data/trace.csv b/data/cloudphyicsIO.csv similarity index 100% rename from data/trace.csv rename to data/cloudphyicsIO.csv diff --git a/data/trace.oracleGeneral.bin b/data/cloudphyicsIO.oracleGeneral.bin similarity index 100% rename from data/trace.oracleGeneral.bin rename to data/cloudphyicsIO.oracleGeneral.bin diff --git a/data/trace.txt b/data/cloudphyicsIO.txt similarity index 100% rename from data/trace.txt rename to data/cloudphyicsIO.txt diff --git a/data/trace.vscsi b/data/cloudphyicsIO.vscsi similarity index 100% rename from data/trace.vscsi rename to data/cloudphyicsIO.vscsi From db6bd314c9e16aa096f7cf4096851d79f6ec5fd2 Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:35:00 -0500 Subject: [PATCH 10/28] allow the algorithm name to be S3-FIFO (#44) --- libCacheSim/bin/cachesim/cache_init.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libCacheSim/bin/cachesim/cache_init.h b/libCacheSim/bin/cachesim/cache_init.h index b4c2668b..44822e93 100644 --- a/libCacheSim/bin/cachesim/cache_init.h +++ b/libCacheSim/bin/cachesim/cache_init.h @@ -121,7 +121,7 @@ static inline cache_t *create_cache(const char *trace_path, cache = Sieve_Belady_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "s3lru") == 0) { cache = S3LRU_init(cc_params, eviction_params); - } else if (strcasecmp(eviction_algo, "s3fifo") == 0) { + } else if (strcasecmp(eviction_algo, "s3fifo") == 0 || strcasecmp(eviction_algo, "s3-fifo") == 0) { cache = S3FIFO_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "s3fifod") == 0) { cache = S3FIFOd_init(cc_params, eviction_params); From 290101ddf7939fd00911cdac53337a54398fa99a Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:51:17 -0500 Subject: [PATCH 11/28] update trace name and fix tests (#45) --- data/{cloudphyicsIO.csv => cloudPhysicsIO.csv} | 0 ...ral.bin => cloudPhysicsIO.oracleGeneral.bin} | Bin data/{cloudphyicsIO.txt => cloudPhysicsIO.txt} | 0 ...cloudphyicsIO.vscsi => cloudPhysicsIO.vscsi} | Bin scripts/install_dependency.sh | 4 ++-- test/common.h | 16 ++++++++-------- 6 files changed, 10 insertions(+), 10 deletions(-) rename data/{cloudphyicsIO.csv => cloudPhysicsIO.csv} (100%) rename data/{cloudphyicsIO.oracleGeneral.bin => cloudPhysicsIO.oracleGeneral.bin} (100%) rename data/{cloudphyicsIO.txt => cloudPhysicsIO.txt} (100%) rename data/{cloudphyicsIO.vscsi => cloudPhysicsIO.vscsi} (100%) diff --git a/data/cloudphyicsIO.csv b/data/cloudPhysicsIO.csv similarity index 100% rename from data/cloudphyicsIO.csv rename to data/cloudPhysicsIO.csv diff --git a/data/cloudphyicsIO.oracleGeneral.bin b/data/cloudPhysicsIO.oracleGeneral.bin similarity index 100% rename from data/cloudphyicsIO.oracleGeneral.bin rename to data/cloudPhysicsIO.oracleGeneral.bin diff --git a/data/cloudphyicsIO.txt b/data/cloudPhysicsIO.txt similarity index 100% rename from data/cloudphyicsIO.txt rename to data/cloudPhysicsIO.txt diff --git a/data/cloudphyicsIO.vscsi b/data/cloudPhysicsIO.vscsi similarity index 100% rename from data/cloudphyicsIO.vscsi rename to data/cloudPhysicsIO.vscsi diff --git a/scripts/install_dependency.sh b/scripts/install_dependency.sh index 653c61ba..611aece0 100644 --- a/scripts/install_dependency.sh +++ b/scripts/install_dependency.sh @@ -55,7 +55,6 @@ setup_zstd() { sudo make install } - CURR_DIR=$(pwd) if [ -n "$(uname -a | grep Ubuntu)" ]; then @@ -66,10 +65,11 @@ else setup_centos fi +setup_zstd + if [[ ! $GITHUB_ACTIONS == "true" ]]; then setup_xgboost setup_lightgbm fi -setup_zstd cd $CURR_DIR diff --git a/test/common.h b/test/common.h index abb0bbed..9f78522b 100644 --- a/test/common.h +++ b/test/common.h @@ -60,7 +60,7 @@ static void _detect_data_path(char *data_path, char *data_name) { static reader_t *setup_oracleGeneralBin_reader(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.oracleGeneral.bin"); + _detect_data_path(data_path, "cloudPhysicsIO.oracleGeneral.bin"); reader_t *reader_oracle = setup_reader(data_path, ORACLE_GENERAL_TRACE, NULL); return reader_oracle; } @@ -86,7 +86,7 @@ static reader_t *setup_vscsi_reader_with_ignored_obj_size(void) { char data_path[1024]; reader_init_param_t *init_params = g_new0(reader_init_param_t, 1); init_params->ignore_obj_size = true; - _detect_data_path(data_path, "trace.vscsi"); + _detect_data_path(data_path, "cloudPhysicsIO.vscsi"); reader_t *reader_vscsi = setup_reader(data_path, VSCSI_TRACE, init_params); g_free(init_params); return reader_vscsi; @@ -94,14 +94,14 @@ static reader_t *setup_vscsi_reader_with_ignored_obj_size(void) { static reader_t *setup_vscsi_reader(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.vscsi"); + _detect_data_path(data_path, "cloudPhysicsIO.vscsi"); reader_t *reader_vscsi = setup_reader(data_path, VSCSI_TRACE, NULL); return reader_vscsi; } static reader_t *setup_binary_reader(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.vscsi"); + _detect_data_path(data_path, "cloudPhysicsIO.vscsi"); reader_init_param_t *init_params_bin = g_new0(reader_init_param_t, 1); init_params_bin->binary_fmt_str = "obj_size_field = 2; @@ -115,7 +115,7 @@ static reader_t *setup_binary_reader(void) { static reader_t *setup_csv_reader_obj_str(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.csv"); + _detect_data_path(data_path, "cloudPhysicsIO.csv"); reader_init_param_t *init_params_csv = g_new0(reader_init_param_t, 1); init_params_csv->delimiter = ','; init_params_csv->time_field = 2; @@ -130,7 +130,7 @@ static reader_t *setup_csv_reader_obj_str(void) { static reader_t *setup_csv_reader_obj_num(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.csv"); + _detect_data_path(data_path, "cloudPhysicsIO.csv"); reader_init_param_t *init_params_csv = g_new0(reader_init_param_t, 1); init_params_csv->delimiter = ','; init_params_csv->time_field = 2; @@ -145,14 +145,14 @@ static reader_t *setup_csv_reader_obj_num(void) { static reader_t *setup_plaintxt_reader_num(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.txt"); + _detect_data_path(data_path, "cloudPhysicsIO.txt"); reader_init_param_t init_params = {.obj_id_is_num = true}; return setup_reader(data_path, PLAIN_TXT_TRACE, &init_params); } static reader_t *setup_plaintxt_reader_str(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.txt"); + _detect_data_path(data_path, "cloudPhysicsIO.txt"); reader_init_param_t init_params = {.obj_id_is_num = false}; return setup_reader(data_path, PLAIN_TXT_TRACE, &init_params); } From 4d47e2074d6f570d1723ea0303fdd01e37c2ab1d Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Wed, 3 Jan 2024 23:24:45 -0500 Subject: [PATCH 12/28] minor updates with several bug fixes (#46) 1. fix warnings 2. fix turning off zstd not working 3. fix bugs at several places 4. it is still challenging to build on macos --- CMakeLists.txt | 34 ++- libCacheSim/bin/cachesim/main.c | 2 +- libCacheSim/cache/admission/adaptsize.cpp | 2 +- libCacheSim/cache/admission/size.c | 2 +- libCacheSim/cache/cache.c | 7 +- libCacheSim/cache/eviction/ARC.c | 14 +- libCacheSim/cache/eviction/ARCv0.c | 13 +- libCacheSim/cache/eviction/Cacheus.c | 5 +- libCacheSim/cache/eviction/Clock.c | 8 +- libCacheSim/cache/eviction/FIFO_Merge.c | 4 +- libCacheSim/cache/eviction/FIFO_Reinsertion.c | 3 +- libCacheSim/cache/eviction/LFU.c | 1 - libCacheSim/cache/eviction/LFUDA.c | 1 - libCacheSim/cache/eviction/LHD/CMakeLists.txt | 9 +- libCacheSim/cache/eviction/LHD/candidate.hpp | 5 +- libCacheSim/cache/eviction/LIRS.c | 288 +++++++++--------- libCacheSim/cache/eviction/LRU.c | 9 +- libCacheSim/cache/eviction/LeCaR.c | 4 +- libCacheSim/cache/eviction/LeCaRv0.c | 1 + libCacheSim/cache/eviction/QDLP.c | 1 - libCacheSim/cache/eviction/S3FIFO.c | 12 +- libCacheSim/cache/eviction/S3FIFOd.c | 8 +- libCacheSim/cache/eviction/SLRU.c | 3 +- libCacheSim/cache/eviction/SLRUv0.c | 3 +- libCacheSim/cache/eviction/SR_LRU.c | 2 +- libCacheSim/cache/eviction/Sieve.c | 4 - libCacheSim/cache/eviction/TwoQ.c | 1 - libCacheSim/cache/eviction/WTinyLFU.c | 2 - .../cache/eviction/belady/FIFO_Belady.c | 4 +- .../cache/eviction/belady/LRU_Belady.c | 2 +- .../cache/eviction/cpp/abstractRank.hpp | 6 +- libCacheSim/cache/eviction/fifo/LP_ARC.c | 8 +- libCacheSim/cache/eviction/fifo/LP_SFIFO.c | 3 +- libCacheSim/cache/eviction/fifo/LP_TwoQ.c | 1 - libCacheSim/cache/eviction/fifo/SFIFO.c | 3 +- libCacheSim/cache/eviction/fifo/SFIFOv0.c | 9 +- libCacheSim/cache/eviction/other/S3LRU.c | 16 +- libCacheSim/cache/eviction/other/flashProb.c | 2 - libCacheSim/cache/eviction/priv/MClock.c | 6 +- libCacheSim/cache/eviction/priv/MyClock.c | 2 +- libCacheSim/cache/eviction/priv/S3FIFOdv2.c | 46 +-- libCacheSim/cache/eviction/priv/myMQv1.c | 19 +- libCacheSim/cache/prefetch/Mithril.c | 38 +-- libCacheSim/dataStructure/bloom.c | 67 ++-- libCacheSim/dataStructure/bloom.h | 2 +- libCacheSim/dataStructure/hash/CMakeLists.txt | 2 +- .../hashtable/chainedHashTableV2.c | 12 +- .../dataStructure/minimalIncrementCBF.c | 3 +- .../dataStructure/minimalIncrementCBF.h | 2 +- libCacheSim/include/libCacheSim/cache.h | 7 +- libCacheSim/include/libCacheSim/reader.h | 22 +- libCacheSim/include/libCacheSim/request.h | 10 +- libCacheSim/include/libCacheSim/sampling.h | 2 +- libCacheSim/profiler/dist.c | 17 +- libCacheSim/profiler/simulator.c | 4 +- libCacheSim/traceAnalyzer/analyzer.cpp | 2 +- libCacheSim/traceReader/CMakeLists.txt | 4 +- .../customizedReader/binaryUtils.h | 12 +- .../traceReader/generalReader/binary.c | 4 +- libCacheSim/traceReader/generalReader/csv.c | 7 +- libCacheSim/traceReader/generalReader/lcs.c | 8 +- libCacheSim/traceReader/reader.c | 4 +- libCacheSim/traceReader/sampling/spatial.c | 2 +- libCacheSim/traceReader/sampling/temporal.c | 2 +- libCacheSim/utils/include/mymath.h | 2 +- libCacheSim/utils/include/mysys.h | 2 +- libCacheSim/utils/mysys.c | 6 +- scripts/install_dependency.sh | 4 +- 68 files changed, 400 insertions(+), 422 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d802d6a0..cf8cf395 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.2) +cmake_minimum_required(VERSION 3.12) project(libCacheSim) set(DESCRIPTION "a high performance cache simulation library") set(PROJECT_WEB "http://cachesim.com") @@ -9,6 +9,9 @@ set(${PROJECT_NAME}_VERSION_PATCH 0) set(${PROJECT_NAME}_RELEASE_VERSION ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}) set(${PROJECT_NAME}_VERSION ${${PROJECT_NAME}_RELEASE_VERSION}.${${PROJECT_NAME}_VERSION_PATCH}) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED On) +set(CMAKE_CXX_EXTENSIONS Off) ######################################## # define options # @@ -19,7 +22,7 @@ option(USE_HUGEPAGE "use transparent hugepage" ON) option(ENABLE_TESTS "whether enable test" ON) option(ENABLE_GLCACHE "enable group-learned cache" OFF) option(SUPPORT_TTL "whether support TTL" OFF) -option(SUPPORT_ZSTD_TRACE "whether support zstd trace" ON) +option(OPT_SUPPORT_ZSTD_TRACE "whether support zstd trace" ON) option(ENABLE_LRB "enable LRB" OFF) set(LOG_LEVEL NONE CACHE STRING "change the logging level") set_property(CACHE LOG_LEVEL PROPERTY STRINGS INFO WARN ERROR DEBUG VERBOSE VVERBOSE VVVERBOSE) @@ -102,7 +105,7 @@ endif() message(STATUS "CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG} CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO} CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}") # string( REPLACE "/DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") -message(STATUS "SUPPORT TTL ${SUPPORT_TTL}, USE_HUGEPAGE ${USE_HUGEPAGE}, LOGLEVEL ${LOG_LEVEL}, ENABLE_GLCACHE ${ENABLE_GLCACHE}, ENABLE_LRB ${ENABLE_LRB}, SUPPORT_ZSTD_TRACE ${SUPPORT_ZSTD_TRACE}") +message(STATUS "SUPPORT TTL ${SUPPORT_TTL}, USE_HUGEPAGE ${USE_HUGEPAGE}, LOGLEVEL ${LOG_LEVEL}, ENABLE_GLCACHE ${ENABLE_GLCACHE}, ENABLE_LRB ${ENABLE_LRB}, OPT_SUPPORT_ZSTD_TRACE ${OPT_SUPPORT_ZSTD_TRACE}") # add_compile_options(-fsanitize=address) # add_link_options(-fsanitize=address) @@ -114,13 +117,12 @@ message(STATUS "SUPPORT TTL ${SUPPORT_TTL}, USE_HUGEPAGE ${USE_HUGEPAGE}, LOGLEV set(CFLAGS "$ENV{CFLAGS} " "-Wall -Wshadow -Winline " "-Wno-unused " - "-Wstrict-prototypes -Wmissing-prototypes " + "-Wstrict-prototypes " + # "-Wmissing-prototypes " "-Wmissing-declarations " "-Wredundant-decls " "-Wunused-value -Wunused-variable " - "-std=c11 " "-fno-strict-aliasing " - "-O0 " ) @@ -147,7 +149,7 @@ set(LIBS ${LIBS} ${GLib_LIBRARY}) # link_libraries(${GLib2_LDFLAGS}) #endif() -if (SUPPORT_ZSTD_TRACE) +if (OPT_SUPPORT_ZSTD_TRACE) add_compile_definitions(SUPPORT_ZSTD_TRACE=1) find_package(ZSTD) if ("${ZSTD_LIBRARIES}" STREQUAL "") @@ -157,7 +159,7 @@ if (SUPPORT_ZSTD_TRACE) message(STATUS "ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIRS}, ZSTD_LIBRARIES ${ZSTD_LIBRARIES}") else() remove_definitions(SUPPORT_ZSTD_TRACE) -endif(SUPPORT_ZSTD_TRACE) +endif(OPT_SUPPORT_ZSTD_TRACE) # libgoogle-perftools-dev google-perftools @@ -311,10 +313,20 @@ if (ENABLE_LRB) ) endif(ENABLE_LRB) -file(GLOB reader_source - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/*.c - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/*.c +set(reader_source + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/reader.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/binary.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/csv.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/lcs.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/libcsv.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/txt.c ) +if (OPT_SUPPORT_ZSTD_TRACE) + set (reader_source + ${reader_source} ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/zstdReader.c + ) +endif(OPT_SUPPORT_ZSTD_TRACE) + file(GLOB dataStructure_source ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/*.c ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/hashtable/*.c diff --git a/libCacheSim/bin/cachesim/main.c b/libCacheSim/bin/cachesim/main.c index e83bf76a..9652d1ef 100644 --- a/libCacheSim/bin/cachesim/main.c +++ b/libCacheSim/bin/cachesim/main.c @@ -63,7 +63,7 @@ int main(int argc, char **argv) { "%s %32s cache size %8ld%s, %lld req, miss ratio %.4lf, byte miss " "ratio %.4lf\n", output_filename, result[i].cache_name, - (long)result[i].cache_size / size_unit, size_unit_str, + (long)(result[i].cache_size / size_unit), size_unit_str, (long long)result[i].n_req, (double)result[i].n_miss / (double)result[i].n_req, (double)result[i].n_miss_byte / (double)result[i].n_req_byte); diff --git a/libCacheSim/cache/admission/adaptsize.cpp b/libCacheSim/cache/admission/adaptsize.cpp index ad5de582..6fe3fadd 100644 --- a/libCacheSim/cache/admission/adaptsize.cpp +++ b/libCacheSim/cache/admission/adaptsize.cpp @@ -23,7 +23,7 @@ static void adaptsize_admissioner_parse_params( const char *init_params, adaptsize_admission_params_t *pa) { if (init_params == NULL) { pa->adaptsize_threshold = INT64_MAX; - INFO("use default adaptsize admission: %ld", pa->adaptsize_threshold); + INFO("use default adaptsize admission: %ld", (long) pa->adaptsize_threshold); } else { char *params_str = strdup(init_params); char *old_params_str = params_str; diff --git a/libCacheSim/cache/admission/size.c b/libCacheSim/cache/admission/size.c index a868e77b..e3f80022 100644 --- a/libCacheSim/cache/admission/size.c +++ b/libCacheSim/cache/admission/size.c @@ -26,7 +26,7 @@ static void size_admissioner_parse_params(const char *init_params, size_admission_params_t *pa) { if (init_params == NULL) { pa->size_threshold = INT64_MAX; - INFO("use default size admission: %ld", pa->size_threshold); + INFO("use default size admission: %ld", (long)pa->size_threshold); } else { char *params_str = strdup(init_params); char *old_params_str = params_str; diff --git a/libCacheSim/cache/cache.c b/libCacheSim/cache/cache.c index 3d527592..04a5023b 100644 --- a/libCacheSim/cache/cache.c +++ b/libCacheSim/cache/cache.c @@ -2,9 +2,8 @@ // Created by Juncheng Yang on 6/20/20. // -#include "../include/libCacheSim/cache.h" - #include "../dataStructure/hashtable/hashtable.h" +#include "../include/libCacheSim/cache.h" #include "../include/libCacheSim/prefetchAlgo.h" /** this file contains both base function, which should be called by all @@ -145,7 +144,7 @@ bool cache_can_insert_default(cache_t *cache, const request_t *req) { if (admissioner->admit(admissioner, req) == false) { DEBUG_ONCE( "admission algorithm does not admit: req %ld, obj %lu, size %lu\n", - cache->n_req, (unsigned long)req->obj_id, + (long)cache->n_req, (unsigned long)req->obj_id, (unsigned long)req->obj_size); return false; } @@ -153,7 +152,7 @@ bool cache_can_insert_default(cache_t *cache, const request_t *req) { if (req->obj_size + cache->obj_md_size > cache->cache_size) { WARN_ONCE("%ld req, obj %lu, size %lu larger than cache size %lu\n", - cache->n_req, (unsigned long)req->obj_id, + (long)cache->n_req, (unsigned long)req->obj_id, (unsigned long)req->obj_size, (unsigned long)cache->cache_size); return false; } diff --git a/libCacheSim/cache/eviction/ARC.c b/libCacheSim/cache/eviction/ARC.c index ec7b6a1e..a719f0fb 100644 --- a/libCacheSim/cache/eviction/ARC.c +++ b/libCacheSim/cache/eviction/ARC.c @@ -180,7 +180,6 @@ static void ARC_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool ARC_get(cache_t *cache, const request_t *req) { - ARC_params_t *params = (ARC_params_t *)(cache->eviction_params); #ifdef DEBUG_MODE return ARC_get_debug(cache, req); #else @@ -520,7 +519,6 @@ static void _ARC_evict_L2_ghost(cache_t *cache, const request_t *req) { static void _ARC_replace(cache_t *cache, const request_t *req) { ARC_params_t *params = (ARC_params_t *)(cache->eviction_params); - cache_obj_t *obj = NULL; bool cond1 = params->L1_data_size > 0; bool cond2 = params->L1_data_size > params->p; @@ -613,13 +611,12 @@ static void ARC_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, * key and value are separated by = */ char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); + // char *value = strsep((char **)¶ms_str, ","); // skip the white space while (params_str != NULL && *params_str == ' ') { @@ -649,7 +646,7 @@ static void print_cache(cache_t *cache) { cache_obj_t *obj = params->L1_data_head; printf("T1: "); while (obj != NULL) { - printf("%ld ", obj->obj_id); + printf("%ld ", (long)obj->obj_id); obj = obj->queue.next; } printf("\n"); @@ -657,7 +654,7 @@ static void print_cache(cache_t *cache) { obj = params->L1_ghost_head; printf("B1: "); while (obj != NULL) { - printf("%ld ", obj->obj_id); + printf("%ld ", (long)obj->obj_id); obj = obj->queue.next; } printf("\n"); @@ -665,7 +662,7 @@ static void print_cache(cache_t *cache) { obj = params->L2_data_head; printf("T2: "); while (obj != NULL) { - printf("%ld ", obj->obj_id); + printf("%ld ", (long)obj->obj_id); obj = obj->queue.next; } printf("\n"); @@ -673,7 +670,7 @@ static void print_cache(cache_t *cache) { obj = params->L2_ghost_head; printf("B2: "); while (obj != NULL) { - printf("%ld ", obj->obj_id); + printf("%ld ", (long)obj->obj_id); obj = obj->queue.next; } printf("\n"); @@ -773,7 +770,6 @@ static inline void _ARC_sanity_check_full(cache_t *cache, } static bool ARC_get_debug(cache_t *cache, const request_t *req) { - ARC_params_t *params = (ARC_params_t *)(cache->eviction_params); cache->n_req += 1; diff --git a/libCacheSim/cache/eviction/ARCv0.c b/libCacheSim/cache/eviction/ARCv0.c index 19fb5e0f..1d61bfd2 100644 --- a/libCacheSim/cache/eviction/ARCv0.c +++ b/libCacheSim/cache/eviction/ARCv0.c @@ -93,7 +93,8 @@ static bool ARCv0_get_debug(cache_t *cache, const request_t *req); */ cache_t *ARCv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("ARCv0", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("ARCv0", ccache_params, cache_specific_params); cache->cache_init = ARCv0_init; cache->cache_free = ARCv0_free; cache->get = ARCv0_get; @@ -186,7 +187,7 @@ static void ARCv0_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool ARCv0_get(cache_t *cache, const request_t *req) { - ARCv0_params_t *params = (ARCv0_params_t *)(cache->eviction_params); + // ARCv0_params_t *params = (ARCv0_params_t *)(cache->eviction_params); #ifdef DEBUG_MODE return ARCv0_get_debug(cache, req); #else @@ -257,7 +258,7 @@ static cache_obj_t *ARCv0_find(cache_t *cache, const request_t *req, } #ifdef QUICK_DEMOTION // params->p = MIN(params->p, cache->cache_size/10); - params->p = cache->cache_size/10; + params->p = cache->cache_size / 10; #endif } else { // cache hit, case I: x in L1_data or L2_data @@ -534,13 +535,12 @@ static void ARCv0_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, * key and value are separated by = */ char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); + // char *value = strsep((char **)¶ms_str, ","); // skip the white space while (params_str != NULL && *params_str == ' ') { @@ -581,7 +581,8 @@ static bool ARCv0_get_debug(cache_t *cache, const request_t *req) { cache->n_req += 1; - printf("%ld obj_id %ld: p %.2lf\n", cache->n_req, req->obj_id, params->p); + printf("%ld obj_id %ld: p %.2lf\n", (long)cache->n_req, (long)req->obj_id, + params->p); print_cache(cache); printf("==================================\n"); diff --git a/libCacheSim/cache/eviction/Cacheus.c b/libCacheSim/cache/eviction/Cacheus.c index a590393a..c8bbf8a7 100644 --- a/libCacheSim/cache/eviction/Cacheus.c +++ b/libCacheSim/cache/eviction/Cacheus.c @@ -273,7 +273,6 @@ static void Cacheus_evict(cache_t *cache, const request_t *req) { cache_t *lfu = params->LFU; cache_t *lru_g = params->LRU_g; cache_t *lfu_g = params->LFU_g; - SR_LRU_params_t *params_LRU = (SR_LRU_params_t *)(lru->eviction_params); // If two voters decide the same: cache_obj_t *lru_to_evict = lru->to_evict(lru, req); @@ -390,8 +389,8 @@ static void update_lr(cache_t *cache, const request_t *req) { // self.learning_rate = 0.9 // elif self.learning_rate <= 0.001: // self.learning_rate = 0.005 - if (params->lr + sign * abs(params->lr * delta_lr) > 0.001) - params->lr = params->lr + sign * abs(params->lr * delta_lr); + if (params->lr + sign * fabs(params->lr * delta_lr) > 0.001) + params->lr = params->lr + sign * fabs(params->lr * delta_lr); else params->lr = 0.0001; params->unlearn_count = 0; diff --git a/libCacheSim/cache/eviction/Clock.c b/libCacheSim/cache/eviction/Clock.c index 5da554b1..2f3faa13 100644 --- a/libCacheSim/cache/eviction/Clock.c +++ b/libCacheSim/cache/eviction/Clock.c @@ -53,7 +53,8 @@ static bool Clock_remove(cache_t *cache, const obj_id_t obj_id); */ cache_t *Clock_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("Clock", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("Clock", ccache_params, cache_specific_params); cache->cache_init = Clock_init; cache->cache_free = Clock_free; cache->get = Clock_get; @@ -231,7 +232,7 @@ static void Clock_evict(cache_t *cache, const request_t *req) { move_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj_to_evict); obj_to_evict = params->q_tail; } - + remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj_to_evict); cache_evict_base(cache, obj_to_evict, true); } @@ -291,8 +292,7 @@ static bool Clock_remove(cache_t *cache, const obj_id_t obj_id) { static const char *Clock_current_params(cache_t *cache, Clock_params_t *params) { static __thread char params_str[128]; - int n = - snprintf(params_str, 128, "n-bit-counter=%d\n", params->n_bit_counter); + snprintf(params_str, 128, "n-bit-counter=%d\n", params->n_bit_counter); return params_str; } diff --git a/libCacheSim/cache/eviction/FIFO_Merge.c b/libCacheSim/cache/eviction/FIFO_Merge.c index 338f998e..83aba917 100644 --- a/libCacheSim/cache/eviction/FIFO_Merge.c +++ b/libCacheSim/cache/eviction/FIFO_Merge.c @@ -261,7 +261,7 @@ static void FIFO_Merge_evict(cache_t *cache, const request_t *req) { if (cache->n_obj <= params->n_exam_obj) { // just evict one object - this is fifo - cache_obj_t *cache_obj = params->q_tail; + cache_obj = params->q_tail; params->next_to_exam = NULL; remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, cache_obj); cache_evict_base(cache, cache_obj, true); @@ -451,6 +451,8 @@ static double retain_metric(cache_t *cache, cache_obj_t *cache_obj) { default: break; } + abort(); + return -1; } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/FIFO_Reinsertion.c b/libCacheSim/cache/eviction/FIFO_Reinsertion.c index 566c3d1c..b2fdd0d6 100644 --- a/libCacheSim/cache/eviction/FIFO_Reinsertion.c +++ b/libCacheSim/cache/eviction/FIFO_Reinsertion.c @@ -270,7 +270,7 @@ static void FIFO_Reinsertion_evict(cache_t *cache, const request_t *req) { if (cache->n_obj <= params->n_exam_obj) { // just evict one object - cache_obj_t *cache_obj = params->next_to_merge->queue.prev; + cache_obj = params->next_to_merge->queue.prev; FIFO_Reinsertion_remove_obj(cache, params->next_to_merge); params->next_to_merge = cache_obj; @@ -473,6 +473,7 @@ static double retain_metric(cache_t *cache, cache_obj_t *cache_obj) { default: break; } + abort(); } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/LFU.c b/libCacheSim/cache/eviction/LFU.c index e93e81b8..bd9705f8 100644 --- a/libCacheSim/cache/eviction/LFU.c +++ b/libCacheSim/cache/eviction/LFU.c @@ -325,7 +325,6 @@ void LFU_remove_obj(cache_t *cache, cache_obj_t *obj) { * cache */ bool LFU_remove(cache_t *cache, obj_id_t obj_id) { - LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); if (obj == NULL) { return false; diff --git a/libCacheSim/cache/eviction/LFUDA.c b/libCacheSim/cache/eviction/LFUDA.c index 8e7ad8f4..747a026d 100644 --- a/libCacheSim/cache/eviction/LFUDA.c +++ b/libCacheSim/cache/eviction/LFUDA.c @@ -311,7 +311,6 @@ static void LFUDA_remove_obj(cache_t *cache, cache_obj_t *obj) { * cache */ static bool LFUDA_remove(cache_t *cache, obj_id_t obj_id) { - LFUDA_params_t *params = (LFUDA_params_t *)(cache->eviction_params); cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); if (obj == NULL) { return false; diff --git a/libCacheSim/cache/eviction/LHD/CMakeLists.txt b/libCacheSim/cache/eviction/LHD/CMakeLists.txt index 19974a1d..b049c97b 100644 --- a/libCacheSim/cache/eviction/LHD/CMakeLists.txt +++ b/libCacheSim/cache/eviction/LHD/CMakeLists.txt @@ -1,11 +1,4 @@ file(GLOB src *.cpp) add_library (LHD ${src}) -set_target_properties(LHD - PROPERTIES - CXX_STANDARD 11 - CXX_STANDARD_REQUIRED YES - CXX_EXTENSIONS NO - ) - - +set_property(LHD CXX_STANDARD 17) diff --git a/libCacheSim/cache/eviction/LHD/candidate.hpp b/libCacheSim/cache/eviction/LHD/candidate.hpp index d4b120df..17a062fa 100644 --- a/libCacheSim/cache/eviction/LHD/candidate.hpp +++ b/libCacheSim/cache/eviction/LHD/candidate.hpp @@ -18,7 +18,10 @@ struct candidate_t { #endif static candidate_t make(const request_t* req) { - return candidate_t{DEFAULT_APP_ID, static_cast(req->obj_id)}; + candidate_t c; + c.appId = DEFAULT_APP_ID; + c.id = static_cast(req->obj_id); + return c; } inline bool operator==(const candidate_t& that) const { diff --git a/libCacheSim/cache/eviction/LIRS.c b/libCacheSim/cache/eviction/LIRS.c index 3b4cf169..6c989dad 100644 --- a/libCacheSim/cache/eviction/LIRS.c +++ b/libCacheSim/cache/eviction/LIRS.c @@ -1,6 +1,6 @@ // // LIRS cache eviction policy implemented by multiple LRUs -// +// // LIRS.c // libcachesim // @@ -15,7 +15,6 @@ extern "C" { // #define DEBUG_MODE 1 - typedef struct LIRS_params { cache_t *LRU_s; cache_t *LRU_q; @@ -73,45 +72,47 @@ static void LIRS_print_cache_compared_to_cacheus(cache_t *cache); */ cache_t *LIRS_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("LIRS", ccache_params, cache_specific_params); - cache->cache_init = LIRS_init; - cache->cache_free = LIRS_free; - cache->get = LIRS_get; - cache->find = LIRS_find; - cache->can_insert = LIRS_can_insert; - cache->insert = LIRS_insert; - cache->evict = LIRS_evict; - cache->remove = LIRS_remove; - cache->to_evict = LIRS_to_evict; - - if (ccache_params.consider_obj_metadata) { - cache->obj_md_size = 8 * 2; - } else { - cache->obj_md_size = 0; - } + cache_t *cache = + cache_struct_init("LIRS", ccache_params, cache_specific_params); + cache->cache_init = LIRS_init; + cache->cache_free = LIRS_free; + cache->get = LIRS_get; + cache->find = LIRS_find; + cache->can_insert = LIRS_can_insert; + cache->insert = LIRS_insert; + cache->evict = LIRS_evict; + cache->remove = LIRS_remove; + cache->to_evict = LIRS_to_evict; + + if (ccache_params.consider_obj_metadata) { + cache->obj_md_size = 8 * 2; + } else { + cache->obj_md_size = 0; + } + + cache->eviction_params = (LIRS_params_t *)malloc(sizeof(LIRS_params_t)); + LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); - cache->eviction_params = (LIRS_params_t *)malloc(sizeof(LIRS_params_t)); - LIRS_params_t * params = (LIRS_params_t *)(cache->eviction_params); - - params->hirs_ratio = 0.01; - params->hirs_limit = MAX(1, (uint64_t)(params->hirs_ratio * cache->cache_size)); - params->lirs_limit = cache->cache_size - params->hirs_limit; - params->hirs_count = 0; - params->lirs_count = 0; - params->nonresident = 0; - - // initialize LRU for stack S and stack Q - common_cache_params_t ccache_params_s = ccache_params; - ccache_params_s.cache_size = params->lirs_limit; - common_cache_params_t ccache_params_q = ccache_params; - ccache_params_q.cache_size = params->hirs_limit; - common_cache_params_t ccache_params_nh = ccache_params; - - params->LRU_s = LRU_init(ccache_params_s, NULL); - params->LRU_q = LRU_init(ccache_params_q, NULL); - params->LRU_nh = LRU_init(ccache_params_nh, NULL); - - return cache; + params->hirs_ratio = 0.01; + params->hirs_limit = + MAX(1, (uint64_t)(params->hirs_ratio * cache->cache_size)); + params->lirs_limit = cache->cache_size - params->hirs_limit; + params->hirs_count = 0; + params->lirs_count = 0; + params->nonresident = 0; + + // initialize LRU for stack S and stack Q + common_cache_params_t ccache_params_s = ccache_params; + ccache_params_s.cache_size = params->lirs_limit; + common_cache_params_t ccache_params_q = ccache_params; + ccache_params_q.cache_size = params->hirs_limit; + common_cache_params_t ccache_params_nh = ccache_params; + + params->LRU_s = LRU_init(ccache_params_s, NULL); + params->LRU_q = LRU_init(ccache_params_q, NULL); + params->LRU_nh = LRU_init(ccache_params_nh, NULL); + + return cache; } /** @@ -120,12 +121,12 @@ cache_t *LIRS_init(const common_cache_params_t ccache_params, * @param cache */ static void LIRS_free(cache_t *cache) { - LIRS_params_t * params = (LIRS_params_t *)(cache->eviction_params); - params->LRU_s->cache_free(params->LRU_s); - params->LRU_q->cache_free(params->LRU_q); - params->LRU_nh->cache_free(params->LRU_nh); - my_free(sizeof(LIRS_params_t), params); - cache_struct_free(cache); + LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); + params->LRU_s->cache_free(params->LRU_s); + params->LRU_q->cache_free(params->LRU_q); + params->LRU_nh->cache_free(params->LRU_nh); + my_free(sizeof(LIRS_params_t), params); + cache_struct_free(cache); } /** @@ -153,8 +154,8 @@ static bool LIRS_get(cache_t *cache, const request_t *req) { limitStack(cache); - #ifdef DEBUG_MODE - LIRS_params_t * params = (LIRS_params_t *)(cache->eviction_params); +#ifdef DEBUG_MODE + LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_t *LRU_s = params->LRU_s; if (cache->n_req >= 2) { // printf("obj:%lu size:%ld ", req->obj_id, req->obj_size); @@ -168,13 +169,16 @@ static bool LIRS_get(cache_t *cache, const request_t *req) { printf("number of requests:%ld \n", cache->n_req); printf("number of objects in S:%ld \n", LRU_s->n_obj); - printf("S(%ld):%ld %ld\n", params->lirs_limit, LRU_s->occupied_byte, params->lirs_count); - printf("Q(%ld): %ld %ld \n", params->hirs_limit, params->LRU_q->occupied_byte, params->hirs_count); - printf("NH: %ld %ld \n", params->LRU_nh->occupied_byte, params->nonresident); + printf("S(%ld):%ld %ld\n", params->lirs_limit, LRU_s->occupied_byte, + params->lirs_count); + printf("Q(%ld): %ld %ld \n", params->hirs_limit, + params->LRU_q->occupied_byte, params->hirs_count); + printf("NH: %ld %ld \n", params->LRU_nh->occupied_byte, + params->nonresident); printf("\n\n"); } - #endif - +#endif + return res; } @@ -196,17 +200,17 @@ static bool LIRS_get(cache_t *cache, const request_t *req) { */ static cache_obj_t *LIRS_find(cache_t *cache, const request_t *req, const bool update_cache) { - LIRS_params_t * params = (LIRS_params_t *)(cache->eviction_params); + LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_obj_t *obj_s = NULL; cache_obj_t *obj_q = NULL; - // find the object in S and Q, + // find the object in S and Q, // and they will be promoted to the top of the stacks obj_s = params->LRU_s->find(params->LRU_s, req, update_cache); obj_q = params->LRU_q->find(params->LRU_q, req, update_cache); - - cache_obj_t *res = NULL; + + // cache_obj_t *res = NULL; if (obj_s != NULL) { if (obj_s->LIRS.is_LIR) { // accessing an LIR block (hit) @@ -215,22 +219,22 @@ static cache_obj_t *LIRS_find(cache_t *cache, const request_t *req, } else { // accessing an HIR block in S (resident and non-resident) if (obj_s->LIRS.in_cache) { - return hit_RD_HIRinS(cache, obj_s, obj_q); //hit + return hit_RD_HIRinS(cache, obj_s, obj_q); // hit } else { return NULL; } - } + } } else if (obj_q != NULL) { // accessing an HIR blocks in Q (resident) // LRU_q->find already did the movement if (obj_q->LIRS.in_cache) { - return hit_RD_HIRinQ(cache, obj_q); //hit + return hit_RD_HIRinQ(cache, obj_q); // hit } else { assert(false); return NULL; } } else { - return NULL; //miss + return NULL; // miss } } @@ -251,10 +255,9 @@ static cache_obj_t *LIRS_insert(cache_t *cache, const request_t *req) { cache_obj_t *obj_s = params->LRU_s->find(params->LRU_s, req, false); cache_obj_t *obj_q = params->LRU_q->find(params->LRU_q, req, false); - //Upon accessing an HIR non-resident in S - if (obj_s != NULL && obj_s->LIRS.is_LIR == false - && obj_s->LIRS.in_cache == false) { - + // Upon accessing an HIR non-resident in S + if (obj_s != NULL && obj_s->LIRS.is_LIR == false && + obj_s->LIRS.in_cache == false) { // change status of the block to be LIR (obj_s is already promoted to top) obj_s->LIRS.is_LIR = true; obj_s->LIRS.in_cache = true; @@ -265,7 +268,6 @@ static cache_obj_t *LIRS_insert(cache_t *cache, const request_t *req) { // Upon accessing an HIR non-resident in Q if (obj_s == NULL && obj_q != NULL && obj_q->LIRS.in_cache == false) { - // insert the req into S and place it on the top of S cache_obj_t *inserted_obj_s = params->LRU_s->insert(params->LRU_s, req); inserted_obj_s->LIRS.is_LIR = false; @@ -278,12 +280,11 @@ static cache_obj_t *LIRS_insert(cache_t *cache, const request_t *req) { params->hirs_count += inserted_obj_s->obj_size; cache->occupied_byte += inserted_obj_s->obj_size + cache->obj_md_size; cache->n_obj += 1; - } // Upon accessing blocks neither in S nor Q if (obj_s == NULL && obj_q == NULL) { - // when LIR block set is not full, + // when LIR block set is not full, // all reference blocks are given an LIR status if (params->lirs_count + req->obj_size <= params->lirs_limit) { cache_obj_t *inserted_obj_s = params->LRU_s->insert(params->LRU_s, req); @@ -292,7 +293,7 @@ static cache_obj_t *LIRS_insert(cache_t *cache, const request_t *req) { params->lirs_count += inserted_obj_s->obj_size; cache->occupied_byte += inserted_obj_s->obj_size + cache->obj_md_size; } else if (params->hirs_count + req->obj_size <= params->hirs_limit) { - // when LIR block set is full, + // when LIR block set is full, // all reference blocks are given an HIR status cache_obj_t *inserted_obj_s = params->LRU_s->insert(params->LRU_s, req); inserted_obj_s->LIRS.is_LIR = false; @@ -352,11 +353,11 @@ static void LIRS_evict(cache_t *cache, const request_t *req) { cache_obj_t *obj_s = params->LRU_s->find(params->LRU_s, req, false); cache_obj_t *obj_q = params->LRU_q->find(params->LRU_q, req, false); - //Upon accessing an HIR non-resident in S - if (obj_s != NULL && obj_s->LIRS.is_LIR == false - && obj_s->LIRS.in_cache == false) { - //remove the HIR resident at the front of Q - while (params->hirs_count >= params->hirs_limit){ + // Upon accessing an HIR non-resident in S + if (obj_s != NULL && obj_s->LIRS.is_LIR == false && + obj_s->LIRS.in_cache == false) { + // remove the HIR resident at the front of Q + while (params->hirs_count >= params->hirs_limit) { evictHIR(cache); } @@ -364,22 +365,21 @@ static void LIRS_evict(cache_t *cache, const request_t *req) { evictLIR(cache); } - // Upon accessing an HIR non-resident in Q if (obj_s == NULL && obj_q != NULL && obj_q->LIRS.in_cache == false) { - //remove the HIR resident at the front of Q - while (params->hirs_count >= params->hirs_limit){ + // remove the HIR resident at the front of Q + while (params->hirs_count >= params->hirs_limit) { evictHIR(cache); } } // Upon accessing blocks neither in S nor Q if (obj_s == NULL && obj_q == NULL) { - if (params->lirs_count + req->obj_size > params->lirs_limit - && params->hirs_count + req->obj_size > params->hirs_limit) { + if (params->lirs_count + req->obj_size > params->lirs_limit && + params->hirs_count + req->obj_size > params->hirs_limit) { // when both LIR and HIR block sets are full, // the curcumstance is same as accessing an HIR non-resident not in S - //remove the HIR resident at the front of Q + // remove the HIR resident at the front of Q evictHIR(cache); } } @@ -408,11 +408,11 @@ static bool LIRS_remove(cache_t *cache, obj_id_t obj_id) { // object in S stack (or in Q stack) if (obj_s != NULL) { if (obj_s->LIRS.is_LIR) { - params->LRU_s->remove(params->LRU_s, obj_id); - params->lirs_count -= obj_s->obj_size; - cache->occupied_byte -= obj_s->obj_size; - cache->n_obj--; - LIRS_prune(cache); + params->LRU_s->remove(params->LRU_s, obj_id); + params->lirs_count -= obj_s->obj_size; + cache->occupied_byte -= obj_s->obj_size; + cache->n_obj--; + LIRS_prune(cache); } else { if (obj_s->LIRS.in_cache) { params->LRU_s->remove(params->LRU_s, obj_id); @@ -424,7 +424,7 @@ static bool LIRS_remove(cache_t *cache, obj_id_t obj_id) { params->nonresident -= obj_s->obj_size; } if (obj_q != NULL) { - params->LRU_q->remove(params->LRU_q, obj_id); + params->LRU_q->remove(params->LRU_q, obj_id); } } } else if (obj_q != NULL) { @@ -453,11 +453,10 @@ static bool LIRS_remove(cache_t *cache, obj_id_t obj_id) { // *********************************************************************** /* LIRS cannot insert an object larger than Q stack size. -* This function also promise there will be enough space for the object -* to be inserted into the cache. + * This function also promise there will be enough space for the object + * to be inserted into the cache. */ bool LIRS_can_insert(cache_t *cache, const request_t *req) { - bool can_insert = cache_can_insert_default(cache, req); if (can_insert == false) { return false; @@ -467,14 +466,14 @@ bool LIRS_can_insert(cache_t *cache, const request_t *req) { cache_obj_t *obj_q = params->LRU_q->find(params->LRU_q, req, false); // accessing an HIR non-resident in S - if (obj_s != NULL && obj_s->LIRS.is_LIR == false - && obj_s->LIRS.in_cache == false) { + if (obj_s != NULL && obj_s->LIRS.is_LIR == false && + obj_s->LIRS.in_cache == false) { while (params->lirs_count + obj_s->obj_size > params->lirs_limit) { evictLIR(cache); } bool res = params->LRU_nh->remove(params->LRU_nh, obj_s->obj_id); - if (res){ + if (res) { params->nonresident -= obj_s->obj_size; } @@ -483,13 +482,14 @@ bool LIRS_can_insert(cache_t *cache, const request_t *req) { // accessing blocks neither in S nor Q if (obj_s == NULL && obj_q == NULL) { - if (req->obj_size > params->lirs_limit || req->obj_size > params->hirs_limit) { + if (req->obj_size > params->lirs_limit || + req->obj_size > params->hirs_limit) { WARN_ONCE("object size too large\n"); // printf("request num: %ld\n", cache->n_req); return false; } - if (params->lirs_count + req->obj_size > params->lirs_limit - && params->hirs_count + req->obj_size > params->hirs_limit) { + if (params->lirs_count + req->obj_size > params->lirs_limit && + params->hirs_count + req->obj_size > params->hirs_limit) { // when both LIR and HIR block sets are full, // the curcumstance is same as accessing an HIR non-resident not in S while (params->hirs_count + req->obj_size > params->hirs_limit) { @@ -500,19 +500,18 @@ bool LIRS_can_insert(cache_t *cache, const request_t *req) { return true; } - INFO("LIRS_can_insert: should not reach here. n_req = %ld\n", cache->n_req); + INFO("LIRS_can_insert: should not reach here. n_req = %ld\n", + (long)cache->n_req); assert(false); } - static void LIRS_prune(cache_t *cache) { - LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_t *LRU_s = params->LRU_s; LRU_params_t *lru_s_params = (LRU_params_t *)(LRU_s->eviction_params); cache_obj_t *obj_to_remove = lru_s_params->q_tail; - + while (obj_to_remove != lru_s_params->q_head) { if (obj_to_remove->LIRS.is_LIR) { break; @@ -521,7 +520,7 @@ static void LIRS_prune(cache_t *cache) { // remove obj from LRU_nh if (obj_to_remove->LIRS.in_cache == false) { bool res = params->LRU_nh->remove(params->LRU_nh, obj_to_remove->obj_id); - if (res){ + if (res) { params->nonresident -= obj_to_remove->obj_size; } } @@ -556,9 +555,9 @@ static cache_obj_t *hit_RD_HIRinS(cache_t *cache, cache_obj_t *cache_obj_s, static cache_obj_t *hit_NR_HIRinS(cache_t *cache, cache_obj_t *cache_obj_s) { LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); - + bool res = params->LRU_nh->remove(params->LRU_nh, cache_obj_s->obj_id); - if (res){ + if (res) { params->nonresident -= cache_obj_s->obj_size; } return NULL; @@ -566,7 +565,7 @@ static cache_obj_t *hit_NR_HIRinS(cache_t *cache, cache_obj_t *cache_obj_s) { static cache_obj_t *hit_RD_HIRinQ(cache_t *cache, cache_obj_t *cache_obj_q) { LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); - + static __thread request_t *req_local = NULL; if (req_local == NULL) { req_local = new_request(); @@ -577,14 +576,15 @@ static cache_obj_t *hit_RD_HIRinQ(cache_t *cache, cache_obj_t *cache_obj_q) { evictLIR(cache); } params->LRU_s->insert(params->LRU_s, req_local); - cache_obj_t *obj_to_update = params->LRU_s->find(params->LRU_s, req_local, false); + cache_obj_t *obj_to_update = + params->LRU_s->find(params->LRU_s, req_local, false); obj_to_update->LIRS.is_LIR = false; obj_to_update->LIRS.in_cache = true; return obj_to_update; } -static void evictLIR(cache_t *cache){ +static void evictLIR(cache_t *cache) { LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_obj_t *obj_to_evict = params->LRU_s->to_evict(params->LRU_s, NULL); @@ -602,10 +602,11 @@ static void evictLIR(cache_t *cache){ if (req_local->obj_size <= params->hirs_limit) { while (params->hirs_count + req_local->obj_size > params->hirs_limit) { evictHIR(cache); - } + } params->LRU_q->insert(params->LRU_q, req_local); - - cache_obj_t *obj_to_update = params->LRU_q->find(params->LRU_q, req_local, false); + + cache_obj_t *obj_to_update = + params->LRU_q->find(params->LRU_q, req_local, false); obj_to_update->LIRS.is_LIR = false; obj_to_update->LIRS.in_cache = true; @@ -617,12 +618,12 @@ static void evictLIR(cache_t *cache){ LIRS_prune(cache); } -static bool evictHIR(cache_t *cache){ +static bool evictHIR(cache_t *cache) { LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_obj_t *obj_to_evict = params->LRU_q->to_evict(params->LRU_q, NULL); - //update the corresponding block in S to be non-resident + // update the corresponding block in S to be non-resident static __thread request_t *req_local = NULL; if (req_local == NULL) { req_local = new_request(); @@ -632,7 +633,8 @@ static bool evictHIR(cache_t *cache){ params->hirs_count -= obj_to_evict->obj_size; params->LRU_q->evict(params->LRU_q, NULL); - cache_obj_t *obj_to_update = params->LRU_s->find(params->LRU_s, req_local, false); + cache_obj_t *obj_to_update = + params->LRU_s->find(params->LRU_s, req_local, false); if (obj_to_update != NULL) { obj_to_update->LIRS.in_cache = false; params->LRU_nh->insert(params->LRU_nh, req_local); @@ -654,7 +656,7 @@ static void limitStack(cache_t *cache) { params->nonresident -= obj_to_evict->obj_size; params->LRU_s->remove(params->LRU_s, obj_to_evict->obj_id); params->LRU_nh->evict(params->LRU_nh, NULL); - } else{ + } else { break; } } @@ -667,80 +669,68 @@ static void limitStack(cache_t *cache) { static void LIRS_print_cache(cache_t *cache) { LIRS_params_t *params = (LIRS_params_t *)cache->eviction_params; - printf("S Stack: %lu:%lu %lu:%lu \n", - params->lirs_limit, params->lirs_count, - params->hirs_limit, params->hirs_count); - cache_obj_t *obj = - ((LRU_params_t *)params->LRU_s->eviction_params)->q_head; + printf("S Stack: %lu:%lu %lu:%lu \n", (unsigned long)params->lirs_limit, + (unsigned long)params->lirs_count, (unsigned long)params->hirs_limit, + (unsigned long)params->hirs_count); + cache_obj_t *obj = ((LRU_params_t *)params->LRU_s->eviction_params)->q_head; while (obj) { - printf("%ld(%u, %s, %s)->", obj->obj_id, obj->obj_size, - obj->LIRS.in_cache ? "R" : "N", - obj->LIRS.is_LIR ? "L" : "H"); + printf("%ld(%u, %s, %s)->", (long)obj->obj_id, obj->obj_size, + obj->LIRS.in_cache ? "R" : "N", obj->LIRS.is_LIR ? "L" : "H"); obj = obj->queue.next; } printf("\n"); printf("Q Stack: \n"); - cache_obj_t *obj_q = - ((LRU_params_t *)params->LRU_q->eviction_params)->q_head; + cache_obj_t *obj_q = ((LRU_params_t *)params->LRU_q->eviction_params)->q_head; while (obj_q) { - printf("%ld(%u, %s, %s)->", obj_q->obj_id, obj_q->obj_size, - obj_q->LIRS.in_cache ? "R" : "N", - obj_q->LIRS.is_LIR ? "L" : "H"); + printf("%ld(%u, %s, %s)->", (long)obj_q->obj_id, obj_q->obj_size, + obj_q->LIRS.in_cache ? "R" : "N", obj_q->LIRS.is_LIR ? "L" : "H"); obj_q = obj_q->queue.next; } printf("\n"); printf("NH Stack: \n"); cache_obj_t *obj_nh = - ((LRU_params_t *)params->LRU_nh->eviction_params)->q_head; + ((LRU_params_t *)params->LRU_nh->eviction_params)->q_head; while (obj_nh) { - printf("%ld(%u, %s, %s)->", obj_nh->obj_id, obj_nh->obj_size, - obj_nh->LIRS.in_cache ? "R" : "N", - obj_nh->LIRS.is_LIR ? "L" : "H"); + printf("%ld(%u, %s, %s)->", (long)obj_nh->obj_id, obj_nh->obj_size, + obj_nh->LIRS.in_cache ? "R" : "N", obj_nh->LIRS.is_LIR ? "L" : "H"); obj_nh = obj_nh->queue.next; } printf("\n\n"); - } static void LIRS_print_cache_compared_to_cacheus(cache_t *cache) { - LIRS_params_t *params = (LIRS_params_t *)cache->eviction_params; + LIRS_params_t *params = (LIRS_params_t *)cache->eviction_params; printf("S:\n"); - cache_obj_t *obj = - ((LRU_params_t *)params->LRU_s->eviction_params)->q_tail; + cache_obj_t *obj = ((LRU_params_t *)params->LRU_s->eviction_params)->q_tail; while (obj) { - printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", - obj->obj_id, - obj->LIRS.is_LIR ? "True" : "False", - obj->LIRS.in_cache ? "True" : "False"); + printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", (long)obj->obj_id, + obj->LIRS.is_LIR ? "True" : "False", + obj->LIRS.in_cache ? "True" : "False"); obj = obj->queue.prev; } printf("Q:\n"); - cache_obj_t *obj_q = - ((LRU_params_t *)params->LRU_q->eviction_params)->q_tail; + cache_obj_t *obj_q = ((LRU_params_t *)params->LRU_q->eviction_params)->q_tail; while (obj_q) { - printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", - obj_q->obj_id, - obj_q->LIRS.is_LIR ? "True" : "False", - obj_q->LIRS.in_cache ? "True" : "False"); + printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", (long)obj_q->obj_id, + obj_q->LIRS.is_LIR ? "True" : "False", + obj_q->LIRS.in_cache ? "True" : "False"); obj_q = obj_q->queue.prev; } printf("NH:\n"); cache_obj_t *obj_nh = - ((LRU_params_t *)params->LRU_nh->eviction_params)->q_tail; + ((LRU_params_t *)params->LRU_nh->eviction_params)->q_tail; while (obj_nh) { - printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", - obj_nh->obj_id, - obj_nh->LIRS.is_LIR ? "True" : "False", - obj_nh->LIRS.in_cache ? "True" : "False"); + printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", (long)obj_nh->obj_id, + obj_nh->LIRS.is_LIR ? "True" : "False", + obj_nh->LIRS.in_cache ? "True" : "False"); obj_nh = obj_nh->queue.prev; } printf("\n"); - } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/LRU.c b/libCacheSim/cache/eviction/LRU.c index f7f74c9c..95dc2de6 100644 --- a/libCacheSim/cache/eviction/LRU.c +++ b/libCacheSim/cache/eviction/LRU.c @@ -48,7 +48,8 @@ static void LRU_print_cache(const cache_t *cache); */ cache_t *LRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("LRU", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("LRU", ccache_params, cache_specific_params); cache->cache_init = LRU_init; cache->cache_free = LRU_free; cache->get = LRU_get; @@ -207,8 +208,8 @@ static void LRU_evict(cache_t *cache, const request_t *req) { #if defined(TRACK_DEMOTION) if (cache->track_demotion) - printf("%ld demote %ld %ld\n", cache->n_req, obj_to_evict->create_time, - obj_to_evict->misc.next_access_vtime); + printf("%ld demote %ld %ld\n", cache->n_req, obj_to_evict->create_time, + obj_to_evict->misc.next_access_vtime); #endif cache_evict_base(cache, obj_to_evict, true); @@ -273,7 +274,7 @@ static void LRU_print_cache(const cache_t *cache) { return; } while (cur != NULL) { - printf("%lu->", cur->obj_id); + printf("%lu->", (unsigned long)cur->obj_id); cur = cur->queue.next; } printf("END\n"); diff --git a/libCacheSim/cache/eviction/LeCaR.c b/libCacheSim/cache/eviction/LeCaR.c index 5cba6bf5..915dfdc7 100644 --- a/libCacheSim/cache/eviction/LeCaR.c +++ b/libCacheSim/cache/eviction/LeCaR.c @@ -585,8 +585,8 @@ bool LeCaR_remove(cache_t *cache, obj_id_t obj_id) { static const char *LeCaR_current_params(cache_t *cache, LeCaR_params_t *params) { static __thread char params_str[128]; - int n = snprintf(params_str, 128, "update-weight=%d,lru-weight=%.lf", - params->update_weight, params->w_lru); + snprintf(params_str, 128, "update-weight=%d,lru-weight=%.lf", + params->update_weight, params->w_lru); return params_str; } diff --git a/libCacheSim/cache/eviction/LeCaRv0.c b/libCacheSim/cache/eviction/LeCaRv0.c index c38a9641..07e36100 100644 --- a/libCacheSim/cache/eviction/LeCaRv0.c +++ b/libCacheSim/cache/eviction/LeCaRv0.c @@ -247,6 +247,7 @@ static cache_obj_t *LeCaRv0_to_evict(cache_t *cache, const request_t *req) { } else { params->LFU->to_evict(params->LFU, req); } + return NULL; } /** diff --git a/libCacheSim/cache/eviction/QDLP.c b/libCacheSim/cache/eviction/QDLP.c index 43e57147..8c52eed9 100644 --- a/libCacheSim/cache/eviction/QDLP.c +++ b/libCacheSim/cache/eviction/QDLP.c @@ -440,7 +440,6 @@ static void QDLP_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/S3FIFO.c b/libCacheSim/cache/eviction/S3FIFO.c index d80f80f6..8fcaf490 100644 --- a/libCacheSim/cache/eviction/S3FIFO.c +++ b/libCacheSim/cache/eviction/S3FIFO.c @@ -297,7 +297,7 @@ static cache_obj_t *S3FIFO_insert(cache_t *cache, const request_t *req) { obj->create_time = cache->n_req; #endif - obj->S3FIFO.freq == 0; + obj->S3FIFO.freq = 0; return obj; } @@ -371,8 +371,8 @@ static void S3FIFO_evict_fifo(cache_t *cache, const request_t *req) { static void S3FIFO_evict_main(cache_t *cache, const request_t *req) { S3FIFO_params_t *params = (S3FIFO_params_t *)cache->eviction_params; - cache_t *fifo = params->fifo; - cache_t *ghost = params->fifo_ghost; + // cache_t *fifo = params->fifo; + // cache_t *ghost = params->fifo_ghost; cache_t *main = params->main_cache; // evict from main cache @@ -408,7 +408,7 @@ static void S3FIFO_evict_main(cache_t *cache, const request_t *req) { // main->evict(main, req); bool removed = main->remove(main, obj_to_evict->obj_id); if (!removed) { - ERROR("cannot remove obj %ld\n", obj_to_evict->obj_id); + ERROR("cannot remove obj %ld\n", (long)obj_to_evict->obj_id); } has_evicted = true; @@ -429,7 +429,7 @@ static void S3FIFO_evict(cache_t *cache, const request_t *req) { S3FIFO_params_t *params = (S3FIFO_params_t *)cache->eviction_params; cache_t *fifo = params->fifo; - cache_t *ghost = params->fifo_ghost; + // cache_t *ghost = params->fifo_ghost; cache_t *main = params->main_cache; if (main->get_occupied_byte(main) > main->cache_size || @@ -499,7 +499,7 @@ static void S3FIFO_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/S3FIFOd.c b/libCacheSim/cache/eviction/S3FIFOd.c index a9ec28e3..28859b69 100644 --- a/libCacheSim/cache/eviction/S3FIFOd.c +++ b/libCacheSim/cache/eviction/S3FIFOd.c @@ -358,7 +358,7 @@ static cache_obj_t *S3FIFOd_insert(cache_t *cache, const request_t *req) { /* insert into the ARC */ params->hit_on_ghost = false; params->main_cache->get(params->main_cache, req); - obj = obj = params->main_cache->find(params->main_cache, req, false); + obj = params->main_cache->find(params->main_cache, req, false); } else { /* insert into the fifo */ obj = params->fifo->insert(params->fifo, req); @@ -435,7 +435,7 @@ static void S3FIFOd_evict(cache_t *cache, const request_t *req) { while (main->get_occupied_byte(main) > main->cache_size) { // evict from main cache - cache_obj_t *obj = main->to_evict(main, req); + obj = main->to_evict(main, req); copy_cache_obj_to_request(params->req_local, obj); params->main_cache_eviction->get(params->main_cache_eviction, params->req_local); @@ -463,7 +463,7 @@ static void S3FIFOd_evict(cache_t *cache, const request_t *req) { while (main->get_occupied_byte(main) > main->cache_size) { // evict from main cache - cache_obj_t *obj = main->to_evict(main, req); + obj = main->to_evict(main, req); copy_cache_obj_to_request(params->req_local, obj); params->main_cache_eviction->get(params->main_cache_eviction, params->req_local); @@ -536,7 +536,7 @@ static void S3FIFOd_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/SLRU.c b/libCacheSim/cache/eviction/SLRU.c index 5f51b6ef..cee57918 100644 --- a/libCacheSim/cache/eviction/SLRU.c +++ b/libCacheSim/cache/eviction/SLRU.c @@ -230,6 +230,7 @@ static bool SLRU_get(cache_t *cache, const request_t *req) { #else bool ck = cache_get_base(cache, req); #endif + return ck; } /** @@ -546,9 +547,9 @@ static void _SLRU_verify_lru_size(cache_t *cache) { } bool SLRU_get_debug(cache_t *cache, const request_t *req) { + // SLRU_params_t *params = (SLRU_params_t *)(cache->eviction_params); cache->n_req += 1; - SLRU_params_t *params = (SLRU_params_t *)(cache->eviction_params); DEBUG_PRINT_CACHE_STATE(cache, params, req); bool cache_hit = cache->find(cache, req, true) != NULL; diff --git a/libCacheSim/cache/eviction/SLRUv0.c b/libCacheSim/cache/eviction/SLRUv0.c index 7b1be481..0530ba5b 100644 --- a/libCacheSim/cache/eviction/SLRUv0.c +++ b/libCacheSim/cache/eviction/SLRUv0.c @@ -259,6 +259,7 @@ static cache_obj_t *SLRUv0_to_evict(cache_t *cache, const request_t *req) { return lru->to_evict(lru, req); } } + return NULL; } /** @@ -432,7 +433,7 @@ static void SLRUv0_print_cache(cache_t *cache) { cache_obj_t *obj = ((LRU_params_t *)params->LRUs[i]->eviction_params)->q_head; while (obj) { - printf("%ld(%u)->", obj->obj_id, obj->obj_size); + printf("%ld(%u)->", (long) obj->obj_id, obj->obj_size); obj = obj->queue.next; } printf(" | "); diff --git a/libCacheSim/cache/eviction/SR_LRU.c b/libCacheSim/cache/eviction/SR_LRU.c index 47fc6e4c..254885e2 100644 --- a/libCacheSim/cache/eviction/SR_LRU.c +++ b/libCacheSim/cache/eviction/SR_LRU.c @@ -114,7 +114,7 @@ static void SR_LRU_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool SR_LRU_get(cache_t *cache, const request_t *req) { - SR_LRU_params_t *params = (SR_LRU_params_t *)(cache->eviction_params); + // SR_LRU_params_t *params = (SR_LRU_params_t *)(cache->eviction_params); bool cache_hit = SR_LRU_find(cache, req, true) != NULL; if (!cache_hit) { diff --git a/libCacheSim/cache/eviction/Sieve.c b/libCacheSim/cache/eviction/Sieve.c index 6764904c..ab256931 100644 --- a/libCacheSim/cache/eviction/Sieve.c +++ b/libCacheSim/cache/eviction/Sieve.c @@ -102,10 +102,7 @@ static void Sieve_free(cache_t *cache) { */ static bool Sieve_get(cache_t *cache, const request_t *req) { - Sieve_params_t *params = (Sieve_params_t *)cache->eviction_params; - bool ck_hit = cache_get_base(cache, req); - return ck_hit; } @@ -170,7 +167,6 @@ static cache_obj_t *Sieve_to_evict_with_freq(cache_t *cache, int to_evict_freq) { Sieve_params_t *params = cache->eviction_params; cache_obj_t *pointer = params->pointer; - cache_obj_t *old_pointer = pointer; /* if we have run one full around or first eviction */ if (pointer == NULL) pointer = params->q_tail; diff --git a/libCacheSim/cache/eviction/TwoQ.c b/libCacheSim/cache/eviction/TwoQ.c index 4032eb34..1110b839 100644 --- a/libCacheSim/cache/eviction/TwoQ.c +++ b/libCacheSim/cache/eviction/TwoQ.c @@ -342,7 +342,6 @@ static void TwoQ_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/WTinyLFU.c b/libCacheSim/cache/eviction/WTinyLFU.c index 68f83104..386daa72 100644 --- a/libCacheSim/cache/eviction/WTinyLFU.c +++ b/libCacheSim/cache/eviction/WTinyLFU.c @@ -347,8 +347,6 @@ static void WTinyLFU_parse_params(cache_t *cache, // params->max_request_num = 32 * cache->cache_size; // 32 * cache_size char *params_str = strdup(cache_specific_params); - char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, * key and value are separated by = */ diff --git a/libCacheSim/cache/eviction/belady/FIFO_Belady.c b/libCacheSim/cache/eviction/belady/FIFO_Belady.c index 3b679443..9841116b 100644 --- a/libCacheSim/cache/eviction/belady/FIFO_Belady.c +++ b/libCacheSim/cache/eviction/belady/FIFO_Belady.c @@ -159,7 +159,7 @@ static bool should_insert(cache_t *cache, int64_t next_access_vtime) { */ static cache_obj_t *FIFO_Belady_find(cache_t *cache, const request_t *req, const bool update_cache) { - FIFO_Belady_params_t *params = (FIFO_Belady_params_t *)cache->eviction_params; + cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); return cache_obj; @@ -280,7 +280,7 @@ static void FIFO_Belady_print_cache(const cache_t *cache) { return; } while (cur != NULL) { - printf("%lu->", cur->obj_id); + printf("%ld->", (long)cur->obj_id); cur = cur->queue.next; } printf("END\n"); diff --git a/libCacheSim/cache/eviction/belady/LRU_Belady.c b/libCacheSim/cache/eviction/belady/LRU_Belady.c index 495fa80f..5aa449fe 100644 --- a/libCacheSim/cache/eviction/belady/LRU_Belady.c +++ b/libCacheSim/cache/eviction/belady/LRU_Belady.c @@ -295,7 +295,7 @@ static void LRU_Belady_print_cache(const cache_t *cache) { return; } while (cur != NULL) { - printf("%lu->", cur->obj_id); + printf("%ld->", (long)cur->obj_id); cur = cur->queue.next; } printf("END\n"); diff --git a/libCacheSim/cache/eviction/cpp/abstractRank.hpp b/libCacheSim/cache/eviction/cpp/abstractRank.hpp index 60fb63c1..a309c562 100644 --- a/libCacheSim/cache/eviction/cpp/abstractRank.hpp +++ b/libCacheSim/cache/eviction/cpp/abstractRank.hpp @@ -12,9 +12,9 @@ #include #include +#include "../../../dataStructure/hashtable/hashtable.h" #include "../../../include/libCacheSim/cache.h" #include "../../../include/libCacheSim/cacheObj.h" -#include "../../../dataStructure/hashtable/hashtable.h" using namespace std; @@ -34,8 +34,8 @@ struct pq_node_type { : obj(obj), priority(priority), last_request_vtime(last_request_vtime){}; void print() const { - printf("obj %lu, priority %f, last_request_vtime %ld\n", obj->obj_id, - priority, last_request_vtime); + printf("obj %lu, priority %f, last_request_vtime %ld\n", + (unsigned long)obj->obj_id, priority, (long)last_request_vtime); } bool operator<(const pq_node_type &rhs) const { diff --git a/libCacheSim/cache/eviction/fifo/LP_ARC.c b/libCacheSim/cache/eviction/fifo/LP_ARC.c index 7bdc8b5d..b1f68127 100644 --- a/libCacheSim/cache/eviction/fifo/LP_ARC.c +++ b/libCacheSim/cache/eviction/fifo/LP_ARC.c @@ -159,7 +159,7 @@ static void LP_ARC_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool LP_ARC_get(cache_t *cache, const request_t *req) { - LP_ARC_params_t *params = (LP_ARC_params_t *)(cache->eviction_params); + // LP_ARC_params_t *params = (LP_ARC_params_t *)(cache->eviction_params); return LP_ARC_get_debug(cache, req); // return cache_get_base(cache, req); @@ -485,13 +485,13 @@ static void LP_ARC_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, * key and value are separated by = */ char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); + // char *value = strsep((char **)¶ms_str, ","); // skip the white space while (params_str != NULL && *params_str == ' ') { @@ -528,7 +528,7 @@ static void print_cache(cache_t *cache) { } static bool LP_ARC_get_debug(cache_t *cache, const request_t *req) { - LP_ARC_params_t *params = (LP_ARC_params_t *)(cache->eviction_params); + // LP_ARC_params_t *params = (LP_ARC_params_t *)(cache->eviction_params); cache->n_req += 1; diff --git a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c index 6f903bec..72a77a18 100644 --- a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c +++ b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c @@ -137,7 +137,7 @@ static void LP_SFIFO_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool LP_SFIFO_get(cache_t *cache, const request_t *req) { - LP_SFIFO_params_t *params = (LP_SFIFO_params_t *)(cache->eviction_params); + // LP_SFIFO_params_t *params = (LP_SFIFO_params_t *)(cache->eviction_params); // printf("%ld %ld %ld %ld %ld\n", cache->n_req, // params->fifos[0]->get_n_obj(params->fifos[0]), // params->fifos[1]->get_n_obj(params->fifos[1]), @@ -243,6 +243,7 @@ static cache_obj_t *LP_SFIFO_to_evict(cache_t *cache, const request_t *req) { return params->fifos[i]->to_evict(params->fifos[i], req); } } + return NULL; } /** diff --git a/libCacheSim/cache/eviction/fifo/LP_TwoQ.c b/libCacheSim/cache/eviction/fifo/LP_TwoQ.c index 66310389..612348e1 100644 --- a/libCacheSim/cache/eviction/fifo/LP_TwoQ.c +++ b/libCacheSim/cache/eviction/fifo/LP_TwoQ.c @@ -348,7 +348,6 @@ static void LP_TwoQ_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/fifo/SFIFO.c b/libCacheSim/cache/eviction/fifo/SFIFO.c index 36c13d20..6ec113da 100644 --- a/libCacheSim/cache/eviction/fifo/SFIFO.c +++ b/libCacheSim/cache/eviction/fifo/SFIFO.c @@ -291,6 +291,7 @@ static cache_obj_t *SFIFO_to_evict(cache_t *cache, const request_t *req) { return params->fifo_tails[i]; } } + return NULL; } /** @@ -512,7 +513,7 @@ static void _SFIFO_verify_fifo_size(cache_t *cache) { bool SFIFO_get_debug(cache_t *cache, const request_t *req) { cache->n_req += 1; - SFIFO_params_t *params = (SFIFO_params_t *)(cache->eviction_params); + // SFIFO_params_t *params = (SFIFO_params_t *)(cache->eviction_params); cache->last_request_metadata = "none"; DEBUG_PRINT_CACHE_STATE(cache, params, req); diff --git a/libCacheSim/cache/eviction/fifo/SFIFOv0.c b/libCacheSim/cache/eviction/fifo/SFIFOv0.c index f21a60b3..c08102df 100644 --- a/libCacheSim/cache/eviction/fifo/SFIFOv0.c +++ b/libCacheSim/cache/eviction/fifo/SFIFOv0.c @@ -11,7 +11,7 @@ // // // we notice that adding belady is not helpful -// +// // // libCacheSim // @@ -72,7 +72,8 @@ static inline int64_t SFIFOv0_get_n_obj(const cache_t *cache); */ cache_t *SFIFOv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("SFIFOv0", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("SFIFOv0", ccache_params, cache_specific_params); cache->cache_init = SFIFOv0_init; cache->cache_free = SFIFOv0_free; cache->get = SFIFOv0_get; @@ -272,6 +273,7 @@ static cache_obj_t *SFIFOv0_to_evict(cache_t *cache, const request_t *req) { } } #endif + return NULL; } /** @@ -303,7 +305,6 @@ static void SFIFOv0_evict(cache_t *cache, const request_t *req) { fifo->evict(fifo, req); #else - int nth_seg_to_evict = 0; for (int i = 0; i < params->n_queues; i++) { cache_t *fifo = params->FIFOs[i]; if (fifo->get_occupied_byte(fifo) > 0) { @@ -459,7 +460,7 @@ static void SFIFOv0_print_cache(cache_t *cache) { cache_obj_t *obj = ((FIFO_params_t *)params->FIFOs[i]->eviction_params)->q_head; while (obj) { - printf("%ld(%u)->", obj->obj_id, obj->obj_size); + printf("%ld(%u)->", (long)obj->obj_id, (unsigned int)obj->obj_size); obj = obj->queue.next; } printf(" | "); diff --git a/libCacheSim/cache/eviction/other/S3LRU.c b/libCacheSim/cache/eviction/other/S3LRU.c index 5b9e495d..5cf499fe 100644 --- a/libCacheSim/cache/eviction/other/S3LRU.c +++ b/libCacheSim/cache/eviction/other/S3LRU.c @@ -288,8 +288,8 @@ static cache_obj_t *S3LRU_insert(cache_t *cache, const request_t *req) { } else { /* insert into the LRU */ if (req->obj_size > params->LRU->cache_size) { - WARN("object size %ld larger than small cache size %ld\n", req->obj_size, - params->LRU->cache_size); + WARN("object size %ld larger than small cache size %ld\n", + (long)req->obj_size, (long)params->LRU->cache_size); return NULL; } obj = params->LRU->insert(params->LRU, req); @@ -303,7 +303,7 @@ static cache_obj_t *S3LRU_insert(cache_t *cache, const request_t *req) { obj->create_time = cache->n_req; #endif - obj->S3FIFO.freq == 0; + obj->S3FIFO.freq = 0; return obj; } @@ -362,8 +362,8 @@ static void S3LRU_evict_LRU(cache_t *cache, const request_t *req) { static void S3LRU_evict_main(cache_t *cache, const request_t *req) { S3LRU_params_t *params = (S3LRU_params_t *)cache->eviction_params; - cache_t *LRU = params->LRU; - cache_t *ghost = params->LRU_ghost; + // cache_t *LRU = params->LRU; + // cache_t *ghost = params->LRU_ghost; cache_t *main = params->main_cache; // evict from main cache @@ -373,7 +373,7 @@ static void S3LRU_evict_main(cache_t *cache, const request_t *req) { DEBUG_ASSERT(obj_to_evict != NULL); bool removed = main->remove(main, obj_to_evict->obj_id); if (!removed) { - ERROR("cannot remove obj %ld\n", obj_to_evict->obj_id); + ERROR("cannot remove obj %ld\n", (long)obj_to_evict->obj_id); } has_evicted = true; @@ -393,7 +393,7 @@ static void S3LRU_evict(cache_t *cache, const request_t *req) { S3LRU_params_t *params = (S3LRU_params_t *)cache->eviction_params; cache_t *LRU = params->LRU; - cache_t *ghost = params->LRU_ghost; + // cache_t *ghost = params->LRU_ghost; cache_t *main = params->main_cache; if (main->get_occupied_byte(main) > main->cache_size || @@ -465,7 +465,7 @@ static void S3LRU_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/other/flashProb.c b/libCacheSim/cache/eviction/other/flashProb.c index d625c439..99e61912 100644 --- a/libCacheSim/cache/eviction/other/flashProb.c +++ b/libCacheSim/cache/eviction/other/flashProb.c @@ -179,7 +179,6 @@ static void flashProb_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool flashProb_get(cache_t *cache, const request_t *req) { - flashProb_params_t *params = (flashProb_params_t *)cache->eviction_params; bool cache_hit = cache_get_base(cache, req); @@ -368,7 +367,6 @@ static void flashProb_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/priv/MClock.c b/libCacheSim/cache/eviction/priv/MClock.c index e8dea550..f5ae1e38 100644 --- a/libCacheSim/cache/eviction/priv/MClock.c +++ b/libCacheSim/cache/eviction/priv/MClock.c @@ -224,7 +224,7 @@ static void print_curr_hand_pos(cache_t *cache) { int64_t obj_idx = 0; while (obj != NULL) { if (obj == params->hands[hand_idx]) { - printf("hand %d: %ld %p %p %p\n", hand_idx, obj_idx, obj, obj->queue.prev, + printf("hand %d: %ld %p %p %p\n", hand_idx, (long) obj_idx, obj, obj->queue.prev, obj->queue.next); hand_idx++; } @@ -392,13 +392,13 @@ static bool MClock_remove(cache_t *cache, const obj_id_t obj_id) { } static int64_t MClock_get_occupied_byte(const cache_t *cache) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; + // MClock_params_t *params = (MClock_params_t *)cache->eviction_params; int64_t occupied_byte = cache->occupied_byte; return occupied_byte; } static int64_t MClock_get_n_obj(const cache_t *cache) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; + // MClock_params_t *params = (MClock_params_t *)cache->eviction_params; int64_t n_obj = cache->n_obj; return n_obj; } diff --git a/libCacheSim/cache/eviction/priv/MyClock.c b/libCacheSim/cache/eviction/priv/MyClock.c index 1ae65936..7768d364 100644 --- a/libCacheSim/cache/eviction/priv/MyClock.c +++ b/libCacheSim/cache/eviction/priv/MyClock.c @@ -189,7 +189,7 @@ static cache_obj_t *MyClock_to_evict_with_freq(cache_t *cache, int to_evict_freq) { MyClock_params_t *params = cache->eviction_params; cache_obj_t *pointer = params->pointer; - cache_obj_t *old_pointer = pointer; + // cache_obj_t *old_pointer = pointer; /* if we have run one full around or first eviction */ if (pointer == NULL) pointer = params->q_tail; diff --git a/libCacheSim/cache/eviction/priv/S3FIFOdv2.c b/libCacheSim/cache/eviction/priv/S3FIFOdv2.c index f9a0509a..6598a07e 100644 --- a/libCacheSim/cache/eviction/priv/S3FIFOdv2.c +++ b/libCacheSim/cache/eviction/priv/S3FIFOdv2.c @@ -81,18 +81,18 @@ static const char *DEFAULT_CACHE_PARAMS = // **** **** // *********************************************************************** cache_t *S3FIFOdv2_init(const common_cache_params_t ccache_params, - const char *cache_specific_params); + const char *cache_specific_params); static void S3FIFOdv2_free(cache_t *cache); static bool S3FIFOdv2_get(cache_t *cache, const request_t *req); -static bool S3FIFOdv2_rebalance(cache_t *cache); +static void S3FIFOdv2_rebalance(cache_t *cache); static void S3FIFOdv2_choose_threshold(cache_t *cache); static void S3FIFOdv2_update_main_reinsert_threshold(cache_t *cache); static void S3FIFOdv2_update_main_reinsert_threshold_incorrect(cache_t *cache); static void S3FIFOdv2_update_small_reinsert_threshold(cache_t *cache); static cache_obj_t *S3FIFOdv2_find(cache_t *cache, const request_t *req, - const bool update_cache); + const bool update_cache); static cache_obj_t *S3FIFOdv2_insert(cache_t *cache, const request_t *req); static cache_obj_t *S3FIFOdv2_to_evict(cache_t *cache, const request_t *req); static void S3FIFOdv2_evict(cache_t *cache, const request_t *req); @@ -101,7 +101,7 @@ static inline int64_t S3FIFOdv2_get_occupied_byte(const cache_t *cache); static inline int64_t S3FIFOdv2_get_n_obj(const cache_t *cache); static inline bool S3FIFOdv2_can_insert(cache_t *cache, const request_t *req); static void S3FIFOdv2_parse_params(cache_t *cache, - const char *cache_specific_params); + const char *cache_specific_params); static void S3FIFOdv2_evict_fifo(cache_t *cache, const request_t *req); static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req); @@ -113,7 +113,7 @@ static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req); // *********************************************************************** cache_t *S3FIFOdv2_init(const common_cache_params_t ccache_params, - const char *cache_specific_params) { + const char *cache_specific_params) { cache_t *cache = cache_struct_init("S3FIFOdv2", ccache_params, cache_specific_params); cache->cache_init = S3FIFOdv2_init; @@ -294,7 +294,7 @@ static void S3FIFOdv2_update_main_reinsert_threshold_incorrect(cache_t *cache) { n_obj += params->main_freq_cnt[i]; } if (n_obj != n_main_obj) - ERROR("n_obj %ld != n_main_obj %d\n", n_obj, n_main_obj); + ERROR("n_obj %ld != n_main_obj %d\n", (long)n_obj, n_main_obj); // if (params->move_to_main_threshold > 1) { // printf("set threshold to %d, ", params->move_to_main_threshold); @@ -335,7 +335,7 @@ static void S3FIFOdv2_update_main_reinsert_threshold(cache_t *cache) { n_obj += params->main_freq_cnt[i]; } if (n_obj != n_main_obj) - ERROR("n_obj %ld != n_main_obj %d\n", n_obj, n_main_obj); + ERROR("n_obj %ld != n_main_obj %d\n", (long)n_obj, n_main_obj); // if (params->main_reinsert_threshold > 1) { // printf("set threshold to %d, ", params->main_reinsert_threshold); @@ -376,7 +376,7 @@ static void S3FIFOdv2_update_small_reinsert_threshold(cache_t *cache) { n_obj += params->small_freq_cnt[i]; } if (n_obj != n_small_obj) - ERROR("n_obj %ld != n_small_obj %d\n", n_obj, n_small_obj); + ERROR("n_obj %ld != n_small_obj %d\n", (long)n_obj, n_small_obj); // if (params->move_to_main_threshold > 1) { // printf("set threshold to %d, ", params->move_to_main_threshold); @@ -414,8 +414,8 @@ static void S3FIFOdv2_choose_threshold(cache_t *cache) { if (params->move_to_main_threshold != 1) { for (int i = 0; i < MAX_FREQ_THRESHOLD; i++) { - printf("%8ld/%-4d ", params->eviction_freq_cnt[i], - params->threshold_cnt[i]); + printf("%8ld/%-4d ", (long)params->eviction_freq_cnt[i], + (int)params->threshold_cnt[i]); } printf("move_to_main_threshold: %d\n", params->move_to_main_threshold); @@ -470,18 +470,18 @@ static bool S3FIFOdv2_rebalance0(cache_t *cache) { } memset(params->first_access_cnt_over_age_fifo, 0, - sizeof(params->first_access_cnt_over_age_fifo)); + sizeof(*params->first_access_cnt_over_age_fifo)); memset(params->first_access_cnt_over_age_main, 0, - sizeof(params->first_access_cnt_over_age_main)); + sizeof(*params->first_access_cnt_over_age_main)); return true; } -static bool S3FIFOdv2_rebalance(cache_t *cache) { +static void S3FIFOdv2_rebalance(cache_t *cache) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - double s = params->n_obj_admit_to_fifo + params->n_obj_admit_to_main + - params->n_obj_move_to_main; + // double s = params->n_obj_admit_to_fifo + params->n_obj_admit_to_main + + // params->n_obj_move_to_main; size_t n_slots = MIN(params->fifo->cache_size, params->main_cache->cache_size) / 100 + 1; @@ -526,7 +526,7 @@ static bool S3FIFOdv2_rebalance(cache_t *cache) { * @return the object or NULL if not found */ static cache_obj_t *S3FIFOdv2_find(cache_t *cache, const request_t *req, - const bool update_cache) { + const bool update_cache) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; // if update cache is false, we only check the fifo and main caches @@ -635,7 +635,7 @@ static cache_obj_t *S3FIFOdv2_insert(cache_t *cache, const request_t *req) { obj->create_time = cache->n_req; #endif - obj->S3FIFO.freq == 0; + obj->S3FIFO.freq = 0; return obj; } @@ -716,8 +716,8 @@ static void S3FIFOdv2_evict_fifo(cache_t *cache, const request_t *req) { static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - cache_t *fifo = params->fifo; - cache_t *ghost = params->fifo_ghost; + // cache_t *fifo = params->fifo; + // cache_t *ghost = params->fifo_ghost; cache_t *main = params->main_cache; // evict from main cache @@ -755,7 +755,7 @@ static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req) { // main->evict(main, req); bool removed = main->remove(main, obj_to_evict->obj_id); if (!removed) { - ERROR("cannot remove obj %ld\n", obj_to_evict->obj_id); + ERROR("cannot remove obj %ld\n", (long)obj_to_evict->obj_id); } has_evicted = true; @@ -776,7 +776,7 @@ static void S3FIFOdv2_evict(cache_t *cache, const request_t *req) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; cache_t *fifo = params->fifo; - cache_t *ghost = params->fifo_ghost; + // cache_t *ghost = params->fifo_ghost; cache_t *main = params->main_cache; if (main->get_occupied_byte(main) > main->cache_size || @@ -841,12 +841,12 @@ static const char *S3FIFOdv2_current_params(S3FIFOdv2_params_t *params) { } static void S3FIFOdv2_parse_params(cache_t *cache, - const char *cache_specific_params) { + const char *cache_specific_params) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)(cache->eviction_params); char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/priv/myMQv1.c b/libCacheSim/cache/eviction/priv/myMQv1.c index 55655564..0dde2cc7 100644 --- a/libCacheSim/cache/eviction/priv/myMQv1.c +++ b/libCacheSim/cache/eviction/priv/myMQv1.c @@ -2,8 +2,8 @@ // multi queue // each queue has a specified size and a specified ghost size // promotion upon hit -// -// in progress, this is not finished +// +// in progress, this is not finished // // // myMQv1.c @@ -60,7 +60,6 @@ static int64_t myMQv1_get_n_obj(const cache_t *cache); // **** **** // *********************************************************************** - /** * @brief initialize the cache * @@ -70,7 +69,8 @@ static int64_t myMQv1_get_n_obj(const cache_t *cache); */ cache_t *myMQv1_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("myMQv1", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("myMQv1", ccache_params, cache_specific_params); cache->cache_init = myMQv1_init; cache->cache_free = myMQv1_free; cache->get = myMQv1_get; @@ -169,7 +169,6 @@ static cache_obj_t *myMQv1_find(cache_t *cache, const request_t *req, cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); ERROR("todo"); - return cache_obj; } @@ -185,12 +184,11 @@ static cache_obj_t *myMQv1_find(cache_t *cache, const request_t *req, * @return the inserted object */ static cache_obj_t *myMQv1_insert(cache_t *cache, const request_t *req) { - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; + // myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; cache_obj_t *obj = cache_insert_base(cache, req); ERROR("todo"); - return obj; } @@ -219,11 +217,9 @@ static cache_obj_t *myMQv1_to_evict(cache_t *cache, const request_t *req) { * @param evicted_obj if not NULL, return the evicted object to caller */ static void myMQv1_evict(cache_t *cache, const request_t *req) { - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; + // myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; ERROR("todo"); - - } /** @@ -278,7 +274,7 @@ static const char *myMQv1_current_params(cache_t *cache, myMQv1_params_t *params) { static __thread char params_str[128]; int n = snprintf(params_str, 128, "n-caches=%ld;cache-size-ratio=%d", - params->n_caches, + (long)params->n_caches, (int)(params->cache_sizes[0] * 100 / cache->cache_size)); for (int i = 1; i < params->n_caches; i++) { @@ -342,7 +338,6 @@ static void myMQv1_parse_params(cache_t *cache, } } else if (strcasecmp(key, "ghost-size-ratio") == 0) { int n_ghost_caches = 0; - int64_t cache_size_sum = 0; int64_t ghost_size_array[myMQv1_MAX_N_cache]; char *v = strsep((char **)&value, ":"); while (v != NULL) { diff --git a/libCacheSim/cache/prefetch/Mithril.c b/libCacheSim/cache/prefetch/Mithril.c index 9678340a..ff1a4726 100644 --- a/libCacheSim/cache/prefetch/Mithril.c +++ b/libCacheSim/cache/prefetch/Mithril.c @@ -8,8 +8,6 @@ // Created by Zhelong on 23/8/15. // Copyright © 2023 Zhelong. All rights reserved. // -#include "../../include/libCacheSim/prefetchAlgo/Mithril.h" - #include #include #include @@ -19,6 +17,7 @@ #include #include "../../include/libCacheSim/prefetchAlgo.h" +#include "../../include/libCacheSim/prefetchAlgo/Mithril.h" #include "glibconfig.h" #define TRACK_BLOCK 192618l @@ -529,7 +528,7 @@ static inline void _Mithril_rec_min_support_one(cache_t *cache, printf("\n"); else printf(", block at old_pos %ld\n", - *(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1)); + (long)*(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1)); } else { gint64 b = TRACK_BLOCK; @@ -537,8 +536,9 @@ static inline void _Mithril_rec_min_support_one(cache_t *cache, g_hash_table_lookup(rmtable->hashtable, GINT_TO_POINTER(b))); if (old_pos != 0) { ERROR("ts %lu, checking %ld, %ld is found at pos %d\n", - Mithril_params->ts, TRACK_BLOCK, - *(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1), old_pos); + (unsigned long)Mithril_params->ts, (long)TRACK_BLOCK, + (long)*(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1), + old_pos); abort(); } } @@ -571,7 +571,7 @@ static inline void _Mithril_rec_min_support_one(cache_t *cache, GET_ROW_IN_MTABLE(Mithril_params, rmtable->mining_table->len - 1); if (req->obj_id != row_in_mtable[0]) { ERROR("after inserting, hashtable mining not consistent %ld %ld\n", - req->obj_id, row_in_mtable[0]); + (long)req->obj_id, (long)row_in_mtable[0]); abort(); } #endif @@ -582,7 +582,8 @@ static inline void _Mithril_rec_min_support_one(cache_t *cache, #ifdef SANITY_CHECK if (req->obj_id != row_in_mtable[0]) { ERROR("ts %lu, hashtable mining found position not correct %ld %ld\n", - Mithril_params->ts, req->obj_id, row_in_mtable[0]); + (unsigned long)Mithril_params->ts, (long)req->obj_id, + (long)row_in_mtable[0]); abort(); } #endif @@ -683,7 +684,7 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { "remove old entry from recording table, " "but it is not in recording hashtable, " "block %ld, recording table pos %ld, ts %ld ", - row_in_rtable[0], (long)rmtable->rtable_cur_row, + (long)row_in_rtable[0], (long)rmtable->rtable_cur_row, (long)Mithril_params->ts); long temp = rmtable->rtable_cur_row - 1; @@ -719,7 +720,7 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { "inconsistent entry in mtable " "and mining hashtable current request %ld, " "mining table %ld\n", - req->obj_id, row_in_mtable[0]); + (long)req->obj_id, (long)row_in_mtable[0]); abort(); } #endif @@ -767,7 +768,7 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { #ifdef SANITY_CHECK if (req->obj_id != row_in_rtable[0]) { ERROR("Hashtable recording found position not correct %ld %ld\n", - req->obj_id, row_in_rtable[0]); + (long)req->obj_id, (long) row_in_rtable[0]); abort(); } #endif @@ -819,9 +820,9 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { #ifdef SANITY_CHECK if (row_in_rtable == cur_row_in_rtable) ERROR("FOUND SRC DEST same, ts %ld %p %p %ld %ld %d %ld\n", - Mithril_params->ts, row_in_rtable, cur_row_in_rtable, - *row_in_rtable, *cur_row_in_rtable, index, - rmtable->rtable_cur_row - 1); + (long)Mithril_params->ts, row_in_rtable, cur_row_in_rtable, + (long)*row_in_rtable, (long)*cur_row_in_rtable, index, + (long)rmtable->rtable_cur_row - 1); #endif memcpy(row_in_rtable, cur_row_in_rtable, sizeof(TS_REPRESENTATION) * rmtable->rtable_row_len); @@ -843,7 +844,7 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { #ifdef SANITY_CHECK if (inserted_row_in_mtable[0] != (gint64)req->obj_id) { ERROR("current block %ld, moving mining row block %ld\n", - (gint64)req->obj_id, inserted_row_in_mtable[0]); + (long)req->obj_id, (long)inserted_row_in_mtable[0]); abort(); } #endif @@ -906,7 +907,7 @@ void print_one_line(gpointer key, gpointer value, gpointer user_data) { (Mithril_params->pf_list_size + 1); printf("src %d, prefetch ", src_key); for (int i = 1; i < Mithril_params->pf_list_size + 1; i++) { - printf("%ld ", Mithril_params->ptable_array[dim1][dim2 + i]); + printf("%ld ", (long)Mithril_params->ptable_array[dim1][dim2 + i]); } printf("\n"); } @@ -1011,7 +1012,8 @@ static void _Mithril_mining(cache_t *cache) { #ifdef PROFILING printf("ts: %lu, clearing training data takes %lf seconds\n", - Mithril_params->ts, g_timer_elapsed(timer, µsecond)); + (unsigned long)Mithril_params->ts, + g_timer_elapsed(timer, µsecond)); g_timer_stop(timer); g_timer_destroy(timer); #endif @@ -1056,8 +1058,8 @@ static void _Mithril_add_to_prefetch_table(cache_t *cache, gpointer gp1, #ifdef SANITY_CHECK if (Mithril_params->ptable_array[dim1][dim2] != GPOINTER_TO_INT(gp1)) { fprintf(stderr, "ERROR prefetch table pos wrong %d %ld, dim %d %d\n", - GPOINTER_TO_INT(gp1), Mithril_params->ptable_array[dim1][dim2], - dim1, dim2); + GPOINTER_TO_INT(gp1), + (long)Mithril_params->ptable_array[dim1][dim2], dim1, dim2); exit(1); } #endif diff --git a/libCacheSim/dataStructure/bloom.c b/libCacheSim/dataStructure/bloom.c index ae27b58f..14878857 100644 --- a/libCacheSim/dataStructure/bloom.c +++ b/libCacheSim/dataStructure/bloom.c @@ -27,19 +27,16 @@ #include "bloom.h" #include "hash/hash.h" #define XXH_INLINE_ALL -#include "hash/xxhash.h" #include "hash/xxh3.h" - +#include "hash/xxhash.h" #define MAKESTRING(n) STRING(n) #define STRING(n) #n - -inline static int test_bit_set_bit(unsigned char * buf, - unsigned int x, int set_bit) -{ +inline static int test_bit_set_bit(unsigned char *buf, unsigned int x, + int set_bit) { unsigned int byte = x >> 3; - unsigned char c = buf[byte]; // expensive memory access + unsigned char c = buf[byte]; // expensive memory access unsigned int mask = 1 << (x % 8); if (c & mask) { @@ -52,25 +49,23 @@ inline static int test_bit_set_bit(unsigned char * buf, } } - -static int bloom_check_add(struct bloom * bloom, - const void * buffer, int len, int add) -{ +static int bloom_check_add(struct bloom *bloom, const void *buffer, int len, + int add) { if (bloom->ready == 0) { printf("bloom at %p not initialized!\n", (void *)bloom); return -1; } int hits = 0; -// register unsigned int a = murmurhash2(buffer, len, 0x9747b28c); -// register unsigned int b = murmurhash2(buffer, len, a); + // register unsigned int a = murmurhash2(buffer, len, 0x9747b28c); + // register unsigned int b = murmurhash2(buffer, len, a); register unsigned int a = XXH64(buffer, len, HASH_SEED0); register unsigned int b = XXH64(buffer, len, HASH_SEED1); register unsigned int x; register unsigned int i; for (i = 0; i < bloom->hashes; i++) { - x = (a + i*b) % bloom->bits; + x = (a + i * b) % bloom->bits; if (test_bit_set_bit(bloom->bf, x, add)) { hits++; } else if (!add) { @@ -80,22 +75,18 @@ static int bloom_check_add(struct bloom * bloom, } if (hits == bloom->hashes) { - return 1; // 1 == element already in (or collision) + return 1; // 1 == element already in (or collision) } return 0; } - -int bloom_init_size(struct bloom * bloom, int entries, double error, - unsigned int cache_size) -{ +int bloom_init_size(struct bloom *bloom, int entries, double error, + unsigned int cache_size) { return bloom_init(bloom, entries, error); } - -int bloom_init(struct bloom * bloom, int entries, double error) -{ +int bloom_init(struct bloom *bloom, int entries, double error) { bloom->ready = 0; if (entries < 1000 || error == 0) { @@ -106,7 +97,7 @@ int bloom_init(struct bloom * bloom, int entries, double error) bloom->error = error; double num = log(bloom->error); - double denom = 0.480453013918201; // ln(2)^2 + double denom = 0.480453013918201; // ln(2)^2 bloom->bpe = -(num / denom); double dentries = (double)entries; @@ -121,29 +112,23 @@ int bloom_init(struct bloom * bloom, int entries, double error) bloom->hashes = (int)ceil(0.693147180559945 * bloom->bpe); // ln(2) bloom->bf = (unsigned char *)calloc(bloom->bytes, sizeof(unsigned char)); - if (bloom->bf == NULL) { // LCOV_EXCL_START + if (bloom->bf == NULL) { // LCOV_EXCL_START return 1; - } // LCOV_EXCL_STOP + } // LCOV_EXCL_STOP bloom->ready = 1; return 0; } - -int bloom_check(struct bloom * bloom, const void * buffer, int len) -{ +int bloom_check(struct bloom *bloom, const void *buffer, int len) { return bloom_check_add(bloom, buffer, len, 0); } - -int bloom_add(struct bloom * bloom, const void * buffer, int len) -{ +int bloom_add(struct bloom *bloom, const void *buffer, int len) { return bloom_check_add(bloom, buffer, len, 1); } - -void bloom_print(struct bloom * bloom) -{ +void bloom_print(struct bloom *bloom) { printf("bloom at %p\n", (void *)bloom); printf(" ->entries = %d\n", bloom->entries); printf(" ->error = %f\n", bloom->error); @@ -153,25 +138,17 @@ void bloom_print(struct bloom * bloom) printf(" ->hash functions = %d\n", bloom->hashes); } - -void bloom_free(struct bloom * bloom) -{ +void bloom_free(struct bloom *bloom) { if (bloom->ready) { free(bloom->bf); } bloom->ready = 0; } - -int bloom_reset(struct bloom * bloom) -{ +int bloom_reset(struct bloom *bloom) { if (!bloom->ready) return 1; memset(bloom->bf, 0, bloom->bytes); return 0; } - -const char * bloom_version() -{ - return MAKESTRING(BLOOM_VERSION); -} \ No newline at end of file +const char *bloom_version(void) { return MAKESTRING(BLOOM_VERSION); } \ No newline at end of file diff --git a/libCacheSim/dataStructure/bloom.h b/libCacheSim/dataStructure/bloom.h index 3156f087..ddd3d61c 100644 --- a/libCacheSim/dataStructure/bloom.h +++ b/libCacheSim/dataStructure/bloom.h @@ -164,7 +164,7 @@ int bloom_reset(struct bloom * bloom); * Return: version string * */ -const char * bloom_version(); +const char * bloom_version(void); #ifdef __cplusplus } diff --git a/libCacheSim/dataStructure/hash/CMakeLists.txt b/libCacheSim/dataStructure/hash/CMakeLists.txt index 579c4bf8..f636a56e 100644 --- a/libCacheSim/dataStructure/hash/CMakeLists.txt +++ b/libCacheSim/dataStructure/hash/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(. DIR_LIB_SRCS) add_library (ds_hash ${DIR_LIB_SRCS}) - +# add_library (ds_hash murmur3.c) diff --git a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c index 5cc736bd..f5b5beff 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c +++ b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c @@ -23,8 +23,6 @@ extern "C" { #endif -#include "chainedHashTableV2.h" - #include #include #include @@ -36,6 +34,7 @@ extern "C" { #include "../../include/libCacheSim/macro.h" #include "../../utils/include/mymath.h" #include "../hash/hash.h" +#include "chainedHashTableV2.h" #define OBJ_EMPTY(cache_obj) ((cache_obj)->obj_size == 0) #define NEXT_OBJ(cur_obj) (((cache_obj_t *)(cur_obj))->hash_next) @@ -255,7 +254,7 @@ bool chained_hashtable_try_delete_v2(hashtable_t *hashtable, * return true. * - if the object is not in the hash table * return false. - * + * * @method chained_hashtable_delete_obj_id_v2 * @date 2023-11-28 * @param hashtable [Handle to the hashtable] @@ -267,7 +266,7 @@ bool chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); cache_obj_t *cur_obj = hashtable->ptr_table[hv]; // the hash bucket is empty - if(cur_obj == NULL) return false; + if (cur_obj == NULL) return false; // the object to remove is the first object in the hash bucket if (cur_obj->obj_id == obj_id) { @@ -282,7 +281,7 @@ bool chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, do { prev_obj = cur_obj; cur_obj = cur_obj->hash_next; - } while(cur_obj != NULL && cur_obj->obj_id != obj_id); + } while (cur_obj != NULL && cur_obj->obj_id != obj_id); // the object to remove is in the hash bucket if (cur_obj != NULL) { @@ -377,7 +376,8 @@ static int count_n_obj_in_bucket(cache_obj_t *curr_obj) { obj_id_arr[chain_len] = curr_obj->obj_id; for (int i = 0; i < chain_len; i++) { if (obj_id_arr[i] == curr_obj->obj_id) { - ERROR("obj_id %lu is duplicated in hashtable\n", curr_obj->obj_id); + ERROR("obj_id %lu is duplicated in hashtable\n", + (unsigned long)curr_obj->obj_id); abort(); } } diff --git a/libCacheSim/dataStructure/minimalIncrementCBF.c b/libCacheSim/dataStructure/minimalIncrementCBF.c index cbeea2ea..7fbd347f 100644 --- a/libCacheSim/dataStructure/minimalIncrementCBF.c +++ b/libCacheSim/dataStructure/minimalIncrementCBF.c @@ -69,6 +69,7 @@ int minimalIncrementCBF_init(struct minimalIncrementCBF * CBF, int entries, doub int minimalIncrementCBF_init_size(struct minimalIncrementCBF * CBF, int entries, double error, unsigned int cache_size) { minimalIncrementCBF_init(CBF, entries, error); + return 0; } static int minimalIncrementCBF_check_add(struct minimalIncrementCBF * CBF, @@ -167,6 +168,6 @@ int minimalIncrementCBF_decay(struct minimalIncrementCBF * CBF) { return 0; } -const char * minimalIncrementCBF_version() { +const char * minimalIncrementCBF_version(void) { return MAKESTRING(VERSION); } \ No newline at end of file diff --git a/libCacheSim/dataStructure/minimalIncrementCBF.h b/libCacheSim/dataStructure/minimalIncrementCBF.h index ada2227b..5022cb9b 100644 --- a/libCacheSim/dataStructure/minimalIncrementCBF.h +++ b/libCacheSim/dataStructure/minimalIncrementCBF.h @@ -142,7 +142,7 @@ int minimalIncrementCBF_decay(struct minimalIncrementCBF * bloom); * Return: version string * */ -const char * minimalIncrementCBF_version(); +const char * minimalIncrementCBF_version(void); #ifdef __cplusplus } diff --git a/libCacheSim/include/libCacheSim/cache.h b/libCacheSim/include/libCacheSim/cache.h index bfe43f45..bd979367 100644 --- a/libCacheSim/include/libCacheSim/cache.h +++ b/libCacheSim/include/libCacheSim/cache.h @@ -307,9 +307,10 @@ static inline void print_cache_stat(const cache_t *cache) { printf( "%s cache size %ld, occupied size %ld, n_req %ld, n_obj %ld, default TTL " "%ld, per_obj_metadata_size %d\n", - cache->cache_name, cache->cache_size, cache->get_occupied_byte(cache), - cache->n_req, cache->get_n_obj(cache), cache->default_ttl, - cache->obj_md_size); + cache->cache_name, (long)cache->cache_size, + (long)cache->get_occupied_byte(cache), (long)cache->n_req, + (long)cache->get_n_obj(cache), (long)cache->default_ttl, + (int)cache->obj_md_size); } /** diff --git a/libCacheSim/include/libCacheSim/reader.h b/libCacheSim/include/libCacheSim/reader.h index 8a9305ff..f20502a0 100644 --- a/libCacheSim/include/libCacheSim/reader.h +++ b/libCacheSim/include/libCacheSim/reader.h @@ -98,7 +98,7 @@ typedef struct reader { /************* used by binary trace *************/ /* mmap the file, this should not change during runtime */ char *mapped_file; - uint64_t mmap_offset; + size_t mmap_offset; struct zstd_reader *zstd_reader_p; bool is_zstd_file; /* the size of one request in binary trace */ @@ -117,8 +117,8 @@ typedef struct reader { /* if true, ignore the obj_size in the trace, and use size one */ bool ignore_obj_size; - /* this is used when - * a) the reader splits a large req into multiple chunked requests + /* this is used when + * a) the reader splits a large req into multiple chunked requests * b) the trace file uses a count field */ int n_req_left; int64_t last_req_clock_time; @@ -180,8 +180,9 @@ reader_t *setup_reader(const char *trace_path, trace_type_e trace_type, const reader_init_param_t *reader_init_param); /* this is the same function as setup_reader */ -static inline reader_t *open_trace(const char *path, const trace_type_e type, - const reader_init_param_t *reader_init_param) { +static inline reader_t *open_trace( + const char *path, const trace_type_e type, + const reader_init_param_t *reader_init_param) { return setup_reader(path, type, reader_init_param); } @@ -272,11 +273,12 @@ static inline void print_reader(reader_t *reader) { "obj_id_is_num: %d, ignore_size_zero_req: %d, ignore_obj_size: %d, " "n_req_left: %d, last_req_clock_time: %ld\n", trace_type_str[reader->trace_type], reader->trace_path, - reader->trace_start_offset, reader->mmap_offset, reader->is_zstd_file, - reader->item_size, reader->file, reader->line_buf, reader->line_buf_size, - reader->csv_delimiter, reader->csv_has_header, reader->obj_id_is_num, - reader->ignore_size_zero_req, reader->ignore_obj_size, - reader->n_req_left, reader->last_req_clock_time); + reader->trace_start_offset, (long)reader->mmap_offset, + reader->is_zstd_file, reader->item_size, reader->file, reader->line_buf, + reader->line_buf_size, reader->csv_delimiter, reader->csv_has_header, + reader->obj_id_is_num, reader->ignore_size_zero_req, + reader->ignore_obj_size, reader->n_req_left, + (long)reader->last_req_clock_time); } #ifdef __cplusplus diff --git a/libCacheSim/include/libCacheSim/request.h b/libCacheSim/include/libCacheSim/request.h index 33e9d944..a32c4337 100644 --- a/libCacheSim/include/libCacheSim/request.h +++ b/libCacheSim/include/libCacheSim/request.h @@ -20,7 +20,7 @@ extern "C" { /* need to optimize this for CPU cacheline */ typedef struct request { int64_t clock_time; /* use uint64_t because vscsi uses microsec timestamp */ - uint64_t hv; /* hash value, used when offloading hash to reader */ + uint64_t hv; /* hash value, used when offloading hash to reader */ obj_id_t obj_id; int64_t obj_size; int32_t ttl; @@ -53,10 +53,10 @@ typedef struct request { /* used in trace analysis */ int64_t vtime_since_last_access; int64_t rtime_since_last_access; - int64_t prev_size; /* prev size */ + int64_t prev_size; /* prev size */ int32_t create_rtime; - bool compulsory_miss; /* use this field only when it is set */ - bool overwrite; // this request overwrites a previous object + bool compulsory_miss; /* use this field only when it is set */ + bool overwrite; // this request overwrites a previous object bool first_seen_in_window; /* the first time see in the time window */ /* used in trace analysis */ @@ -68,7 +68,7 @@ typedef struct request { * allocate a new request_t struct and fill in necessary field * @return */ -static inline request_t *new_request() { +static inline request_t *new_request(void) { request_t *req = my_malloc(request_t); memset(req, 0, sizeof(request_t)); req->obj_size = 1; diff --git a/libCacheSim/include/libCacheSim/sampling.h b/libCacheSim/include/libCacheSim/sampling.h index 6c679f9f..d54c6b31 100644 --- a/libCacheSim/include/libCacheSim/sampling.h +++ b/libCacheSim/include/libCacheSim/sampling.h @@ -13,7 +13,7 @@ typedef bool (*trace_sampling_func)(struct sampler *sampler, request_t *req); typedef struct sampler *(*clone_sampler_func)(const struct sampler *sampler); -typedef struct sampler *(*free_sampler_func)(struct sampler *sampler); +typedef void (*free_sampler_func)(struct sampler *sampler); enum sampler_type { SPATIAL_SAMPLER, diff --git a/libCacheSim/profiler/dist.c b/libCacheSim/profiler/dist.c index 913ad29f..21a82539 100644 --- a/libCacheSim/profiler/dist.c +++ b/libCacheSim/profiler/dist.c @@ -134,7 +134,7 @@ int32_t *get_stack_dist(reader_t *reader, const dist_type_e dist_type, stack_dist = get_stack_dist_add_req(req, &splay_tree, hash_table, curr_ts, &last_access_ts); if (stack_dist > (int64_t)UINT32_MAX) { - ERROR("stack distance %ld is larger than UINT32_MAX\n", stack_dist); + ERROR("stack distance %ld is larger than UINT32_MAX\n", (long)stack_dist); abort(); } if (dist_type == STACK_DIST) { @@ -159,7 +159,8 @@ int32_t *get_stack_dist(reader_t *reader, const dist_type_e dist_type, return stack_dist_array; } -int32_t *get_access_dist(reader_t *reader, const dist_type_e dist_type, int64_t *array_size) { +int32_t *get_access_dist(reader_t *reader, const dist_type_e dist_type, + int64_t *array_size) { int64_t curr_ts = 0; int64_t dist = 0; request_t *req = new_request(); @@ -174,7 +175,7 @@ int32_t *get_access_dist(reader_t *reader, const dist_type_e dist_type, int64_t while (req->valid) { dist = get_access_dist_add_req(req, hash_table, curr_ts, dist_type); if (dist > (int64_t)UINT32_MAX) { - ERROR("access distance %ld is larger than UINT32_MAX\n", dist); + ERROR("access distance %ld is larger than UINT32_MAX\n", (long)dist); abort(); } @@ -203,8 +204,8 @@ void save_dist(reader_t *const reader, const int32_t *dist_array, } void save_dist_txt(reader_t *const reader, const int32_t *dist_array, - int64_t array_size, const char *const ofilepath, - const dist_type_e dist_type) { + int64_t array_size, const char *const ofilepath, + const dist_type_e dist_type) { char *file_path = (char *)malloc(strlen(ofilepath) + 128); sprintf(file_path, "%s.%s.txt", ofilepath, dist_type_str[dist_type]); FILE *file = fopen(file_path, "w"); @@ -254,12 +255,12 @@ void _write_dist_cnt(gpointer key, gpointer value, gpointer user_data) { int64_t dist = (int64_t)GPOINTER_TO_SIZE(key); int64_t cnt = (int64_t)GPOINTER_TO_SIZE(value); FILE *file = (FILE *)user_data; - fprintf(file, "%ld:%ld, ", dist, cnt); + fprintf(file, "%ld:%ld, ", (long)dist, (long)cnt); } void save_dist_as_cnt_txt(reader_t *const reader, const int32_t *dist_array, - const int64_t array_size, const char *const ofilepath, - const dist_type_e dist_type) { + const int64_t array_size, const char *const ofilepath, + const dist_type_e dist_type) { assert(get_num_of_req(reader) == array_size); char *file_path = (char *)malloc(strlen(ofilepath) + 128); diff --git a/libCacheSim/profiler/simulator.c b/libCacheSim/profiler/simulator.c index 91471bb0..a53f7cdd 100644 --- a/libCacheSim/profiler/simulator.c +++ b/libCacheSim/profiler/simulator.c @@ -51,7 +51,7 @@ static void _simulate(gpointer data, gpointer user_data) { reader_t *warmup_cloned_reader = clone_reader(params->warmup_reader); read_one_req(warmup_cloned_reader, req); while (req->valid) { - bool ck = local_cache->get(local_cache, req); + local_cache->get(local_cache, req); result[idx].n_warmup_req += 1; read_one_req(warmup_cloned_reader, req); } @@ -72,7 +72,7 @@ static void _simulate(gpointer data, gpointer user_data) { while (req->valid && (n_warmup < params->n_warmup_req || req->clock_time - start_ts < params->warmup_sec)) { req->clock_time -= start_ts; - bool ck = local_cache->get(local_cache, req); + local_cache->get(local_cache, req); n_warmup += 1; read_one_req(cloned_reader, req); } diff --git a/libCacheSim/traceAnalyzer/analyzer.cpp b/libCacheSim/traceAnalyzer/analyzer.cpp index eab48568..03b7c5a1 100644 --- a/libCacheSim/traceAnalyzer/analyzer.cpp +++ b/libCacheSim/traceAnalyzer/analyzer.cpp @@ -111,7 +111,7 @@ void traceAnalyzer::TraceAnalyzer::run() { ERROR( "The data is not ordered by time, please sort the trace first!" "Current time %ld requested object %lu\n", - req->clock_time + start_ts_, req->obj_id); + (long) req->clock_time + start_ts_, (unsigned long) req->obj_id); } DEBUG_ASSERT(curr_time_window_idx == time_to_window_idx(req->clock_time)); diff --git a/libCacheSim/traceReader/CMakeLists.txt b/libCacheSim/traceReader/CMakeLists.txt index 25e9e3d9..7b749902 100644 --- a/libCacheSim/traceReader/CMakeLists.txt +++ b/libCacheSim/traceReader/CMakeLists.txt @@ -10,9 +10,9 @@ set(source sampling/temporal.c ) -if (SUPPORT_ZSTD_TRACE) +if (OPT_SUPPORT_ZSTD_TRACE) set(source ${source} generalReader/zstdReader.c) -endif (SUPPORT_ZSTD_TRACE) +endif (OPT_SUPPORT_ZSTD_TRACE) add_library(traceReader ${source}) diff --git a/libCacheSim/traceReader/customizedReader/binaryUtils.h b/libCacheSim/traceReader/customizedReader/binaryUtils.h index 59b2c5ef..780cabc1 100644 --- a/libCacheSim/traceReader/customizedReader/binaryUtils.h +++ b/libCacheSim/traceReader/customizedReader/binaryUtils.h @@ -4,6 +4,12 @@ #include "../generalReader/zstdReader.h" #endif +#include "../../include/libCacheSim/reader.h" + +#ifdef __cplusplus +extern "C" { +#endif + /* read decompressed data file */ static inline char *_read_bytes(reader_t *reader) { if (reader->mmap_offset >= reader->file_size) { @@ -44,4 +50,8 @@ static inline char *read_bytes(reader_t *reader) { start = _read_bytes(reader); } return start; -} \ No newline at end of file +} + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/traceReader/generalReader/binary.c b/libCacheSim/traceReader/generalReader/binary.c index 86a11fab..78571eb6 100644 --- a/libCacheSim/traceReader/generalReader/binary.c +++ b/libCacheSim/traceReader/generalReader/binary.c @@ -56,8 +56,8 @@ int binaryReader_setup(reader_t *const reader) { params->fmt_str = strdup(init_params->binary_fmt_str + 1); const char *fmt_str = params->fmt_str; - uint32_t field_size = 0; - uint32_t curr_offset = 0; + // uint32_t field_size = 0; + // uint32_t curr_offset = 0; params->n_fields = strlen(fmt_str); diff --git a/libCacheSim/traceReader/generalReader/csv.c b/libCacheSim/traceReader/generalReader/csv.c index b872eb3f..82348cc1 100644 --- a/libCacheSim/traceReader/generalReader/csv.c +++ b/libCacheSim/traceReader/generalReader/csv.c @@ -79,7 +79,7 @@ static int read_first_line(const reader_t *reader, char *in_buf, */ static char csv_detect_delimiter(const reader_t *reader) { char first_line[1024] = {0}; - int _buf_size = read_first_line(reader, first_line, 1024); + read_first_line(reader, first_line, 1024); char possible_delims[4] = {'\t', ',', '|', ':'}; int possible_delim_counts[4] = {0, 0, 0, 0}; @@ -168,7 +168,7 @@ bool check_delimiter(const reader_t *reader, char delimiter) { bool is_delimiter_correct = true; size_t n = 0; - ssize_t read_size = getline(&buf, &n, ifile); + getline(&buf, &n, ifile); #define N_TEST 1024 for (int i = 0; i < N_TEST; i++) { if (strchr(buf, delimiter) == NULL) { @@ -338,8 +338,7 @@ void csv_reset_reader(reader_t *reader) { csv_set_delim(csv_params->csv_parser, csv_params->delimiter); if (csv_params->has_header) { - ssize_t _read_size = - getline(&reader->line_buf, &reader->line_buf_size, reader->file); + getline(&reader->line_buf, &reader->line_buf_size, reader->file); } } diff --git a/libCacheSim/traceReader/generalReader/lcs.c b/libCacheSim/traceReader/generalReader/lcs.c index 1b65c406..5886d3ba 100644 --- a/libCacheSim/traceReader/generalReader/lcs.c +++ b/libCacheSim/traceReader/generalReader/lcs.c @@ -1,9 +1,8 @@ -#include "lcs.h" - #include #include "../customizedReader/binaryUtils.h" +#include "lcs.h" #include "readerInternal.h" #ifdef __cplusplus @@ -24,12 +23,13 @@ const char *print_lcs_trace_format(lcs_trace_header_t *header) { bool verify_LCS_trace_header(lcs_trace_header_t *header) { if (header->start_magic != LCS_TRACE_START_MAGIC) { ERROR("invalid trace file, start magic is wrong 0x%lx\n", - header->start_magic); + (unsigned long)header->start_magic); return false; } if (header->end_magic != LCS_TRACE_END_MAGIC) { - ERROR("invalid trace file, end magic is wrong 0x%lx\n", header->end_magic); + ERROR("invalid trace file, end magic is wrong 0x%lx\n", + (unsigned long)header->end_magic); return false; } diff --git a/libCacheSim/traceReader/reader.c b/libCacheSim/traceReader/reader.c index 190e7dbc..6680f410 100644 --- a/libCacheSim/traceReader/reader.c +++ b/libCacheSim/traceReader/reader.c @@ -52,8 +52,8 @@ reader_t *setup_reader(const char *const trace_path, * currently zstd reader only supports a few binary trace */ reader->is_zstd_file = false; reader->zstd_reader_p = NULL; - size_t slen = strlen(trace_path); #ifdef SUPPORT_ZSTD_TRACE + size_t slen = strlen(trace_path); if (strncmp(trace_path + (slen - 4), ".zst", 4) == 0 || strncmp(trace_path + (slen - 7), ".zst.22", 7) == 0) { reader->is_zstd_file = true; @@ -281,7 +281,7 @@ int read_one_req(reader_t *const reader, request_t *const req) { if (reader->cap_at_n_req > 1 && reader->n_read_req >= reader->cap_at_n_req) { DEBUG("read_one_req: processed %ld requests capped by the user\n", - reader->n_read_req); + (long) reader->n_read_req); req->valid = false; return 1; } diff --git a/libCacheSim/traceReader/sampling/spatial.c b/libCacheSim/traceReader/sampling/spatial.c index c3f57ad1..236e9e9e 100644 --- a/libCacheSim/traceReader/sampling/spatial.c +++ b/libCacheSim/traceReader/sampling/spatial.c @@ -28,7 +28,7 @@ sampler_t *clone_spatial_sampler(const sampler_t *sampler) { return cloned_sampler; } -sampler_t *free_spatial_sampler(sampler_t *sampler) { free(sampler); } +void free_spatial_sampler(sampler_t *sampler) { free(sampler); } sampler_t *create_spatial_sampler(double sampling_ratio) { if (sampling_ratio > 1 || sampling_ratio <= 0) { diff --git a/libCacheSim/traceReader/sampling/temporal.c b/libCacheSim/traceReader/sampling/temporal.c index 098242fd..f626a9dd 100644 --- a/libCacheSim/traceReader/sampling/temporal.c +++ b/libCacheSim/traceReader/sampling/temporal.c @@ -33,7 +33,7 @@ sampler_t *clone_temporal_sampler(const sampler_t *sampler) { return cloned_sampler; } -sampler_t *free_temporal_sampler(sampler_t *sampler) { +void free_temporal_sampler(sampler_t *sampler) { free(sampler->other_params); free(sampler); } diff --git a/libCacheSim/utils/include/mymath.h b/libCacheSim/utils/include/mymath.h index 6b9137ae..03ce862c 100644 --- a/libCacheSim/utils/include/mymath.h +++ b/libCacheSim/utils/include/mymath.h @@ -24,7 +24,7 @@ void set_rand_seed(uint64_t seed); * random number generator from Knuth MMIX * @return */ -static inline uint64_t next_rand() { +static inline uint64_t next_rand(void) { rand_seed = 6364136223846793005 * rand_seed + 1442695040888963407; return rand_seed; } diff --git a/libCacheSim/utils/include/mysys.h b/libCacheSim/utils/include/mysys.h index eae3335a..f5786c6f 100644 --- a/libCacheSim/utils/include/mysys.h +++ b/libCacheSim/utils/include/mysys.h @@ -18,7 +18,7 @@ int get_n_cores(void); int n_cores(void); -double gettime(); +double gettime(void); void print_cwd(void); diff --git a/libCacheSim/utils/mysys.c b/libCacheSim/utils/mysys.c index 7af0efdb..893d47be 100644 --- a/libCacheSim/utils/mysys.c +++ b/libCacheSim/utils/mysys.c @@ -58,7 +58,7 @@ int get_n_cores(void) { return get_nprocs(); #else - return sysconf(_SC_NPROCESSORS_ONLN) + return sysconf(_SC_NPROCESSORS_ONLN); #endif WARN("Unknown system, use 4 threads as default\n"); return 4; @@ -88,9 +88,9 @@ void print_glib_ver(void) { // return ebx; // } -int n_cores() { return get_n_cores(); } +int n_cores(void) { return get_n_cores(); } -double gettime() { +double gettime(void) { struct timeval tv; gettimeofday(&tv, NULL); diff --git a/scripts/install_dependency.sh b/scripts/install_dependency.sh index 611aece0..c6e82450 100644 --- a/scripts/install_dependency.sh +++ b/scripts/install_dependency.sh @@ -2,7 +2,7 @@ setup_ubuntu() { sudo apt update - sudo apt install -yqq libglib2.0-dev libgoogle-perftools-dev build-essential cmake google-perftools + sudo apt install -yqq libglib2.0-dev libgoogle-perftools-dev build-essential cmake google-perftools xxhash } setup_centos() { @@ -10,7 +10,7 @@ setup_centos() { } setup_macOS() { - brew install glib google-perftools + brew install glib google-perftools argp-standalone } setup_xgboost() { From 938165f27b3060e95b906043603f3db9fdafd93f Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Thu, 4 Jan 2024 10:30:55 -0500 Subject: [PATCH 13/28] Minor fix (#47) * add SIEVE to be the main cache of TinyLFU algorithm * log fix * Add options for trace print formatting * add a note how to run the caffeine simulator * Update table of contents in README.md --- README.md | 42 +++++++++++-------- libCacheSim/bin/traceUtils/cli_parser.cpp | 25 ++++++++++- libCacheSim/bin/traceUtils/internal.hpp | 3 ++ libCacheSim/bin/traceUtils/tracePrintMain.cpp | 32 ++++++++------ libCacheSim/cache/eviction/WTinyLFU.c | 2 + libCacheSim/traceAnalyzer/analyzer.cpp | 5 ++- scripts/note | 9 ++++ 7 files changed, 86 insertions(+), 32 deletions(-) create mode 100644 scripts/note diff --git a/README.md b/README.md index 4d4cf1cd..84bdbe6c 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,30 @@ -* [What is libCacheSim](#what-is-libcachesim) -* [libCacheSim features ](#libcachesim-features) -* [Supported algorithms](#supported-algorithms) -* [Build and Install libCacheSim](#build-and-install-libcachesim) - + [One-line install](#one-line-install) - + [Install dependency](#install-dependency) - + [Build libCacheSim](#build-libcachesim) -* [Usage](#usage) - + [cachesim (a high-performance cache simulator)](#cachesim-a-high-performance-cache-simulator) - + [Trace analysis](#trace-analysis) - + [Using libCacheSim as a library ](#using-libcachesim-as-a-library) - + [Extending libCacheSim (new algorithms and trace types)](#extending-libcachesim-new-algorithms-and-trace-types) -* [Open source cache traces](#open-source-cache-traces) -* [Questions? ](#questions) -* [Reference](#reference) -* [License](#license) +- [libCacheSim - building and running cache simulations](#libcachesim---building-and-running-cache-simulations) + - [News](#news) + - [What is libCacheSim](#what-is-libcachesim) + - [libCacheSim features](#libcachesim-features) + - [Supported algorithms](#supported-algorithms) + - [Build and Install libCacheSim](#build-and-install-libcachesim) + - [One-line install](#one-line-install) + - [Install dependency](#install-dependency) + - [Build libCacheSim](#build-libcachesim) + - [Usage](#usage) + - [cachesim (a high-performance cache simulator)](#cachesim-a-high-performance-cache-simulator) + - [basic usage](#basic-usage) + - [Run a single cache simulation](#run-a-single-cache-simulation) + - [Run multiple cache simulations with different cache sizes](#run-multiple-cache-simulations-with-different-cache-sizes) + - [Plot miss ratio curve](#plot-miss-ratio-curve) + - [Trace analysis](#trace-analysis) + - [Using libCacheSim as a library](#using-libcachesim-as-a-library) + - [Extending libCacheSim (new algorithms and trace types)](#extending-libcachesim-new-algorithms-and-trace-types) + - [Open source cache traces](#open-source-cache-traces) + - [Questions?](#questions) + - [Contributions](#contributions) + - [Reference](#reference) + - [License](#license) + - [Related](#related) @@ -251,7 +259,7 @@ The compressed traces can be used with libCacheSim without decompression. And li | Tencent Photo | 2018 | object | [link](http://iotta.snia.org/traces/parallel?only=27476) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/tencentPhoto/) | | WikiCDN | 2019 | object | [link](https://wikitech.wikimedia.org/wiki/Analytics/Data_Lake/Traffic/Caching) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/wiki/) | | Tencent CBS | 2020 | block | [link](http://iotta.snia.org/traces/parallel?only=27917) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/tencentBlock/) | -| Alibaba CBS | 2020 | block | [link](https://github.com/alibaba/block-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/alibabaBlock/) | +| Alibaba Block | 2020 | block | [link](https://github.com/alibaba/block-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/alibabaBlock/) | | Twitter | 2020 | key-value | [link](https://github.com/twitter/cache-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/twitter/) | | MetaKV | 2022 | key-value | [link](https://cachelib.org/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval/#list-of-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/metaKV/) | | MetaCDN | 2023 | object | [link](https://cachelib.org/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval/#list-of-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/metaCDN/) | diff --git a/libCacheSim/bin/traceUtils/cli_parser.cpp b/libCacheSim/bin/traceUtils/cli_parser.cpp index de885268..7e869a55 100644 --- a/libCacheSim/bin/traceUtils/cli_parser.cpp +++ b/libCacheSim/bin/traceUtils/cli_parser.cpp @@ -30,6 +30,9 @@ enum argp_option_short { // trace print OPTION_NUM_REQ = 'n', + OPTION_FIELD_DELIMITER = 0x201, + OPTION_OBJ_ID_ONLY = 0x202, + OPTION_OBJ_ID_32bit = 0x204, // trace filter OPTION_FILTER_TYPE = 0x301, @@ -63,6 +66,11 @@ static struct argp_option options[] = { {0, 0, 0, 0, "tracePrint options:"}, {"num-req", OPTION_NUM_REQ, "-1", 0, "Number of requests to process, -1 means all requests in the trace", 6}, + {"field-delimiter", OPTION_FIELD_DELIMITER, ",", 0, + "The delimiter formatting the trace", 6}, + {"obj-id-only", OPTION_OBJ_ID_ONLY, "0", 0, "Only to print object id", 6}, + {"obj-id-32bit", OPTION_OBJ_ID_32bit, "0", 0, + "Print object id as 32-bit int", 6}, {0, 0, 0, 0, "traceFilter options:"}, {"filter-type", OPTION_FILTER_TYPE, "FIFO", 0, @@ -106,6 +114,15 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case OPTION_NUM_REQ: arguments->n_req = atoll(arg); break; + case OPTION_FIELD_DELIMITER: + arguments->delimiter = arg[0]; + break; + case OPTION_OBJ_ID_ONLY: + arguments->print_obj_id_only = atol(arg); + break; + case OPTION_OBJ_ID_32bit: + arguments->print_obj_id_32bit = atol(arg); + break; case OPTION_FILTER_TYPE: arguments->cache_name = arg; break; @@ -146,6 +163,8 @@ static char doc[] = "tracePrint: utility to print binary trace in human-readable format\n" "traceConv: utility to convert a trace to oracleGeneral format\n\n" "traceFilter: utility to filter a trace\n\n" + "example usage: ./tracePrint /trace/path oracleGeneral -n 20 " + "--obj-id-only=1\n\n" "example usage: ./traceConv /trace/path csv -o " "/path/new_trace.oracleGeneral -t " "\"obj-id-col=5,time-col=2,obj-size-col=4\"\n\n" @@ -171,6 +190,9 @@ static void init_arg(struct arguments *args) { args->remove_size_change = false; args->cache_name = NULL; args->cache_size = 0; + args->delimiter = ','; + args->print_obj_id_only = false; + args->print_obj_id_32bit = false; } static void print_parsed_arg(struct arguments *args) { @@ -229,7 +251,8 @@ void parse_cmd(int argc, char *argv[], struct arguments *args) { assert(N_ARGS == 2); args->reader = create_reader(args->trace_type_str, args->trace_path, - args->trace_type_params, args->n_req, args->ignore_obj_size, 0); + args->trace_type_params, args->n_req, + args->ignore_obj_size, 0); print_parsed_arg(args); } diff --git a/libCacheSim/bin/traceUtils/internal.hpp b/libCacheSim/bin/traceUtils/internal.hpp index f3715231..b2ef443d 100644 --- a/libCacheSim/bin/traceUtils/internal.hpp +++ b/libCacheSim/bin/traceUtils/internal.hpp @@ -31,6 +31,9 @@ struct arguments { /* trace print */ int64_t num_req; /* number of requests to print */ + char delimiter; + bool print_obj_id_only; + bool print_obj_id_32bit; /* trace filter */ char *cache_name; diff --git a/libCacheSim/bin/traceUtils/tracePrintMain.cpp b/libCacheSim/bin/traceUtils/tracePrintMain.cpp index 1c41c009..733fb867 100644 --- a/libCacheSim/bin/traceUtils/tracePrintMain.cpp +++ b/libCacheSim/bin/traceUtils/tracePrintMain.cpp @@ -8,7 +8,6 @@ #include "../../include/libCacheSim/reader.h" #include "internal.hpp" - int main(int argc, char *argv[]) { struct arguments args; @@ -16,22 +15,32 @@ int main(int argc, char *argv[]) { request_t *req = new_request(); read_one_req(args.reader, req); - + bool trace_has_next_access_vtime = req->next_access_vtime != -2; - if (trace_has_next_access_vtime) { - printf("# time, object, size, next_access_vtime\n"); - } else { - printf("# time, object, size\n"); + if (!args.print_obj_id_only) { + if (trace_has_next_access_vtime) { + printf("# time,object,size,next_access_vtime\n"); + } else { + printf("# time,object,size\n"); + } } while (req->valid) { - printf("%ld, %lu, %d", (long)req->clock_time, (unsigned long)req->obj_id, - (int)req->obj_size); - if (trace_has_next_access_vtime) { - printf(", %ld\n", (long)req->next_access_vtime); + if (args.print_obj_id_32bit) { + req->obj_id = (uint32_t)req->obj_id; + } + + if (args.print_obj_id_only) { + printf("%lu\n", (unsigned long)req->obj_id); } else { - printf("\n"); + printf("%ld%c%lu%c%d", (long)req->clock_time, args.delimiter, + (unsigned long)req->obj_id, args.delimiter, (int)req->obj_size); + if (trace_has_next_access_vtime) { + printf("%c%ld\n", args.delimiter, (long)req->next_access_vtime); + } else { + printf("\n"); + } } read_one_req(args.reader, req); } @@ -40,4 +49,3 @@ int main(int argc, char *argv[]) { return 0; } - diff --git a/libCacheSim/cache/eviction/WTinyLFU.c b/libCacheSim/cache/eviction/WTinyLFU.c index 386daa72..aa1d4424 100644 --- a/libCacheSim/cache/eviction/WTinyLFU.c +++ b/libCacheSim/cache/eviction/WTinyLFU.c @@ -138,6 +138,8 @@ cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, params->main_cache = Hyperbolic_init(ccache_params_local, NULL); } else if (strcasecmp(params->main_cache_type, "LHD") == 0) { params->main_cache = LHD_init(ccache_params_local, NULL); + } else if (strcasecmp(params->main_cache_type, "SIEVE") == 0) { + params->main_cache = Sieve_init(ccache_params_local, NULL); } else { ERROR("WTinyLFU does not support %s \n", params->main_cache_type); } diff --git a/libCacheSim/traceAnalyzer/analyzer.cpp b/libCacheSim/traceAnalyzer/analyzer.cpp index 03b7c5a1..aefd8ae5 100644 --- a/libCacheSim/traceAnalyzer/analyzer.cpp +++ b/libCacheSim/traceAnalyzer/analyzer.cpp @@ -110,8 +110,9 @@ void traceAnalyzer::TraceAnalyzer::run() { if (curr_time_window_idx != time_to_window_idx(req->clock_time)) { ERROR( "The data is not ordered by time, please sort the trace first!" - "Current time %ld requested object %lu\n", - (long) req->clock_time + start_ts_, (unsigned long) req->obj_id); + "Current time %ld requested object %lu, obj size %lu\n", + (long)req->clock_time + start_ts_, (unsigned long)req->obj_id, + req->obj_size); } DEBUG_ASSERT(curr_time_window_idx == time_to_window_idx(req->clock_time)); diff --git a/scripts/note b/scripts/note new file mode 100644 index 00000000..5688d7c9 --- /dev/null +++ b/scripts/note @@ -0,0 +1,9 @@ +## How to run Caffeine simulator +``` +git clone https://github.com/ben-manes/caffeine.git +./gradlew build +export GRADLE_OPTS="-Xmx204800m" +# modify simulator/src/main/resources/reference.conf, the lirs format is txt with only the object id +./gradlew run simulator:run + +``` From a98824e9c66ffd4f5dc4e1874896bfa3432ee201 Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Fri, 5 Jan 2024 23:36:08 -0500 Subject: [PATCH 14/28] many fixes (#48) * fix compile warnning and errors * fix bugs in CMakeLists * remove xxhash from source and use system-level xxhash so that macos can compile * change global variable names * add n_core function for macos --- CMakeLists.txt | 35 +++--- cmake/Modules/Findargp.cmake | 84 ++++++++++++++ libCacheSim/bin/cachesim/cli_parser.c | 4 +- libCacheSim/bin/cli_reader_utils.c | 11 +- libCacheSim/bin/traceAnalyzer/cli_parser.cpp | 6 +- libCacheSim/bin/traceUtils/cli_parser.cpp | 2 +- libCacheSim/bin/traceUtils/traceConv.cpp | 8 +- libCacheSim/bin/traceUtils/traceConvMain.cpp | 2 +- .../bin/traceUtils/traceFilterMain.cpp | 2 +- libCacheSim/cache/eviction/CR_LFU.c | 3 +- libCacheSim/cache/eviction/LIRS.c | 2 +- libCacheSim/cache/eviction/WTinyLFU.c | 1 + libCacheSim/cache/eviction/nop.c | 13 ++- libCacheSim/cache/eviction/priv/QDLPv0.c | 1 + libCacheSim/dataStructure/hash/CMakeLists.txt | 6 +- libCacheSim/dataStructure/robin_hood.h | 2 +- libCacheSim/include/libCacheSim/dist.h | 2 +- libCacheSim/include/libCacheSim/enum.h | 2 +- libCacheSim/include/libCacheSim/reader.h | 2 +- libCacheSim/profiler/dist.c | 6 +- libCacheSim/traceAnalyzer/analyzer.cpp | 4 +- scripts/install_dependency.sh | 2 +- test/common.h | 37 ++++-- test/test_evictionAlgo.c | 107 +++++++++--------- test/test_prefetchAlgo.c | 20 ++-- test/test_simulator.c | 16 +-- 26 files changed, 239 insertions(+), 141 deletions(-) create mode 100644 cmake/Modules/Findargp.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index cf8cf395..3aaa37d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,10 +32,10 @@ set_property(CACHE LOG_LEVEL PROPERTY STRINGS INFO WARN ERROR DEBUG VERBOSE VVER # detect platform # ######################################## if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + message(STATUS "Mac OS X detected, version ${CMAKE_SYSTEM_VERSION}") add_definitions(-DOS_DARWIN) - if (NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Debug) - endif() + set(CMAKE_INCLUDE_PATH ${CMAKE_INCLUDE_PATH} "/opt/homebrew/include/") + set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/opt/homebrew/") elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") # set(CFLAGS "$ENV{CFLAGS} " "-Wl,--export-dynamic ") @@ -121,7 +121,6 @@ set(CFLAGS "$ENV{CFLAGS} " # "-Wmissing-prototypes " "-Wmissing-declarations " "-Wredundant-decls " - "-Wunused-value -Wunused-variable " "-fno-strict-aliasing " ) @@ -142,21 +141,22 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") find_package(GLib REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) set(LIBS ${LIBS} ${GLib_LIBRARY}) -#if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -# pkg_check_modules(GLib2 REQUIRED Glib-2.0) -# include_directories(${GLib2_INCLUDE_DIRS}) -# # add_compile_options("${LDFLAGS}") -# link_libraries(${GLib2_LDFLAGS}) -#endif() + + +find_package(argp REQUIRED) +include_directories(${ARGP_INCLUDE_DIRS}) +set(LIBS ${LIBS} ${ARGP_LIBRARY}) if (OPT_SUPPORT_ZSTD_TRACE) add_compile_definitions(SUPPORT_ZSTD_TRACE=1) find_package(ZSTD) + # https://stackoverflow.com/questions/61377055/cannot-find-gflags-gflags-h-while-building-library-osx/61379123#61379123 + include_directories(${ZSTD_INCLUDE_DIR}) if ("${ZSTD_LIBRARIES}" STREQUAL "") message(FATAL_ERROR "zstd not found") endif() link_libraries(${ZSTD_LIBRARIES}) - message(STATUS "ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIRS}, ZSTD_LIBRARIES ${ZSTD_LIBRARIES}") + message(STATUS "ZSTD_INCLUDE_DIR ${ZSTD_INCLUDE_DIR}, ZSTD_LIBRARIES ${ZSTD_LIBRARIES}") else() remove_definitions(SUPPORT_ZSTD_TRACE) endif(OPT_SUPPORT_ZSTD_TRACE) @@ -224,7 +224,7 @@ message(STATUS "<<++=====------------------/\\------------------=====++>>") message(STATUS "================== dependency related ==================") message(STATUS "glib found? ${GLib_FOUND} - LIBS=${GLib_LIBRARY}, header =${GLib_INCLUDE_DIRS}") message(STATUS "tcmalloc found? ${Tcmalloc_FOUND} - LIBS=${Tcmalloc_LIBRARIES}, header=${Tcmalloc_INCLUDE_DIRS}") -message(STATUS "ZSTD found? ${ZSTD_FOUND} - LIBS=${ZSTD_LIBRARIES}, header=${ZSTD_INCLUDE_DIRS}") +message(STATUS "ZSTD found? ${ZSTD_FOUND} - LIBS=${ZSTD_LIBRARIES}, header=${ZSTD_INCLUDE_DIR}") message(STATUS "==================== CMake related =====================") message(STATUS "platform = ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_VERSION}") @@ -330,7 +330,7 @@ endif(OPT_SUPPORT_ZSTD_TRACE) file(GLOB dataStructure_source ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/*.c ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/hashtable/*.c - ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/hash/*.c + ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/hash/murmur3.c ) file(GLOB profiler_source @@ -347,7 +347,8 @@ set(LIB_SOURCE ${LIB_SOURCE} ${cache_source} ${reader_source} ${dataStructure_so # # https://stackoverflow.com/questions/32469953/why-is-cmake-designed-so-that-it-removes-runtime-path-when-installing SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) -add_library(${PROJECT_NAME} SHARED ${LIB_SOURCE}) +add_library(${PROJECT_NAME} ${LIB_SOURCE}) +# add_library(${PROJECT_NAME} SHARED ${LIB_SOURCE}) set_target_properties(${PROJECT_NAME} PROPERTIES VERSION ${${PROJECT_NAME}_VERSION}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE C) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/libCacheSim/include/libCacheSim.h) @@ -366,12 +367,6 @@ install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - - -# #target_compile_definitions(binBenchmark PRIVATE HASH=0xb2) -# #add_executable(binBenchmark libCacheSim/benchmark/main.c) -# #target_link_libraries(binBenchmark benchmark cachelib evictionAlgos traceReader profiler dataStructure utils -# # ${LIBS}) ############################# diff --git a/cmake/Modules/Findargp.cmake b/cmake/Modules/Findargp.cmake new file mode 100644 index 00000000..0d780260 --- /dev/null +++ b/cmake/Modules/Findargp.cmake @@ -0,0 +1,84 @@ +# - try to find the argp library/component of glibc +# +# Users may optionally supply: +# ARGP_ROOT_DIR - a prefix to start searching. +# +# Cache Variables: (probably not for direct use in your scripts) +# ARGP_INCLUDE_DIR +# ARGP_LIBRARY, only defined if linking to an extra library is required +# +# Non-cache variables you might use in your CMakeLists.txt: +# ARGP_FOUND +# ARGP_INCLUDE_DIRS +# ARGP_LIBRARIES +# +# Requires these CMake modules: +# FindPackageHandleStandardArgs (known included with CMake >=2.6.2) +# +# Original Author: +# 2009-2010 Rylie Pavlik +# https://ryliepavlik.com/ +# Iowa State University HCI Graduate Program/VRAC +# +# Copyright 2009-2010, Iowa State University +# +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) +# +# SPDX-License-Identifier: BSL-1.0 + +set(ARGP_ROOT_DIR + "${ARGP_ROOT_DIR}" + CACHE + PATH + "Path to search for ARGP library") + +### +# Configure ARGP +### +set(_check ARGP_INCLUDE_DIR) + +find_path(ARGP_INCLUDE_DIR + NAMES + argp.h + HINTS + "${ARGP_ROOT_DIR}" + PATHS + /usr/local + /opt/local + /sw) +mark_as_advanced(ARGP_INCLUDE_DIR) + +include(CheckFunctionExists) +check_function_exists(argp_parse ARGP_BUILTIN) + +if(NOT ARGP_BUILTIN) + find_library(ARGP_LIBRARY + NAMES + argp + HINTS + "${ARGP_ROOT_DIR}" + PATH_SUFFIXES + lib + lib64 + PATHS + /usr/local + /opt/local + /sw) + list(APPEND _check ARGP_LIBRARY) + mark_as_advanced(ARGP_LIBRARY) +endif() + +# handle the QUIETLY and REQUIRED arguments and set xxx_FOUND to TRUE if +# all listed variables are TRUE +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(argp + DEFAULT_MSG + ${_check}) + +if(ARGP_FOUND) + set(ARGP_INCLUDE_DIRS "${ARGP_INCLUDE_DIR}") + set(ARGP_LIBRARIES "${ARGP_LIBRARY}") + mark_as_advanced(ARGP_ROOT_DIR) +endif() \ No newline at end of file diff --git a/libCacheSim/bin/cachesim/cli_parser.c b/libCacheSim/bin/cachesim/cli_parser.c index a2299bfc..72de683a 100644 --- a/libCacheSim/bin/cachesim/cli_parser.c +++ b/libCacheSim/bin/cachesim/cli_parser.c @@ -477,7 +477,7 @@ static void set_cache_size(struct arguments *args, reader_t *reader) { args->n_cache_size = n_cache_sizes; if (args->n_cache_size == 0) { - printf("working set %ld too small\n", wss); + printf("working set %ld too small\n", (long) wss); exit(0); } } @@ -494,7 +494,7 @@ void print_parsed_args(struct arguments *args) { output_str, OUTPUT_STR_LEN - 1, "trace path: %s, trace_type %s, ofilepath " "%s, %d threads, warmup %d sec, total %d algo x %d size = %d caches", - args->trace_path, trace_type_str[args->trace_type], args->ofilepath, + args->trace_path, g_trace_type_name[args->trace_type], args->ofilepath, args->n_thread, args->warmup_sec, args->n_eviction_algo, args->n_cache_size, args->n_eviction_algo * args->n_cache_size); diff --git a/libCacheSim/bin/cli_reader_utils.c b/libCacheSim/bin/cli_reader_utils.c index 748ad548..a38cde0c 100644 --- a/libCacheSim/bin/cli_reader_utils.c +++ b/libCacheSim/bin/cli_reader_utils.c @@ -1,13 +1,12 @@ #define _GNU_SOURCE -#include "cli_reader_utils.h" - #include #include #include "../include/libCacheSim/reader.h" #include "../utils/include/mystr.h" +#include "cli_reader_utils.h" #ifdef __cplusplus extern "C" { @@ -60,11 +59,12 @@ trace_type_e trace_type_str_to_enum(const char *trace_type_str, return ORACLE_CF1_TRACE; } else if (strcasecmp(trace_type_str, "oracleSysTwrNS") == 0) { return ORACLE_SYS_TWRNS_TRACE; - } else if (strcasecmp(trace_type_str, "valpinTrace") == 0){ + } else if (strcasecmp(trace_type_str, "valpinTrace") == 0) { return VALPIN_TRACE; } else { ERROR("unsupported trace type: %s\n", trace_type_str); } + return UNKNOWN_TRACE; } bool is_true(const char *arg) { @@ -215,7 +215,7 @@ trace_type_e detect_trace_type(const char *trace_path) { trace_type = UNKNOWN_TRACE; } - INFO("detecting trace type: %s\n", trace_type_str[trace_type]); + INFO("detecting trace type: %s\n", g_trace_type_name[trace_type]); return trace_type; } @@ -276,7 +276,8 @@ void cal_working_set_size(reader_t *reader, int64_t *wss_obj, } *wss_obj *= scaling_factor; *wss_byte *= scaling_factor; - INFO("working set size: %ld object %ld byte\n", *wss_obj, *wss_byte); + INFO("working set size: %ld object %ld byte\n", (long)*wss_obj, + (long)*wss_byte); free_request(req); reset_reader(reader); diff --git a/libCacheSim/bin/traceAnalyzer/cli_parser.cpp b/libCacheSim/bin/traceAnalyzer/cli_parser.cpp index d7a59940..a694bcb9 100644 --- a/libCacheSim/bin/traceAnalyzer/cli_parser.cpp +++ b/libCacheSim/bin/traceAnalyzer/cli_parser.cpp @@ -61,9 +61,9 @@ enum argp_option_short { static struct argp_option options[] = { {NULL, 0, NULL, 0, "trace reader related parameters:", 0}, {"trace-type-params", OPTION_TRACE_TYPE_PARAMS, - "time-col=1,obj-id-col=2,obj-size-col=3,delimiter=,", NULL, + "time-col=1,obj-id-col=2,obj-size-col=3,delimiter=,", 0, "Parameters used for csv trace", 1}, - {"num-req", OPTION_NUM_REQ, 0, NULL, + {"num-req", OPTION_NUM_REQ, 0, 0, "Num of requests to process, default -1 means all requests in the trace", 1}, @@ -112,7 +112,7 @@ static struct argp_option options[] = { {NULL, 0, NULL, 0, "common parameters:", 0}, - {"output", OPTION_OUTPUT_PATH, "", NULL, "Output path", 8}, + {"output", OPTION_OUTPUT_PATH, "", OPTION_ARG_OPTIONAL, "Output path", 8}, {"verbose", OPTION_VERBOSE, NULL, OPTION_ARG_OPTIONAL, "Produce verbose output", 8}, {0}}; diff --git a/libCacheSim/bin/traceUtils/cli_parser.cpp b/libCacheSim/bin/traceUtils/cli_parser.cpp index 7e869a55..39233c45 100644 --- a/libCacheSim/bin/traceUtils/cli_parser.cpp +++ b/libCacheSim/bin/traceUtils/cli_parser.cpp @@ -200,7 +200,7 @@ static void print_parsed_arg(struct arguments *args) { int n = 0; char output_str[OUTPUT_STR_LEN]; n = snprintf(output_str, OUTPUT_STR_LEN - 1, "trace path: %s, trace_type %s", - args->trace_path, trace_type_str[args->trace_type]); + args->trace_path, g_trace_type_name[args->trace_type]); if (args->trace_type_params != NULL) n += snprintf(output_str + n, OUTPUT_STR_LEN - n - 1, diff --git a/libCacheSim/bin/traceUtils/traceConv.cpp b/libCacheSim/bin/traceUtils/traceConv.cpp index 404324c1..26236177 100644 --- a/libCacheSim/bin/traceUtils/traceConv.cpp +++ b/libCacheSim/bin/traceUtils/traceConv.cpp @@ -96,7 +96,7 @@ void convert_to_oracleGeneral(reader_t *reader, std::string ofilepath, "%s: %ld M requests (%.2lf GB), trace time %ld, working set %lld " "object, %lld B (%.2lf GB)\n", reader->trace_path, (long)(n_req_curr / 1e6), - (double)total_bytes / GiB, start_ts - req->clock_time, + (double)total_bytes / GiB, (long)(start_ts - req->clock_time), (long long)n_obj, (long long)unique_bytes, (double)unique_bytes / GiB); } @@ -118,8 +118,8 @@ void convert_to_oracleGeneral(reader_t *reader, std::string ofilepath, "%s: %ld M requests (%.2lf GB), trace time %ld, working set %lld " "object, %lld B (%.2lf GB), reversing output...\n", reader->trace_path, (long)(n_req_curr / 1e6), (double)total_bytes / GiB, - start_ts - req->clock_time, (long long)n_obj, (long long)unique_bytes, - (double)unique_bytes / GiB); + (long)(start_ts - req->clock_time), (long long)n_obj, + (long long)unique_bytes, (double)unique_bytes / GiB); struct trace_stat stat; stat.n_req = n_req_curr; @@ -251,6 +251,6 @@ static void _reverse_file(std::string ofilepath, struct trace_stat stat, remove((ofilepath + ".reverse").c_str()); INFO("trace conversion finished, %ld requests %ld objects, output %s\n", - n_req, stat.n_obj, ofilepath.c_str()); + (long) n_req, (long) stat.n_obj, ofilepath.c_str()); } } // namespace traceConv \ No newline at end of file diff --git a/libCacheSim/bin/traceUtils/traceConvMain.cpp b/libCacheSim/bin/traceUtils/traceConvMain.cpp index 23198212..a2883c11 100644 --- a/libCacheSim/bin/traceUtils/traceConvMain.cpp +++ b/libCacheSim/bin/traceUtils/traceConvMain.cpp @@ -30,7 +30,7 @@ int main(int argc, char *argv[]) { struct arguments args; cli::parse_cmd(argc, argv, &args); - if (args.ofilepath == NULL) { + if (strlen(args.ofilepath) == 0) { snprintf(args.ofilepath, OFILEPATH_LEN, "%s.oracleGeneral", args.trace_path); } diff --git a/libCacheSim/bin/traceUtils/traceFilterMain.cpp b/libCacheSim/bin/traceUtils/traceFilterMain.cpp index a6eb46c0..e601d268 100644 --- a/libCacheSim/bin/traceUtils/traceFilterMain.cpp +++ b/libCacheSim/bin/traceUtils/traceFilterMain.cpp @@ -53,7 +53,7 @@ void filter(reader_t *reader, cache_t *cache, std::string ofilepath) { read_one_req(reader, req); } - INFO("write %ld/%ld %.4lf requests to file %s\n", n_written_req, n_req, + INFO("write %ld/%ld %.4lf requests to file %s\n", (long) n_written_req, (long) n_req, (double)n_written_req / n_req, ofilepath.c_str()); free_request(req); output_file.close(); diff --git a/libCacheSim/cache/eviction/CR_LFU.c b/libCacheSim/cache/eviction/CR_LFU.c index 33ae33df..d3759a1e 100644 --- a/libCacheSim/cache/eviction/CR_LFU.c +++ b/libCacheSim/cache/eviction/CR_LFU.c @@ -46,7 +46,8 @@ static void free_list_node(void *list_node) { */ cache_t *CR_LFU_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("CR_LFU", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("CR_LFU", ccache_params, cache_specific_params); cache->cache_init = CR_LFU_init; cache->cache_free = CR_LFU_free; cache->get = CR_LFU_get; diff --git a/libCacheSim/cache/eviction/LIRS.c b/libCacheSim/cache/eviction/LIRS.c index 6c989dad..c34ba0f4 100644 --- a/libCacheSim/cache/eviction/LIRS.c +++ b/libCacheSim/cache/eviction/LIRS.c @@ -502,7 +502,7 @@ bool LIRS_can_insert(cache_t *cache, const request_t *req) { INFO("LIRS_can_insert: should not reach here. n_req = %ld\n", (long)cache->n_req); - assert(false); + abort(); } static void LIRS_prune(cache_t *cache) { diff --git a/libCacheSim/cache/eviction/WTinyLFU.c b/libCacheSim/cache/eviction/WTinyLFU.c index aa1d4424..42bf9255 100644 --- a/libCacheSim/cache/eviction/WTinyLFU.c +++ b/libCacheSim/cache/eviction/WTinyLFU.c @@ -253,6 +253,7 @@ cache_obj_t *WTinyLFU_insert(cache_t *cache, const request_t *req) { static cache_obj_t *WTinyLFU_to_evict(cache_t *cache, const request_t *req) { // Warning: don't use this function DEBUG_ASSERT(false); + return NULL; } static void WTinyLFU_evict(cache_t *cache, const request_t *req) { diff --git a/libCacheSim/cache/eviction/nop.c b/libCacheSim/cache/eviction/nop.c index bbf5bba7..5cf78d60 100644 --- a/libCacheSim/cache/eviction/nop.c +++ b/libCacheSim/cache/eviction/nop.c @@ -113,9 +113,9 @@ static bool nop_get(cache_t *cache, const request_t *req) { */ static cache_obj_t *nop_find(cache_t *cache, const request_t *req, const bool update_cache) { -// cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); + // cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); -// return cache_obj; + // return cache_obj; return NULL; } @@ -130,9 +130,9 @@ static cache_obj_t *nop_find(cache_t *cache, const request_t *req, * @return the inserted object */ static cache_obj_t *nop_insert(cache_t *cache, const request_t *req) { -// cache_obj_t *obj = cache_insert_base(cache, req); + // cache_obj_t *obj = cache_insert_base(cache, req); -// return obj; + // return obj; return NULL; } @@ -147,7 +147,8 @@ static cache_obj_t *nop_insert(cache_t *cache, const request_t *req) { * @return the object to be evicted */ static cache_obj_t *nop_to_evict(cache_t *cache, const request_t *req) { - assert(false); + assert(false); + return NULL; } /** @@ -159,7 +160,7 @@ static cache_obj_t *nop_to_evict(cache_t *cache, const request_t *req) { * @param req not used */ static void nop_evict(cache_t *cache, const request_t *req) { -// cache_evict_base(cache, obj_to_evict, true); + // cache_evict_base(cache, obj_to_evict, true); } /** diff --git a/libCacheSim/cache/eviction/priv/QDLPv0.c b/libCacheSim/cache/eviction/priv/QDLPv0.c index 7b5b102b..f21e9166 100644 --- a/libCacheSim/cache/eviction/priv/QDLPv0.c +++ b/libCacheSim/cache/eviction/priv/QDLPv0.c @@ -320,6 +320,7 @@ static cache_obj_t *QDLPv0_to_evict(cache_t *cache, const request_t *req) { // not implemented, we need to evict from the clock cache assert(0); } + return NULL; } /** diff --git a/libCacheSim/dataStructure/hash/CMakeLists.txt b/libCacheSim/dataStructure/hash/CMakeLists.txt index f636a56e..e32a49bd 100644 --- a/libCacheSim/dataStructure/hash/CMakeLists.txt +++ b/libCacheSim/dataStructure/hash/CMakeLists.txt @@ -1,5 +1,5 @@ -aux_source_directory(. DIR_LIB_SRCS) -add_library (ds_hash ${DIR_LIB_SRCS}) +# aux_source_directory(. DIR_LIB_SRCS) +# add_library (ds_hash ${DIR_LIB_SRCS}) -# add_library (ds_hash murmur3.c) +add_library (ds_hash murmur3.c) diff --git a/libCacheSim/dataStructure/robin_hood.h b/libCacheSim/dataStructure/robin_hood.h index 1a5c294d..7ee5d323 100644 --- a/libCacheSim/dataStructure/robin_hood.h +++ b/libCacheSim/dataStructure/robin_hood.h @@ -207,7 +207,7 @@ static Counts& counts() { // workaround missing "is_trivially_copyable" in g++ < 5.0 // See https://stackoverflow.com/a/31798726/48181 #if defined(__GNUC__) && __GNUC__ < 5 -# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__) +# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __is_trivially_copyable(__VA_ARGS__) #else # define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value #endif diff --git a/libCacheSim/include/libCacheSim/dist.h b/libCacheSim/include/libCacheSim/dist.h index 82c090ab..9b1daf9c 100644 --- a/libCacheSim/include/libCacheSim/dist.h +++ b/libCacheSim/include/libCacheSim/dist.h @@ -19,7 +19,7 @@ typedef enum { FUTURE_STACK_DIST, } dist_type_e; -static char *dist_type_str[] = { +static char *g_dist_type_name[] = { "DIST_SINCE_LAST_ACCESS", "DIST_SINCE_FIRST_ACCESS", "STACK_DIST", diff --git a/libCacheSim/include/libCacheSim/enum.h b/libCacheSim/include/libCacheSim/enum.h index c05c9b0a..fbc9900b 100644 --- a/libCacheSim/include/libCacheSim/enum.h +++ b/libCacheSim/include/libCacheSim/enum.h @@ -65,7 +65,7 @@ typedef enum { UNKNOWN_TRACE, } __attribute__((__packed__)) trace_type_e; -static char *trace_type_str[UNKNOWN_TRACE + 2] = { +static char *g_trace_type_name[UNKNOWN_TRACE + 2] = { "CSV_TRACE", "BIN_TRACE", "PLAIN_TXT_TRACE", diff --git a/libCacheSim/include/libCacheSim/reader.h b/libCacheSim/include/libCacheSim/reader.h index f20502a0..644fb667 100644 --- a/libCacheSim/include/libCacheSim/reader.h +++ b/libCacheSim/include/libCacheSim/reader.h @@ -272,7 +272,7 @@ static inline void print_reader(reader_t *reader) { "%p, line_buf_size: %zu, csv_delimiter: %c, csv_has_header: %d, " "obj_id_is_num: %d, ignore_size_zero_req: %d, ignore_obj_size: %d, " "n_req_left: %d, last_req_clock_time: %ld\n", - trace_type_str[reader->trace_type], reader->trace_path, + g_trace_type_name[reader->trace_type], reader->trace_path, reader->trace_start_offset, (long)reader->mmap_offset, reader->is_zstd_file, reader->item_size, reader->file, reader->line_buf, reader->line_buf_size, reader->csv_delimiter, reader->csv_has_header, diff --git a/libCacheSim/profiler/dist.c b/libCacheSim/profiler/dist.c index 21a82539..1309c1dd 100644 --- a/libCacheSim/profiler/dist.c +++ b/libCacheSim/profiler/dist.c @@ -196,7 +196,7 @@ void save_dist(reader_t *const reader, const int32_t *dist_array, int64_t array_size, const char *const ofilepath, const dist_type_e dist_type) { char *file_path = (char *)malloc(strlen(ofilepath) + 128); - sprintf(file_path, "%s.%s", ofilepath, dist_type_str[dist_type]); + sprintf(file_path, "%s.%s", ofilepath, g_dist_type_name[dist_type]); FILE *file = fopen(file_path, "wb"); fwrite(dist_array, sizeof(int32_t), get_num_of_req(reader), file); fclose(file); @@ -207,7 +207,7 @@ void save_dist_txt(reader_t *const reader, const int32_t *dist_array, int64_t array_size, const char *const ofilepath, const dist_type_e dist_type) { char *file_path = (char *)malloc(strlen(ofilepath) + 128); - sprintf(file_path, "%s.%s.txt", ofilepath, dist_type_str[dist_type]); + sprintf(file_path, "%s.%s.txt", ofilepath, g_dist_type_name[dist_type]); FILE *file = fopen(file_path, "w"); for (int i = 0; i < array_size; i++) { fprintf(file, "%d\n", dist_array[i]); @@ -264,7 +264,7 @@ void save_dist_as_cnt_txt(reader_t *const reader, const int32_t *dist_array, assert(get_num_of_req(reader) == array_size); char *file_path = (char *)malloc(strlen(ofilepath) + 128); - sprintf(file_path, "%s.%s.cnt", ofilepath, dist_type_str[dist_type]); + sprintf(file_path, "%s.%s.cnt", ofilepath, g_dist_type_name[dist_type]); FILE *file = fopen(file_path, "w"); GHashTable *hash_table = diff --git a/libCacheSim/traceAnalyzer/analyzer.cpp b/libCacheSim/traceAnalyzer/analyzer.cpp index aefd8ae5..e2b83dba 100644 --- a/libCacheSim/traceAnalyzer/analyzer.cpp +++ b/libCacheSim/traceAnalyzer/analyzer.cpp @@ -111,8 +111,8 @@ void traceAnalyzer::TraceAnalyzer::run() { ERROR( "The data is not ordered by time, please sort the trace first!" "Current time %ld requested object %lu, obj size %lu\n", - (long)req->clock_time + start_ts_, (unsigned long)req->obj_id, - req->obj_size); + (long)(req->clock_time + start_ts_), (unsigned long)req->obj_id, + (long)req->obj_size); } DEBUG_ASSERT(curr_time_window_idx == time_to_window_idx(req->clock_time)); diff --git a/scripts/install_dependency.sh b/scripts/install_dependency.sh index c6e82450..a155d749 100644 --- a/scripts/install_dependency.sh +++ b/scripts/install_dependency.sh @@ -10,7 +10,7 @@ setup_centos() { } setup_macOS() { - brew install glib google-perftools argp-standalone + brew install glib google-perftools argp-standalone xxhash } setup_xgboost() { diff --git a/test/common.h b/test/common.h index 9f78522b..cc8a9dc2 100644 --- a/test/common.h +++ b/test/common.h @@ -10,7 +10,12 @@ #include #include #include +#ifdef __linux__ #include +#elif __APPLE__ +#include +#include +#endif #include "../libCacheSim/include/libCacheSim.h" #include "../libCacheSim/include/libCacheSim/prefetchAlgo.h" @@ -27,20 +32,30 @@ #define DEFAULT_TTL (300 * 86400) -static inline unsigned int _n_cores0() { - unsigned int eax = 11, ebx = 0, ecx = 1, edx = 0; +// static inline unsigned int _n_cores0() { +// unsigned int eax = 11, ebx = 0, ecx = 1, edx = 0; - asm volatile("cpuid" - : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) - : "0"(eax), "2"(ecx) - :); - // printf("Cores: %d\nThreads: %d\nActual thread: %d\n", eax, ebx, edx); - return ebx; -} +// asm volatile("cpuid" +// : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) +// : "0"(eax), "2"(ecx) +// :); +// // printf("Cores: %d\nThreads: %d\nActual thread: %d\n", eax, ebx, edx); +// return ebx; +// } -static inline unsigned int _n_cores() { - return get_nprocs(); +#ifdef __linux__ +static inline unsigned int _n_cores(void) { return get_nprocs(); } +#elif __APPLE__ +static inline unsigned int _n_cores(void) { + int count; + size_t count_len = sizeof(count); + sysctlbyname("hw.physicalcpu", &count, &count_len, NULL, 0); + // fprintf(stderr, "you have %i cpu cores", count); + return count; } +#else +#error "what platform is this" +#endif static void _detect_data_path(char *data_path, char *data_name) { sprintf(data_path, "data/%s", data_name); diff --git a/test/test_evictionAlgo.c b/test/test_evictionAlgo.c index 774f7b14..476c9603 100644 --- a/test/test_evictionAlgo.c +++ b/test/test_evictionAlgo.c @@ -5,8 +5,7 @@ #include "../libCacheSim/utils/include/mymath.h" #include "common.h" -// static const uint64_t req_cnt_true = 113872, req_byte_true = 4205978112; -static const uint64_t req_cnt_true = 113872, req_byte_true = 4368040448; +static const uint64_t g_req_cnt_true = 113872, g_req_byte_true = 4368040448; static void _verify_profiler_results(const cache_stat_t *res, uint64_t num_of_sizes, @@ -24,23 +23,23 @@ static void _verify_profiler_results(const cache_stat_t *res, static void print_results(const cache_t *cache, const cache_stat_t *res) { printf("%s uint64_t cache_size[] = {", cache->cache_name); - printf("%ld", res[0].cache_size); + printf("%ld", (long)res[0].cache_size); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].cache_size); + printf(", %ld", (long)res[i].cache_size); } printf("};\n"); printf("uint64_t miss_cnt_true[] = {"); - printf("%ld", res[0].n_miss); + printf("%ld", (long)res[0].n_miss); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].n_miss); + printf(", %ld", (long)res[i].n_miss); } printf("};\n"); printf("uint64_t miss_byte_true[] = {"); - printf("%ld", res[0].n_miss_byte); + printf("%ld", (long)res[0].n_miss_byte); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].n_miss_byte); + printf(", %ld", (long)res[i].n_miss_byte); } printf("};\n"); } @@ -60,8 +59,8 @@ static void test_LRU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -82,8 +81,8 @@ static void test_Clock(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -103,8 +102,8 @@ static void test_FIFO(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -128,8 +127,8 @@ static void test_Belady(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -153,8 +152,8 @@ static void test_BeladySize(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -174,8 +173,8 @@ static void test_Random(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -195,8 +194,8 @@ static void test_LFU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); g_free(res); } @@ -216,8 +215,8 @@ static void test_LFUCpp(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -237,8 +236,8 @@ static void test_GDSF(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -258,8 +257,8 @@ static void test_LHD(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -279,8 +278,8 @@ static void test_Hyperbolic(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -300,8 +299,8 @@ static void test_LeCaR(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -321,8 +320,8 @@ static void test_Cacheus(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -342,8 +341,8 @@ static void test_SR_LRU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -363,8 +362,8 @@ static void test_CR_LFU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -384,8 +383,8 @@ static void test_LFUDA(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -405,8 +404,8 @@ static void test_MRU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -426,8 +425,8 @@ static void test_ARC(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -447,8 +446,8 @@ static void test_SLRU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -468,8 +467,8 @@ static void test_QDLP_FIFO(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -489,8 +488,8 @@ static void test_S3FIFO(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -510,8 +509,8 @@ static void test_Sieve(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -535,8 +534,8 @@ static void test_LIRS(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } diff --git a/test/test_prefetchAlgo.c b/test/test_prefetchAlgo.c index 190c6ec0..188cf17a 100644 --- a/test/test_prefetchAlgo.c +++ b/test/test_prefetchAlgo.c @@ -4,8 +4,8 @@ #include "../libCacheSim/utils/include/mymath.h" #include "common.h" -// static const uint64_t req_cnt_true = 113872, req_byte_true = 4205978112; -static const uint64_t req_cnt_true = 113872, req_byte_true = 4368040448; + +static const uint64_t g_req_cnt_true = 113872, g_req_byte_true = 4368040448; static void _verify_profiler_results(const cache_stat_t *res, uint64_t num_of_sizes, @@ -23,23 +23,23 @@ static void _verify_profiler_results(const cache_stat_t *res, static void print_results(const cache_t *cache, const cache_stat_t *res) { printf("%s uint64_t cache_size[] = {", cache->cache_name); - printf("%ld", res[0].cache_size); + printf("%ld", (long)res[0].cache_size); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].cache_size); + printf(", %ld", (long)res[i].cache_size); } printf("};\n"); printf("uint64_t miss_cnt_true[] = {"); - printf("%ld", res[0].n_miss); + printf("%ld", (long)res[0].n_miss); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].n_miss); + printf(", %ld", (long)res[i].n_miss); } printf("};\n"); printf("uint64_t miss_byte_true[] = {"); - printf("%ld", res[0].n_miss_byte); + printf("%ld", (long)res[0].n_miss_byte); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].n_miss_byte); + printf(", %ld", (long)res[i].n_miss_byte); } printf("};\n"); } @@ -59,8 +59,8 @@ static void test_Mithril(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } diff --git a/test/test_simulator.c b/test/test_simulator.c index 937cb959..8921d274 100644 --- a/test/test_simulator.c +++ b/test/test_simulator.c @@ -106,7 +106,8 @@ static void test_simulator(gconstpointer user_data) { g_assert_true(caches[i] != NULL); } - res = simulate_with_multi_caches(reader, caches, 4, NULL, 0, 0, _n_cores(), false); + res = simulate_with_multi_caches(reader, caches, 4, NULL, 0, 0, _n_cores(), + false); g_assert_cmpuint(res[0].cache_size, ==, STEP_SIZE); g_assert_cmpuint(res[1].n_req_byte, ==, req_byte_true); g_assert_cmpuint(res[3].n_req, ==, req_cnt_true); @@ -205,9 +206,10 @@ static void test_simulator_with_ttl(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); for (uint64_t i = 0; i < CACHE_SIZE / STEP_SIZE; i++) { - printf("cache size: %lu, n_req: %lu, n_req_byte: %lu, n_miss: %8lu %16lu\n", - res[i].cache_size, res[i].n_req, res[i].n_req_byte, res[i].n_miss, - res[i].n_miss_byte); + printf("cache size: %lu, n_req: %ld, n_req_byte: %ld, n_miss: %8ld %16ld\n", + (unsigned long)res[i].cache_size, (long)res[i].n_req, + (long)res[i].n_req_byte, (long)res[i].n_miss, + (long)res[i].n_miss_byte); g_assert_cmpuint(res[i].cache_size, ==, STEP_SIZE * (i + 1)); g_assert_cmpuint(res[i].n_req, ==, req_cnt_true); g_assert_cmpuint(res[i].n_req_byte, ==, req_byte_true); @@ -224,13 +226,11 @@ int main(int argc, char *argv[]) { reader_t *reader; reader = setup_plaintxt_reader_num(); - g_test_add_data_func_full("/libCacheSim/simulator_no_size_plain_num", - reader, + g_test_add_data_func_full("/libCacheSim/simulator_no_size_plain_num", reader, test_simulator_no_size, test_teardown); reader = setup_plaintxt_reader_str(); - g_test_add_data_func_full("/libCacheSim/simulator_no_size_plain_str", - reader, + g_test_add_data_func_full("/libCacheSim/simulator_no_size_plain_str", reader, test_simulator_no_size, test_teardown); reader = setup_csv_reader_obj_num(); From 6439066dfd36f0d9fd27bfbb0bfc936c6fe9c970 Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Sat, 6 Jan 2024 23:39:14 -0500 Subject: [PATCH 15/28] add random two and remove myclock (it has renamed to sieve now) (#49) --- libCacheSim/bin/cachesim/cache_init.h | 13 +- libCacheSim/cache/eviction/ARCv0.c | 8 - libCacheSim/cache/eviction/CMakeLists.txt | 2 +- libCacheSim/cache/eviction/LeCaRv0.c | 8 - libCacheSim/cache/eviction/RandomTwo.c | 204 +++++++++++ libCacheSim/cache/eviction/TwoQ.c | 7 - .../cache/eviction/belady/Sieve_Belady.c | 18 +- libCacheSim/cache/eviction/priv/MyClock.c | 327 ------------------ libCacheSim/cache/eviction/priv/QDLPv0.c | 2 +- libCacheSim/include/libCacheSim/cacheObj.h | 14 +- .../include/libCacheSim/evictionAlgo.h | 6 +- libCacheSim/traceReader/generalReader/csv.c | 5 +- test/test_evictionAlgo.c | 1 - 13 files changed, 234 insertions(+), 381 deletions(-) create mode 100644 libCacheSim/cache/eviction/RandomTwo.c delete mode 100644 libCacheSim/cache/eviction/priv/MyClock.c diff --git a/libCacheSim/bin/cachesim/cache_init.h b/libCacheSim/bin/cachesim/cache_init.h index 44822e93..d0c8700a 100644 --- a/libCacheSim/bin/cachesim/cache_init.h +++ b/libCacheSim/bin/cachesim/cache_init.h @@ -39,6 +39,10 @@ static inline cache_t *create_cache(const char *trace_path, cache = ARCv0_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "lhd") == 0) { cache = LHD_init(cc_params, eviction_params); + } else if (strcasecmp(eviction_algo, "random") == 0) { + cache = Random_init(cc_params, eviction_params); + } else if (strcasecmp(eviction_algo, "randomTwo") == 0) { + cache = RandomTwo_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "lfu") == 0) { cache = LFU_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "gdsf") == 0) { @@ -102,8 +106,8 @@ static inline cache_t *create_cache(const char *trace_path, } else if (strcasecmp(eviction_algo, "fifomerge") == 0 || strcasecmp(eviction_algo, "fifo-merge") == 0) { cache = FIFO_Merge_init(cc_params, eviction_params); - // } else if (strcasecmp(eviction_algo, "fifo-reinsertion") == 0) { - // cache = FIFO_Reinsertion_init(cc_params, eviction_params); + // } else if (strcasecmp(eviction_algo, "fifo-reinsertion") == 0) { + // cache = FIFO_Reinsertion_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "flashProb") == 0) { // used to measure application level write amp cache = flashProb_init(cc_params, eviction_params); @@ -121,7 +125,8 @@ static inline cache_t *create_cache(const char *trace_path, cache = Sieve_Belady_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "s3lru") == 0) { cache = S3LRU_init(cc_params, eviction_params); - } else if (strcasecmp(eviction_algo, "s3fifo") == 0 || strcasecmp(eviction_algo, "s3-fifo") == 0) { + } else if (strcasecmp(eviction_algo, "s3fifo") == 0 || + strcasecmp(eviction_algo, "s3-fifo") == 0) { cache = S3FIFO_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "s3fifod") == 0) { cache = S3FIFOd_init(cc_params, eviction_params); @@ -139,8 +144,6 @@ static inline cache_t *create_cache(const char *trace_path, cache = LRB_init(cc_params, eviction_params); #endif #ifdef INCLUDE_PRIV - } else if (strcasecmp(eviction_algo, "myclock") == 0) { - cache = MyClock_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "mclock") == 0) { cache = MClock_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "lp-sfifo") == 0) { diff --git a/libCacheSim/cache/eviction/ARCv0.c b/libCacheSim/cache/eviction/ARCv0.c index 1d61bfd2..1c3c6488 100644 --- a/libCacheSim/cache/eviction/ARCv0.c +++ b/libCacheSim/cache/eviction/ARCv0.c @@ -30,8 +30,6 @@ extern "C" { // #define LAZY_PROMOTION // #define QUICK_DEMOTION -// #define USE_MYCLOCK - typedef struct ARCv0_params { // L1_data is T1 in the paper, L1_ghost is B1 in the paper cache_t *T1; @@ -141,12 +139,6 @@ cache_t *ARCv0_init(const common_cache_params_t ccache_params, snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "ARCv0-QD"); #endif -#ifdef USE_MYCLOCK - params->T2->cache_free(params->T2); - params->T2 = MyClock_init(ccache_params_local, NULL); - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "ARCv0-myclock"); -#endif - return cache; } diff --git a/libCacheSim/cache/eviction/CMakeLists.txt b/libCacheSim/cache/eviction/CMakeLists.txt index 45dd821f..0a9a695a 100644 --- a/libCacheSim/cache/eviction/CMakeLists.txt +++ b/libCacheSim/cache/eviction/CMakeLists.txt @@ -1,6 +1,7 @@ set(sourceC MRU.c Random.c + RandomTwo.c LFU.c LFUDA.c ARC.c @@ -43,7 +44,6 @@ set(sourceC if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/priv") set(sourceC ${sourceC} - priv/MyClock.c priv/QDLPv0.c diff --git a/libCacheSim/cache/eviction/LeCaRv0.c b/libCacheSim/cache/eviction/LeCaRv0.c index 07e36100..1f57679c 100644 --- a/libCacheSim/cache/eviction/LeCaRv0.c +++ b/libCacheSim/cache/eviction/LeCaRv0.c @@ -18,8 +18,6 @@ extern "C" { #endif -// #define USE_MYCLOCK - typedef struct LeCaRv0_params { cache_t *LRU; // LRU cache_t *LRU_g; // eviction history of LRU @@ -118,12 +116,6 @@ cache_t *LeCaRv0_init(const common_cache_params_t ccache_params, params->LRU_g = LRU_init(ccache_params_g, NULL); params->LFU_g = LRU_init(ccache_params_g, NULL); -#ifdef USE_MYCLOCK - params->LRU->cache_free(params->LRU); - params->LRU = MyClock_init(ccache_params, NULL); - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "LeCaRv0-myclock"); -#endif - return cache; } diff --git a/libCacheSim/cache/eviction/RandomTwo.c b/libCacheSim/cache/eviction/RandomTwo.c new file mode 100644 index 00000000..1fd933b1 --- /dev/null +++ b/libCacheSim/cache/eviction/RandomTwo.c @@ -0,0 +1,204 @@ +// +// RandomTwo.c +// libCacheSim +// +// Picks two objects at random and evicts the one that is the least recently +// used RandomTwo eviction +// +// Created by Juncheng on 8/2/16. +// Copyright © 2016 Juncheng. All rights reserved. +// + +#include "../../dataStructure/hashtable/hashtable.h" +#include "../../include/libCacheSim/evictionAlgo.h" +#include "../../include/libCacheSim/macro.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// *********************************************************************** +// **** **** +// **** function declarations **** +// **** **** +// *********************************************************************** + +static void RandomTwo_free(cache_t *cache); +static bool RandomTwo_get(cache_t *cache, const request_t *req); +static cache_obj_t *RandomTwo_find(cache_t *cache, const request_t *req, + const bool update_cache); +static cache_obj_t *RandomTwo_insert(cache_t *cache, const request_t *req); +static cache_obj_t *RandomTwo_to_evict(cache_t *cache, const request_t *req); +static void RandomTwo_evict(cache_t *cache, const request_t *req); +static bool RandomTwo_remove(cache_t *cache, const obj_id_t obj_id); + +// *********************************************************************** +// **** **** +// **** end user facing functions **** +// **** **** +// **** init, free, get **** +// *********************************************************************** +/** + * @brief initialize a RandomTwo cache + * + * @param ccache_params some common cache parameters + * @param cache_specific_params RandomTwo specific parameters, should be NULL + */ +cache_t *RandomTwo_init(const common_cache_params_t ccache_params, + const char *cache_specific_params) { + common_cache_params_t ccache_params_copy = ccache_params; + ccache_params_copy.hashpower = MAX(12, ccache_params_copy.hashpower - 8); + + cache_t *cache = + cache_struct_init("RandomTwo", ccache_params, cache_specific_params); + cache->cache_init = RandomTwo_init; + cache->cache_free = RandomTwo_free; + cache->get = RandomTwo_get; + cache->find = RandomTwo_find; + cache->insert = RandomTwo_insert; + cache->to_evict = RandomTwo_to_evict; + cache->evict = RandomTwo_evict; + cache->remove = RandomTwo_remove; + + return cache; +} + +/** + * free resources used by this cache + * + * @param cache + */ +static void RandomTwo_free(cache_t *cache) { cache_struct_free(cache); } + +/** + * @brief this function is the user facing API + * it performs the following logic + * + * ``` + * if obj in cache: + * update_metadata + * return true + * else: + * if cache does not have enough space: + * evict until it has space to insert + * insert the object + * return false + * ``` + * + * @param cache + * @param req + * @return true if cache hit, false if cache miss + */ +static bool RandomTwo_get(cache_t *cache, const request_t *req) { + return cache_get_base(cache, req); +} + +// *********************************************************************** +// **** **** +// **** developer facing APIs (used by cache developer) **** +// **** **** +// *********************************************************************** + +/** + * @brief check whether an object is in the cache + * + * @param cache + * @param req + * @param update_cache whether to update the cache, + * if true, the object is promoted + * and if the object is expired, it is removed from the cache + * @return true on hit, false on miss + */ +static cache_obj_t *RandomTwo_find(cache_t *cache, const request_t *req, + const bool update_cache) { + cache_obj_t *obj = cache_find_base(cache, req, update_cache); + if (obj != NULL && update_cache) { + obj->RandomTwo.last_access_vtime = cache->n_req; + } + + return obj; +} + +/** + * @brief insert an object into the cache, + * update the hash table and cache metadata + * this function assumes the cache has enough space + * and eviction is not part of this function + * + * @param cache + * @param req + * @return the inserted object + */ +static cache_obj_t *RandomTwo_insert(cache_t *cache, const request_t *req) { + cache_obj_t *obj = cache_insert_base(cache, req); + obj->RandomTwo.last_access_vtime = cache->n_req; + + return obj; +} + +/** + * @brief find the object to be evicted + * this function does not actually evict the object or update metadata + * not all eviction algorithms support this function + * because the eviction logic cannot be decoupled from finding eviction + * candidate, so use assert(false) if you cannot support this function + * + * @param cache the cache + * @return the object to be evicted + */ +static cache_obj_t *RandomTwo_to_evict(cache_t *cache, const request_t *req) { + cache_obj_t *obj_to_evict1 = hashtable_rand_obj(cache->hashtable); + cache_obj_t *obj_to_evict2 = hashtable_rand_obj(cache->hashtable); + DEBUG_ASSERT(obj_to_evict->obj_size != 0); + if (obj_to_evict1->RandomTwo.last_access_vtime < + obj_to_evict2->RandomTwo.last_access_vtime) + return obj_to_evict1; + else + return obj_to_evict2; +} + +/** + * @brief evict an object from the cache + * it needs to call cache_evict_base before returning + * which updates some metadata such as n_obj, occupied size, and hash table + * + * @param cache + * @param req not used + */ +static void RandomTwo_evict(cache_t *cache, const request_t *req) { + cache_obj_t *obj_to_evict1 = hashtable_rand_obj(cache->hashtable); + cache_obj_t *obj_to_evict2 = hashtable_rand_obj(cache->hashtable); + DEBUG_ASSERT(obj_to_evict->obj_size != 0); + if (obj_to_evict1->RandomTwo.last_access_vtime < + obj_to_evict2->RandomTwo.last_access_vtime) + cache_evict_base(cache, obj_to_evict1, true); + else + cache_evict_base(cache, obj_to_evict2, true); +} + +/** + * @brief remove an object from the cache + * this is different from cache_evict because it is used to for user trigger + * remove, and eviction is used by the cache to make space for new objects + * + * it needs to call cache_remove_obj_base before returning + * which updates some metadata such as n_obj, occupied size, and hash table + * + * @param cache + * @param obj_id + * @return true if the object is removed, false if the object is not in the + * cache + */ +static bool RandomTwo_remove(cache_t *cache, const obj_id_t obj_id) { + cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); + if (obj == NULL) { + return false; + } + cache_remove_obj_base(cache, obj, true); + + return true; +} + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/cache/eviction/TwoQ.c b/libCacheSim/cache/eviction/TwoQ.c index 1110b839..f9933880 100644 --- a/libCacheSim/cache/eviction/TwoQ.c +++ b/libCacheSim/cache/eviction/TwoQ.c @@ -19,8 +19,6 @@ extern "C" { #endif -// #define USE_MYCLOCK - typedef struct { cache_t *Ain; cache_t *Aout; @@ -110,11 +108,6 @@ cache_t *TwoQ_init(const common_cache_params_t ccache_params, ccache_params_local.cache_size = params->Am_cache_size; params->Am = LRU_init(ccache_params_local, NULL); -#ifdef USE_MYCLOCK - params->Am->cache_free(params->Am); - params->Am = MyClock_init(ccache_params_local, NULL); - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "TwoQ-myclock"); -#endif return cache; } diff --git a/libCacheSim/cache/eviction/belady/Sieve_Belady.c b/libCacheSim/cache/eviction/belady/Sieve_Belady.c index 05301114..fd66bcc5 100644 --- a/libCacheSim/cache/eviction/belady/Sieve_Belady.c +++ b/libCacheSim/cache/eviction/belady/Sieve_Belady.c @@ -173,7 +173,7 @@ static cache_obj_t *Sieve_Belady_find(cache_t *cache, const request_t *req, const bool update_cache) { cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); if (cache_obj != NULL && update_cache) { - cache_obj->myclock.freq = 1; + cache_obj->sieve.freq = 1; } return cache_obj; @@ -199,14 +199,14 @@ static cache_obj_t *Sieve_Belady_insert(cache_t *cache, const request_t *req) { if (should_insert(cache, req->next_access_vtime)) { obj = cache_insert_base(cache, req); prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - obj->myclock.freq = 0; - obj->myclock.new_obj = true; + obj->sieve.freq = 0; + // obj->sieve.new_obj = true; } #else obj = cache_insert_base(cache, req); prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - obj->myclock.freq = 0; - obj->myclock.new_obj = true; + obj->sieve.freq = 0; + // obj->sieve.new_obj = true; #endif return obj; @@ -232,8 +232,8 @@ static void Sieve_Belady_evict(cache_t *cache, const request_t *req) { /* find the first untouched */ while (obj != NULL && should_insert(cache, obj->misc.next_access_vtime)) { - // while (obj != NULL && obj->myclock.freq > 0) { - obj->myclock.freq -= 1; + // while (obj != NULL && obj->sieve.freq > 0) { + obj->sieve.freq -= 1; obj = obj->queue.prev; } @@ -241,8 +241,8 @@ static void Sieve_Belady_evict(cache_t *cache, const request_t *req) { if (obj == NULL) { obj = params->q_tail; while (obj != NULL && should_insert(cache, obj->misc.next_access_vtime)) { - // while (obj != NULL && obj->myclock.freq > 0) { - obj->myclock.freq -= 1; + // while (obj != NULL && obj->sieve.freq > 0) { + obj->sieve.freq -= 1; obj = obj->queue.prev; } } diff --git a/libCacheSim/cache/eviction/priv/MyClock.c b/libCacheSim/cache/eviction/priv/MyClock.c deleted file mode 100644 index 7768d364..00000000 --- a/libCacheSim/cache/eviction/priv/MyClock.c +++ /dev/null @@ -1,327 +0,0 @@ - - -#include "../../../dataStructure/hashtable/hashtable.h" -#include "../../../include/libCacheSim/cache.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - cache_obj_t *q_head; - cache_obj_t *q_tail; - - cache_obj_t *pointer; - - int64_t n_miss; - int64_t n_new_obj; // used to track the number of objects since last pointer - // went back to the tail (the oldest) -} MyClock_params_t; - -// *********************************************************************** -// **** **** -// **** function declarations **** -// **** **** -// *********************************************************************** - -static void MyClock_free(cache_t *cache); -static bool MyClock_get(cache_t *cache, const request_t *req); -static cache_obj_t *MyClock_find(cache_t *cache, const request_t *req, - const bool update_cache); -static cache_obj_t *MyClock_insert(cache_t *cache, const request_t *req); -static cache_obj_t *MyClock_to_evict(cache_t *cache, const request_t *req); -static void MyClock_evict(cache_t *cache, const request_t *req); -static bool MyClock_remove(cache_t *cache, const obj_id_t obj_id); - -static void MyClock_verify(cache_t *cache); - -// *********************************************************************** -// **** **** -// **** end user facing functions **** -// **** **** -// **** init, free, get **** -// *********************************************************************** - -/** - * @brief initialize cache - * - * @param ccache_params some common cache parameters - * @param cache_specific_params cache specific parameters, see parse_params - * function or use -e "print" with the cachesim binary - */ -cache_t *MyClock_init(const common_cache_params_t ccache_params, - const char *cache_specific_params) { - cache_t *cache = - cache_struct_init("MyClock", ccache_params, cache_specific_params); - cache->cache_init = MyClock_init; - cache->cache_free = MyClock_free; - cache->get = MyClock_get; - cache->find = MyClock_find; - cache->insert = MyClock_insert; - cache->evict = MyClock_evict; - cache->remove = MyClock_remove; - cache->to_evict = MyClock_to_evict; - - if (ccache_params.consider_obj_metadata) { - cache->obj_md_size = 1; - } else { - cache->obj_md_size = 0; - } - - cache->eviction_params = my_malloc(MyClock_params_t); - memset(cache->eviction_params, 0, sizeof(MyClock_params_t)); - MyClock_params_t *params = (MyClock_params_t *)cache->eviction_params; - params->pointer = NULL; - params->q_head = NULL; - params->q_tail = NULL; - params->n_miss = 0; - - return cache; -} - -/** - * free resources used by this cache - * - * @param cache - */ -static void MyClock_free(cache_t *cache) { - free(cache->eviction_params); - cache_struct_free(cache); -} - -/** - * @brief this function is the user facing API - * it performs the following logic - * - * ``` - * if obj in cache: - * update_metadata - * return true - * else: - * if cache does not have enough space: - * evict until it has space to insert - * insert the object - * return false - * ``` - * - * @param cache - * @param req - * @return true if cache hit, false if cache miss - */ - -static bool MyClock_get(cache_t *cache, const request_t *req) { - MyClock_params_t *params = (MyClock_params_t *)cache->eviction_params; - - bool ck_hit = cache_get_base(cache, req); - if (!ck_hit) { - params->n_miss++; - } - - // if (cache->n_req % 1000 == 0) { - // printf("sieve %ld %ld %lf\n", cache->cache_size, cache->n_req, - // (double)params->n_new_obj / (double)cache->cache_size); - // } - - return ck_hit; -} - -// *********************************************************************** -// **** **** -// **** developer facing APIs (used by cache developer) **** -// **** **** -// *********************************************************************** - -/** - * @brief find an object in the cache - * - * @param cache - * @param req - * @param update_cache whether to update the cache, - * if true, the object is promoted - * and if the object is expired, it is removed from the cache - * @return the object or NULL if not found - */ -static cache_obj_t *MyClock_find(cache_t *cache, const request_t *req, - const bool update_cache) { - cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); - if (cache_obj != NULL && update_cache) { - cache_obj->myclock.freq = 1; - // cache_obj->myclock.freq += 1; - } - - return cache_obj; -} - -/** - * @brief insert an object into the cache, - * update the hash table and cache metadata - * this function assumes the cache has enough space - * eviction should be - * performed before calling this function - * - * @param cache - * @param req - * @return the inserted object - */ -static cache_obj_t *MyClock_insert(cache_t *cache, const request_t *req) { - MyClock_params_t *params = cache->eviction_params; - cache_obj_t *obj = cache_insert_base(cache, req); - prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - obj->myclock.freq = 0; - obj->myclock.new_obj = true; - params->n_new_obj++; - - return obj; -} - -/** - * @brief find the object to be evicted - * this function does not actually evict the object or update metadata - * not all eviction algorithms support this function - * because the eviction logic cannot be decoupled from finding eviction - * candidate, so use assert(false) if you cannot support this function - * - * @param cache the cache - * @return the object to be evicted - */ -static cache_obj_t *MyClock_to_evict_with_freq(cache_t *cache, - const request_t *req, - int to_evict_freq) { - MyClock_params_t *params = cache->eviction_params; - cache_obj_t *pointer = params->pointer; - // cache_obj_t *old_pointer = pointer; - - /* if we have run one full around or first eviction */ - if (pointer == NULL) pointer = params->q_tail; - - /* find the first untouched */ - while (pointer != NULL && pointer->myclock.freq > to_evict_freq) { - pointer = pointer->queue.prev; - } - - /* if we have finished one around, start from the tail */ - if (pointer == NULL) { - pointer = params->q_tail; - while (pointer != NULL && pointer->myclock.freq > to_evict_freq) { - pointer = pointer->queue.prev; - } - } - - if (pointer == NULL) return NULL; - - return pointer; -} - -static cache_obj_t *MyClock_to_evict(cache_t *cache, const request_t *req) { - // because we do not change the frequency of the object, - // if all objects have frequency 1, we may return NULL - int to_evict_freq = 0; - - cache_obj_t *obj_to_evict = - MyClock_to_evict_with_freq(cache, req, to_evict_freq); - - while (obj_to_evict == NULL) { - to_evict_freq += 1; - - obj_to_evict = MyClock_to_evict_with_freq(cache, req, to_evict_freq); - } - - return obj_to_evict; -} - -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - * @param evicted_obj if not NULL, return the evicted object to caller - */ -static void MyClock_evict(cache_t *cache, const request_t *req) { - MyClock_params_t *params = cache->eviction_params; - cache_obj_t *obj = params->pointer; - - /* if we have run one full around or first eviction */ - if (obj == NULL) { - obj = params->q_tail; - } - - /* find the first untouched */ - while ((obj != NULL && obj->myclock.freq > 0)) { - obj->myclock.freq -= 1; - if (obj->myclock.new_obj) params->n_new_obj -= 1; - - obj->myclock.new_obj = false; - obj = obj->queue.prev; - } - - /* if we have finished one around, start from the tail */ - if (obj == NULL) { - obj = params->q_tail; - while (obj != NULL && obj->myclock.freq > 0) { - obj->myclock.freq -= 1; - obj->myclock.new_obj = false; - obj = obj->queue.prev; - } - } - - if (obj->myclock.new_obj) params->n_new_obj -= 1; - params->pointer = obj->queue.prev; - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - cache_evict_base(cache, obj, true); -} - -static void MyClock_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove) { - DEBUG_ASSERT(obj_to_remove != NULL); - MyClock_params_t *params = cache->eviction_params; - if (obj_to_remove == params->pointer) { - params->pointer = obj_to_remove->queue.prev; - } - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj_to_remove); - cache_remove_obj_base(cache, obj_to_remove, true); -} - -/** - * @brief remove an object from the cache - * this is different from cache_evict because it is used to for user trigger - * remove, and eviction is used by the cache to make space for new objects - * - * it needs to call cache_remove_obj_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param obj_id - * @return true if the object is removed, false if the object is not in the - * cache - */ -static bool MyClock_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; - } - - MyClock_remove_obj(cache, obj); - - return true; -} - -static void MyClock_verify(cache_t *cache) { - MyClock_params_t *params = cache->eviction_params; - int64_t n_obj = 0, n_byte = 0; - cache_obj_t *obj = params->q_head; - - while (obj != NULL) { - assert(hashtable_find_obj_id(cache->hashtable, obj->obj_id) != NULL); - n_obj++; - n_byte += obj->obj_size; - obj = obj->queue.next; - } - - assert(n_obj == cache->get_n_obj(cache)); - assert(n_byte == cache->get_occupied_byte(cache)); -} - -#ifdef __cplusplus -} -#endif diff --git a/libCacheSim/cache/eviction/priv/QDLPv0.c b/libCacheSim/cache/eviction/priv/QDLPv0.c index f21e9166..0e0f8bf9 100644 --- a/libCacheSim/cache/eviction/priv/QDLPv0.c +++ b/libCacheSim/cache/eviction/priv/QDLPv0.c @@ -2,7 +2,7 @@ * @file QDLP.c * @brief Implementation of QDLPv0 eviction algorithm * QDLPv0: lazy promotion and quick demotion - * it uses a probationary FIFO queue with a myclock cache + * it uses a probationary FIFO queue with a sieve cache * objects are first inserted into the FIFO queue, and moved to the clock cache * when evicting from the FIFO queue if it has been accessed * if the cache is full, evict from the FIFO cache diff --git a/libCacheSim/include/libCacheSim/cacheObj.h b/libCacheSim/include/libCacheSim/cacheObj.h index 86a378fd..918e1a56 100644 --- a/libCacheSim/include/libCacheSim/cacheObj.h +++ b/libCacheSim/include/libCacheSim/cacheObj.h @@ -98,6 +98,10 @@ typedef struct { int lru_id; } SLRU_obj_metadata_t; +typedef struct { + int64_t last_access_vtime; +} RandomTwo_obj_metadata_t; + typedef struct { int64_t last_access_vtime; int32_t freq; @@ -126,14 +130,6 @@ typedef struct { int32_t freq; } __attribute__((packed)) misc_metadata_t; -typedef struct { - int32_t clock_id; - int32_t freq; - int32_t n_miss; - bool visited; - bool new_obj; -} __attribute__((packed)) myclock_obj_params_t; - // ############################## cache obj ################################### struct cache_obj; typedef struct cache_obj { @@ -165,6 +161,7 @@ typedef struct cache_obj { SR_LRU_obj_metadata_t SR_LRU; CR_LFU_obj_metadata_t CR_LFU; Hyperbolic_obj_metadata_t hyperbolic; + RandomTwo_obj_metadata_t RandomTwo; Belady_obj_metadata_t Belady; FIFO_Merge_obj_metadata_t FIFO_Merge; FIFO_Reinsertion_obj_metadata_t FIFO_Reinsertion; @@ -173,7 +170,6 @@ typedef struct cache_obj { QDLP_obj_metadata_t QDLP; LIRS_obj_metadata_t LIRS; S3FIFO_obj_metadata_t S3FIFO; - myclock_obj_params_t myclock; Sieve_obj_params_t sieve; #if defined(ENABLE_GLCACHE) && ENABLE_GLCACHE == 1 diff --git a/libCacheSim/include/libCacheSim/evictionAlgo.h b/libCacheSim/include/libCacheSim/evictionAlgo.h index 2fd35c4e..f343bbeb 100644 --- a/libCacheSim/include/libCacheSim/evictionAlgo.h +++ b/libCacheSim/include/libCacheSim/evictionAlgo.h @@ -97,6 +97,9 @@ cache_t *MRU_init(const common_cache_params_t ccache_params, cache_t *Random_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *RandomTwo_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); + cache_t *SLRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); @@ -177,9 +180,6 @@ cache_t *LP_ARC_init(const common_cache_params_t ccache_params, cache_t *LP_TwoQ_init(const common_cache_params_t ccache_params, const char *cache_specific_params); -cache_t *MyClock_init(const common_cache_params_t ccache_params, - const char *cache_specific_params); - cache_t *QDLPv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); diff --git a/libCacheSim/traceReader/generalReader/csv.c b/libCacheSim/traceReader/generalReader/csv.c index 82348cc1..acdc133b 100644 --- a/libCacheSim/traceReader/generalReader/csv.c +++ b/libCacheSim/traceReader/generalReader/csv.c @@ -168,7 +168,7 @@ bool check_delimiter(const reader_t *reader, char delimiter) { bool is_delimiter_correct = true; size_t n = 0; - getline(&buf, &n, ifile); + size_t _n = getline(&buf, &n, ifile); #define N_TEST 1024 for (int i = 0; i < N_TEST; i++) { if (strchr(buf, delimiter) == NULL) { @@ -338,7 +338,8 @@ void csv_reset_reader(reader_t *reader) { csv_set_delim(csv_params->csv_parser, csv_params->delimiter); if (csv_params->has_header) { - getline(&reader->line_buf, &reader->line_buf_size, reader->file); + size_t _n = + getline(&reader->line_buf, &reader->line_buf_size, reader->file); } } diff --git a/test/test_evictionAlgo.c b/test/test_evictionAlgo.c index 476c9603..59a0befe 100644 --- a/test/test_evictionAlgo.c +++ b/test/test_evictionAlgo.c @@ -66,7 +66,6 @@ static void test_LRU(gconstpointer user_data) { } static void test_Clock(gconstpointer user_data) { - /* myclock */ uint64_t miss_cnt_true[] = {93313, 89775, 83411, 81328, 74815, 72283, 71927, 64456}; uint64_t miss_byte_true[] = {4213887488, 4064512000, 3762650624, 3644467200, From eeae9891003dee054ea6df2b32e3a6dff7107663 Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Sat, 6 Jan 2024 23:59:10 -0500 Subject: [PATCH 16/28] update docker file to support ARM CPU (#50) --- dockerfile | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/dockerfile b/dockerfile index 17af5b3f..9cb5f708 100644 --- a/dockerfile +++ b/dockerfile @@ -1,33 +1,27 @@ FROM ubuntu:22.04 as base MAINTAINER Juncheng -LABEL version="0.0.1" +LABEL version="0.0.2" WORKDIR / # install dependency -RUN apt update && apt-get install -yqq libglib2.0-dev libgoogle-perftools-dev cmake git sudo wget gcc g++ - -# install dependency zstd -RUN wget https://github.com/facebook/zstd/releases/download/v1.5.0/zstd-1.5.0.tar.gz > /dev/null 2>&1 && \ -tar xvf zstd-1.5.0.tar.gz >/dev/null && \ -cd zstd-1.5.0/build/cmake/ && mkdir _build && cd _build/ && \ -cmake .. && make -j && make install && \ -cd .. +# RUN apt update && apt-get install -yqq libglib2.0-dev libgoogle-perftools-dev cmake git sudo wget gcc g++ xxhash +RUN apt update && apt-get install -yqq cmake git sudo wget # clone repo RUN git clone https://github.com/1a1a11a/libCacheSim -b develop # build libCacheSim WORKDIR /libCacheSim/ -RUN mkdir _build; -# RUN bash ./scripts/install_dependency.sh -# RUN bash ./scripts/install_libcachesim.sh +# RUN mkdir _build; +RUN bash ./scripts/install_dependency.sh +RUN bash ./scripts/install_libcachesim.sh WORKDIR /libCacheSim/_build/ -RUN cmake -DSUPPORT_ZSTD_TRACE=on .. && make -j && sudo make install +# RUN cmake -DSUPPORT_ZSTD_TRACE=on .. && make -j && sudo make install WORKDIR /libCacheSim/_build/bin/ @@ -38,8 +32,9 @@ WORKDIR /libCacheSim/_build/bin/ # sudo docker build -t 1a1a11a/libcachesim -f dockerfile . # push to docker hub -# sudo docker tag 1a1a11a/libcachesim:latest 1a1a11a/libcachesim:0.0.1 +# sudo docker tag 1a1a11a/libcachesim:latest 1a1a11a/libcachesim:0.0.2 # sudo docker push 1a1a11a/libcachesim:latest +# sudo docker push 1a1a11a/libcachesim:0.0.2 # use the container # sudo docker run -v /local/data/path:/data -it 1a1a11a/libcachesim:latest bash @@ -58,5 +53,3 @@ WORKDIR /libCacheSim/_build/bin/ # sudo apt update && sudo apt-get install -yqq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - - From 61c77f75d9b63b4e7ace8a96b9e1c6d3b1dca535 Mon Sep 17 00:00:00 2001 From: Nathaniel Filardo <105816689+nwf-msr@users.noreply.github.com> Date: Sun, 14 Jan 2024 00:52:14 +0000 Subject: [PATCH 17/28] Fix two set but unused variables (#52) FIXES #51 --- libCacheSim/cache/eviction/Random.c | 3 ++- libCacheSim/cache/eviction/RandomTwo.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libCacheSim/cache/eviction/Random.c b/libCacheSim/cache/eviction/Random.c index c89e3ab7..94f9d374 100644 --- a/libCacheSim/cache/eviction/Random.c +++ b/libCacheSim/cache/eviction/Random.c @@ -48,7 +48,8 @@ cache_t *Random_init(const common_cache_params_t ccache_params, common_cache_params_t ccache_params_copy = ccache_params; ccache_params_copy.hashpower = MAX(12, ccache_params_copy.hashpower - 8); - cache_t *cache = cache_struct_init("Random", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("Random", ccache_params_copy, cache_specific_params); cache->cache_init = Random_init; cache->cache_free = Random_free; cache->get = Random_get; diff --git a/libCacheSim/cache/eviction/RandomTwo.c b/libCacheSim/cache/eviction/RandomTwo.c index 1fd933b1..9c432aeb 100644 --- a/libCacheSim/cache/eviction/RandomTwo.c +++ b/libCacheSim/cache/eviction/RandomTwo.c @@ -50,7 +50,7 @@ cache_t *RandomTwo_init(const common_cache_params_t ccache_params, ccache_params_copy.hashpower = MAX(12, ccache_params_copy.hashpower - 8); cache_t *cache = - cache_struct_init("RandomTwo", ccache_params, cache_specific_params); + cache_struct_init("RandomTwo", ccache_params_copy, cache_specific_params); cache->cache_init = RandomTwo_init; cache->cache_free = RandomTwo_free; cache->get = RandomTwo_get; From 921c112b17282ce651dfa48ad0d0e7b375161219 Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:24:10 -0500 Subject: [PATCH 18/28] update example (#53) --- example/cacheCluster/CMakeLists.txt | 24 +++--- example/cacheCluster/main.cpp | 25 +++--- example/cacheHierarchy/CMakeLists.txt | 40 ++++----- .../cmake/Modules/FindGLib.cmake | 83 ------------------- example/cacheSimulator/CMakeLists.txt | 22 ++--- .../cmake/Modules/FindGLib.cmake | 83 ------------------- .../cacheSimulatorConcurrent/CMakeLists.txt | 17 +++- example/cacheSimulatorConcurrent/README.md | 2 - .../cmake/Modules/FindGLib.cmake | 83 ------------------- example/cacheSimulatorConcurrent/main.cpp | 6 +- 10 files changed, 67 insertions(+), 318 deletions(-) delete mode 100644 example/cacheHierarchy/cmake/Modules/FindGLib.cmake delete mode 100644 example/cacheSimulator/cmake/Modules/FindGLib.cmake delete mode 100644 example/cacheSimulatorConcurrent/cmake/Modules/FindGLib.cmake diff --git a/example/cacheCluster/CMakeLists.txt b/example/cacheCluster/CMakeLists.txt index 40d6c36f..3bf99e2d 100644 --- a/example/cacheCluster/CMakeLists.txt +++ b/example/cacheCluster/CMakeLists.txt @@ -1,29 +1,29 @@ cmake_minimum_required (VERSION 3.2) project (cacheCluster) -set(CMAKE_BUILD_TYPE Debug) -set(CMAKE_CXX_STANDARD 11) +# set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -# clear an error when using cmake in macOS -# if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -# set(CMAKE_MACOSX_RPATH 1) -# # Fix linking on 10.14+. See https://stackoverflow.com/questions/54068035 -# LINK_DIRECTORIES(/usr/local/lib) -# # a temporary fix for mac -# SET(CMAKE_C_FLAGS_DEBUG " -L/usr/local/lib -lglib-2.0 -lintl -Wno-unused-command-line-argument") -# endif() - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../../cmake/Modules/") find_package(GLib "2.40" REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) message(STATUS "glib found? " ${GLib_FOUND} ", GLIB = " ${GLib_LIBRARY} ", header = " ${GLib_INCLUDE_DIRS}) + +find_package(ZSTD) +# https://stackoverflow.com/questions/61377055/cannot-find-gflags-gflags-h-while-building-library-osx/61379123#61379123 +include_directories(${ZSTD_INCLUDE_DIR}) +if ("${ZSTD_LIBRARIES}" STREQUAL "") + message(FATAL_ERROR "zstd not found") +endif() + + ################ this mode compiles the external cache replacement algorithm together with the simulator ############ message(STATUS "project_source dir = " ${PROJECT_SOURCE_DIR}) file(GLOB ALL_CXX_SRC ${PROJECT_SOURCE_DIR}/*.cpp) file(GLOB ALL_C_SRC ${PROJECT_SOURCE_DIR}/*.c) add_executable(cacheCluster ${ALL_CXX_SRC} ${ALL_C_SRC}) -target_link_libraries(cacheCluster libCacheSim m ${GLib_LIBRARY} pthread dl) +target_link_libraries(cacheCluster libCacheSim m ${GLib_LIBRARY} pthread dl ${ZSTD_LIBRARIES}) diff --git a/example/cacheCluster/main.cpp b/example/cacheCluster/main.cpp index 008d2882..747e1ac7 100644 --- a/example/cacheCluster/main.cpp +++ b/example/cacheCluster/main.cpp @@ -13,9 +13,11 @@ namespace CDNSimulator { void simulate(int argc, char *argv[]) { - const char *data_path = "../../../data/trace.csv"; + const char *data_path = "../../../data/twitter_cluster52.csv"; if (argc > 1) { data_path = argv[1]; + } else { + printf("use default data at ../../../data/twitter_cluster52.csv\n"); } if (access(data_path, F_OK) == -1) { @@ -25,9 +27,9 @@ void simulate(int argc, char *argv[]) { /* setup a csv reader */ reader_init_param_t init_params = default_reader_init_params(); - init_params.obj_id_field = 5; - init_params.obj_size_field = 4; - init_params.time_field = 2; + init_params.obj_id_field = 2; + init_params.obj_size_field = 3; + init_params.time_field = 1; init_params.has_header_set = true; init_params.has_header = true; init_params.delimiter = ','; @@ -36,12 +38,17 @@ void simulate(int argc, char *argv[]) { reader_t *reader = open_trace(data_path, CSV_TRACE, &init_params); const uint64_t n_server = 10; - const uint64_t server_dram_cache_size = 8 * GiB; - const uint64_t server_disk_cache_size = 100 * GiB; + const uint64_t server_dram_cache_size = 1 * MiB; + const uint64_t server_disk_cache_size = 10 * MiB; const std::string algo = "lru"; // each cache holds 2 ** 20 objects, this is for performance optimization // you can specify a smaller number to save memory const uint32_t hashpower = 20; + printf( + "setting up a cluster of %lu servers, each server has %lu MB DRAM cache " + "and %lu MB disk cache, using %s as cache replacement algorithm\n", + (unsigned long)n_server, (unsigned long)(server_dram_cache_size / MiB), + (unsigned long)(server_disk_cache_size / MiB), algo.c_str()); CacheCluster cluster(0); @@ -68,10 +75,8 @@ void simulate(int argc, char *argv[]) { n_req_byte += req->obj_size; } - std::cout << n_req << " requests, " << n_miss - << " misses, miss ratio: " << (double)n_miss / n_req - << ", byte miss ratio: " << (double)n_miss_byte / n_req_byte - << std::endl; + std::cout << n_req << " requests, " << n_miss << " misses, miss ratio: " << (double)n_miss / n_req + << ", byte miss ratio: " << (double)n_miss_byte / n_req_byte << std::endl; close_trace(reader); } diff --git a/example/cacheHierarchy/CMakeLists.txt b/example/cacheHierarchy/CMakeLists.txt index cb4fa113..19a62c31 100644 --- a/example/cacheHierarchy/CMakeLists.txt +++ b/example/cacheHierarchy/CMakeLists.txt @@ -1,33 +1,28 @@ cmake_minimum_required (VERSION 3.2) project (layeredCache) - -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) message("****************** this example only works after libCacheSim has been installed ******************") -#### -## export C_INCLUDE_PATH=/usr/local/include -## export CPLUS_INCLUDE_PATH=/usr/local/include - -# clear an error when using cmake in macOS -if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(CMAKE_MACOSX_RPATH 1) - # Fix linking on 10.14+. See https://stackoverflow.com/questions/54068035 - LINK_DIRECTORIES(/usr/local/lib) - # a temporary fix for mac - SET(CMAKE_C_FLAGS_DEBUG " -L/usr/local/lib -lglib-2.0 -lintl -Wno-unused-command-line-argument") -endif() - - -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") -# find libglib2.0-dev +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../../cmake/Modules/") find_package(GLib "2.40" REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) message(STATUS "glib found? " ${GLib_FOUND} ", GLIB = " ${GLib_LIBRARY} ", header = " ${GLib_INCLUDE_DIRS}) -# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "/usr/local/share/cmake/yaml-cpp/") +find_package(ZSTD) +include_directories(${ZSTD_INCLUDE_DIR}) +if ("${ZSTD_LIBRARIES}" STREQUAL "") + message(FATAL_ERROR "zstd not found") +endif() + + +# https://github.com/jbeder/yaml-cpp +# wget https://github.com/jbeder/yaml-cpp/archive/refs/tags/0.8.0.tar.gz +# tar -xzf 0.8.0.tar.gz; mkdir yaml-cpp-0.8.0/_build; cd yaml-cpp-0.8.0/_build; +# cmake -DBUILD_SHARED_LIBS=ON ..; +# make -j; sudo make install find_package(YAML-CPP REQUIRED) # set(YAML_CPP_INCLUDE_DIR /usr/local/include/) # set(YAML_CPP_LIBRARIES /usr/local/lib/) @@ -38,10 +33,5 @@ message(STATUS ${YAML_CPP_INCLUDE_DIR} ";" ${YAML_CPP_LIBRARIES}) file(GLOB ALL_SRC ${PROJECT_SOURCE_DIR}/*.cpp) message(STATUS "all sources " ${ALL_SRC}) add_executable(layeredCache ${ALL_SRC}) -target_link_libraries(layeredCache yaml-cpp libCacheSim m ${GLib_LIBRARY}) - - - - - +target_link_libraries(layeredCache yaml-cpp libCacheSim dl m ${GLib_LIBRARY} ${ZSTD_LIBRARIES} pthread) diff --git a/example/cacheHierarchy/cmake/Modules/FindGLib.cmake b/example/cacheHierarchy/cmake/Modules/FindGLib.cmake deleted file mode 100644 index 8939d857..00000000 --- a/example/cacheHierarchy/cmake/Modules/FindGLib.cmake +++ /dev/null @@ -1,83 +0,0 @@ -# FindGLib.cmake -# -# -# CMake support for GLib/GObject/GIO. -# -# License: -# -# Copyright (c) 2016 Evan Nemerson -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, copy, -# modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -find_package(PkgConfig) - -if(PKG_CONFIG_FOUND) - pkg_search_module(GLib_PKG glib-2.0) -endif() - -find_library(GLib_LIBRARY glib-2.0 HINTS ${GLib_PKG_LIBRARY_DIRS}) -set(GLib glib-2.0) - -if(GLib_LIBRARY AND NOT GLib_FOUND) - add_library(${GLib} SHARED IMPORTED) - set_property(TARGET ${GLib} PROPERTY IMPORTED_LOCATION "${GLib_LIBRARY}") - set_property(TARGET ${GLib} PROPERTY INTERFACE_COMPILE_OPTIONS "${GLib_PKG_CFLAGS_OTHER}") - - find_path(GLib_INCLUDE_DIRS "glib.h" - HINTS ${GLib_PKG_INCLUDE_DIRS} - PATH_SUFFIXES "glib-2.0") - - get_filename_component(GLib_LIBDIR "${GLib}" DIRECTORY) - find_path(GLib_CONFIG_INCLUDE_DIR "glibconfig.h" - HINTS - ${GLib_LIBDIR} - ${GLib_PKG_INCLUDE_DIRS} - PATHS - "${CMAKE_LIBRARY_PATH}" - PATH_SUFFIXES - "glib-2.0/include" - "glib-2.0") - unset(GLib_LIBDIR) - - if(GLib_CONFIG_INCLUDE_DIR) - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MAJOR_VERSION REGEX "^#define GLIB_MAJOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MAJOR_VERSION ([0-9]+)$" "\\1" GLib_MAJOR_VERSION "${GLib_MAJOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MINOR_VERSION REGEX "^#define GLIB_MINOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MINOR_VERSION ([0-9]+)$" "\\1" GLib_MINOR_VERSION "${GLib_MINOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MICRO_VERSION REGEX "^#define GLIB_MICRO_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MICRO_VERSION ([0-9]+)$" "\\1" GLib_MICRO_VERSION "${GLib_MICRO_VERSION}") - set(GLib_VERSION "${GLib_MAJOR_VERSION}.${GLib_MINOR_VERSION}.${GLib_MICRO_VERSION}") - unset(GLib_MAJOR_VERSION) - unset(GLib_MINOR_VERSION) - unset(GLib_MICRO_VERSION) - - list(APPEND GLib_INCLUDE_DIRS ${GLib_CONFIG_INCLUDE_DIR}) - set_property(TARGET ${GLib} PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${GLib_INCLUDE_DIRS}") - endif() -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(GLib - REQUIRED_VARS - GLib_LIBRARY - GLib_INCLUDE_DIRS - VERSION_VAR - GLib_VERSION) diff --git a/example/cacheSimulator/CMakeLists.txt b/example/cacheSimulator/CMakeLists.txt index 49775ead..fbf63fcb 100644 --- a/example/cacheSimulator/CMakeLists.txt +++ b/example/cacheSimulator/CMakeLists.txt @@ -4,27 +4,23 @@ project (cacheSimulator) message("****************** this example only works after libCacheSim has been installed ******************") - -# clear an error when using cmake in macOS -# if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -# set(CMAKE_MACOSX_RPATH 1) -# # Fix linking on 10.14+. See https://stackoverflow.com/questions/54068035 -# LINK_DIRECTORIES(/usr/local/lib) -# # a temporary fix for mac -# SET(CMAKE_C_FLAGS_DEBUG " -L/usr/local/lib -lglib-2.0 -lintl -Wno-unused-command-line-argument") -# endif() - find_library(libCacheSim libCacheSim) find_path(libCacheSimInclude libCacheSim) message(STATUS "libCacheSim found? ${libCacheSim} ${libCacheSimInclude}") include_directories(${libCacheSimInclude}) - -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../../cmake/Modules/") find_package(GLib "2.40" REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) message(STATUS "glib found? ${GLib_FOUND}, lib = ${GLib_LIBRARY}, header = ${GLib_INCLUDE_DIRS}") +find_package(ZSTD) +# https://stackoverflow.com/questions/61377055/cannot-find-gflags-gflags-h-while-building-library-osx/61379123#61379123 +include_directories(${ZSTD_INCLUDE_DIR}) +if ("${ZSTD_LIBRARIES}" STREQUAL "") + message(FATAL_ERROR "zstd not found") +endif() + add_executable(cacheSimulator main.c) -target_link_libraries(cacheSimulator libCacheSim m ${GLib_LIBRARY}) +target_link_libraries(cacheSimulator libCacheSim m ${GLib_LIBRARY} ${ZSTD_LIBRARIES}) diff --git a/example/cacheSimulator/cmake/Modules/FindGLib.cmake b/example/cacheSimulator/cmake/Modules/FindGLib.cmake deleted file mode 100644 index 8939d857..00000000 --- a/example/cacheSimulator/cmake/Modules/FindGLib.cmake +++ /dev/null @@ -1,83 +0,0 @@ -# FindGLib.cmake -# -# -# CMake support for GLib/GObject/GIO. -# -# License: -# -# Copyright (c) 2016 Evan Nemerson -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, copy, -# modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -find_package(PkgConfig) - -if(PKG_CONFIG_FOUND) - pkg_search_module(GLib_PKG glib-2.0) -endif() - -find_library(GLib_LIBRARY glib-2.0 HINTS ${GLib_PKG_LIBRARY_DIRS}) -set(GLib glib-2.0) - -if(GLib_LIBRARY AND NOT GLib_FOUND) - add_library(${GLib} SHARED IMPORTED) - set_property(TARGET ${GLib} PROPERTY IMPORTED_LOCATION "${GLib_LIBRARY}") - set_property(TARGET ${GLib} PROPERTY INTERFACE_COMPILE_OPTIONS "${GLib_PKG_CFLAGS_OTHER}") - - find_path(GLib_INCLUDE_DIRS "glib.h" - HINTS ${GLib_PKG_INCLUDE_DIRS} - PATH_SUFFIXES "glib-2.0") - - get_filename_component(GLib_LIBDIR "${GLib}" DIRECTORY) - find_path(GLib_CONFIG_INCLUDE_DIR "glibconfig.h" - HINTS - ${GLib_LIBDIR} - ${GLib_PKG_INCLUDE_DIRS} - PATHS - "${CMAKE_LIBRARY_PATH}" - PATH_SUFFIXES - "glib-2.0/include" - "glib-2.0") - unset(GLib_LIBDIR) - - if(GLib_CONFIG_INCLUDE_DIR) - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MAJOR_VERSION REGEX "^#define GLIB_MAJOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MAJOR_VERSION ([0-9]+)$" "\\1" GLib_MAJOR_VERSION "${GLib_MAJOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MINOR_VERSION REGEX "^#define GLIB_MINOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MINOR_VERSION ([0-9]+)$" "\\1" GLib_MINOR_VERSION "${GLib_MINOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MICRO_VERSION REGEX "^#define GLIB_MICRO_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MICRO_VERSION ([0-9]+)$" "\\1" GLib_MICRO_VERSION "${GLib_MICRO_VERSION}") - set(GLib_VERSION "${GLib_MAJOR_VERSION}.${GLib_MINOR_VERSION}.${GLib_MICRO_VERSION}") - unset(GLib_MAJOR_VERSION) - unset(GLib_MINOR_VERSION) - unset(GLib_MICRO_VERSION) - - list(APPEND GLib_INCLUDE_DIRS ${GLib_CONFIG_INCLUDE_DIR}) - set_property(TARGET ${GLib} PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${GLib_INCLUDE_DIRS}") - endif() -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(GLib - REQUIRED_VARS - GLib_LIBRARY - GLib_INCLUDE_DIRS - VERSION_VAR - GLib_VERSION) diff --git a/example/cacheSimulatorConcurrent/CMakeLists.txt b/example/cacheSimulatorConcurrent/CMakeLists.txt index d5f60a93..1494124d 100644 --- a/example/cacheSimulatorConcurrent/CMakeLists.txt +++ b/example/cacheSimulatorConcurrent/CMakeLists.txt @@ -1,18 +1,27 @@ -cmake_minimum_required (VERSION 2.8) +cmake_minimum_required (VERSION 3.2) project (cacheSimulatorMultiSize) -# set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") find_library(libCacheSim libCacheSim) find_path(libCacheSimInclude libCacheSim) include_directories(${libCacheSimInclude}) message(STATUS "libCacheSim found? ${libCacheSim}, ${libCacheSimInclude}") + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../../cmake/Modules/") find_package(GLib "2.40" REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) message(STATUS "glib found? ${GLib_FOUND}, lib = ${GLib_LIBRARY}, header = ${GLib_INCLUDE_DIRS}") +find_package(ZSTD) +# https://stackoverflow.com/questions/61377055/cannot-find-gflags-gflags-h-while-building-library-osx/61379123#61379123 +include_directories(${ZSTD_INCLUDE_DIR}) +if ("${ZSTD_LIBRARIES}" STREQUAL "") + message(FATAL_ERROR "zstd not found") +endif() add_executable(cacheSimulatorMultiSize main.cpp) -target_link_libraries(cacheSimulatorMultiSize libCacheSim) +target_link_libraries(cacheSimulatorMultiSize libCacheSim m ${GLib_LIBRARY} ${ZSTD_LIBRARIES}) + diff --git a/example/cacheSimulatorConcurrent/README.md b/example/cacheSimulatorConcurrent/README.md index 73619b35..4093815f 100644 --- a/example/cacheSimulatorConcurrent/README.md +++ b/example/cacheSimulatorConcurrent/README.md @@ -1,7 +1,5 @@ ## Building a concurrent cache simulator -> WARN -> this has not been updated This tutorial shows how to build a concurrent cache simulator using libCacheSim. diff --git a/example/cacheSimulatorConcurrent/cmake/Modules/FindGLib.cmake b/example/cacheSimulatorConcurrent/cmake/Modules/FindGLib.cmake deleted file mode 100644 index 8939d857..00000000 --- a/example/cacheSimulatorConcurrent/cmake/Modules/FindGLib.cmake +++ /dev/null @@ -1,83 +0,0 @@ -# FindGLib.cmake -# -# -# CMake support for GLib/GObject/GIO. -# -# License: -# -# Copyright (c) 2016 Evan Nemerson -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, copy, -# modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -find_package(PkgConfig) - -if(PKG_CONFIG_FOUND) - pkg_search_module(GLib_PKG glib-2.0) -endif() - -find_library(GLib_LIBRARY glib-2.0 HINTS ${GLib_PKG_LIBRARY_DIRS}) -set(GLib glib-2.0) - -if(GLib_LIBRARY AND NOT GLib_FOUND) - add_library(${GLib} SHARED IMPORTED) - set_property(TARGET ${GLib} PROPERTY IMPORTED_LOCATION "${GLib_LIBRARY}") - set_property(TARGET ${GLib} PROPERTY INTERFACE_COMPILE_OPTIONS "${GLib_PKG_CFLAGS_OTHER}") - - find_path(GLib_INCLUDE_DIRS "glib.h" - HINTS ${GLib_PKG_INCLUDE_DIRS} - PATH_SUFFIXES "glib-2.0") - - get_filename_component(GLib_LIBDIR "${GLib}" DIRECTORY) - find_path(GLib_CONFIG_INCLUDE_DIR "glibconfig.h" - HINTS - ${GLib_LIBDIR} - ${GLib_PKG_INCLUDE_DIRS} - PATHS - "${CMAKE_LIBRARY_PATH}" - PATH_SUFFIXES - "glib-2.0/include" - "glib-2.0") - unset(GLib_LIBDIR) - - if(GLib_CONFIG_INCLUDE_DIR) - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MAJOR_VERSION REGEX "^#define GLIB_MAJOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MAJOR_VERSION ([0-9]+)$" "\\1" GLib_MAJOR_VERSION "${GLib_MAJOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MINOR_VERSION REGEX "^#define GLIB_MINOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MINOR_VERSION ([0-9]+)$" "\\1" GLib_MINOR_VERSION "${GLib_MINOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MICRO_VERSION REGEX "^#define GLIB_MICRO_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MICRO_VERSION ([0-9]+)$" "\\1" GLib_MICRO_VERSION "${GLib_MICRO_VERSION}") - set(GLib_VERSION "${GLib_MAJOR_VERSION}.${GLib_MINOR_VERSION}.${GLib_MICRO_VERSION}") - unset(GLib_MAJOR_VERSION) - unset(GLib_MINOR_VERSION) - unset(GLib_MICRO_VERSION) - - list(APPEND GLib_INCLUDE_DIRS ${GLib_CONFIG_INCLUDE_DIR}) - set_property(TARGET ${GLib} PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${GLib_INCLUDE_DIRS}") - endif() -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(GLib - REQUIRED_VARS - GLib_LIBRARY - GLib_INCLUDE_DIRS - VERSION_VAR - GLib_VERSION) diff --git a/example/cacheSimulatorConcurrent/main.cpp b/example/cacheSimulatorConcurrent/main.cpp index f05f0673..1b583102 100644 --- a/example/cacheSimulatorConcurrent/main.cpp +++ b/example/cacheSimulatorConcurrent/main.cpp @@ -3,7 +3,7 @@ #include #define NUM_SIZES 8 -const char *TRACE_PATH = "../../../data/trace.csv"; +const char *TRACE_PATH = "../../../data/cloudPhysicsIO.csv"; void run_one_cache_multiple_sizes(cache_t *cache, reader_t *reader) { uint64_t cache_sizes[NUM_SIZES] = {100, 200, 400, 800, @@ -44,7 +44,7 @@ void run_multiple_caches(reader_t *reader) { cache_t *caches[8] = { LRU_init(cc_params, nullptr), LFU_init(cc_params, nullptr), - FIFO_init(cc_params, nullptr), SLRU_init(cc_params, nullptr), + FIFO_init(cc_params, nullptr), Sieve_init(cc_params, nullptr), LHD_init(cc_params, nullptr), LeCaR_init(cc_params, nullptr), ARC_init(cc_params, nullptr), NULL}; @@ -55,7 +55,7 @@ void run_multiple_caches(reader_t *reader) { cache_stat_t *result = simulate_with_multi_caches( reader, caches, 8, nullptr, 0.0, 0, - static_cast(std::thread::hardware_concurrency())); + static_cast(std::thread::hardware_concurrency()), 0); printf( " cache name cache size num_miss num_req" From f6043c41df46f10b52f698e6d9e8d8b96ac9a92f Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Mon, 15 Jan 2024 21:25:04 +0000 Subject: [PATCH 19/28] update README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 84bdbe6c..4506cb16 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ ## News * **2023 June**: **QDLP** is available now, see [our paper](https://dl.acm.org/doi/10.1145/3593856.3595887) for details. -* **2023 Oct**: **[S3-FIFO](https://dl.acm.org/doi/10.1145/3600006.3613147)** and **SIEVE** are available! These are very simple algorithms that are very effective in reducing cache misses. Try them out in libCacheSim and your production! +* **2023 Oct**: **[S3-FIFO](https://dl.acm.org/doi/10.1145/3600006.3613147)** and **SIEVE(https://sievecache.com)** are available! These are very simple algorithms that are very effective in reducing cache misses. Try them out in libCacheSim and your production! +* **2024 Jan**: We compiled a list of open-source cache datasets at the bottom of this page --- From f3f6ba831067528e7562cff93d407cca0db54d6c Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 26 Jan 2024 18:41:59 -0500 Subject: [PATCH 20/28] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4506cb16..1dabe0e3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![build](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml/badge.svg)](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml) +The main development of libCacheSim is at [https://github.com/1a1a11a/libCacheSim](https://github.com/1a1a11a/libCacheSim), the [cachemon](https://github.com/cachemon/libCacheSim) repo is a mirror of the stable branch. - [libCacheSim - building and running cache simulations](#libcachesim---building-and-running-cache-simulations) From 1ce9ea91ff20082b970c3ea82f5ee1481fefa82c Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 26 Jan 2024 20:07:56 -0500 Subject: [PATCH 21/28] minor fix --- libCacheSim/cache/eviction/RandomTwo.c | 2 -- libCacheSim/include/libCacheSim/evictionAlgo.h | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/libCacheSim/cache/eviction/RandomTwo.c b/libCacheSim/cache/eviction/RandomTwo.c index 9c432aeb..8392d0c4 100644 --- a/libCacheSim/cache/eviction/RandomTwo.c +++ b/libCacheSim/cache/eviction/RandomTwo.c @@ -149,7 +149,6 @@ static cache_obj_t *RandomTwo_insert(cache_t *cache, const request_t *req) { static cache_obj_t *RandomTwo_to_evict(cache_t *cache, const request_t *req) { cache_obj_t *obj_to_evict1 = hashtable_rand_obj(cache->hashtable); cache_obj_t *obj_to_evict2 = hashtable_rand_obj(cache->hashtable); - DEBUG_ASSERT(obj_to_evict->obj_size != 0); if (obj_to_evict1->RandomTwo.last_access_vtime < obj_to_evict2->RandomTwo.last_access_vtime) return obj_to_evict1; @@ -168,7 +167,6 @@ static cache_obj_t *RandomTwo_to_evict(cache_t *cache, const request_t *req) { static void RandomTwo_evict(cache_t *cache, const request_t *req) { cache_obj_t *obj_to_evict1 = hashtable_rand_obj(cache->hashtable); cache_obj_t *obj_to_evict2 = hashtable_rand_obj(cache->hashtable); - DEBUG_ASSERT(obj_to_evict->obj_size != 0); if (obj_to_evict1->RandomTwo.last_access_vtime < obj_to_evict2->RandomTwo.last_access_vtime) cache_evict_base(cache, obj_to_evict1, true); diff --git a/libCacheSim/include/libCacheSim/evictionAlgo.h b/libCacheSim/include/libCacheSim/evictionAlgo.h index f343bbeb..31589001 100644 --- a/libCacheSim/include/libCacheSim/evictionAlgo.h +++ b/libCacheSim/include/libCacheSim/evictionAlgo.h @@ -132,6 +132,7 @@ cache_t *flashProb_init(const common_cache_params_t ccache_params, cache_t *LRU_Prob_init(const common_cache_params_t ccache_params, const char *cache_specific_params); + cache_t *SFIFOv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); From 25369d4642f886dd8704b425651aee701a1fe665 Mon Sep 17 00:00:00 2001 From: Zhelong Zhao <50295684+zztaki@users.noreply.github.com> Date: Tue, 20 Feb 2024 23:33:09 +0800 Subject: [PATCH 22/28] feat(prefetch): add OBL (#57) Signed-off-by: zztaki --- libCacheSim/bin/cachesim/cli_parser.c | 2 +- libCacheSim/cache/prefetch/CMakeLists.txt | 2 +- libCacheSim/cache/prefetch/OBL.c | 178 ++++++++++++++++++ .../include/libCacheSim/prefetchAlgo.h | 7 + .../include/libCacheSim/prefetchAlgo/OBL.h | 29 +++ test/common.h | 3 + test/test_prefetchAlgo.c | 18 ++ 7 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 libCacheSim/cache/prefetch/OBL.c create mode 100644 libCacheSim/include/libCacheSim/prefetchAlgo/OBL.h diff --git a/libCacheSim/bin/cachesim/cli_parser.c b/libCacheSim/bin/cachesim/cli_parser.c index 72de683a..6f0d62c8 100644 --- a/libCacheSim/bin/cachesim/cli_parser.c +++ b/libCacheSim/bin/cachesim/cli_parser.c @@ -72,7 +72,7 @@ static struct argp_option options[] = { {"admission-params", OPTION_ADMISSION_PARAMS, "\"prob=0.8\"", 0, "params for admission algorithm", 4}, {"prefetch", OPTION_PREFETCH_ALGO, "Mithril", 0, - "Prefetching algorithm: Mithril", 4}, + "Prefetching algorithm: Mithril/OBL/PG/AMP", 4}, {"prefetch-params", OPTION_PREFETCH_PARAMS, "\"block-size=65536\"", 0, "optional params for each prefetching algorithm, e.g., block-size=65536", 4}, diff --git a/libCacheSim/cache/prefetch/CMakeLists.txt b/libCacheSim/cache/prefetch/CMakeLists.txt index 025b6409..c066e2e1 100644 --- a/libCacheSim/cache/prefetch/CMakeLists.txt +++ b/libCacheSim/cache/prefetch/CMakeLists.txt @@ -1,5 +1,5 @@ -add_library(prefetchC Mithril.c) +add_library(prefetchC Mithril.c OBL.c) add_library(prefetch INTERFACE) target_link_libraries(prefetch INTERFACE prefetchC) diff --git a/libCacheSim/cache/prefetch/OBL.c b/libCacheSim/cache/prefetch/OBL.c new file mode 100644 index 00000000..a802b0c1 --- /dev/null +++ b/libCacheSim/cache/prefetch/OBL.c @@ -0,0 +1,178 @@ +// +// an OBL module that supports sequential prefetching for block storage. Each +// object (logical block address) should be uniform in size. +// +// +// OBL.c +// libCacheSim +// +// Created by Zhelong on 24/1/29. +// Copyright © 2024 Zhelong. All rights reserved. +// +#include "../../include/libCacheSim/prefetchAlgo/OBL.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "../../include/libCacheSim/prefetchAlgo.h" +// #define DEBUG + +#ifdef __cplusplus +extern "C" { +#endif + +// *********************************************************************** +// **** **** +// **** helper function declarations **** +// **** **** +// *********************************************************************** + +const char *OBL_default_params(void) { return "block-size=512, sequential-confidence-k=4"; } + +static void set_OBL_default_init_params(OBL_init_params_t *init_params) { + init_params->block_size = 512; + init_params->sequential_confidence_k = 4; +} + +static void OBL_parse_init_params(const char *cache_specific_params, OBL_init_params_t *init_params) { + char *params_str = strdup(cache_specific_params); + + while (params_str != NULL && params_str[0] != '\0') { + char *key = strsep((char **)¶ms_str, "="); + char *value = strsep((char **)¶ms_str, ","); + while (params_str != NULL && *params_str == ' ') { + params_str++; + } + if (strcasecmp(key, "block-size") == 0) { + init_params->block_size = atoi(value); + } else if (strcasecmp(key, "sequential-confidence-k") == 0) { + init_params->sequential_confidence_k = atoi(value); + } else { + ERROR("OBL does not have parameter %s\n", key); + printf("default params: %s\n", OBL_default_params()); + exit(1); + } + } +} + +static void set_OBL_params(OBL_params_t *OBL_params, OBL_init_params_t *init_params, uint64_t cache_size) { + OBL_params->block_size = init_params->block_size; + OBL_params->sequential_confidence_k = init_params->sequential_confidence_k; + OBL_params->do_prefetch = false; + if (OBL_params->sequential_confidence_k <= 0) { + printf("sequential_confidence_k should be positive\n"); + exit(1); + } + OBL_params->prev_access_block = (obj_id_t *)malloc(OBL_params->sequential_confidence_k * sizeof(obj_id_t)); + for (int i = 0; i < OBL_params->sequential_confidence_k; i++) { + OBL_params->prev_access_block[i] = UINT64_MAX; + } + OBL_params->curr_idx = 0; +} + +/************************************************************************** + ** prefetcher interfaces + ** + ** create, free, clone, handle_find, handle_insert, handle_evict, prefetch + **************************************************************************/ +prefetcher_t *create_OBL_prefetcher(const char *init_params, uint64_t cache_size); +/** + check if the previous access is sequential. If true, set do_prefetch to true. + +@param cache the cache struct +@param req the request containing the request +@return +*/ +static void OBL_handle_find(cache_t *cache, const request_t *req, bool hit) { + OBL_params_t *OBL_params = (OBL_params_t *)(cache->prefetcher->params); + int32_t sequential_confidence_k = OBL_params->sequential_confidence_k; + + // assert(req->obj_size == OBL_params->block_size); + bool flag = true; + for (int i = 0; i < sequential_confidence_k; i++) { + if (OBL_params->prev_access_block[(OBL_params->curr_idx + 1 + i) % sequential_confidence_k] != + req->obj_id - sequential_confidence_k + i) { + flag = false; + break; + } + } + OBL_params->do_prefetch = flag; + OBL_params->curr_idx = (OBL_params->curr_idx + 1) % sequential_confidence_k; + OBL_params->prev_access_block[OBL_params->curr_idx] = req->obj_id; +} + +/** + prefetch next block if the previous access is sequential + + @param cache the cache struct + @param req the request containing the request + @return + */ +void OBL_prefetch(cache_t *cache, const request_t *req) { + OBL_params_t *OBL_params = (OBL_params_t *)(cache->prefetcher->params); + + if (OBL_params->do_prefetch) { + OBL_params->do_prefetch = false; + request_t *new_req = new_request(); + new_req->obj_size = OBL_params->block_size; + new_req->obj_id = req->obj_id + 1; + if (cache->find(cache, new_req, false)) { + free_request(new_req); + return; + } + while (cache->get_occupied_byte(cache) + OBL_params->block_size > cache->cache_size) { + cache->evict(cache, req); + } + cache->insert(cache, new_req); + free_request(new_req); + } +} + +void free_OBL_prefetcher(prefetcher_t *prefetcher) { + OBL_params_t *OBL_params = (OBL_params_t *)prefetcher->params; + free(OBL_params->prev_access_block); + + my_free(sizeof(OBL_params_t), OBL_params); + if (prefetcher->init_params) { + free(prefetcher->init_params); + } + my_free(sizeof(prefetcher_t), prefetcher); +} + +prefetcher_t *clone_OBL_prefetcher(prefetcher_t *prefetcher, uint64_t cache_size) { + return create_OBL_prefetcher(prefetcher->init_params, cache_size); +} + +prefetcher_t *create_OBL_prefetcher(const char *init_params, uint64_t cache_size) { + OBL_init_params_t *OBL_init_params = my_malloc(OBL_init_params_t); + memset(OBL_init_params, 0, sizeof(OBL_init_params_t)); + + set_OBL_default_init_params(OBL_init_params); + if (init_params != NULL) { + OBL_parse_init_params(init_params, OBL_init_params); + } + + OBL_params_t *OBL_params = my_malloc(OBL_params_t); + set_OBL_params(OBL_params, OBL_init_params, cache_size); + + prefetcher_t *prefetcher = (prefetcher_t *)my_malloc(prefetcher_t); + memset(prefetcher, 0, sizeof(prefetcher_t)); + prefetcher->params = OBL_params; + prefetcher->prefetch = OBL_prefetch; + prefetcher->handle_find = OBL_handle_find; + prefetcher->handle_insert = NULL; + prefetcher->handle_evict = NULL; + prefetcher->free = free_OBL_prefetcher; + prefetcher->clone = clone_OBL_prefetcher; + if (init_params) { + prefetcher->init_params = strdup(init_params); + } + + my_free(sizeof(OBL_init_params_t), OBL_init_params); + return prefetcher; +} \ No newline at end of file diff --git a/libCacheSim/include/libCacheSim/prefetchAlgo.h b/libCacheSim/include/libCacheSim/prefetchAlgo.h index 5a0c3c78..2c486357 100644 --- a/libCacheSim/include/libCacheSim/prefetchAlgo.h +++ b/libCacheSim/include/libCacheSim/prefetchAlgo.h @@ -1,6 +1,8 @@ #ifndef PREFETCHINGALGO_H #define PREFETCHINGALGO_H +#include + #include "cache.h" #include "request.h" @@ -13,6 +15,7 @@ typedef struct prefetcher *(*prefetcher_create_func_ptr)(const char *); typedef void (*prefetcher_prefetch_func_ptr)(cache_t *, const request_t *); typedef void (*prefetcher_handle_find_func_ptr)(cache_t *, const request_t *, bool); +typedef void (*prefetcher_handle_insert_func_ptr)(cache_t *, const request_t *); typedef void (*prefetcher_handle_evict_func_ptr)(cache_t *, const request_t *); typedef void (*prefetcher_free_func_ptr)(struct prefetcher *); typedef struct prefetcher *(*prefetcher_clone_func_ptr)(struct prefetcher *, @@ -23,6 +26,7 @@ typedef struct prefetcher { void *init_params; prefetcher_prefetch_func_ptr prefetch; prefetcher_handle_find_func_ptr handle_find; + prefetcher_handle_insert_func_ptr handle_insert; prefetcher_handle_evict_func_ptr handle_evict; prefetcher_free_func_ptr free; prefetcher_clone_func_ptr clone; @@ -30,6 +34,7 @@ typedef struct prefetcher { prefetcher_t *create_Mithril_prefetcher(const char *init_paramsm, uint64_t cache_size); +prefetcher_t *create_OBL_prefetcher(const char *init_paramsm, uint64_t cache_size); static inline prefetcher_t *create_prefetcher(const char *prefetching_algo, const char *prefetching_params, @@ -37,6 +42,8 @@ static inline prefetcher_t *create_prefetcher(const char *prefetching_algo, prefetcher_t *prefetcher = NULL; if (strcasecmp(prefetching_algo, "Mithril") == 0) { prefetcher = create_Mithril_prefetcher(prefetching_params, cache_size); + } else if (strcasecmp(prefetching_algo, "OBL") == 0) { + prefetcher = create_OBL_prefetcher(prefetching_params, cache_size); } else { ERROR("prefetching algo %s not supported\n", prefetching_algo); } diff --git a/libCacheSim/include/libCacheSim/prefetchAlgo/OBL.h b/libCacheSim/include/libCacheSim/prefetchAlgo/OBL.h new file mode 100644 index 00000000..b926cf71 --- /dev/null +++ b/libCacheSim/include/libCacheSim/prefetchAlgo/OBL.h @@ -0,0 +1,29 @@ +#ifndef OBL_H +#define OBL_H +#include +#include +#include +#include +#include +#include + +#include "../cache.h" + +typedef struct OBL_params { + int32_t block_size; + bool do_prefetch; + + uint32_t curr_idx; // current index in the prev_access_block + int32_t sequential_confidence_k; // number of prev sequential accesses to be + // considered as a sequential access + obj_id_t* prev_access_block; // prev k accessed +} OBL_params_t; + +typedef struct OBL_init_params { + int32_t block_size; + int32_t sequential_confidence_k; +} OBL_init_params_t; + +#define DO_PREFETCH(id) + +#endif \ No newline at end of file diff --git a/test/common.h b/test/common.h index cc8a9dc2..2024fbff 100644 --- a/test/common.h +++ b/test/common.h @@ -258,6 +258,9 @@ static cache_t *create_test_cache(const char *alg_name, cache = LRU_init(cc_params, NULL); cache->prefetcher = create_prefetcher("Mithril", NULL, cc_params.cache_size); + } else if (strcasecmp(alg_name, "OBL") == 0) { + cache = LRU_init(cc_params, NULL); + cache->prefetcher = create_prefetcher("OBL", NULL, cc_params.cache_size); } else { printf("cannot recognize algorithm %s\n", alg_name); exit(1); diff --git a/test/test_prefetchAlgo.c b/test/test_prefetchAlgo.c index 188cf17a..9046ca4b 100644 --- a/test/test_prefetchAlgo.c +++ b/test/test_prefetchAlgo.c @@ -65,6 +65,23 @@ static void test_Mithril(gconstpointer user_data) { my_free(sizeof(cache_stat_t), res); } +static void test_OBL(gconstpointer user_data) { + uint64_t miss_cnt_true[] = {92139, 88548, 82337, 80487, 71259, 70869, 70737, 70469}; + uint64_t miss_byte_true[] = {4213140480, 4060079616, 3776877568, 3659406848, + 3099764736, 3076965888, 3074241024, 3060499968}; + + reader_t *reader = (reader_t *)user_data; + common_cache_params_t cc_params = {.cache_size = CACHE_SIZE, .hashpower = 20, .default_ttl = DEFAULT_TTL}; + cache_t *cache = create_test_cache("OBL", cc_params, reader, NULL); + g_assert_true(cache != NULL); + cache_stat_t *res = simulate_at_multi_sizes_with_step_size(reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); + + print_results(cache, res); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, miss_cnt_true, g_req_byte_true, miss_byte_true); + cache->cache_free(cache); + my_free(sizeof(cache_stat_t), res); +} + int main(int argc, char *argv[]) { g_test_init(&argc, &argv, NULL); srand(0); // for reproducibility @@ -80,6 +97,7 @@ int main(int argc, char *argv[]) { reader = setup_oracleGeneralBin_reader(); // reader = setup_vscsi_reader_with_ignored_obj_size(); g_test_add_data_func("/libCacheSim/cacheAlgo_Mithril", reader, test_Mithril); + g_test_add_data_func("/libCacheSim/cacheAlgo_OBL", reader, test_OBL); return g_test_run(); } \ No newline at end of file From 3dc3f678722b6d89ae8eb3e1699ad81a8679b060 Mon Sep 17 00:00:00 2001 From: Juncheng Yang <1a1a11a@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:39:23 -0500 Subject: [PATCH 23/28] Update README.md (#58) * Update README.md * Update README.md * Update README.md --- README.md | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 1dabe0e3..7220b409 100644 --- a/README.md +++ b/README.md @@ -2,33 +2,27 @@ [![build](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml/badge.svg)](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml) -The main development of libCacheSim is at [https://github.com/1a1a11a/libCacheSim](https://github.com/1a1a11a/libCacheSim), the [cachemon](https://github.com/cachemon/libCacheSim) repo is a mirror of the stable branch. +#### The main development of libCacheSim is at [https://github.com/1a1a11a/libCacheSim](https://github.com/1a1a11a/libCacheSim), the [cachemon](https://github.com/cachemon/libCacheSim) repo is a mirror of the stable branch. Please fork and submit PR to this repo. -- [libCacheSim - building and running cache simulations](#libcachesim---building-and-running-cache-simulations) - - [News](#news) - - [What is libCacheSim](#what-is-libcachesim) - - [libCacheSim features](#libcachesim-features) - - [Supported algorithms](#supported-algorithms) - - [Build and Install libCacheSim](#build-and-install-libcachesim) - - [One-line install](#one-line-install) - - [Install dependency](#install-dependency) - - [Build libCacheSim](#build-libcachesim) - - [Usage](#usage) - - [cachesim (a high-performance cache simulator)](#cachesim-a-high-performance-cache-simulator) - - [basic usage](#basic-usage) - - [Run a single cache simulation](#run-a-single-cache-simulation) - - [Run multiple cache simulations with different cache sizes](#run-multiple-cache-simulations-with-different-cache-sizes) - - [Plot miss ratio curve](#plot-miss-ratio-curve) - - [Trace analysis](#trace-analysis) - - [Using libCacheSim as a library](#using-libcachesim-as-a-library) - - [Extending libCacheSim (new algorithms and trace types)](#extending-libcachesim-new-algorithms-and-trace-types) - - [Open source cache traces](#open-source-cache-traces) - - [Questions?](#questions) - - [Contributions](#contributions) - - [Reference](#reference) - - [License](#license) - - [Related](#related) +- [What is libCacheSim](#what-is-libcachesim) +- [libCacheSim features](#libcachesim-features) +- [Supported algorithms](#supported-algorithms) +- [Build and Install libCacheSim](#build-and-install-libcachesim) + - [One-line install](#one-line-install) + - [Install dependency](#install-dependency) + - [Build libCacheSim](#build-libcachesim) +- [Usage](#usage) + - [cachesim (a high-performance cache simulator)](#cachesim-a-high-performance-cache-simulator) + - [Trace analysis](#trace-analysis) + - [Using libCacheSim as a library](#using-libcachesim-as-a-library) + - [Extending libCacheSim (new algorithms and trace types)](#extending-libcachesim-new-algorithms-and-trace-types) +- [Open source cache traces](#open-source-cache-traces) +- [Questions?](#questions) +- [Contributions](#contributions) +- [Reference](#reference) +- [License](#license) +- [Related](#related) From e075986198f4e288b0b77fa34e9bd7662423a0ee Mon Sep 17 00:00:00 2001 From: Zhelong Zhao <50295684+zztaki@users.noreply.github.com> Date: Thu, 22 Feb 2024 09:16:43 +0800 Subject: [PATCH 24/28] feat(prefetch): add pg (#59) Signed-off-by: zztaki --- libCacheSim/cache/prefetch/CMakeLists.txt | 2 +- libCacheSim/cache/prefetch/PG.c | 380 ++++++++++++++++++ .../include/libCacheSim/prefetchAlgo.h | 3 + .../include/libCacheSim/prefetchAlgo/PG.h | 81 ++++ test/common.h | 3 + test/test_prefetchAlgo.c | 18 + 6 files changed, 486 insertions(+), 1 deletion(-) create mode 100644 libCacheSim/cache/prefetch/PG.c create mode 100644 libCacheSim/include/libCacheSim/prefetchAlgo/PG.h diff --git a/libCacheSim/cache/prefetch/CMakeLists.txt b/libCacheSim/cache/prefetch/CMakeLists.txt index c066e2e1..e41aa166 100644 --- a/libCacheSim/cache/prefetch/CMakeLists.txt +++ b/libCacheSim/cache/prefetch/CMakeLists.txt @@ -1,5 +1,5 @@ -add_library(prefetchC Mithril.c OBL.c) +add_library(prefetchC Mithril.c OBL.c PG.c) add_library(prefetch INTERFACE) target_link_libraries(prefetch INTERFACE prefetchC) diff --git a/libCacheSim/cache/prefetch/PG.c b/libCacheSim/cache/prefetch/PG.c new file mode 100644 index 00000000..0eea9f5b --- /dev/null +++ b/libCacheSim/cache/prefetch/PG.c @@ -0,0 +1,380 @@ +// +// a PG module that supports different obj size +// +// +// PG.c +// libCacheSim +// +// Created by Juncheng on 11/20/16. +// Copyright © 2016 Juncheng. All rights reserved. +// +// Modified by Zhelong on 2/21/24. + +#include +#include +#include +#include +#include +#include + +#include "../../include/libCacheSim/prefetchAlgo.h" + +#define TRACK_BLOCK 192618l +#define SANITY_CHECK 1 +#define PROFILING +// #define DEBUG + +#include "../../include/libCacheSim/prefetchAlgo/PG.h" +#include "glibconfig.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// *********************************************************************** +// **** **** +// **** helper function declarations **** +// **** **** +// *********************************************************************** +static inline void _graphNode_destroy(gpointer data); +static inline void _PG_add_to_graph(cache_t *cache, const request_t *req); +static inline GList *_PG_get_prefetch_list(cache_t *cache, const request_t *req); + +const char *PG_default_params(void) { + return "lookahead-range=20, " + "block-size=1, max-metadata-size=0.1, " + "prefetch-threshold=0.05"; +} + +static void set_PG_default_init_params(PG_init_params_t *init_params) { + init_params->lookahead_range = 20; + init_params->block_size = 1; // for general use + init_params->max_metadata_size = 0.1; + init_params->prefetch_threshold = 0.05; +} + +static void PG_parse_init_params(const char *cache_specific_params, PG_init_params_t *init_params) { + char *params_str = strdup(cache_specific_params); + + while (params_str != NULL && params_str[0] != '\0') { + char *key = strsep((char **)¶ms_str, "="); + char *value = strsep((char **)¶ms_str, ","); + while (params_str != NULL && *params_str == ' ') { + params_str++; + } + if (strcasecmp(key, "lookahead-range") == 0) { + init_params->lookahead_range = atoi(value); + } else if (strcasecmp(key, "block-size") == 0) { + init_params->block_size = (unsigned long)atoi(value); + } else if (strcasecmp(key, "max-metadata-size") == 0) { + init_params->max_metadata_size = atof(value); + } else if (strcasecmp(key, "prefetch-threshold") == 0) { + init_params->prefetch_threshold = atof(value); + } else if (strcasecmp(key, "print") == 0 || strcasecmp(key, "default") == 0) { + printf("default params: %s\n", PG_default_params()); + exit(0); + } else { + ERROR("pg does not have parameter %s\n", key); + printf("default params: %s\n", PG_default_params()); + exit(1); + } + } +} + +static void set_PG_params(PG_params_t *PG_params, PG_init_params_t *init_params, uint64_t cache_size) { + PG_params->lookahead_range = init_params->lookahead_range; + PG_params->block_size = init_params->block_size; + PG_params->cur_metadata_size = 0; + PG_params->max_metadata_size = (uint64_t)(init_params->block_size * cache_size * init_params->max_metadata_size); + PG_params->prefetch_threshold = init_params->prefetch_threshold; + + PG_params->stop_recording = FALSE; + + PG_params->graph = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, _graphNode_destroy); + PG_params->prefetched = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + PG_params->past_requests = g_new0(guint64, PG_params->lookahead_range); + + PG_params->past_request_pointer = 0; + PG_params->num_of_hit = 0; + PG_params->num_of_prefetch = 0; + + PG_params->cache_size_map = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); +} + +// *********************************************************************** +// **** **** +// **** prefetcher interfaces **** +// **** **** +// **** create, free, clone, handle_find, handle_evict, prefetch **** +// *********************************************************************** +prefetcher_t *create_PG_prefetcher(const char *init_params, uint64_t cache_size); +/** + 1. record the request in cache_size_map for being aware of prefetching object's + size in the future. + 2. call `_PG_add_to_graph` to update graph. + + @param cache the cache struct + @param req the request containing the request + @return +*/ +static void PG_handle_find(cache_t *cache, const request_t *req, bool hit) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + + /*use cache_size_map to record the current requested obj's size*/ + g_hash_table_insert(PG_params->cache_size_map, GINT_TO_POINTER(req->obj_id), GINT_TO_POINTER(req->obj_size)); + + _PG_add_to_graph(cache, req); + + if (g_hash_table_contains(PG_params->prefetched, GINT_TO_POINTER(req->obj_id))) { + PG_params->num_of_hit++; + g_hash_table_remove(PG_params->prefetched, GINT_TO_POINTER(req->obj_id)); + if (g_hash_table_contains(PG_params->prefetched, GINT_TO_POINTER(req->obj_id))) { + fprintf(stderr, "ERROR found prefetch\n"); + } + } +} + +/** + remove this obj from `prefetched` if it was previously prefetched into cache. + + @param cache the cache struct + @param req the request containing the request + @return +*/ +void PG_handle_evict(cache_t *cache, const request_t *check_req) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + + g_hash_table_remove(PG_params->prefetched, GINT_TO_POINTER(check_req->obj_id)); +} + +/** + prefetch some objects which are from `_PG_get_prefetch_list` + + @param cache the cache struct + @param req the request containing the request + @return + */ +void PG_prefetch(cache_t *cache, const request_t *req) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + + // begin prefetching + GList *prefetch_list = _PG_get_prefetch_list(cache, req); + if (prefetch_list) { + GList *node = prefetch_list; + request_t *new_req = my_malloc(request_t); + memcpy(new_req, req, sizeof(request_t)); + while (node) { + new_req->obj_id = GPOINTER_TO_INT(node->data); + new_req->obj_size = + GPOINTER_TO_INT(g_hash_table_lookup(PG_params->cache_size_map, GINT_TO_POINTER(new_req->obj_id))); + if (!cache->find(cache, new_req, false)) { + while ((long)cache->get_occupied_byte(cache) + new_req->obj_size + cache->obj_md_size > + (long)cache->cache_size) { + cache->evict(cache, new_req); + } + cache->insert(cache, new_req); + + PG_params->num_of_prefetch += 1; + + g_hash_table_insert(PG_params->prefetched, GINT_TO_POINTER(new_req->obj_id), GINT_TO_POINTER(1)); + } + node = node->next; + } + + my_free(sizeof(request_t), new_req); + g_list_free(prefetch_list); + } +} + +void free_PG_prefetcher(prefetcher_t *prefetcher) { + PG_params_t *PG_params = (PG_params_t *)prefetcher->params; + + g_hash_table_destroy(PG_params->cache_size_map); + g_hash_table_destroy(PG_params->graph); + g_hash_table_destroy(PG_params->prefetched); + + g_free(PG_params->past_requests); + + my_free(sizeof(PG_params_t), PG_params); + if (prefetcher->init_params) { + free(prefetcher->init_params); + } + my_free(sizeof(prefetcher_t), prefetcher); +} + +prefetcher_t *clone_PG_prefetcher(prefetcher_t *prefetcher, uint64_t cache_size) { + return create_PG_prefetcher(prefetcher->init_params, cache_size); +} + +prefetcher_t *create_PG_prefetcher(const char *init_params, uint64_t cache_size) { + PG_init_params_t *PG_init_params = my_malloc(PG_init_params_t); + memset(PG_init_params, 0, sizeof(PG_init_params_t)); + + set_PG_default_init_params(PG_init_params); + if (init_params != NULL) { + PG_parse_init_params(init_params, PG_init_params); + check_params((PG_init_params)); + } + + PG_params_t *PG_params = my_malloc(PG_params_t); + + set_PG_params(PG_params, PG_init_params, cache_size); + + prefetcher_t *prefetcher = (prefetcher_t *)my_malloc(prefetcher_t); + memset(prefetcher, 0, sizeof(prefetcher_t)); + prefetcher->params = PG_params; + prefetcher->prefetch = PG_prefetch; + prefetcher->handle_find = PG_handle_find; + prefetcher->handle_insert = NULL; + prefetcher->handle_evict = PG_handle_evict; + prefetcher->free = free_PG_prefetcher; + prefetcher->clone = clone_PG_prefetcher; + if (init_params) { + prefetcher->init_params = strdup(init_params); + } + + my_free(sizeof(PG_init_params_t), PG_init_params); + return prefetcher; +} + +/******************** PG help function ********************/ +static inline void _graphNode_destroy(gpointer data) { + graphNode_t *graphNode = (graphNode_t *)data; + g_hash_table_destroy(graphNode->graph); + pqueue_free(graphNode->pq); + g_free(graphNode); +} + +/** + 1. insert the `req->obj_id` to the past_request_pointer. + 2. update the graph using `past_requests[past_request_pointer]` as the + node and `node->past_requests[i]` as the directed arc. + + @param cache the cache struct + @param req the request containing the request + @return + */ +static inline void _PG_add_to_graph(cache_t *cache, const request_t *req) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + guint64 block, current_block = 0; + char req_lbl[MAX_OBJ_ID_LEN], current_req_lbl[MAX_OBJ_ID_LEN]; + graphNode_t *graphNode = NULL; + + current_block = get_Nth_past_request_l(PG_params, PG_params->past_request_pointer); + if (current_block) { + graphNode = (graphNode_t *)g_hash_table_lookup(PG_params->graph, GINT_TO_POINTER(current_block)); + } + + // now update past requests + set_Nth_past_request_l(PG_params, PG_params->past_request_pointer++, (guint64)(req->obj_id)); + + PG_params->past_request_pointer = PG_params->past_request_pointer % PG_params->lookahead_range; + + if (!(current_req_lbl[0] || current_block)) { + // this is the first request + return; + } + + if (graphNode == NULL) { + if (!PG_params->stop_recording) { + // current block is not in graph, insert + gpointer key = GINT_TO_POINTER(current_block); + graphNode = g_new0(graphNode_t, 1); + graphNode->graph = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, g_free); + graphNode->pq = pqueue_init(2); + graphNode->total_count = 0; + g_hash_table_insert(PG_params->graph, key, graphNode); + PG_params->cur_metadata_size += (8 + 8 * 3); + } else { + // no space for meta data + return; + } + } + + for (int i = 0; i < PG_params->lookahead_range; i++) { + graphNode->total_count++; + + block = get_Nth_past_request_l(PG_params, i); + if (block == 0) break; + + pq_node_t *pq_node = (pq_node_t *)g_hash_table_lookup(graphNode->graph, GINT_TO_POINTER(block)); + if (pq_node) { + // relation already exists + pq_node->pri.pri++; + pqueue_change_priority(graphNode->pq, pq_node->pri, pq_node); + +#ifdef SANITY_CHECK + if (pq_node->obj_id != block) { + ERROR("pq node content not equal block\n"); + } +#endif + + } else { + // there is no probability between current_block->block + if (!PG_params->stop_recording) { + pq_node_t *pq_node = g_new0(pq_node_t, 1); + pq_node->obj_id = block; + pq_node->pri.pri = 1; + pqueue_insert(graphNode->pq, pq_node); + g_hash_table_insert(graphNode->graph, GINT_TO_POINTER(pq_node->obj_id), pq_node); + PG_params->cur_metadata_size += (8 + 8 * 3); + } else { + // no space for meta data + return; + } + } + } + + if (PG_params->max_metadata_size <= PG_params->cur_metadata_size) { + PG_params->stop_recording = TRUE; + } +} + +/** + get some objs which are associated with req->obj_id and their probability + is higher than `prefetch_threshold`. + + @param cache the cache struct + @param req the request containing the request + @return list containing all objs that should be prefetched + */ +static inline GList *_PG_get_prefetch_list(cache_t *cache, const request_t *req) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + GList *list = NULL; + graphNode_t *graphNode = g_hash_table_lookup(PG_params->graph, GINT_TO_POINTER(req->obj_id)); + + if (graphNode == NULL) { + return list; + } + + GList *pq_node_list = NULL; + while (1) { + pq_node_t *pqNode = pqueue_pop(graphNode->pq); + if (pqNode == NULL) { + break; + } + if ((double)(pqNode->pri.pri) / (graphNode->total_count) > PG_params->prefetch_threshold) { + list = g_list_prepend(list, GINT_TO_POINTER(pqNode->obj_id)); + pq_node_list = g_list_prepend(pq_node_list, pqNode); + } else { + // printf("threshold %lf\n", + // (double)(pqNode->pri)/(graphNode->total_count)); + break; + } + } + + if (pq_node_list) { + GList *node = pq_node_list; + while (node) { + pqueue_insert(graphNode->pq, node->data); + node = node->next; + } + } + g_list_free(pq_node_list); + + return list; +} + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/include/libCacheSim/prefetchAlgo.h b/libCacheSim/include/libCacheSim/prefetchAlgo.h index 2c486357..89b25f9c 100644 --- a/libCacheSim/include/libCacheSim/prefetchAlgo.h +++ b/libCacheSim/include/libCacheSim/prefetchAlgo.h @@ -35,6 +35,7 @@ typedef struct prefetcher { prefetcher_t *create_Mithril_prefetcher(const char *init_paramsm, uint64_t cache_size); prefetcher_t *create_OBL_prefetcher(const char *init_paramsm, uint64_t cache_size); +prefetcher_t *create_PG_prefetcher(const char *init_paramsm, uint64_t cache_size); static inline prefetcher_t *create_prefetcher(const char *prefetching_algo, const char *prefetching_params, @@ -44,6 +45,8 @@ static inline prefetcher_t *create_prefetcher(const char *prefetching_algo, prefetcher = create_Mithril_prefetcher(prefetching_params, cache_size); } else if (strcasecmp(prefetching_algo, "OBL") == 0) { prefetcher = create_OBL_prefetcher(prefetching_params, cache_size); + } else if (strcasecmp(prefetching_algo, "PG") == 0) { + prefetcher = create_PG_prefetcher(prefetching_params, cache_size); } else { ERROR("prefetching algo %s not supported\n", prefetching_algo); } diff --git a/libCacheSim/include/libCacheSim/prefetchAlgo/PG.h b/libCacheSim/include/libCacheSim/prefetchAlgo/PG.h new file mode 100644 index 00000000..98adbc2e --- /dev/null +++ b/libCacheSim/include/libCacheSim/prefetchAlgo/PG.h @@ -0,0 +1,81 @@ +// +// PG.h +// mimircache +// +// Created by Juncheng on 6/2/16. +// Copyright © 2016 Juncheng. All rights reserved. +// +// Modified by Zhelong on 2/21/24. + +/** since this is sequence based prefetching, we will use gint64 for block + * number **/ + +#ifndef PG_h +#define PG_h + +#include + +#include "../../../dataStructure/pqueue.h" +#include "../../../traceReader/generalReader/readerInternal.h" +#include "../cache.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* check params vaild */ +#define check_params(params) \ + (assert(params->lookahead_range > 0 && params->lookahead_range <= 100 && params->max_metadata_size > 0 && \ + params->max_metadata_size < 1 && params->prefetch_threshold > 0 && params->prefetch_threshold < 1 && \ + params->block_size > 0)) + +#define PG_getPage(x) ((struct PG_Page*)g_hashtable_lookup()) + +// #define get_Nth_past_request_c(PG_params, n) +// ((char**)((PG_params)->past_requests))[(n)] +#define get_Nth_past_request_c(PG_params, n, des) strcpy((des), ((char**)((PG_params)->past_requests))[(n)]) + +#define get_Nth_past_request_l(PG_params, n) ((guint64*)((PG_params)->past_requests))[(n)] +#define set_Nth_past_request_c(PG_params, n, v) strcpy(((char**)((PG_params)->past_requests))[(n)], (char*)(v)) +#define set_Nth_past_request_l(PG_params, n, v) ((guint64*)((PG_params)->past_requests))[(n)] = (v) + +typedef struct { + uint8_t lookahead_range; + uint block_size; // In the PG algorithm, the existence of block_size, like Mithril, is to correct the maximum + // metadata size while ignoring object size + uint64_t cur_metadata_size; + uint64_t max_metadata_size; // unit byte + double prefetch_threshold; + + gboolean stop_recording; + + GHashTable* graph; // key -> graphNode_t + GHashTable* prefetched; + void* past_requests; // past requests, using array instead of queue to avoid + // frequent memory allocation + + uint32_t past_request_pointer; + gint64 num_of_prefetch; + gint64 num_of_hit; + + GHashTable* cache_size_map; // key -> size +} PG_params_t; + +typedef struct { + GHashTable* graph; // key -> pq_node_t + pqueue_t* pq; + uint64_t total_count; +} graphNode_t; + +typedef struct { + uint8_t lookahead_range; + uint block_size; + double max_metadata_size; + double prefetch_threshold; +} PG_init_params_t; + +#ifdef __cplusplus +} +#endif + +#endif /* PG_H */ diff --git a/test/common.h b/test/common.h index 2024fbff..6d89057c 100644 --- a/test/common.h +++ b/test/common.h @@ -261,6 +261,9 @@ static cache_t *create_test_cache(const char *alg_name, } else if (strcasecmp(alg_name, "OBL") == 0) { cache = LRU_init(cc_params, NULL); cache->prefetcher = create_prefetcher("OBL", NULL, cc_params.cache_size); + } else if (strcasecmp(alg_name, "PG") == 0) { + cache = LRU_init(cc_params, NULL); + cache->prefetcher = create_prefetcher("PG", NULL, cc_params.cache_size); } else { printf("cannot recognize algorithm %s\n", alg_name); exit(1); diff --git a/test/test_prefetchAlgo.c b/test/test_prefetchAlgo.c index 9046ca4b..37b2397e 100644 --- a/test/test_prefetchAlgo.c +++ b/test/test_prefetchAlgo.c @@ -82,6 +82,23 @@ static void test_OBL(gconstpointer user_data) { my_free(sizeof(cache_stat_t), res); } +static void test_PG(gconstpointer user_data) { + uint64_t miss_cnt_true[] = {92786, 89494, 83403, 81564, 72360, 71973, 71842, 71574}; + uint64_t miss_byte_true[] = {4195964416, 4054977024, 3776220672, 3659069952, + 3100251136, 3077595648, 3074874880, 3061133824}; + + reader_t *reader = (reader_t *)user_data; + common_cache_params_t cc_params = {.cache_size = CACHE_SIZE, .hashpower = 20, .default_ttl = DEFAULT_TTL}; + cache_t *cache = create_test_cache("PG", cc_params, reader, NULL); + g_assert_true(cache != NULL); + cache_stat_t *res = simulate_at_multi_sizes_with_step_size(reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); + + print_results(cache, res); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, miss_cnt_true, g_req_byte_true, miss_byte_true); + cache->cache_free(cache); + my_free(sizeof(cache_stat_t), res); +} + int main(int argc, char *argv[]) { g_test_init(&argc, &argv, NULL); srand(0); // for reproducibility @@ -98,6 +115,7 @@ int main(int argc, char *argv[]) { // reader = setup_vscsi_reader_with_ignored_obj_size(); g_test_add_data_func("/libCacheSim/cacheAlgo_Mithril", reader, test_Mithril); g_test_add_data_func("/libCacheSim/cacheAlgo_OBL", reader, test_OBL); + g_test_add_data_func("/libCacheSim/cacheAlgo_PG", reader, test_PG); return g_test_run(); } \ No newline at end of file From d27a7fdd33b8c3275b70b24a8582740bc4de3a54 Mon Sep 17 00:00:00 2001 From: Zhelong Zhao <50295684+zztaki@users.noreply.github.com> Date: Fri, 23 Feb 2024 09:51:16 +0800 Subject: [PATCH 25/28] chore: improve copying request (#60) Signed-off-by: zztaki --- libCacheSim/cache/prefetch/Mithril.c | 4 ++-- libCacheSim/cache/prefetch/PG.c | 2 +- libCacheSim/include/libCacheSim/request.h | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libCacheSim/cache/prefetch/Mithril.c b/libCacheSim/cache/prefetch/Mithril.c index ff1a4726..6f237854 100644 --- a/libCacheSim/cache/prefetch/Mithril.c +++ b/libCacheSim/cache/prefetch/Mithril.c @@ -340,7 +340,7 @@ void Mithril_prefetch(cache_t *cache, const request_t *req) { (Mithril_params->pf_list_size + 1); request_t *new_req = my_malloc(request_t); - memcpy(new_req, req, sizeof(request_t)); + copy_request(new_req, req); if (prefetch_table_index) { int i; @@ -497,7 +497,7 @@ static inline bool _Mithril_check_sequential(cache_t *cache, if (Mithril_params->sequential_K == 0) return FALSE; request_t *new_req = my_malloc(request_t); - memcpy(new_req, req, sizeof(request_t)); + copy_request(new_req, req); bool is_sequential = TRUE; gint sequential_K = Mithril_params->sequential_K; if (sequential_K == -1) { /* when use AMP, this is -1 */ diff --git a/libCacheSim/cache/prefetch/PG.c b/libCacheSim/cache/prefetch/PG.c index 0eea9f5b..db16c0e2 100644 --- a/libCacheSim/cache/prefetch/PG.c +++ b/libCacheSim/cache/prefetch/PG.c @@ -162,7 +162,7 @@ void PG_prefetch(cache_t *cache, const request_t *req) { if (prefetch_list) { GList *node = prefetch_list; request_t *new_req = my_malloc(request_t); - memcpy(new_req, req, sizeof(request_t)); + copy_request(new_req, req); while (node) { new_req->obj_id = GPOINTER_TO_INT(node->data); new_req->obj_size = diff --git a/libCacheSim/include/libCacheSim/request.h b/libCacheSim/include/libCacheSim/request.h index a32c4337..23cc7298 100644 --- a/libCacheSim/include/libCacheSim/request.h +++ b/libCacheSim/include/libCacheSim/request.h @@ -87,7 +87,7 @@ static inline request_t *new_request(void) { * @param req_dest * @param req_src */ -static inline void copy_request(request_t *req_dest, request_t *req_src) { +static inline void copy_request(request_t *req_dest, const request_t *req_src) { memcpy(req_dest, req_src, sizeof(request_t)); } @@ -96,7 +96,7 @@ static inline void copy_request(request_t *req_dest, request_t *req_src) { * @param req * @return */ -static inline request_t *clone_request(request_t *req) { +static inline request_t *clone_request(const request_t *req) { request_t *req_new = my_malloc(request_t); copy_request(req_new, req); return req_new; From 3080f2c2643b7a2b03bcfb7c5837f6fff3aa8908 Mon Sep 17 00:00:00 2001 From: Zhelong Zhao <50295684+zztaki@users.noreply.github.com> Date: Fri, 23 Feb 2024 09:52:27 +0800 Subject: [PATCH 26/28] docs: update admission and prefetch (#61) Signed-off-by: zztaki --- README.md | 14 ++++++++++++-- doc/quickstart_cachesim.md | 11 +++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7220b409..56c0edaa 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ ## libCacheSim features * **High performance** - over 20M requests/sec for a realistic trace replay. * **High memory efficiency** - predictable and small memory footprint. -* **State-of-the-art algorithms** - eviction algorithms, admission algorithms, sampling techniques, approximate miss ratio computation, see [here](/doc/quickstart_cachesim.md). +* **State-of-the-art algorithms** - eviction algorithms, admission algorithms, prefetching algorithms, sampling techniques, approximate miss ratio computation, see [here](/doc/quickstart_cachesim.md). * Parallelism out-of-the-box - uses the many CPU cores to speed up trace analysis and cache simulations. * **The ONLY feature-rich trace analyzer** - all types of trace analysis you need, see [here](/doc/quickstart_traceAnalyzer.md). * **Simple API** - easy to build cache clusters, multi-layer caching, etc.; see [here](/doc/API.md). @@ -55,6 +55,7 @@ ## Supported algorithms cachesim supports the following algorithms: +### Eviction algorithms * [FIFO](/libCacheSim/cache/eviction/FIFO.c), [LRU](/libCacheSim/cache/eviction/LRU.c), [Clock](/libCacheSim/cache/eviction/Clock.c), [SLRU](/libCacheSim/cache/eviction/SLRU.c) * [LFU](/libCacheSim/cache/eviction/LFU.c), [LFU with dynamic aging](/libCacheSim/cache/eviction/LFUDA.c) * [ARC](/libCacheSim/cache/eviction/ARC.c), [TwoQ](/libCacheSim/cache/eviction/TwoQ.c) @@ -66,10 +67,19 @@ cachesim supports the following algorithms: * [LHD](/libCacheSim/cache/eviction/LHD/LHD_Interface.cpp) * [LRB](/libCacheSim/cache/eviction/LRB/LRB_Interface.cpp) * [GLCache](/libCacheSim/cache/eviction/GLCache/GLCache.c) -* [TinyLFU](/libCacheSim/cache/eviction/TinyLFU.c) +* [WTinyLFU](/libCacheSim/cache/eviction/WTinyLFU.c) * [QD-LP](/libCacheSim/cache/eviction/QDLP.c) * [S3-FIFO](/libCacheSim/cache/eviction/S3FIFO.c) * [Sieve](/libCacheSim/cache/eviction/Sieve.c) +### Admission algorithms +* [Adaptsize](/libCacheSim/cache/admission/adaptsize.c) +* [Bloomfilter](/libCacheSim/cache/admission/bloomfilter.c) +* [Prob](/libCacheSim/cache/admission/prob.c) +* [Size](/libCacheSim/cache/admission/size.c) +### Prefetching algorithms +* [OBL](/libCacheSim/cache/prefetch/OBL.c) +* [Mithril](/libCacheSim/cache/prefetch/Mithril.c) +* [PG](/libCacheSim/cache/prefetch/PG.c) --- diff --git a/doc/quickstart_cachesim.md b/doc/quickstart_cachesim.md index d22556d0..6f22f605 100644 --- a/doc/quickstart_cachesim.md +++ b/doc/quickstart_cachesim.md @@ -71,7 +71,7 @@ cachesim supports the following algorithms: * [ARC](/libCacheSim/cache/eviction/ARC.c) * [SLRU](/libCacheSim/cache/eviction/SLRU.c) * [GDSF](/libCacheSim/cache/eviction/GDSF.c) -* [TinyLFU](/libCacheSim/cache/eviction/TinyLFU.c) +* [WTinyLFU](/libCacheSim/cache/eviction/WTinyLFU.c) * [LeCaR](/libCacheSim/cache/eviction/LeCaR.c) * [Cacheus](/libCacheSim/cache/eviction/Cacheus.c) * [Hyperbolic](/libCacheSim/cache/eviction/Hyperbolic.c) @@ -148,12 +148,19 @@ Some eviction algorithms have parameters, you can set the parameters by using `- ### Admission algorithm cachesim supports the following admission algorithms: size, probabilistic, bloomFilter, adaptSize. -You can use `-a` or `--admission-algo` to set the admission algorithm. +You can use `-a` or `--admission` to set the admission algorithm. ```bash # add a bloom filter to filter out objects on first access ./cachesim ../data/trace.vscsi vscsi lru 1gb -a bloomFilter ``` +### Prefetching algorithm +cachesim supports the following prefetching algorithms: OBL, Mithril, PG (and AMP is on the way). +You can use `-p` or `--prefetch` to set the prefetching algorithm. +```bash +# add a mithril to record object association information and fetch objests that are likely to be accessed in the future +./cachesim ../data/trace.vscsi vscsi lru 1gb -p Mithril +``` ### Advanced features ```bash From 0252dcfc0c9f8f5ee400369a5bbb9265b7535261 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 17 Mar 2024 17:07:52 -0400 Subject: [PATCH 27/28] minor updates update documentation of traceConv; add better description in Belady algorithm --- doc/quickstart_traceUtils.md | 11 ++++++----- libCacheSim/bin/cachesim/cache_init.h | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/quickstart_traceUtils.md b/doc/quickstart_traceUtils.md index 774239f7..e837772d 100644 --- a/doc/quickstart_traceUtils.md +++ b/doc/quickstart_traceUtils.md @@ -7,21 +7,22 @@ Print requests from a trace. ```bash # print 10 requests from a trace -./tracePrint ../data/trace.vscsi vscsi -n 10 +./bin/tracePrint ../data/cloudPhysicsIO.vscsi vscsi -n 10 ``` ### traceConv -Convert a trace to libCacheSim format so it has a smaller size, contains next request time (oracle information) and runs faster. +Convert a trace to oracleGeneral format so you can run it faster (10x speedup) using less memory. Meanwhile, the generated trace has a smaller size, contains next request time. ```bash -./traceConv ../data/trace.vscsi vscsi ../data/trace.vscsi.bin +# the first parameter is the input trace, the second parameter is trace type, the third parameter is the output trace path +./bin/traceConv ../data/cloudPhysicsIO.txt txt cloudPhysicsIO.oracleGeneral.bin ``` -Note that the conversion supports all trace types including csv trace. Note that if the object id is numeric, add the option -e `obj_id_is_num=1` to the command line, which reduces memory usage. +Note that the conversion supports all trace types including csv trace. Moreover, if the object id is numeric, add the option `-e obj_id_is_num=1` to the command line, which reduces memory usage. We can also sample a trace to reduce the size of the trace. ```bash # sample 1% of the trace -./traceConv ../data/trace.vscsi vscsi ../data/trace.vscsi.bin -s 0.01 +./bin/traceConv ../data/cloudPhysicsIO.vscsi vscsi ../data/cloudPhysicsIO.oracleGeneral -s 0.01 ``` diff --git a/libCacheSim/bin/cachesim/cache_init.h b/libCacheSim/bin/cachesim/cache_init.h index d0c8700a..6c372117 100644 --- a/libCacheSim/bin/cachesim/cache_init.h +++ b/libCacheSim/bin/cachesim/cache_init.h @@ -87,6 +87,10 @@ static inline cache_t *create_cache(const char *trace_path, } else if (strcasecmp(eviction_algo, "belady") == 0) { if (strcasestr(trace_path, "oracleGeneral") == NULL) { WARN("belady is only supported for oracleGeneral trace\n"); + WARN("to convert a trace to oracleGeneral format\n"); + WARN("./bin/traceConv input_trace trace_format output_trace\n"); + WARN("./bin/traceConv ../data/cloudPhysicsIO.txt txt cloudPhysicsIO.oracleGeneral.bin\n"); + exit(1); } cache = Belady_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "nop") == 0) { From a726b5667a163dc8a0fedcf1f27319808c19f2c7 Mon Sep 17 00:00:00 2001 From: ChaosD Date: Mon, 29 Apr 2024 15:06:27 +0800 Subject: [PATCH 28/28] add concurrent support --- libCacheSim/bin/cachesim/cli_parser.c | 6 +- libCacheSim/bin/cachesim/internal.h | 2 +- libCacheSim/bin/cachesim/main.c | 19 +- libCacheSim/bin/cachesim/sim.c | 3 +- libCacheSim/cache/cache.c | 95 ++++--- libCacheSim/cache/cacheObj.c | 43 ++- libCacheSim/cache/eviction/Clock.c | 93 ++++--- libCacheSim/cache/eviction/FIFO.c | 70 ++--- libCacheSim/cache/eviction/LFU.c | 76 +++--- libCacheSim/cache/eviction/LRU.c | 82 +++--- libCacheSim/cache/eviction/Sieve.c | 99 +++---- .../hashtable/cChainedHashTable.c | 255 +++++------------- .../hashtable/cChainedHashTable.h | 2 +- .../dataStructure/hashtable/hashtable.h | 93 +++---- .../test/test_concurrent_hashtable.c | 33 ++- libCacheSim/include/config.h | 2 +- libCacheSim/include/libCacheSim/cache.h | 1 + libCacheSim/include/libCacheSim/cacheObj.h | 32 ++- .../include/libCacheSim/evictionAlgo.h | 3 + libCacheSim/profiler/simulator.c | 81 ++++-- libCacheSim/utils/include/mymutex.h | 7 + 21 files changed, 526 insertions(+), 571 deletions(-) diff --git a/libCacheSim/bin/cachesim/cli_parser.c b/libCacheSim/bin/cachesim/cli_parser.c index 6f0d62c8..caf0a0d8 100644 --- a/libCacheSim/bin/cachesim/cli_parser.c +++ b/libCacheSim/bin/cachesim/cli_parser.c @@ -107,7 +107,8 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case OPTION_NUM_THREAD: arguments->n_thread = atoi(arg); if (arguments->n_thread == 0 || arguments->n_thread == -1) { - arguments->n_thread = n_cores(); + // arguments->n_thread = n_cores(); + arguments->n_thread = 1; } break; case OPTION_TRACE_TYPE_PARAMS: @@ -221,7 +222,8 @@ static void init_arg(struct arguments *args) { args->ignore_obj_size = false; args->consider_obj_metadata = false; args->report_interval = 3600 * 24; - args->n_thread = n_cores(); + // args->n_thread = n_cores(); + args->n_thread = 1; args->warmup_sec = -1; memset(args->ofilepath, 0, OFILEPATH_LEN); args->n_req = -1; diff --git a/libCacheSim/bin/cachesim/internal.h b/libCacheSim/bin/cachesim/internal.h index 115f473d..a40bc89f 100644 --- a/libCacheSim/bin/cachesim/internal.h +++ b/libCacheSim/bin/cachesim/internal.h @@ -56,7 +56,7 @@ void parse_cmd(int argc, char *argv[], struct arguments *args); void free_arg(struct arguments *args); -void simulate(reader_t *reader, cache_t *cache, int report_interval, +void simulate(reader_t *reader, cache_t *cache, int report_interval, int num_of_threads, int warmup_sec, char *ofilepath); void print_parsed_args(struct arguments *args); diff --git a/libCacheSim/bin/cachesim/main.c b/libCacheSim/bin/cachesim/main.c index 9652d1ef..8b3d1f0b 100644 --- a/libCacheSim/bin/cachesim/main.c +++ b/libCacheSim/bin/cachesim/main.c @@ -19,14 +19,14 @@ int main(int argc, char **argv) { if (args.n_cache_size == 0) { ERROR("no cache size found\n"); } + // if (args.n_cache_size * args.n_eviction_algo == 1) { + // INFO("simulate with single cache\n"); + // simulate(args.reader, args.caches[0], args.report_interval, args.warmup_sec, args.n_thread, + // args.ofilepath); - if (args.n_cache_size * args.n_eviction_algo == 1) { - simulate(args.reader, args.caches[0], args.report_interval, args.warmup_sec, - args.ofilepath); - - free_arg(&args); - return 0; - } + // free_arg(&args); + // return 0; + // } // cache_stat_t *result = simulate_at_multi_sizes( // args.reader, args.cache, args.n_cache_size, args.cache_sizes, NULL, 0, @@ -61,12 +61,13 @@ int main(int argc, char **argv) { for (int i = 0; i < args.n_cache_size * args.n_eviction_algo; i++) { snprintf(output_str, 1024, "%s %32s cache size %8ld%s, %lld req, miss ratio %.4lf, byte miss " - "ratio %.4lf\n", + "ratio %.4lf, throughput %.2lf MQPS\n", output_filename, result[i].cache_name, (long)(result[i].cache_size / size_unit), size_unit_str, (long long)result[i].n_req, (double)result[i].n_miss / (double)result[i].n_req, - (double)result[i].n_miss_byte / (double)result[i].n_req_byte); + (double)result[i].n_miss_byte / (double)result[i].n_req_byte, + (double)result[i].n_req / 1000000.0 / result[i].runtime); printf("%s", output_str); fprintf(output_file, "%s", output_str); } diff --git a/libCacheSim/bin/cachesim/sim.c b/libCacheSim/bin/cachesim/sim.c index 56519748..15a5c13e 100644 --- a/libCacheSim/bin/cachesim/sim.c +++ b/libCacheSim/bin/cachesim/sim.c @@ -89,7 +89,8 @@ void simulate(reader_t *reader, cache_t *cache, int report_interval, #if defined(TRACK_EVICTION_V_AGE) while (cache->get_occupied_byte(cache) > 0) { - cache->evict(cache, req); + cache_obj_t* obj_evicted = cache->evict(cache, req); + my_free(sizeof(cache_obj_t), obj_evicted); } #endif diff --git a/libCacheSim/cache/cache.c b/libCacheSim/cache/cache.c index 04a5023b..bbbc064e 100644 --- a/libCacheSim/cache/cache.c +++ b/libCacheSim/cache/cache.c @@ -5,7 +5,7 @@ #include "../dataStructure/hashtable/hashtable.h" #include "../include/libCacheSim/cache.h" #include "../include/libCacheSim/prefetchAlgo.h" - +#include "../utils/include/mymutex.h" /** this file contains both base function, which should be called by all *eviction algorithms, and the queue related functions, which should be called *by algorithm that uses only one queue and needs to update the queue such as @@ -77,7 +77,7 @@ void cache_struct_free(cache_t *cache) { } /** - * @brief create a new cache with the same size as the old cache + * @brief create a new cache with the same size as the old cache. Not thread-safe. * * @param old_cache * @param new_size @@ -101,7 +101,7 @@ cache_t *clone_cache(const cache_t *old_cache) { return cache; } /** - * @brief this function is called by all eviction algorithms to clone old cache + * @brief this function is called by all eviction algorithms to clone old cache. Not thread-safe. * with new size * * @param old_cache @@ -175,7 +175,6 @@ bool cache_can_insert_default(cache_t *cache, const request_t *req) { cache_obj_t *cache_find_base(cache_t *cache, const request_t *req, const bool update_cache) { cache_obj_t *cache_obj = hashtable_find(cache->hashtable, req); - // "update_cache = true" means that it is a real user request, use handle_find // to update prefetcher's state if (cache->prefetcher && cache->prefetcher->handle_find && update_cache) { @@ -196,10 +195,10 @@ cache_obj_t *cache_find_base(cache_t *cache, const request_t *req, if (update_cache) { cache_obj->misc.next_access_vtime = req->next_access_vtime; - cache_obj->misc.freq += 1; + // cache_obj->misc.freq += 1; + fetch_add(&cache_obj->misc.freq, 1); } } - return cache_obj; } @@ -212,8 +211,6 @@ cache_obj_t *cache_find_base(cache_t *cache, const request_t *req, * update_metadata * return true * else: - * if cache does not have enough space: - * evict until it has space to insert * insert the object * return false * ``` @@ -224,32 +221,26 @@ cache_obj_t *cache_find_base(cache_t *cache, const request_t *req, */ bool cache_get_base(cache_t *cache, const request_t *req) { cache->n_req += 1; - VERBOSE("******* %s req %ld, obj %ld, obj_size %ld, cache size %ld/%ld\n", cache->cache_name, cache->n_req, req->obj_id, req->obj_size, cache->get_occupied_byte(cache), cache->cache_size); cache_obj_t *obj = cache->find(cache, req, true); bool hit = (obj != NULL); - if (hit) { VVERBOSE("req %ld, obj %ld --- cache hit\n", cache->n_req, req->obj_id); - } else if (!cache->can_insert(cache, req)) { - VVERBOSE("req %ld, obj %ld --- cache miss cannot insert\n", cache->n_req, - req->obj_id); } else { - while (cache->get_occupied_byte(cache) + req->obj_size + - cache->obj_md_size > - cache->cache_size) { - cache->evict(cache, req); - } + // while (cache->get_occupied_byte(cache) + req->obj_size + + // cache->obj_md_size > + // cache->cache_size) { + // cache->evict(cache, req); + // } cache->insert(cache, req); } - if (cache->prefetcher && cache->prefetcher->prefetch) { cache->prefetcher->prefetch(cache, req); } - + // INFO("req %ld, obj %ld --- cache hit %d\n", cache->n_req, req->obj_id, hit); return hit; } @@ -259,37 +250,65 @@ bool cache_get_base(cache_t *cache, const request_t *req) { * this function assumes the cache has enough space * and eviction is not part of this function * + * ``` + * if cache not full: + * create an object; + * else: + * evict an object; + * reassign the object; + * insert the object; + * return false + * ``` + * * @param cache * @param req * @return the inserted object */ cache_obj_t *cache_insert_base(cache_t *cache, const request_t *req) { - cache_obj_t *cache_obj = hashtable_insert(cache->hashtable, req); - cache->occupied_byte += - (int64_t)cache_obj->obj_size + (int64_t)cache->obj_md_size; - cache->n_obj += 1; + cache_obj_t *new_cache_obj; + if(cache->get_occupied_byte(cache) + req->obj_size + + cache->obj_md_size < cache->cache_size){ + new_cache_obj = create_cache_obj_from_request(req); + } + else{ + new_cache_obj = cache->to_evict(cache, req); + copy_request_to_cache_obj(new_cache_obj, req); + } + + + // cache_obj_t *cache_obj = hashtable_insert(cache->hashtable, req); + hashtable_insert_obj(cache->hashtable, new_cache_obj); + // cache->occupied_byte += + // (int64_t)cache_obj->obj_size + (int64_t)cache->obj_md_size; + fetch_add(&cache->occupied_byte, + (int64_t)new_cache_obj->obj_size + (int64_t)cache->obj_md_size); + + // cache->n_obj += 1; + fetch_add(&cache->n_obj, 1); + #ifdef SUPPORT_TTL if (cache->default_ttl != 0 && req->ttl == 0) { - cache_obj->exp_time = (int32_t)cache->default_ttl + req->clock_time; + uint32_t exp_time = (int32_t)cache->default_ttl + req->clock_time; + new_cache_obj->exp_time = exp_time; } #endif #if defined(TRACK_EVICTION_V_AGE) || defined(TRACK_DEMOTION) || \ defined(TRACK_CREATE_TIME) - cache_obj->create_time = CURR_TIME(cache, req); + new_cache_obj->create_time = CURR_TIME(cache, req); #endif - cache_obj->misc.next_access_vtime = req->next_access_vtime; - cache_obj->misc.freq = 0; + new_cache_obj->misc.next_access_vtime = req->next_access_vtime; + new_cache_obj->misc.freq = 0; - return cache_obj; + return new_cache_obj; } /** * @brief this function is called by all eviction algorithms in the eviction - * function, it updates the cache metadata. Because it frees the object struct, - * it needs to be called at the end of the eviction function. + * function, it updates the cache metadata. It needs to be called at the end + * of the eviction function. * * @param cache the cache * @param obj the object to be removed @@ -313,24 +332,24 @@ void cache_evict_base(cache_t *cache, cache_obj_t *obj, } cache_remove_obj_base(cache, obj, remove_from_hashtable); + // cache->occupied_byte -= (obj->obj_size + cache->obj_md_size); + fetch_sub(&cache->occupied_byte, + obj->obj_size + cache->obj_md_size); } /** * @brief this function is called by all eviction algorithms that * need to remove an object from the cache, it updates the cache metadata, - * because it frees the object struct, it needs to be called at the end of - * the eviction function. + * because it not frees the object struct, while it does not update the + * eviction data structure. This stale item will be removed in the next eviction. * * @param cache the cache * @param obj the object to be removed */ void cache_remove_obj_base(cache_t *cache, cache_obj_t *obj, bool remove_from_hashtable) { - DEBUG_ASSERT(cache->occupied_byte >= obj->obj_size + cache->obj_md_size); - cache->occupied_byte -= (obj->obj_size + cache->obj_md_size); - cache->n_obj -= 1; - if (remove_from_hashtable) { - hashtable_delete(cache->hashtable, obj); + if(hashtable_try_delete(cache->hashtable, obj)){ + fetch_sub(&cache->n_obj, 1); } } diff --git a/libCacheSim/cache/cacheObj.c b/libCacheSim/cache/cacheObj.c index 34fe480e..651628ae 100644 --- a/libCacheSim/cache/cacheObj.c +++ b/libCacheSim/cache/cacheObj.c @@ -7,30 +7,6 @@ #include "../include/libCacheSim/macro.h" #include "../include/libCacheSim/request.h" -/** - * [verify the fingerprint of cache_obj] - * @method verify_cache_obj_fingerprint - * @author Chaos - * @date 2023-11-22 - * @param cache_obj [pointer of cache item] - * @return [is valid or not] - */ -bool verify_cache_obj_fingerprint(const cache_obj_t *cache_obj) { - return cache_obj->obj_id == cache_obj->fingerprint; -} - -/** - * [set the fingerprint of cache_obj] - * @method set_cache_obj_fingerprint - * @author Chaos - * @date 2023-11-22 - * @param cache_obj [pointer of cache item] - */ -void set_cache_obj_fingerprint(cache_obj_t *cache_obj) { - cache_obj->fingerprint = cache_obj->obj_id; -} - - /** * copy the cache_obj to req_dest * @param req_dest @@ -41,7 +17,8 @@ void copy_cache_obj_to_request(request_t *req_dest, req_dest->obj_id = cache_obj->obj_id; req_dest->obj_size = cache_obj->obj_size; req_dest->next_access_vtime = cache_obj->misc.next_access_vtime; - req_dest->valid = verify_cache_obj_fingerprint(cache_obj); + // req_dest->valid = verify_cache_obj_fingerprint(cache_obj); + req_dest->valid = true; } /** @@ -58,7 +35,7 @@ void copy_request_to_cache_obj(cache_obj_t *cache_obj, const request_t *req) { cache_obj->exp_time = 0; #endif cache_obj->obj_id = req->obj_id; - set_cache_obj_fingerprint(cache_obj); + // set_cache_obj_fingerprint(cache_obj); } /** @@ -89,7 +66,7 @@ cache_obj_t *create_cache_obj_from_obj_id(const obj_id_t obj_id) { #ifdef SUPPORT_TTL cache_obj->exp_time = 0; #endif - set_cache_obj_fingerprint(cache_obj); + // set_cache_obj_fingerprint(cache_obj); return cache_obj; } @@ -271,4 +248,16 @@ void append_obj_to_tail(cache_obj_t **head, cache_obj_t **tail, *tail = cache_obj; +} + + +/** + * free the the doubly linked list + * @param head + * @param tail + */ +void free_list(cache_obj_t **head, cache_obj_t **tail) { + cache_obj_t *oldHead = *head; + remove_obj_from_list(head, tail, oldHead); + free_cache_obj(oldHead); } \ No newline at end of file diff --git a/libCacheSim/cache/eviction/Clock.c b/libCacheSim/cache/eviction/Clock.c index 2f3faa13..e60f0d31 100644 --- a/libCacheSim/cache/eviction/Clock.c +++ b/libCacheSim/cache/eviction/Clock.c @@ -99,7 +99,9 @@ cache_t *Clock_init(const common_cache_params_t ccache_params, * @param cache */ static void Clock_free(cache_t *cache) { - free(cache->eviction_params); + Clock_params_t *params = (Clock_params_t *)cache->eviction_params; + free_list(¶ms->q_head, ¶ms->q_tail); + free(params); cache_struct_free(cache); } @@ -158,6 +160,24 @@ static cache_obj_t *Clock_find(cache_t *cache, const request_t *req, return obj; } +void link_at_tail(Clock_params_t *params, cache_obj_t *obj) { + obj->queue.next = NULL; + cache_obj_t *tail = params->q_tail; + + // this is the thread that first makes Tail points to the obj + // other threads must follow this, o.w. oldHead will be nullptr + while (!CAS(¶ms->q_tail, &tail, obj)) {} + + if (tail != NULL) { + tail->queue.next = obj; + } else { + params->q_head = obj; + } +#ifdef USE_BELADY + obj->next_access_vtime = req->next_access_vtime; +#endif +} + /** * @brief insert an object into the cache, * update the hash table and cache metadata @@ -170,15 +190,10 @@ static cache_obj_t *Clock_find(cache_t *cache, const request_t *req, */ static cache_obj_t *Clock_insert(cache_t *cache, const request_t *req) { Clock_params_t *params = (Clock_params_t *)cache->eviction_params; - cache_obj_t *obj = cache_insert_base(cache, req); - prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - obj->clock.freq = 0; -#ifdef USE_BELADY - obj->next_access_vtime = req->next_access_vtime; -#endif - + link_at_tail(params, obj); + cache_obj_set_in_cache(obj, true); return obj; } @@ -194,21 +209,24 @@ static cache_obj_t *Clock_insert(cache_t *cache, const request_t *req) { */ static cache_obj_t *Clock_to_evict(cache_t *cache, const request_t *req) { Clock_params_t *params = (Clock_params_t *)cache->eviction_params; - - int n_round = 0; - cache_obj_t *obj_to_evict = params->q_tail; -#ifdef USE_BELADY - while (obj_to_evict->next_access_vtime != INT64_MAX) { -#else - while (obj_to_evict->clock.freq - n_round >= 1) { -#endif - obj_to_evict = obj_to_evict->queue.prev; - if (obj_to_evict == NULL) { - obj_to_evict = params->q_tail; - n_round += 1; + cache_obj_t *obj_to_evict; + while (true){ + obj_to_evict = params->q_head; + while(!CAS(¶ms->q_head, &obj_to_evict, obj_to_evict->queue.next)) {} + if(obj_to_evict->queue.next != NULL) { + obj_to_evict->queue.next->queue.prev = NULL; + } else { + params->q_tail = NULL; + } + if (obj_to_evict->clock.freq >= 1) { + fetch_sub(&obj_to_evict->clock.freq, 1); + link_at_tail(params, obj_to_evict); + } + else{ + break; } } - + cache_evict_base(cache, obj_to_evict, true); return obj_to_evict; } @@ -222,19 +240,7 @@ static cache_obj_t *Clock_to_evict(cache_t *cache, const request_t *req) { * @param evicted_obj if not NULL, return the evicted object to caller */ static void Clock_evict(cache_t *cache, const request_t *req) { - Clock_params_t *params = (Clock_params_t *)cache->eviction_params; - - cache_obj_t *obj_to_evict = params->q_tail; - while (obj_to_evict->clock.freq >= 1) { - obj_to_evict->clock.freq -= 1; - params->n_obj_rewritten += 1; - params->n_byte_rewritten += obj_to_evict->obj_size; - move_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj_to_evict); - obj_to_evict = params->q_tail; - } - - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj_to_evict); - cache_evict_base(cache, obj_to_evict, true); + Clock_to_evict(cache, req); } /** @@ -253,10 +259,10 @@ static void Clock_evict(cache_t *cache, const request_t *req) { * @param obj */ static void Clock_remove_obj(cache_t *cache, cache_obj_t *obj) { - Clock_params_t *params = (Clock_params_t *)cache->eviction_params; + // Clock_params_t *params = (Clock_params_t *)cache->eviction_params; - DEBUG_ASSERT(obj != NULL); - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); + // DEBUG_ASSERT(obj != NULL); + // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); cache_remove_obj_base(cache, obj, true); } @@ -274,14 +280,13 @@ static void Clock_remove_obj(cache_t *cache, cache_obj_t *obj) { * cache */ static bool Clock_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - - Clock_remove_obj(cache, obj); - - return true; + else + return false; } // *********************************************************************** diff --git a/libCacheSim/cache/eviction/FIFO.c b/libCacheSim/cache/eviction/FIFO.c index 2c5d1f5c..6e0465c7 100644 --- a/libCacheSim/cache/eviction/FIFO.c +++ b/libCacheSim/cache/eviction/FIFO.c @@ -76,7 +76,9 @@ cache_t *FIFO_init(const common_cache_params_t ccache_params, * @param cache */ static void FIFO_free(cache_t *cache) { - free(cache->eviction_params); + FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; + free_list(¶ms->q_head, ¶ms->q_tail); + free(params); cache_struct_free(cache); } @@ -122,7 +124,7 @@ static bool FIFO_get(cache_t *cache, const request_t *req) { static cache_obj_t *FIFO_find(cache_t *cache, const request_t *req, const bool update_cache) { return cache_find_base(cache, req, update_cache); -} +} /** * @brief insert an object into the cache, @@ -137,8 +139,17 @@ static cache_obj_t *FIFO_find(cache_t *cache, const request_t *req, static cache_obj_t *FIFO_insert(cache_t *cache, const request_t *req) { FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; cache_obj_t *obj = cache_insert_base(cache, req); - prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - + // prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); + // Use CAS to update the q_tail + obj->queue.next = NULL; + cache_obj_t *tail = params->q_tail; + while (!CAS(¶ms->q_tail, &tail, obj)) {} + if (tail != NULL) { + tail->queue.next = obj; + } else { + params->q_head = obj; + } + cache_obj_set_in_cache(obj, true); return obj; } @@ -154,7 +165,21 @@ static cache_obj_t *FIFO_insert(cache_t *cache, const request_t *req) { */ static cache_obj_t *FIFO_to_evict(cache_t *cache, const request_t *req) { FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; - return params->q_tail; + cache_obj_t *obj_to_evict = params->q_head; + DEBUG_ASSERT(params->q_head != NULL); + + // we can simply call remove_obj_from_list here, but for the best performance, + // we chose to do it manually + // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); + + while(!CAS(¶ms->q_head, &obj_to_evict, obj_to_evict->queue.next)) {} + if (obj_to_evict->queue.next != NULL) { + obj_to_evict->queue.next->queue.prev = NULL; + } else { + params->q_tail = NULL; + } + cache_evict_base(cache, obj_to_evict, true); + return obj_to_evict; } /** @@ -167,24 +192,7 @@ static cache_obj_t *FIFO_to_evict(cache_t *cache, const request_t *req) { * @param evicted_obj if not NULL, return the evicted object to caller */ static void FIFO_evict(cache_t *cache, const request_t *req) { - FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; - cache_obj_t *obj_to_evict = params->q_tail; - DEBUG_ASSERT(params->q_tail != NULL); - - // we can simply call remove_obj_from_list here, but for the best performance, - // we chose to do it manually - // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - - params->q_tail = params->q_tail->queue.prev; - if (likely(params->q_tail != NULL)) { - params->q_tail->queue.next = NULL; - } else { - /* cache->n_obj has not been updated */ - DEBUG_ASSERT(cache->n_obj == 1); - params->q_head = NULL; - } - - cache_evict_base(cache, obj_to_evict, true); + FIFO_to_evict(cache, req); } /** @@ -201,17 +209,13 @@ static void FIFO_evict(cache_t *cache, const request_t *req) { * cache */ static bool FIFO_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - - FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; - - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - cache_remove_obj_base(cache, obj, true); - - return true; + else + return false; } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/LFU.c b/libCacheSim/cache/eviction/LFU.c index bd9705f8..feee54b8 100644 --- a/libCacheSim/cache/eviction/LFU.c +++ b/libCacheSim/cache/eviction/LFU.c @@ -37,6 +37,7 @@ typedef struct LFU_params { GHashTable *freq_map; uint64_t min_freq; uint64_t max_freq; + pthread_mutex_t mutex_; } LFU_params_t; // *********************************************************************** @@ -108,7 +109,7 @@ cache_t *LFU_init(const common_cache_params_t ccache_params, params->min_freq = 1; params->max_freq = 1; params->freq_one_node = freq_node; - + pthread_mutex_init(¶ms->mutex_, NULL); return cache; } @@ -166,9 +167,11 @@ static bool LFU_get(cache_t *cache, const request_t *req) { static cache_obj_t *LFU_find(cache_t *cache, const request_t *req, const bool update_cache) { LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); + cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); + pthread_mutex_lock(¶ms->mutex_); - if (cache_obj && likely(update_cache)) { + if (cache_obj && likely(update_cache) && cache_obj_in_cache(cache_obj)) { /* freq incr and move to next freq node */ cache_obj->lfu.freq += 1; if (params->max_freq < cache_obj->lfu.freq) { @@ -177,6 +180,7 @@ static cache_obj_t *LFU_find(cache_t *cache, const request_t *req, // find the freq_node this object belongs to and update its info gpointer old_key = GSIZE_TO_POINTER(cache_obj->lfu.freq - 1); + freq_node_t *old_node = g_hash_table_lookup(params->freq_map, old_key); DEBUG_ASSERT(old_node != NULL); DEBUG_ASSERT(old_node->freq == cache_obj->lfu.freq - 1); @@ -215,6 +219,7 @@ static cache_obj_t *LFU_find(cache_t *cache, const request_t *req, } } } + pthread_mutex_unlock(¶ms->mutex_); return cache_obj; } @@ -230,15 +235,20 @@ static cache_obj_t *LFU_find(cache_t *cache, const request_t *req, */ static cache_obj_t *LFU_insert(cache_t *cache, const request_t *req) { LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); + + cache_obj_t *cache_obj = cache_insert_base(cache, req); + pthread_mutex_lock(¶ms->mutex_); + params->min_freq = 1; freq_node_t *freq_one_node = params->freq_one_node; - cache_obj_t *cache_obj = cache_insert_base(cache, req); cache_obj->lfu.freq = 1; freq_one_node->n_obj += 1; append_obj_to_tail(&freq_one_node->first_obj, &freq_one_node->last_obj, cache_obj); + cache_obj_set_in_cache(cache_obj, true); + pthread_mutex_unlock(¶ms->mutex_); return cache_obj; } @@ -255,25 +265,13 @@ static cache_obj_t *LFU_insert(cache_t *cache, const request_t *req) { */ static cache_obj_t *LFU_to_evict(cache_t *cache, const request_t *req) { LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); - freq_node_t *min_freq_node = get_min_freq_node(params); - return min_freq_node->first_obj; -} - -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - */ -static void LFU_evict(cache_t *cache, const request_t *req) { - LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); + pthread_mutex_lock(¶ms->mutex_); freq_node_t *min_freq_node = get_min_freq_node(params); min_freq_node->n_obj--; cache_obj_t *obj_to_evict = min_freq_node->first_obj; + cache_obj_set_in_cache(obj_to_evict, false); remove_obj_from_list(&min_freq_node->first_obj, &min_freq_node->last_obj, obj_to_evict); @@ -287,28 +285,27 @@ static void LFU_evict(cache_t *cache, const request_t *req) { // /* update min freq */ // update_min_freq(params); } - + pthread_mutex_unlock(¶ms->mutex_); + cache_evict_base(cache, obj_to_evict, true); + return obj_to_evict; +} + +/** + * @brief evict an object from the cache + * it needs to call cache_evict_base before returning + * which updates some metadata such as n_obj, occupied size, and hash table + * + * @param cache + * @param req not used + */ +static void LFU_evict(cache_t *cache, const request_t *req) { + LFU_to_evict(cache, req); } void LFU_remove_obj(cache_t *cache, cache_obj_t *obj) { assert(obj != NULL); - LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); - - gpointer key = GSIZE_TO_POINTER(obj->lfu.freq); - freq_node_t *freq_node = g_hash_table_lookup(params->freq_map, key); - DEBUG_ASSERT(freq_node->freq == obj->lfu.freq); - DEBUG_ASSERT(freq_node->n_obj > 0); - - freq_node->n_obj--; - remove_obj_from_list(&freq_node->first_obj, &freq_node->last_obj, obj); - cache_remove_obj_base(cache, obj, true); - - if (freq_node->freq == params->min_freq && freq_node->n_obj == 0) { - /* update min freq */ - update_min_freq(params); - } } /** @@ -325,14 +322,13 @@ void LFU_remove_obj(cache_t *cache, cache_obj_t *obj) { * cache */ bool LFU_remove(cache_t *cache, obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - - LFU_remove_obj(cache, obj); - - return true; + else + return false; } // *********************************************************************** diff --git a/libCacheSim/cache/eviction/LRU.c b/libCacheSim/cache/eviction/LRU.c index 95dc2de6..f5e93d88 100644 --- a/libCacheSim/cache/eviction/LRU.c +++ b/libCacheSim/cache/eviction/LRU.c @@ -76,8 +76,8 @@ cache_t *LRU_init(const common_cache_params_t ccache_params, LRU_params_t *params = malloc(sizeof(LRU_params_t)); params->q_head = NULL; params->q_tail = NULL; + pthread_mutex_init(¶ms->mutex_, NULL); cache->eviction_params = params; - return cache; } @@ -86,7 +86,11 @@ cache_t *LRU_init(const common_cache_params_t ccache_params, * * @param cache */ -static void LRU_free(cache_t *cache) { cache_struct_free(cache); } +static void LRU_free(cache_t *cache) { + LRU_params_t *params = (LRU_params_t *)cache->eviction_params; + free_list(¶ms->q_head, ¶ms->q_tail); + cache_struct_free(cache); +} /** * @brief this function is the user facing API @@ -131,14 +135,18 @@ static cache_obj_t *LRU_find(cache_t *cache, const request_t *req, const bool update_cache) { LRU_params_t *params = (LRU_params_t *)cache->eviction_params; cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); + + pthread_mutex_lock(¶ms->mutex_); + if (cache_obj && likely(update_cache) && cache_obj_in_cache(cache_obj)) { - if (cache_obj && likely(update_cache)) { /* lru_head is the newest, move cur obj to lru_head */ #ifdef USE_BELADY if (req->next_access_vtime != INT64_MAX) #endif move_obj_to_head(¶ms->q_head, ¶ms->q_tail, cache_obj); } + pthread_mutex_unlock(¶ms->mutex_); + return cache_obj; } @@ -156,8 +164,13 @@ static cache_obj_t *LRU_insert(cache_t *cache, const request_t *req) { LRU_params_t *params = (LRU_params_t *)cache->eviction_params; cache_obj_t *obj = cache_insert_base(cache, req); + + pthread_mutex_lock(¶ms->mutex_); prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); + cache_obj_set_in_cache(obj, true); + pthread_mutex_unlock(¶ms->mutex_); + // Now the object is ready for access. return obj; } @@ -173,36 +186,23 @@ static cache_obj_t *LRU_insert(cache_t *cache, const request_t *req) { */ static cache_obj_t *LRU_to_evict(cache_t *cache, const request_t *req) { LRU_params_t *params = (LRU_params_t *)cache->eviction_params; - - DEBUG_ASSERT(params->q_tail != NULL || cache->occupied_byte == 0); - - cache->to_evict_candidate_gen_vtime = cache->n_req; - return params->q_tail; -} - -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - */ -static void LRU_evict(cache_t *cache, const request_t *req) { - LRU_params_t *params = (LRU_params_t *)cache->eviction_params; - cache_obj_t *obj_to_evict = params->q_tail; DEBUG_ASSERT(params->q_tail != NULL); // we can simply call remove_obj_from_list here, but for the best performance, // we chose to do it manually // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj) + pthread_mutex_lock(¶ms->mutex_); + cache_obj_t *obj_to_evict = params->q_tail; + cache_obj_set_in_cache(obj_to_evict, false); + + // Remove the item from the hash table and prefetcher (if existed). params->q_tail = params->q_tail->queue.prev; if (likely(params->q_tail != NULL)) { params->q_tail->queue.next = NULL; } else { /* cache->n_obj has not been updated */ - DEBUG_ASSERT(cache->n_obj == 1); + // DEBUG_ASSERT(cache->n_obj == 1); params->q_head = NULL; } @@ -211,8 +211,21 @@ static void LRU_evict(cache_t *cache, const request_t *req) { printf("%ld demote %ld %ld\n", cache->n_req, obj_to_evict->create_time, obj_to_evict->misc.next_access_vtime); #endif - + pthread_mutex_unlock(¶ms->mutex_); cache_evict_base(cache, obj_to_evict, true); + return obj_to_evict; +} + +/** + * @brief evict an object from the cache + * it needs to call cache_evict_base before returning + * which updates some metadata such as n_obj, occupied size, and hash table + * + * @param cache + * @param req not used + */ +static void LRU_evict(cache_t *cache, const request_t *req) { + LRU_to_evict(cache, req); } /** @@ -232,10 +245,6 @@ static void LRU_evict(cache_t *cache, const request_t *req) { */ static void LRU_remove_obj(cache_t *cache, cache_obj_t *obj) { assert(obj != NULL); - - LRU_params_t *params = (LRU_params_t *)cache->eviction_params; - - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); cache_remove_obj_base(cache, obj, true); } @@ -253,23 +262,23 @@ static void LRU_remove_obj(cache_t *cache, cache_obj_t *obj) { * cache */ static bool LRU_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - LRU_params_t *params = (LRU_params_t *)cache->eviction_params; - - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - cache_remove_obj_base(cache, obj, true); - - return true; + else + return false; } static void LRU_print_cache(const cache_t *cache) { LRU_params_t *params = (LRU_params_t *)cache->eviction_params; + pthread_mutex_lock(¶ms->mutex_); + cache_obj_t *cur = params->q_head; // print from the most recent to the least recent if (cur == NULL) { + pthread_mutex_unlock(¶ms->mutex_); printf("empty\n"); return; } @@ -278,6 +287,7 @@ static void LRU_print_cache(const cache_t *cache) { cur = cur->queue.next; } printf("END\n"); + pthread_mutex_unlock(¶ms->mutex_); } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/Sieve.c b/libCacheSim/cache/eviction/Sieve.c index ab256931..064e6edf 100644 --- a/libCacheSim/cache/eviction/Sieve.c +++ b/libCacheSim/cache/eviction/Sieve.c @@ -12,6 +12,8 @@ typedef struct { cache_obj_t *q_tail; cache_obj_t *pointer; + pthread_mutex_t mutex_; + } Sieve_params_t; // *********************************************************************** @@ -67,6 +69,7 @@ cache_t *Sieve_init(const common_cache_params_t ccache_params, params->pointer = NULL; params->q_head = NULL; params->q_tail = NULL; + pthread_mutex_init(¶ms->mutex_, NULL); return cache; } @@ -77,7 +80,9 @@ cache_t *Sieve_init(const common_cache_params_t ccache_params, * @param cache */ static void Sieve_free(cache_t *cache) { - free(cache->eviction_params); + Sieve_params_t *params = cache->eviction_params; + free_list(¶ms->q_head, ¶ms->q_tail); + free(params); cache_struct_free(cache); } @@ -125,7 +130,7 @@ static bool Sieve_get(cache_t *cache, const request_t *req) { static cache_obj_t *Sieve_find(cache_t *cache, const request_t *req, const bool update_cache) { cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); - if (cache_obj != NULL && update_cache) { + if (cache_obj != NULL && update_cache && cache_obj_in_cache(cache_obj)) { cache_obj->sieve.freq = 1; } @@ -146,64 +151,33 @@ static cache_obj_t *Sieve_find(cache_t *cache, const request_t *req, static cache_obj_t *Sieve_insert(cache_t *cache, const request_t *req) { Sieve_params_t *params = cache->eviction_params; cache_obj_t *obj = cache_insert_base(cache, req); + pthread_mutex_lock(¶ms->mutex_); prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); obj->sieve.freq = 0; + cache_obj_set_in_cache(obj, true); + pthread_mutex_unlock(¶ms->mutex_); return obj; } -/** - * @brief find the object to be evicted - * this function does not actually evict the object or update metadata - * not all eviction algorithms support this function - * because the eviction logic cannot be decoupled from finding eviction - * candidate, so use assert(false) if you cannot support this function - * - * @param cache the cache - * @return the object to be evicted - */ -static cache_obj_t *Sieve_to_evict_with_freq(cache_t *cache, - const request_t *req, - int to_evict_freq) { +static cache_obj_t *Sieve_to_evict(cache_t *cache, const request_t *req) { Sieve_params_t *params = cache->eviction_params; - cache_obj_t *pointer = params->pointer; + pthread_mutex_lock(¶ms->mutex_); /* if we have run one full around or first eviction */ - if (pointer == NULL) pointer = params->q_tail; - - /* find the first untouched */ - while (pointer != NULL && pointer->sieve.freq > to_evict_freq) { - pointer = pointer->queue.prev; - } - - /* if we have finished one around, start from the tail */ - if (pointer == NULL) { - pointer = params->q_tail; - while (pointer != NULL && pointer->sieve.freq > to_evict_freq) { - pointer = pointer->queue.prev; - } - } - - if (pointer == NULL) return NULL; - - return pointer; -} - -static cache_obj_t *Sieve_to_evict(cache_t *cache, const request_t *req) { - // because we do not change the frequency of the object, - // if all objects have frequency 1, we may return NULL - int to_evict_freq = 0; - - cache_obj_t *obj_to_evict = - Sieve_to_evict_with_freq(cache, req, to_evict_freq); - - while (obj_to_evict == NULL) { - to_evict_freq += 1; + cache_obj_t *obj = params->pointer == NULL ? params->q_tail : params->pointer; - obj_to_evict = Sieve_to_evict_with_freq(cache, req, to_evict_freq); + while (obj->sieve.freq > 0) { + obj->sieve.freq -= 1; + obj = obj->queue.prev == NULL ? params->q_tail : obj->queue.prev; } + cache_obj_set_in_cache(obj, false); + params->pointer = obj->queue.prev; + remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); + pthread_mutex_unlock(¶ms->mutex_); - return obj_to_evict; + cache_evict_base(cache, obj, true); + return obj; } /** @@ -216,19 +190,7 @@ static cache_obj_t *Sieve_to_evict(cache_t *cache, const request_t *req) { * @param evicted_obj if not NULL, return the evicted object to caller */ static void Sieve_evict(cache_t *cache, const request_t *req) { - Sieve_params_t *params = cache->eviction_params; - - /* if we have run one full around or first eviction */ - cache_obj_t *obj = params->pointer == NULL ? params->q_tail : params->pointer; - - while (obj->sieve.freq > 0) { - obj->sieve.freq -= 1; - obj = obj->queue.prev == NULL ? params->q_tail : obj->queue.prev; - } - - params->pointer = obj->queue.prev; - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - cache_evict_base(cache, obj, true); + Sieve_to_evict(cache, req); } static void Sieve_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove) { @@ -237,7 +199,7 @@ static void Sieve_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove) { if (obj_to_remove == params->pointer) { params->pointer = obj_to_remove->queue.prev; } - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj_to_remove); + // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj_to_remove); cache_remove_obj_base(cache, obj_to_remove, true); } @@ -255,14 +217,13 @@ static void Sieve_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove) { * cache */ static bool Sieve_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - - Sieve_remove_obj(cache, obj); - - return true; + else + return false; } static void Sieve_verify(cache_t *cache) { diff --git a/libCacheSim/dataStructure/hashtable/cChainedHashTable.c b/libCacheSim/dataStructure/hashtable/cChainedHashTable.c index 5179ff25..6a4f1383 100644 --- a/libCacheSim/dataStructure/hashtable/cChainedHashTable.c +++ b/libCacheSim/dataStructure/hashtable/cChainedHashTable.c @@ -42,128 +42,26 @@ extern "C" { /************************ help func ************************/ -/* free object, called by other functions when iterating through the hashtable - */ -static inline void foreach_free_obj_locked(cache_obj_t *cache_obj, void *user_data) { - my_free(sizeof(cache_obj_t), cache_obj); -} - /** * This function finds an object in the hashtable bucket. - * @method find_in_bucket_locked + * @method find_pointer_locked * @author Chaos - * @date 2023-11-23 + * @date 2024-4-19 * @param hashtable [Handle of the hashtable.] - * @param bucket_id [The id of the hashtable bucket.] - * @param obj_id [The id of the object to find.] - * @return [The pointer to the object. If not found, return NULL.] + * @param bucket_id [Id of the hashtable bucket.] + * @param obj_id [Id of the object to find.] + * @return [Double pointer to the object. (The pointer to 'obj->hash_next')] */ -static inline cache_obj_t* find_in_bucket_locked(const hashtable_t *hashtable, uint64_t bucket_id, obj_id_t obj_id){ - cache_obj_t *cache_obj = hashtable->ptr_table[bucket_id]; - while (cache_obj) { - if (cache_obj->obj_id == obj_id) { - assert(verify_cache_obj_fingerprint(cache_obj)); - return cache_obj; - } - cache_obj = cache_obj->hash_next; - } - // cache_obj is NULL - return cache_obj; -} -/** - * This function adds an object to the hashtable bucket. - * @method add_to_bucket_locked - * @author Chaos - * @date 2023-11-23 - * @param hashtable [Handle of the hashtable.] - * @param bucket_id [The id of the hashtable bucket.] - * @param cache_obj [The pointer to the object to add.] - * @return [The pointer to the object. If the object is already in the hashtable, return the existing one.] - */ -static inline cache_obj_t *add_to_bucket_locked(hashtable_t *hashtable, uint64_t bucket_id, cache_obj_t *cache_obj) { - // If the object is already in the hashtable, free the inserted object and return the existing one. - cache_obj_t* curr_obj = find_in_bucket_locked(hashtable, bucket_id, cache_obj->obj_id); - if(curr_obj != NULL){ - free_cache_obj(cache_obj); - return curr_obj; - } - // If the object is not in the hashtable, insert it to the head of the bucket. - curr_obj = hashtable->ptr_table[bucket_id]; - cache_obj->hash_next = curr_obj; - hashtable->ptr_table[bucket_id] = cache_obj; - __sync_fetch_and_add(&hashtable->n_obj, 1); - return cache_obj; -} - -/** - * This function deletes an object in the hashtable bucket. - * @method delete_obj_id_in_bucket_locked - * @author Chaos - * @date 2023-11-23 - * @param hashtable [Handle of the hashtable.] - * @param bucket_id [The id of the hashtable bucket.] - * @param cache_obj [The pointer to the object to delete.] - * @return [Success or not.] - */ -static inline bool delete_in_bucket_locked(hashtable_t *hashtable, uint64_t bucket_id, cache_obj_t *cache_obj) { - cache_obj_t *curr_obj = hashtable->ptr_table[bucket_id]; - // If the object to delete is NULL, return false. - if(curr_obj == NULL || cache_obj == NULL){ - return false; - } - // If the object to delete is the head of the bucket, delete it and return true. - if (curr_obj == cache_obj) { - hashtable->ptr_table[bucket_id] = cache_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(cache_obj); - __sync_fetch_and_sub(&hashtable->n_obj, 1); - return true; - } - // If the object to delete is not the head of the bucket, find it. - while (curr_obj != NULL && curr_obj->hash_next != cache_obj) { - curr_obj = curr_obj->hash_next; - } - // If the object to delete is in the bucket, delete it and return true. - if (curr_obj != NULL) { - curr_obj->hash_next = cache_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(cache_obj); - __sync_fetch_and_sub(&hashtable->n_obj, 1); - return true; - } - // If the object to delete is not in the bucket, return false. - return false; +static inline cache_obj_t** find_pointer_locked(const hashtable_t *hashtable, uint64_t bucket_id, obj_id_t obj_id){ + cache_obj_t **ptr = &hashtable->ptr_table[bucket_id]; + while(*ptr != NULL && obj_id != (*ptr)->obj_id){ + ptr = &(*ptr)->hash_next; + } + return ptr; } -static inline bool delete_obj_id_in_bucket_locked(hashtable_t *hashtable, uint64_t bucket_id, obj_id_t obj_id) { - cache_obj_t *curr_obj = hashtable->ptr_table[bucket_id]; - cache_obj_t *prev_obj = curr_obj; - - if(curr_obj == NULL) return false; - - if(curr_obj->obj_id == obj_id){ - // the object to remove is the head of the bucket - hashtable->ptr_table[bucket_id] = curr_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(curr_obj); - __sync_fetch_and_sub(&hashtable->n_obj, 1); - return true; - } - // the object to remove is not the head of the bucket - do { - prev_obj = curr_obj; - curr_obj = curr_obj->hash_next; - } while(curr_obj != NULL && curr_obj->obj_id != obj_id); - // the object to remove is in the bucket - if (curr_obj != NULL) { - prev_obj->hash_next = curr_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(curr_obj); - __sync_fetch_and_sub(&hashtable->n_obj, 1); - return true; - } - // the object to remove is not in the bucket (also not in the hashtable) - return false; -} - /************************ hashtable func ************************/ /** * [This function is not thread-safe. Other threads muse wait for the return of this function.] @@ -192,21 +90,31 @@ hashtable_t *create_concurrent_chained_hashtable(const uint16_t hashpower) { hashtable->external_obj = false; hashtable->hashpower = hashpower; hashtable->n_obj = 0; - hashtable->rwlocks_ = init_RWLocks((hashpower>7) ? (hashpower-7) : 0); + hashtable->rwlocks_ = init_RWLocks((hashpower > 10) ? (hashpower-10) : 0); return hashtable; } +/** + * This function finds an object in the hashtable. + * If the object is in the hashtable: + * - return the pointer. + * Else: + * - return NULL. + * @Author Chaos + * @Date 2024-04-18 + */ + cache_obj_t *concurrent_chained_hashtable_find_obj_id(const hashtable_t *hashtable, const obj_id_t obj_id) { uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); /** Add read lock for query */ pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); pthread_rwlock_rdlock(rwlock_); - - cache_obj_t *cache_obj = find_in_bucket_locked(hashtable, hv, obj_id); - + /************within lock******************/ + cache_obj_t* entry = *find_pointer_locked(hashtable, hv, obj_id); + /************out lock******************/ pthread_rwlock_unlock(rwlock_); - return cache_obj; + return entry; } cache_obj_t *concurrent_chained_hashtable_find(const hashtable_t *hashtable, @@ -220,14 +128,14 @@ cache_obj_t *concurrent_chained_hashtable_find_obj(const hashtable_t *hashtable, } /** - * This function inserts an object to the hashtable. - * If the object is not in the hashtable: - * - increase the number of objects in the hashtable - * - return the inserted object. - * Else: - * - return the existing one. + * This function inserts an object to the hashtable. + * If the object is in the hashtable: + * - delete the old one. + * + * Then, increase the number of objects in the hashtable and + * return the inserted object. * @Author Chaos - * @Date 2023-11-22 + * @Date 2024-04-18 */ cache_obj_t *concurrent_chained_hashtable_insert_obj(hashtable_t *hashtable, cache_obj_t *cache_obj) { @@ -236,10 +144,26 @@ cache_obj_t *concurrent_chained_hashtable_insert_obj(hashtable_t *hashtable, /** Add write lock for insertion */ pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); pthread_rwlock_wrlock(rwlock_); - cache_obj_t *inserted_cache_obj = add_to_bucket_locked(hashtable, hv, cache_obj); - // If successfully inserted, increase the number of objects in the hashtable. + + /************within lock******************/ + cache_obj_t** ptr = find_pointer_locked(hashtable, hv, cache_obj->obj_id); + cache_obj_t* old = *ptr; + cache_obj->hash_next = (old == NULL ? NULL : old->hash_next); + // Initialize the in_cache flag. Set to true after the object is inserted into the cache eviction. + cache_obj_set_in_cache(cache_obj, false); + *ptr = cache_obj; + // If overwritten + if (old != NULL) { + cache_obj_set_in_cache(old, false); + } + else{ + // If successfully inserted, increase the number of objects in the hashtable. + hashtable->n_obj += 1; + } + /************out lock******************/ + pthread_rwlock_unlock(rwlock_); - return inserted_cache_obj; + return old; } cache_obj_t *concurrent_chained_hashtable_insert(hashtable_t *hashtable, @@ -258,35 +182,37 @@ cache_obj_t *concurrent_chained_hashtable_insert(hashtable_t *hashtable, * - return false. * @Author Chaos * @Date 2023-11-22 - */bool concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, + */ +cache_obj_t *concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, const obj_id_t obj_id) { uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); /** Add write lock for removal */ pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); pthread_rwlock_wrlock(rwlock_); - - bool res = delete_obj_id_in_bucket_locked(hashtable, hv, obj_id); - + /************within lock******************/ + cache_obj_t** ptr = find_pointer_locked(hashtable, hv, obj_id); + cache_obj_t* result = *ptr; + if (result != NULL) { + cache_obj_set_in_cache(result, false); + *ptr = result->hash_next; + hashtable->n_obj -= 1; + } + /************out lock******************/ pthread_rwlock_unlock(rwlock_); - return res; + return result; } -bool concurrent_chained_hashtable_try_delete(hashtable_t *hashtable, - cache_obj_t *cache_obj) { - uint64_t hv = get_hash_value_int_64(&cache_obj->obj_id) & hashmask(hashtable->hashpower); - /** Add write lock for removal */ - pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); - pthread_rwlock_wrlock(rwlock_); - bool res = delete_in_bucket_locked(hashtable, hv, cache_obj); - pthread_rwlock_unlock(rwlock_); - return res; -} void concurrent_chained_hashtable_delete(hashtable_t *hashtable, cache_obj_t *cache_obj) { - concurrent_chained_hashtable_try_delete(hashtable, cache_obj); + concurrent_chained_hashtable_delete_obj_id(hashtable, cache_obj->obj_id); } +bool concurrent_chained_hashtable_try_delete(hashtable_t *hashtable, + cache_obj_t *cache_obj) { + cache_obj_t* result = concurrent_chained_hashtable_delete_obj_id(hashtable, cache_obj->obj_id); + return result != NULL; +} cache_obj_t *concurrent_chained_hashtable_rand_obj(const hashtable_t *hashtable) { uint64_t pos = next_rand() & hashmask(hashtable->hashpower); @@ -305,51 +231,16 @@ cache_obj_t *concurrent_chained_hashtable_rand_obj(const hashtable_t *hashtable) return hashtable->ptr_table[pos]; } -void concurrent_chained_hashtable_foreach(hashtable_t *hashtable, - hashtable_iter iter_func, void *user_data) { - cache_obj_t *cur_obj, *next_obj; - for (uint64_t i = 0; i < hashsize(hashtable->hashpower); i++) { - /** Write lock for iter_func*/ - pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, i); - pthread_rwlock_wrlock(rwlock_); - cur_obj = hashtable->ptr_table[i]; - while (cur_obj != NULL) { - next_obj = cur_obj->hash_next; - iter_func(cur_obj, user_data); - cur_obj = next_obj; - } - pthread_rwlock_unlock(rwlock_); - } -} - +// The hashtable is not responsible for freeing the objects. The release of objects is managed by the cache eviciton policies. void free_concurrent_chained_hashtable(hashtable_t *hashtable) { - if (!hashtable->external_obj) - concurrent_chained_hashtable_foreach(hashtable, foreach_free_obj_locked, NULL); + // if (!hashtable->external_obj) + // concurrent_chained_hashtable_foreach(hashtable, foreach_free_obj_locked, NULL); my_free(sizeof(cache_obj_t *) * hashsize(hashtable->hashpower), hashtable->ptr_table); destory_RWLocks(hashtable->rwlocks_); } -static int count_n_obj_in_bucket_locked(cache_obj_t *curr_obj) { - obj_id_t obj_id_arr[64]; - int chain_len = 0; - while (curr_obj != NULL) { - obj_id_arr[chain_len] = curr_obj->obj_id; - for (int i = 0; i < chain_len; i++) { - if (obj_id_arr[i] == curr_obj->obj_id) { - ERROR("obj_id %lu is duplicated in hashtable\n", curr_obj->obj_id); - abort(); - } - } - - curr_obj = curr_obj->hash_next; - chain_len += 1; - } - return chain_len; -} - - #ifdef __cplusplus } #endif diff --git a/libCacheSim/dataStructure/hashtable/cChainedHashTable.h b/libCacheSim/dataStructure/hashtable/cChainedHashTable.h index ae28bfe0..50ee4350 100644 --- a/libCacheSim/dataStructure/hashtable/cChainedHashTable.h +++ b/libCacheSim/dataStructure/hashtable/cChainedHashTable.h @@ -47,7 +47,7 @@ bool concurrent_chained_hashtable_try_delete(hashtable_t *hashtable, void concurrent_chained_hashtable_delete(hashtable_t *hashtable, cache_obj_t *cache_obj); -bool concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, +cache_obj_t *concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, const obj_id_t obj_id); cache_obj_t *concurrent_chained_hashtable_rand_obj(const hashtable_t *hashtable); diff --git a/libCacheSim/dataStructure/hashtable/hashtable.h b/libCacheSim/dataStructure/hashtable/hashtable.h index 98f33059..56d2b2a5 100644 --- a/libCacheSim/dataStructure/hashtable/hashtable.h +++ b/libCacheSim/dataStructure/hashtable/hashtable.h @@ -15,54 +15,55 @@ extern "C" { #include "../../utils/include/mymath.h" #include "hashtableStruct.h" -#if HASHTABLE_TYPE == CHAINED_HASHTABLE -#include "chainedHashTable.h" -#define create_hashtable(hashpower) create_chained_hashtable(hashpower) -#define hashtable_find(hashtable, obj_id) \ - chained_hashtable_find(hashtable, obj_id) -#define hashtable_find_req(hashtable, req) \ - chained_hashtable_find(hashtable, req) -#define hashtable_find_obj(hashtable, cache_obj) \ - chained_hashtable_find_obj(hashtable, cache_obj) -#define hashtable_insert(hashtable, req) \ - chained_hashtable_insert(hashtable, req) -#define hashtable_insert_obj(hashtable, cache_obj) assert(0); -#define hashtable_delete(hashtable, cache_obj) \ - chained_hashtable_delete(hashtable, cache_obj) -#define hashtable_rand_obj(hashtable) chained_hashtable_rand_obj(hashtable) -#define hashtable_foreach(hashtable, iter_func, user_data) \ - chained_hashtable_foreach(hashtable, iter_func, user_data) -#define free_hashtable(hashtable) free_chained_hashtable(hashtable) -#define hashtable_add_ptr_to_monitoring(hashtable, ptr) \ - chained_hashtable_add_ptr_to_monitoring(hashtable, ptr) -#define HASHTABLE_VER 1 +// #if HASHTABLE_TYPE == CHAINED_HASHTABLE +// #include "chainedHashTable.h" +// #define create_hashtable(hashpower) create_chained_hashtable(hashpower) +// #define hashtable_find(hashtable, obj_id) \ +// chained_hashtable_find(hashtable, obj_id) +// #define hashtable_find_req(hashtable, req) \ +// chained_hashtable_find(hashtable, req) +// #define hashtable_find_obj(hashtable, cache_obj) \ +// chained_hashtable_find_obj(hashtable, cache_obj) +// #define hashtable_insert(hashtable, req) \ +// chained_hashtable_insert(hashtable, req) +// #define hashtable_insert_obj(hashtable, cache_obj) assert(0); +// #define hashtable_delete(hashtable, cache_obj) \ +// chained_hashtable_delete(hashtable, cache_obj) +// #define hashtable_rand_obj(hashtable) chained_hashtable_rand_obj(hashtable) +// #define hashtable_foreach(hashtable, iter_func, user_data) \ +// chained_hashtable_foreach(hashtable, iter_func, user_data) +// #define free_hashtable(hashtable) free_chained_hashtable(hashtable) +// #define hashtable_add_ptr_to_monitoring(hashtable, ptr) \ +// chained_hashtable_add_ptr_to_monitoring(hashtable, ptr) +// #define HASHTABLE_VER 1 -#elif HASHTABLE_TYPE == CHAINED_HASHTABLEV2 -#include "chainedHashTableV2.h" -#define create_hashtable(hashpower) create_chained_hashtable_v2(hashpower) -#define hashtable_find(hashtable, req) chained_hashtable_find_v2(hashtable, req) -#define hashtable_find_obj_id(hashtable, obj_id) \ - chained_hashtable_find_obj_id_v2(hashtable, obj_id) -#define hashtable_find_obj(hashtable, cache_obj) \ - chained_hashtable_find_obj_v2(hashtable, cache_obj) -#define hashtable_insert(hashtable, req) \ - chained_hashtable_insert_v2(hashtable, req) -#define hashtable_insert_obj(hashtable, cache_obj) \ - chained_hashtable_insert_obj_v2(hashtable, cache_obj) -#define hashtable_delete(hashtable, cache_obj) \ - chained_hashtable_delete_v2(hashtable, cache_obj) -#define hashtable_try_delete(hashtable, cache_obj) \ - chained_hashtable_try_delete_v2(hashtable, cache_obj) -#define hashtable_delete_obj_id(hashtable, obj_id) \ - chained_hashtable_delete_obj_id_v2(hashtable, obj_id) -#define hashtable_rand_obj(hashtable) chained_hashtable_rand_obj_v2(hashtable) -#define hashtable_foreach(hashtable, iter_func, user_data) \ - chained_hashtable_foreach_v2(hashtable, iter_func, user_data) -#define free_hashtable(hashtable) free_chained_hashtable_v2(hashtable) -#define hashtable_add_ptr_to_monitoring(hashtable, ptr) -#define HASHTABLE_VER 2 +// #elif HASHTABLE_TYPE == CHAINED_HASHTABLEV2 +// #include "chainedHashTableV2.h" +// #define create_hashtable(hashpower) create_chained_hashtable_v2(hashpower) +// #define hashtable_find(hashtable, req) chained_hashtable_find_v2(hashtable, req) +// #define hashtable_find_obj_id(hashtable, obj_id) \ +// chained_hashtable_find_obj_id_v2(hashtable, obj_id) +// #define hashtable_find_obj(hashtable, cache_obj) \ +// chained_hashtable_find_obj_v2(hashtable, cache_obj) +// #define hashtable_insert(hashtable, req) \ +// chained_hashtable_insert_v2(hashtable, req) +// #define hashtable_insert_obj(hashtable, cache_obj) \ +// chained_hashtable_insert_obj_v2(hashtable, cache_obj) +// #define hashtable_delete(hashtable, cache_obj) \ +// chained_hashtable_delete_v2(hashtable, cache_obj) +// #define hashtable_try_delete(hashtable, cache_obj) \ +// chained_hashtable_try_delete_v2(hashtable, cache_obj) +// #define hashtable_delete_obj_id(hashtable, obj_id) \ +// chained_hashtable_delete_obj_id_v2(hashtable, obj_id) +// #define hashtable_rand_obj(hashtable) chained_hashtable_rand_obj_v2(hashtable) +// #define hashtable_foreach(hashtable, iter_func, user_data) \ +// chained_hashtable_foreach_v2(hashtable, iter_func, user_data) +// #define free_hashtable(hashtable) free_chained_hashtable_v2(hashtable) +// #define hashtable_add_ptr_to_monitoring(hashtable, ptr) +// #define HASHTABLE_VER 2 -#elif HASHTABLE_TYPE == CONCURRENT_CHAINED_HASHTABLE +// #elif HASHTABLE_TYPE == CONCURRENT_CHAINED_HASHTABLE +#if HASHTABLE_TYPE == CONCURRENT_CHAINED_HASHTABLE #include "cChainedHashTable.h" #define create_hashtable(hashpower) create_concurrent_chained_hashtable(hashpower) #define hashtable_find(hashtable, req) concurrent_chained_hashtable_find(hashtable, req) diff --git a/libCacheSim/dataStructure/test/test_concurrent_hashtable.c b/libCacheSim/dataStructure/test/test_concurrent_hashtable.c index da03555b..3bbab1a4 100644 --- a/libCacheSim/dataStructure/test/test_concurrent_hashtable.c +++ b/libCacheSim/dataStructure/test/test_concurrent_hashtable.c @@ -50,7 +50,7 @@ typedef cache_obj_t* (*func_hashtable_find_obj_id_t)(const hashtable_t *hashtabl typedef cache_obj_t* (*func_hashtable_insert_obj_t)(hashtable_t *hashtable, cache_obj_t *cache_obj); /** Function pointer to remove a cache object in the cache instance. */ -typedef bool (*func_hashtable_delete_obj_id_t)(hashtable_t *hashtable, +typedef cache_obj_t* (*func_hashtable_delete_obj_id_t)(hashtable_t *hashtable, const obj_id_t obj_id); typedef hashtable_t *(*func_create_hashtable_t)(const uint16_t hashpower_init); @@ -168,7 +168,10 @@ void *func_insert(void* arg){ for (uint64_t it = 0; it < range; it++) { cur_obj_id = get_next_rand(cur_obj_id); cache_obj_t* cur_obj = create_cache_obj_from_obj_id(cur_obj_id); - ht_handle.func_insert_obj(table, cur_obj); + cache_obj_t* result = ht_handle.func_insert_obj(table, cur_obj); + if(result){ + free_cache_obj(result); + } inserts++; } /** Update the shared atomic counter */ @@ -206,8 +209,10 @@ void *func_remove(void* arg){ gettimeofday(&start, NULL); for (uint64_t it = 0; it < range; it++) { cur_obj_id = get_next_rand(cur_obj_id); - if(ht_handle.func_delete_obj(table, cur_obj_id)){ + cache_obj_t* result = ht_handle.func_delete_obj(table, cur_obj_id); + if(result){ success_removal++; + free_cache_obj(result); } else{ fail_removal++; @@ -263,7 +268,9 @@ void *func_mixed(void* arg){ for(uint64_t it = 0; it < range; it++){ cur_obj_id = get_next_rand(cur_obj_id); cache_obj_t* cur_obj = create_cache_obj_from_obj_id(cur_obj_id); - ht_handle.func_insert_obj(table, cur_obj); + cache_obj_t* old_item = ht_handle.func_insert_obj(table, cur_obj); + if(old_item) + free_cache_obj(old_item); inserts++; } for(uint64_t rounds = 0; rounds < 10; rounds++){ @@ -278,7 +285,9 @@ void *func_mixed(void* arg){ } for(uint64_t it = 0; it < range; it++){ cur_obj_id = get_next_rand(cur_obj_id); - ht_handle.func_delete_obj(table, cur_obj_id); + cache_obj_t* removed_item = ht_handle.func_delete_obj(table, cur_obj_id); + if(removed_item) + free_cache_obj(removed_item); removals++; } __sync_fetch_and_add(i_counter, inserts); @@ -362,13 +371,13 @@ void stress_test() { ht_handle.func_free_hashtable = free_concurrent_chained_hashtable; } // If hashtable type is chainedHashTableV2. - else if(g_ht_type == 1){ - ht_handle.func_find_obj = chained_hashtable_find_obj_id_v2; - ht_handle.func_insert_obj = chained_hashtable_insert_obj_v2; - ht_handle.func_delete_obj = chained_hashtable_delete_obj_id_v2; - ht_handle.func_create_hashtable = create_chained_hashtable_v2; - ht_handle.func_free_hashtable = free_chained_hashtable_v2; - } + // else if(g_ht_type == 1){ + // ht_handle.func_find_obj = chained_hashtable_find_obj_id_v2; + // ht_handle.func_insert_obj = chained_hashtable_insert_obj_v2; + // ht_handle.func_delete_obj = chained_hashtable_delete_obj_id_v2; + // ht_handle.func_create_hashtable = create_chained_hashtable_v2; + // ht_handle.func_free_hashtable = free_chained_hashtable_v2; + // } // If hashtable type is invalid. else{ diff --git a/libCacheSim/include/config.h b/libCacheSim/include/config.h index 6e89e6f9..44561c8f 100644 --- a/libCacheSim/include/config.h +++ b/libCacheSim/include/config.h @@ -36,7 +36,7 @@ extern "C" { #endif #ifndef HASH_POWER_DEFAULT -#define HASH_POWER_DEFAULT 23 +#define HASH_POWER_DEFAULT 20 #endif #ifndef CHAINED_HASHTABLE_EXPAND_THRESHOLD diff --git a/libCacheSim/include/libCacheSim/cache.h b/libCacheSim/include/libCacheSim/cache.h index bd979367..d7e3979e 100644 --- a/libCacheSim/include/libCacheSim/cache.h +++ b/libCacheSim/include/libCacheSim/cache.h @@ -81,6 +81,7 @@ typedef struct { int64_t n_obj; int64_t occupied_byte; int64_t cache_size; + double runtime; /* current trace time, used to determine obj expiration */ int64_t curr_rtime; diff --git a/libCacheSim/include/libCacheSim/cacheObj.h b/libCacheSim/include/libCacheSim/cacheObj.h index 65b22499..1d174561 100644 --- a/libCacheSim/include/libCacheSim/cacheObj.h +++ b/libCacheSim/include/libCacheSim/cacheObj.h @@ -11,6 +11,7 @@ #include "../config.h" #include "mem.h" +#include "../../utils/include/mymutex.h" #ifdef __cplusplus extern "C" { @@ -132,10 +133,12 @@ typedef struct { // ############################## cache obj ################################### struct cache_obj; + +#define FLAG_IN_CACHE_BIT 0b001 + typedef struct cache_obj { - struct cache_obj *hash_next; + struct cache_obj *hash_next; // The first field to make an aligned pointer obj_id_t obj_id; - uint64_t fingerprint; uint32_t obj_size; struct { struct cache_obj *prev; @@ -144,6 +147,8 @@ typedef struct cache_obj { #ifdef SUPPORT_TTL uint32_t exp_time; #endif + uint8_t flags; + /* age is defined as the time since the object entered the cache */ #if defined(TRACK_EVICTION_V_AGE) || \ defined(TRACK_DEMOTION) || defined(TRACK_CREATE_TIME) @@ -179,6 +184,23 @@ typedef struct cache_obj { }; } __attribute__((packed)) cache_obj_t; +/** + * Set/get flags of the cache_obj atomically + */ + +inline bool cache_obj_in_cache(cache_obj_t *obj) { + return obj->flags & FLAG_IN_CACHE_BIT; +} + +inline void cache_obj_set_in_cache(cache_obj_t *obj, bool in_cache) { + if (in_cache) { + fetch_or(&obj->flags, FLAG_IN_CACHE_BIT); + } else { + fetch_and(&obj->flags, ~FLAG_IN_CACHE_BIT); + } +} + + struct request; /** @@ -281,6 +303,12 @@ void prepend_obj_to_head(cache_obj_t **head, cache_obj_t **tail, */ void append_obj_to_tail(cache_obj_t **head, cache_obj_t **tail, cache_obj_t *cache_obj); +/** + * free the the doubly linked list + * @param head + * @param tail + */ +void free_list(cache_obj_t **head, cache_obj_t **tail); /** * free cache_obj, this is only used when the cache_obj is explicitly * malloced diff --git a/libCacheSim/include/libCacheSim/evictionAlgo.h b/libCacheSim/include/libCacheSim/evictionAlgo.h index 31589001..e87d0e76 100644 --- a/libCacheSim/include/libCacheSim/evictionAlgo.h +++ b/libCacheSim/include/libCacheSim/evictionAlgo.h @@ -1,6 +1,7 @@ #pragma once #include "cache.h" +#include #ifdef __cplusplus extern "C" { @@ -15,6 +16,7 @@ typedef struct { typedef struct { cache_obj_t *q_head; cache_obj_t *q_tail; + pthread_mutex_t mutex_; } LRU_params_t; /* used by LFU related */ @@ -23,6 +25,7 @@ typedef struct freq_node { cache_obj_t *first_obj; cache_obj_t *last_obj; uint32_t n_obj; + pthread_mutex_t mutex_; } freq_node_t; typedef struct { diff --git a/libCacheSim/profiler/simulator.c b/libCacheSim/profiler/simulator.c index a53f7cdd..06de2259 100644 --- a/libCacheSim/profiler/simulator.c +++ b/libCacheSim/profiler/simulator.c @@ -19,7 +19,8 @@ extern "C" { #include "../include/libCacheSim/plugin.h" #include "../utils/include/myprint.h" #include "../utils/include/mystr.h" - +#include "../utils/include/mysys.h" +#include "../utils/include/mymutex.h" typedef struct simulator_multithreading_params { reader_t *reader; ssize_t n_caches; @@ -28,7 +29,7 @@ typedef struct simulator_multithreading_params { reader_t *warmup_reader; int warmup_sec; /* num of seconds of requests used for warming up cache */ cache_stat_t *result; - GMutex mtx; /* prevent simultaneous write to progress */ + // GMutex mtx; /* prevent simultaneous write to progress */ gint *progress; gpointer other_data; bool free_cache_when_finish; @@ -36,8 +37,10 @@ typedef struct simulator_multithreading_params { static void _simulate(gpointer data, gpointer user_data) { sim_mt_params_t *params = (sim_mt_params_t *)user_data; - int idx = GPOINTER_TO_UINT(data) - 1; + int idx = (GPOINTER_TO_UINT(data) - 1) % params->n_caches; + int t_idx = GPOINTER_TO_UINT(data) / params->n_caches; set_rand_seed(0); + // INFO("Start simulation. Thread %d in Cache %s (index=%d)\n", t_idx, params->caches[idx]->cache_name, idx); cache_stat_t *result = params->result; reader_t *cloned_reader = clone_reader(params->reader); @@ -50,10 +53,12 @@ static void _simulate(gpointer data, gpointer user_data) { if (params->warmup_reader) { reader_t *warmup_cloned_reader = clone_reader(params->warmup_reader); read_one_req(warmup_cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; while (req->valid) { local_cache->get(local_cache, req); - result[idx].n_warmup_req += 1; + fetch_add(&result[idx].n_warmup_req, 1); read_one_req(warmup_cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; } close_reader(warmup_cloned_reader); INFO("cache %s (size %" PRIu64 @@ -64,6 +69,7 @@ static void _simulate(gpointer data, gpointer user_data) { } read_one_req(cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; int64_t start_ts = (int64_t)req->clock_time; /* using warmup_frac or warmup_sec of requests from reader to warm up */ @@ -75,8 +81,9 @@ static void _simulate(gpointer data, gpointer user_data) { local_cache->get(local_cache, req); n_warmup += 1; read_one_req(cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; } - result[idx].n_warmup_req += n_warmup; + fetch_add(&result[idx].n_warmup_req, n_warmup); INFO("cache %s (size %" PRIu64 ") finishes warm up using " "with %" PRIu64 " requests, %.2lf hour trace time\n", @@ -84,18 +91,27 @@ static void _simulate(gpointer data, gpointer user_data) { (double)(req->clock_time - start_ts) / 3600.0); } + double start_time = gettime(); while (req->valid) { - result[idx].n_req++; - result[idx].n_req_byte += req->obj_size; + if(result[idx].runtime != 0){ + free_request(req); + close_reader(cloned_reader); + return; + } + fetch_add(&result[idx].n_req, 1); + fetch_add(&result[idx].n_req_byte, req->obj_size); req->clock_time -= start_ts; if (local_cache->get(local_cache, req) == false) { - result[idx].n_miss++; - result[idx].n_miss_byte += req->obj_size; + fetch_add(&result[idx].n_miss, 1); + fetch_add(&result[idx].n_miss_byte, req->obj_size); } read_one_req(cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; } + result[idx].runtime = gettime() - start_time; + /* disabled due to ARC and LeCaR use ghost entries in the hash table */ #if defined(SUPPORT_TTL) && defined(ENABLE_SCAN) /* get expiration information */ @@ -124,14 +140,16 @@ static void _simulate(gpointer data, gpointer user_data) { CACHE_NAME_ARRAY_LEN); // report progress - g_mutex_lock(&(params->mtx)); - (*(params->progress))++; - g_mutex_unlock(&(params->mtx)); + // g_mutex_lock(&(params->mtx)); + // (*(params->progress))++; + // g_mutex_unlock(&(params->mtx)); + + __atomic_fetch_add(params->progress, 1, __ATOMIC_ACQ_REL); // clean up - if (params->free_cache_when_finish) { - local_cache->cache_free(local_cache); - } + // if (params->free_cache_when_finish) { + // local_cache->cache_free(local_cache); + // } free_request(req); close_reader(cloned_reader); } @@ -191,7 +209,7 @@ cache_stat_t *simulate_at_multi_sizes(reader_t *reader, const cache_t *cache, params->result = result; params->free_cache_when_finish = true; params->progress = &progress; - g_mutex_init(&(params->mtx)); + // g_mutex_init(&(params->mtx)); // build the thread pool GThreadPool *gthread_pool = g_thread_pool_new( @@ -225,7 +243,7 @@ cache_stat_t *simulate_at_multi_sizes(reader_t *reader, const cache_t *cache, // clean up g_thread_pool_free(gthread_pool, FALSE, TRUE); - g_mutex_clear(&(params->mtx)); + // g_mutex_clear(&(params->mtx)); my_free(sizeof(cache_t *) * num_of_sizes, params->caches); my_free(sizeof(sim_mt_params_t), params); @@ -252,7 +270,7 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], int num_of_threads, bool free_cache_when_finish) { assert(num_of_caches > 0); - int i, progress = 0; + int i, j, progress = 0; cache_stat_t *result = my_malloc_n(cache_stat_t, num_of_caches); memset(result, 0, sizeof(cache_stat_t) * num_of_caches); @@ -261,6 +279,7 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], sim_mt_params_t *params = my_malloc(sim_mt_params_t); params->reader = reader; params->caches = caches; + params->n_caches = num_of_caches; params->warmup_reader = warmup_reader; params->warmup_sec = warmup_sec; if (warmup_frac > 1e-6) { @@ -272,19 +291,20 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], params->result = result; params->free_cache_when_finish = free_cache_when_finish; params->progress = &progress; - g_mutex_init(&(params->mtx)); + // g_mutex_init(&(params->mtx)); // build the thread pool GThreadPool *gthread_pool = g_thread_pool_new( - (GFunc)_simulate, (gpointer)params, num_of_threads, TRUE, NULL); + (GFunc)_simulate, (gpointer)params, num_of_caches * num_of_threads, TRUE, NULL); ASSERT_NOT_NULL(gthread_pool, "cannot create thread pool in simulator\n"); - // start computation - for (i = 1; i < num_of_caches + 1; i++) { + // start computation + for (i = 1; i < num_of_caches + 1; i++){ result[i - 1].cache_size = caches[i - 1]->cache_size; - - ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i), NULL), + for (j = 0; j < num_of_threads; j++) { + ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i + j * num_of_caches), NULL), "cannot push data into thread_pool in get_miss_ratio\n"); + } } char start_cache_size[64], end_cache_size[64]; @@ -299,13 +319,20 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], num_of_caches, num_of_threads); // wait for all simulations to finish - while (progress < (uint64_t)num_of_caches - 1) { - print_progress((double)progress / (double)(num_of_caches - 1) * 100); + while (progress < (uint64_t)num_of_caches) { + print_progress((double)progress / (double)(num_of_caches) * 100); } // clean up + g_thread_pool_free(gthread_pool, FALSE, TRUE); - g_mutex_clear(&(params->mtx)); + // g_mutex_clear(&(params->mtx)); + for (i = 0; i < num_of_caches; i++){ + if (params->free_cache_when_finish) { + cache_t *local_cache = params->caches[i]; + local_cache->cache_free(local_cache); + } + } my_free(sizeof(sim_mt_params_t), params); // user is responsible for free-ing the result diff --git a/libCacheSim/utils/include/mymutex.h b/libCacheSim/utils/include/mymutex.h index 2b63e467..efeea7a0 100644 --- a/libCacheSim/utils/include/mymutex.h +++ b/libCacheSim/utils/include/mymutex.h @@ -6,6 +6,7 @@ #include #include +#include typedef struct RWLocks { @@ -22,3 +23,9 @@ void destory_RWLocks(RWLocks_t* rwlocks); pthread_rwlock_t* getRWLock(RWLocks_t* rwlocks, uint64_t hash); +#define CAS(ptr, expected, desired) \ + __atomic_compare_exchange_n(ptr, expected, desired, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED) +#define fetch_add(ptr, val) __atomic_add_fetch(ptr, val, __ATOMIC_ACQ_REL) +#define fetch_sub(ptr, val) __atomic_sub_fetch(ptr, val, __ATOMIC_ACQ_REL) +#define fetch_and(ptr, val) __atomic_and_fetch(ptr, val, __ATOMIC_ACQ_REL) +#define fetch_or(ptr, val) __atomic_or_fetch(ptr, val, __ATOMIC_ACQ_REL)