From 1bce12074d034c87fb5458f2f73cef4800394321 Mon Sep 17 00:00:00 2001 From: Cryp Toon Date: Wed, 7 Feb 2024 17:32:54 +0100 Subject: [PATCH] Update cache database code --- bitcoinlib/config/config.py | 12 ++--- bitcoinlib/db_cache.py | 23 +++++----- tests/test_services.py | 87 ++++++++++++++++++++++--------------- tests/test_tools.py | 16 +------ tests/test_wallets.py | 51 +++++----------------- 5 files changed, 79 insertions(+), 110 deletions(-) diff --git a/bitcoinlib/config/config.py b/bitcoinlib/config/config.py index eaeed417..24baf365 100644 --- a/bitcoinlib/config/config.py +++ b/bitcoinlib/config/config.py @@ -211,9 +211,6 @@ # }, ] -# UNITTESTS -UNITTESTS_FULL_DATABASE_TEST = False - # CACHING SERVICE_CACHING_ENABLED = True @@ -235,7 +232,7 @@ def config_get(section, var, fallback, is_boolean=False): global ALLOW_DATABASE_THREADS, DEFAULT_DATABASE_CACHE global BCL_LOG_FILE, LOGLEVEL, ENABLE_BITCOINLIB_LOGGING global TIMEOUT_REQUESTS, DEFAULT_LANGUAGE, DEFAULT_NETWORK, DEFAULT_WITNESS_TYPE - global UNITTESTS_FULL_DATABASE_TEST, SERVICE_CACHING_ENABLED, DATABASE_ENCRYPTION_ENABLED, DB_FIELD_ENCRYPTION_KEY + global SERVICE_CACHING_ENABLED, DATABASE_ENCRYPTION_ENABLED, DB_FIELD_ENCRYPTION_KEY global SERVICE_MAX_ERRORS, BLOCK_COUNT_CACHE_TIME, MAX_TRANSACTIONS # Read settings from Configuration file provided in OS environment~/.bitcoinlib/ directory @@ -262,7 +259,8 @@ def config_get(section, var, fallback, is_boolean=False): DEFAULT_DATABASE = str(Path(BCL_DATABASE_DIR, default_databasefile)) default_databasefile_cache = DEFAULT_DATABASE_CACHE = \ config_get('locations', 'default_databasefile_cache', fallback='bitcoinlib_cache.sqlite') - if not default_databasefile_cache.startswith('postgresql') or default_databasefile_cache.startswith('mysql'): + if not (default_databasefile_cache.startswith('postgresql') or default_databasefile_cache.startswith('mysql') or + default_databasefile_cache.startswith('mariadb')): DEFAULT_DATABASE_CACHE = str(Path(BCL_DATABASE_DIR, default_databasefile_cache)) ALLOW_DATABASE_THREADS = config_get("common", "allow_database_threads", fallback=True, is_boolean=True) SERVICE_CACHING_ENABLED = config_get('common', 'service_caching_enabled', fallback=True, is_boolean=True) @@ -286,10 +284,6 @@ def config_get(section, var, fallback, is_boolean=False): DEFAULT_NETWORK = config_get('common', 'default_network', fallback=DEFAULT_NETWORK) DEFAULT_WITNESS_TYPE = config_get('common', 'default_witness_type', fallback=DEFAULT_WITNESS_TYPE) - full_db_test = os.environ.get('UNITTESTS_FULL_DATABASE_TEST') - if full_db_test in [1, True, 'True', 'true', 'TRUE']: - UNITTESTS_FULL_DATABASE_TEST = True - if not data: return False return True diff --git a/bitcoinlib/db_cache.py b/bitcoinlib/db_cache.py index f339f1de..ec8e85ff 100644 --- a/bitcoinlib/db_cache.py +++ b/bitcoinlib/db_cache.py @@ -27,15 +27,15 @@ _logger = logging.getLogger(__name__) -try: - dbcacheurl_obj = urlparse(DEFAULT_DATABASE_CACHE) - if dbcacheurl_obj.netloc: - dbcacheurl = dbcacheurl_obj.netloc.replace(dbcacheurl_obj.password, 'xxx') - else: - dbcacheurl = dbcacheurl_obj.path - _logger.info("Default Cache Database %s" % dbcacheurl) -except Exception: - _logger.warning("Default Cache Database: unable to parse URL") +# try: +# dbcacheurl_obj = urlparse(DEFAULT_DATABASE_CACHE) +# if dbcacheurl_obj.netloc: +# dbcacheurl = dbcacheurl_obj.netloc.replace(dbcacheurl_obj.password, 'xxx') +# else: +# dbcacheurl = dbcacheurl_obj.path +# _logger.info("Default Cache Database %s" % dbcacheurl) +# except Exception: +# _logger.warning("Default Cache Database: unable to parse URL") Base = declarative_base() @@ -66,8 +66,9 @@ def __init__(self, db_uri=None): db_uri += "&" if "?" in db_uri else "?" db_uri += "check_same_thread=False" if self.o.scheme == 'mysql': - db_uri += "&" if "?" in db_uri else "?" - db_uri += 'binary_prefix=true' + raise NotImplementedError("MySQL does not allow indexing on LargeBinary fields, so caching is not possible") + # db_uri += "&" if "?" in db_uri else "?" + # db_uri += 'binary_prefix=true' self.engine = create_engine(db_uri, isolation_level='READ UNCOMMITTED') Session = sessionmaker(bind=self.engine) diff --git a/tests/test_services.py b/tests/test_services.py index c29ce3ba..fa763e35 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -38,19 +38,22 @@ MAXIMUM_ESTIMATED_FEE_DIFFERENCE = 3.00 # Maximum difference from average estimated fee before test_estimatefee fails. # Use value above >0, and 1 for 100% -DATABASEFILE_CACHE_UNITTESTS = os.path.join(str(BCL_DATABASE_DIR), 'bitcoinlibcache.unittest.sqlite') -DATABASEFILE_CACHE_UNITTESTS2 = os.path.join(str(BCL_DATABASE_DIR), 'bitcoinlibcache2.unittest.sqlite') -DATABASE_CACHE_POSTGRESQL = 'postgresql://postgres:postgres@localhost:5432/bitcoinlibcache.unittest' -# FIXME: MySQL databases are not supported. Not allowed to create indexes/primary keys on binary fields -DATABASE_CACHE_MYSQL = 'mysql://root:root@localhost:3306/bitcoinlibcache.unittest' +CACHE_DBNAME1 = 'bitcoinlib_cache_unittest1' +CACHE_DBNAME2 = 'bitcoinlib_cache_unittest2' +if os.getenv('UNITTEST_DATABASE') == 'mysql' or os.getenv('UNITTEST_DATABASE') == 'mariadb': + DATABASE_CACHE_UNITTESTS = 'mariadb://user:password@localhost:3306/%s' % CACHE_DBNAME1 + DATABASE_CACHE_UNITTESTS2 = 'mariadb://user:password@localhost:3306/%s' % CACHE_DBNAME2 +elif os.getenv('UNITTEST_DATABASE') == 'postgresql': + DATABASE_CACHE_UNITTESTS = 'postgresql://postgres:postgres@localhost:5432/%s' % CACHE_DBNAME1 + DATABASE_CACHE_UNITTESTS2 = 'postgresql://postgres:postgres@localhost:5432/%s' % CACHE_DBNAME2 +else: + DATABASE_CACHE_UNITTESTS = 'sqlite://' + os.path.join(str(BCL_DATABASE_DIR), CACHE_DBNAME1) + '.sqlite' + DATABASE_CACHE_UNITTESTS2 = 'sqlite://' + os.path.join(str(BCL_DATABASE_DIR), CACHE_DBNAME2) + '.sqlite' + DATABASES_CACHE = [ - DATABASEFILE_CACHE_UNITTESTS2, + DATABASE_CACHE_UNITTESTS, + DATABASE_CACHE_UNITTESTS2, ] -if UNITTESTS_FULL_DATABASE_TEST: - DATABASES_CACHE += [ - DATABASE_CACHE_POSTGRESQL, - DATABASE_CACHE_MYSQL - ] TIMEOUT_TEST = 3 @@ -59,7 +62,7 @@ class ServiceTest(Service): def __init__(self, network=DEFAULT_NETWORK, min_providers=1, max_providers=1, providers=None, - timeout=TIMEOUT_TEST, cache_uri=DATABASEFILE_CACHE_UNITTESTS, ignore_priority=True, + timeout=TIMEOUT_TEST, cache_uri=DATABASE_CACHE_UNITTESTS, ignore_priority=True, exclude_providers=None, max_errors=SERVICE_MAX_ERRORS, strict=True): super(self.__class__, self).__init__(network, min_providers, max_providers, providers, timeout, cache_uri, ignore_priority, exclude_providers, max_errors, strict) @@ -803,33 +806,47 @@ def test_service_transaction_unconfirmed(self): class TestServiceCache(unittest.TestCase): - # TODO: Add mysql support @classmethod def setUpClass(cls): + session.close_all_sessions() try: - if os.path.isfile(DATABASEFILE_CACHE_UNITTESTS2): - os.remove(DATABASEFILE_CACHE_UNITTESTS2) - except Exception: - pass - try: - DbCache(DATABASE_CACHE_POSTGRESQL).drop_db() - # DbCache(DATABASEFILE_CACHE_MYSQL).drop_db() - except Exception: + DbCache(CACHE_DBNAME1).drop_db() + DbCache(CACHE_DBNAME2).drop_db() + except: pass - try: - con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True) + if os.getenv('UNITTEST_DATABASE') == 'postgresql': + try: + con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True) + cur = con.cursor() + cur.execute(sql.SQL("CREATE DATABASE {}").format( + sql.Identifier('bitcoinlibcache.unittest')) + ) + cur.close() + con.close() + except Exception: + pass + elif os.getenv('UNITTEST_DATABASE') == 'mysql': + con = mysql.connector.connect(user='user', host='localhost', password='password') cur = con.cursor() - cur.execute(sql.SQL("CREATE DATABASE {}").format( - sql.Identifier('bitcoinlibcache.unittest')) - ) + cur.execute("DROP DATABASE IF EXISTS {}".format(CACHE_DBNAME1)) + cur.execute("DROP DATABASE IF EXISTS {}".format(CACHE_DBNAME2)) + cur.execute("CREATE DATABASE {}".format(CACHE_DBNAME1)) + cur.execute("CREATE DATABASE {}".format(CACHE_DBNAME2)) + con.commit() cur.close() con.close() - except Exception: - pass + else: + if os.path.isfile(DATABASE_CACHE_UNITTESTS): + try: + os.remove(DATABASE_CACHE_UNITTESTS) + os.remove(DATABASE_CACHE_UNITTESTS2) + except: + pass + def test_service_cache_transactions(self): - srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2) + srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2) address = '1JQ7ybfFBoWhPJpjoihezpeAjd2xv9nXaN' # Get 2 transactions, nothing in cache res = srv.gettransactions(address, limit=2) @@ -853,7 +870,7 @@ def test_service_cache_transactions(self): # FIXME: Disabled, lack of providers # def test_service_cache_gettransaction(self): - # srv = ServiceTest(network='litecoin_testnet', cache_uri=DATABASEFILE_CACHE_UNITTESTS2) + # srv = ServiceTest(network='litecoin_testnet', cache_uri=DATABASE_CACHE_UNITTESTS2) # txid = 'b6533d361daac291f64fff32a5c157a4785b423ce36e2eac27117879f93973da' # # t = srv.gettransaction(txid) @@ -878,7 +895,7 @@ def test_service_cache_transactions(self): def test_service_cache_transactions_after_txid(self): # Do not store anything in cache if after_txid is used - srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2, exclude_providers=['mempool']) + srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2, exclude_providers=['mempool']) address = '12spqcvLTFhL38oNJDDLfW1GpFGxLdaLCL' res = srv.gettransactions(address, after_txid='5f31da8f47a5bd92a6929179082c559e8acc270a040b19838230aab26309cf2d') @@ -898,7 +915,7 @@ def test_service_cache_transactions_after_txid(self): self.assertGreaterEqual(srv.results_cache_n, 1) def test_service_cache_transaction_coinbase(self): - srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2, exclude_providers=['bitaps', 'bitgo']) + srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2, exclude_providers=['bitaps', 'bitgo']) t = srv.gettransaction('68104dbd6819375e7bdf96562f89290b41598df7b002089ecdd3c8d999025b13') if t: self.assertGreaterEqual(srv.results_cache_n, 0) @@ -922,7 +939,7 @@ def test_service_cache_transaction_segwit_database(self): self.assertEqual(t.raw_hex(), rawtx) def test_service_cache_with_latest_tx_query(self): - srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2) + srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2) address = 'bc1qxfrgfhs49d7dtcfzlhp7f7cwsp8zpp60hywp0f' after_txid = '13401ad121c8ae91e18b4bb0db5d8f350a2b0b5ddd5ca26165137bf07fefad90' srv.gettransaction('4156e78f347e47d2ccdd4a19614d958c6e4502d09a68f63ed0c72691f63a5028') @@ -932,7 +949,7 @@ def test_service_cache_with_latest_tx_query(self): self.assertGreaterEqual(len(txs), 5) def test_service_cache_correctly_update_spent_info(self): - srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2) + srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2) srv.gettransactions('1KoAvaL3wfpcNvGCQYkqFJG9Ccqm52sZHa', limit=1) txs = srv.gettransactions('1KoAvaL3wfpcNvGCQYkqFJG9Ccqm52sZHa') self.assertTrue(txs[0].outputs[0].spent) @@ -974,7 +991,7 @@ def test_service_cache_disabled(self): def test_service_cache_transaction_p2sh_p2wpkh_input(self): txid = '6ab6432a6b7b04ecc335c6e8adccc45c25f46e33752478f0bcacaf3f1b61ad92' - srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2) + srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2) t = srv.gettransaction(txid) self.assertEqual(t.size, 249) self.assertEqual(srv.results_cache_n, 0) diff --git a/tests/test_tools.py b/tests/test_tools.py index 370d9f54..f3379a81 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -29,36 +29,24 @@ def database_init(dbname=DATABASE_NAME): if os.getenv('UNITTEST_DATABASE') == 'postgresql': con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True) cur = con.cursor() - # try: cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format( sql.Identifier(dbname)) ) cur.execute(sql.SQL("CREATE DATABASE {}").format( sql.Identifier(dbname)) ) - # except Exception: - # pass - # finally: - # cur.close() - # con.close() cur.close() con.close() return 'postgresql://postgres:postgres@localhost:5432/' + dbname elif os.getenv('UNITTEST_DATABASE') == 'mysql': - try: - con = mysql.connector.connect(user='user', host='localhost', password='password') - except: - try: - con = mysql.connector.connect(user='root', host='localhost', password='password') - except: - con = mysql.connector.connect(user='root', host='localhost') + con = mysql.connector.connect(user='user', host='localhost', password='password') cur = con.cursor() cur.execute("DROP DATABASE IF EXISTS {}".format(dbname)) cur.execute("CREATE DATABASE {}".format(dbname)) con.commit() cur.close() con.close() - return 'mysql://root@localhost:3306/' + dbname + return 'mysql://user:password@localhost:3306/' + dbname else: dburi = os.path.join(str(BCL_DATABASE_DIR), '%s.sqlite' % dbname) if os.path.isfile(dburi): diff --git a/tests/test_wallets.py b/tests/test_wallets.py index 0fe0014f..303f6365 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -43,30 +43,13 @@ DATABASE_NAME = 'bitcoinlib_test' DATABASE_NAME_2 = 'bitcoinlib2_test' -# db_uris = ( -# ('sqlite', 'sqlite:///' + DATABASEFILE_UNITTESTS, 'sqlite:///' + DATABASEFILE_UNITTESTS_2),) - print("DATABASE USED: %s" % os.getenv('UNITTEST_DATABASE')) -# if UNITTESTS_FULL_DATABASE_TEST: -# db_uris += ( -# ('mysql', 'mysql://root:root@localhost:3306/' + DATABASE_NAME, -# 'mysql://root:root@localhost:3306/' + DATABASE_NAME_2), -# ('postgresql', 'postgresql://postgres:postgres@localhost:5432/' + DATABASE_NAME, -# 'postgresql://postgres:postgres@localhost:5432/' + DATABASE_NAME_2), -# ) -# -# -# params = (('SCHEMA', 'DATABASE_URI', 'DATABASE_URI_2'), ( -# db_uris -# )) - - def database_init(dbname=DATABASE_NAME): session.close_all_sessions() if os.getenv('UNITTEST_DATABASE') == 'postgresql': - # con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True) - # cur = con.cursor() + con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True) + cur = con.cursor() # try: # cur.execute(sql.SQL("ALTER DATABASE {} allow_connections = off").format(sql.Identifier(dbname))) # cur.execute(sql.SQL("UPDATE pg_database SET datallowconn = 'false' WHERE datname = '{}'").format( @@ -77,7 +60,6 @@ def database_init(dbname=DATABASE_NAME): # except Exception as e: # print(e) # res = cur.execute(sql.SQL("SELECT sum(numbackends) FROM pg_stat_database")) - # print(res) # res = cur.execute(sql.SQL(""" # DO $$ DECLARE # r RECORD; @@ -86,27 +68,14 @@ def database_init(dbname=DATABASE_NAME): # EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; # END LOOP; # END $$;""")) - # print(res) - # try: - # cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(dbname))) - # cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(dbname))) - # except: - # pass - # try: - # # drop all tables - # finally: - # cur.close() - # con.close() - # con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True) - # cur = con.cursor() - # cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(dbname))) - # con.commit() - # cur.close() - # con.close() - # return 'postgresql://postgres:postgres@localhost:5432/' + dbname - # postgresql = testing.postgresql.Postgresql() - # return postgresql.url() - return 'testing.postgresql' + try: + cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(dbname))) + cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(dbname))) + except: + pass + cur.close() + con.close() + return 'postgresql://postgres:postgres@localhost:5432/' + dbname elif os.getenv('UNITTEST_DATABASE') == 'mysql': con = mysql.connector.connect(user='user', host='localhost', password='password') cur = con.cursor()