Skip to content

Commit

Permalink
Update cache database code
Browse files Browse the repository at this point in the history
  • Loading branch information
Cryp Toon committed Feb 7, 2024
1 parent 742b55d commit 1bce120
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 110 deletions.
12 changes: 3 additions & 9 deletions bitcoinlib/config/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,6 @@
# },
]

# UNITTESTS
UNITTESTS_FULL_DATABASE_TEST = False

# CACHING
SERVICE_CACHING_ENABLED = True

Expand All @@ -235,7 +232,7 @@ def config_get(section, var, fallback, is_boolean=False):
global ALLOW_DATABASE_THREADS, DEFAULT_DATABASE_CACHE
global BCL_LOG_FILE, LOGLEVEL, ENABLE_BITCOINLIB_LOGGING
global TIMEOUT_REQUESTS, DEFAULT_LANGUAGE, DEFAULT_NETWORK, DEFAULT_WITNESS_TYPE
global UNITTESTS_FULL_DATABASE_TEST, SERVICE_CACHING_ENABLED, DATABASE_ENCRYPTION_ENABLED, DB_FIELD_ENCRYPTION_KEY
global SERVICE_CACHING_ENABLED, DATABASE_ENCRYPTION_ENABLED, DB_FIELD_ENCRYPTION_KEY
global SERVICE_MAX_ERRORS, BLOCK_COUNT_CACHE_TIME, MAX_TRANSACTIONS

# Read settings from Configuration file provided in OS environment~/.bitcoinlib/ directory
Expand All @@ -262,7 +259,8 @@ def config_get(section, var, fallback, is_boolean=False):
DEFAULT_DATABASE = str(Path(BCL_DATABASE_DIR, default_databasefile))
default_databasefile_cache = DEFAULT_DATABASE_CACHE = \
config_get('locations', 'default_databasefile_cache', fallback='bitcoinlib_cache.sqlite')
if not default_databasefile_cache.startswith('postgresql') or default_databasefile_cache.startswith('mysql'):
if not (default_databasefile_cache.startswith('postgresql') or default_databasefile_cache.startswith('mysql') or
default_databasefile_cache.startswith('mariadb')):
DEFAULT_DATABASE_CACHE = str(Path(BCL_DATABASE_DIR, default_databasefile_cache))
ALLOW_DATABASE_THREADS = config_get("common", "allow_database_threads", fallback=True, is_boolean=True)
SERVICE_CACHING_ENABLED = config_get('common', 'service_caching_enabled', fallback=True, is_boolean=True)
Expand All @@ -286,10 +284,6 @@ def config_get(section, var, fallback, is_boolean=False):
DEFAULT_NETWORK = config_get('common', 'default_network', fallback=DEFAULT_NETWORK)
DEFAULT_WITNESS_TYPE = config_get('common', 'default_witness_type', fallback=DEFAULT_WITNESS_TYPE)

full_db_test = os.environ.get('UNITTESTS_FULL_DATABASE_TEST')
if full_db_test in [1, True, 'True', 'true', 'TRUE']:
UNITTESTS_FULL_DATABASE_TEST = True

if not data:
return False
return True
Expand Down
23 changes: 12 additions & 11 deletions bitcoinlib/db_cache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,15 @@


_logger = logging.getLogger(__name__)
try:
dbcacheurl_obj = urlparse(DEFAULT_DATABASE_CACHE)
if dbcacheurl_obj.netloc:
dbcacheurl = dbcacheurl_obj.netloc.replace(dbcacheurl_obj.password, 'xxx')
else:
dbcacheurl = dbcacheurl_obj.path
_logger.info("Default Cache Database %s" % dbcacheurl)
except Exception:
_logger.warning("Default Cache Database: unable to parse URL")
# try:
# dbcacheurl_obj = urlparse(DEFAULT_DATABASE_CACHE)
# if dbcacheurl_obj.netloc:
# dbcacheurl = dbcacheurl_obj.netloc.replace(dbcacheurl_obj.password, 'xxx')
# else:
# dbcacheurl = dbcacheurl_obj.path
# _logger.info("Default Cache Database %s" % dbcacheurl)
# except Exception:
# _logger.warning("Default Cache Database: unable to parse URL")
Base = declarative_base()


Expand Down Expand Up @@ -66,8 +66,9 @@ def __init__(self, db_uri=None):
db_uri += "&" if "?" in db_uri else "?"
db_uri += "check_same_thread=False"
if self.o.scheme == 'mysql':
db_uri += "&" if "?" in db_uri else "?"
db_uri += 'binary_prefix=true'
raise NotImplementedError("MySQL does not allow indexing on LargeBinary fields, so caching is not possible")
# db_uri += "&" if "?" in db_uri else "?"
# db_uri += 'binary_prefix=true'
self.engine = create_engine(db_uri, isolation_level='READ UNCOMMITTED')

Session = sessionmaker(bind=self.engine)
Expand Down
87 changes: 52 additions & 35 deletions tests/test_services.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,19 +38,22 @@
MAXIMUM_ESTIMATED_FEE_DIFFERENCE = 3.00 # Maximum difference from average estimated fee before test_estimatefee fails.
# Use value above >0, and 1 for 100%

DATABASEFILE_CACHE_UNITTESTS = os.path.join(str(BCL_DATABASE_DIR), 'bitcoinlibcache.unittest.sqlite')
DATABASEFILE_CACHE_UNITTESTS2 = os.path.join(str(BCL_DATABASE_DIR), 'bitcoinlibcache2.unittest.sqlite')
DATABASE_CACHE_POSTGRESQL = 'postgresql://postgres:postgres@localhost:5432/bitcoinlibcache.unittest'
# FIXME: MySQL databases are not supported. Not allowed to create indexes/primary keys on binary fields
DATABASE_CACHE_MYSQL = 'mysql://root:root@localhost:3306/bitcoinlibcache.unittest'
CACHE_DBNAME1 = 'bitcoinlib_cache_unittest1'
CACHE_DBNAME2 = 'bitcoinlib_cache_unittest2'
if os.getenv('UNITTEST_DATABASE') == 'mysql' or os.getenv('UNITTEST_DATABASE') == 'mariadb':
DATABASE_CACHE_UNITTESTS = 'mariadb://user:password@localhost:3306/%s' % CACHE_DBNAME1
DATABASE_CACHE_UNITTESTS2 = 'mariadb://user:password@localhost:3306/%s' % CACHE_DBNAME2
elif os.getenv('UNITTEST_DATABASE') == 'postgresql':
DATABASE_CACHE_UNITTESTS = 'postgresql://postgres:postgres@localhost:5432/%s' % CACHE_DBNAME1
DATABASE_CACHE_UNITTESTS2 = 'postgresql://postgres:postgres@localhost:5432/%s' % CACHE_DBNAME2
else:
DATABASE_CACHE_UNITTESTS = 'sqlite://' + os.path.join(str(BCL_DATABASE_DIR), CACHE_DBNAME1) + '.sqlite'
DATABASE_CACHE_UNITTESTS2 = 'sqlite://' + os.path.join(str(BCL_DATABASE_DIR), CACHE_DBNAME2) + '.sqlite'

DATABASES_CACHE = [
DATABASEFILE_CACHE_UNITTESTS2,
DATABASE_CACHE_UNITTESTS,
DATABASE_CACHE_UNITTESTS2,
]
if UNITTESTS_FULL_DATABASE_TEST:
DATABASES_CACHE += [
DATABASE_CACHE_POSTGRESQL,
DATABASE_CACHE_MYSQL
]

TIMEOUT_TEST = 3

Expand All @@ -59,7 +62,7 @@
class ServiceTest(Service):

def __init__(self, network=DEFAULT_NETWORK, min_providers=1, max_providers=1, providers=None,
timeout=TIMEOUT_TEST, cache_uri=DATABASEFILE_CACHE_UNITTESTS, ignore_priority=True,
timeout=TIMEOUT_TEST, cache_uri=DATABASE_CACHE_UNITTESTS, ignore_priority=True,
exclude_providers=None, max_errors=SERVICE_MAX_ERRORS, strict=True):
super(self.__class__, self).__init__(network, min_providers, max_providers, providers, timeout, cache_uri,
ignore_priority, exclude_providers, max_errors, strict)
Expand Down Expand Up @@ -803,33 +806,47 @@ def test_service_transaction_unconfirmed(self):

class TestServiceCache(unittest.TestCase):

# TODO: Add mysql support
@classmethod
def setUpClass(cls):
session.close_all_sessions()
try:
if os.path.isfile(DATABASEFILE_CACHE_UNITTESTS2):
os.remove(DATABASEFILE_CACHE_UNITTESTS2)
except Exception:
pass
try:
DbCache(DATABASE_CACHE_POSTGRESQL).drop_db()
# DbCache(DATABASEFILE_CACHE_MYSQL).drop_db()
except Exception:
DbCache(CACHE_DBNAME1).drop_db()
DbCache(CACHE_DBNAME2).drop_db()
except:
pass

try:
con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True)
if os.getenv('UNITTEST_DATABASE') == 'postgresql':
try:
con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True)
cur = con.cursor()
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier('bitcoinlibcache.unittest'))
)
cur.close()
con.close()
except Exception:
pass
elif os.getenv('UNITTEST_DATABASE') == 'mysql':
con = mysql.connector.connect(user='user', host='localhost', password='password')
cur = con.cursor()
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier('bitcoinlibcache.unittest'))
)
cur.execute("DROP DATABASE IF EXISTS {}".format(CACHE_DBNAME1))
cur.execute("DROP DATABASE IF EXISTS {}".format(CACHE_DBNAME2))
cur.execute("CREATE DATABASE {}".format(CACHE_DBNAME1))
cur.execute("CREATE DATABASE {}".format(CACHE_DBNAME2))
con.commit()
cur.close()
con.close()
except Exception:
pass
else:
if os.path.isfile(DATABASE_CACHE_UNITTESTS):
try:
os.remove(DATABASE_CACHE_UNITTESTS)
os.remove(DATABASE_CACHE_UNITTESTS2)
except:
pass


def test_service_cache_transactions(self):
srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2)
srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2)
address = '1JQ7ybfFBoWhPJpjoihezpeAjd2xv9nXaN'
# Get 2 transactions, nothing in cache
res = srv.gettransactions(address, limit=2)
Expand All @@ -853,7 +870,7 @@ def test_service_cache_transactions(self):

# FIXME: Disabled, lack of providers
# def test_service_cache_gettransaction(self):
# srv = ServiceTest(network='litecoin_testnet', cache_uri=DATABASEFILE_CACHE_UNITTESTS2)
# srv = ServiceTest(network='litecoin_testnet', cache_uri=DATABASE_CACHE_UNITTESTS2)
# txid = 'b6533d361daac291f64fff32a5c157a4785b423ce36e2eac27117879f93973da'
#
# t = srv.gettransaction(txid)
Expand All @@ -878,7 +895,7 @@ def test_service_cache_transactions(self):

def test_service_cache_transactions_after_txid(self):
# Do not store anything in cache if after_txid is used
srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2, exclude_providers=['mempool'])
srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2, exclude_providers=['mempool'])
address = '12spqcvLTFhL38oNJDDLfW1GpFGxLdaLCL'
res = srv.gettransactions(address,
after_txid='5f31da8f47a5bd92a6929179082c559e8acc270a040b19838230aab26309cf2d')
Expand All @@ -898,7 +915,7 @@ def test_service_cache_transactions_after_txid(self):
self.assertGreaterEqual(srv.results_cache_n, 1)

def test_service_cache_transaction_coinbase(self):
srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2, exclude_providers=['bitaps', 'bitgo'])
srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2, exclude_providers=['bitaps', 'bitgo'])
t = srv.gettransaction('68104dbd6819375e7bdf96562f89290b41598df7b002089ecdd3c8d999025b13')
if t:
self.assertGreaterEqual(srv.results_cache_n, 0)
Expand All @@ -922,7 +939,7 @@ def test_service_cache_transaction_segwit_database(self):
self.assertEqual(t.raw_hex(), rawtx)

def test_service_cache_with_latest_tx_query(self):
srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2)
srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2)
address = 'bc1qxfrgfhs49d7dtcfzlhp7f7cwsp8zpp60hywp0f'
after_txid = '13401ad121c8ae91e18b4bb0db5d8f350a2b0b5ddd5ca26165137bf07fefad90'
srv.gettransaction('4156e78f347e47d2ccdd4a19614d958c6e4502d09a68f63ed0c72691f63a5028')
Expand All @@ -932,7 +949,7 @@ def test_service_cache_with_latest_tx_query(self):
self.assertGreaterEqual(len(txs), 5)

def test_service_cache_correctly_update_spent_info(self):
srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2)
srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2)
srv.gettransactions('1KoAvaL3wfpcNvGCQYkqFJG9Ccqm52sZHa', limit=1)
txs = srv.gettransactions('1KoAvaL3wfpcNvGCQYkqFJG9Ccqm52sZHa')
self.assertTrue(txs[0].outputs[0].spent)
Expand Down Expand Up @@ -974,7 +991,7 @@ def test_service_cache_disabled(self):

def test_service_cache_transaction_p2sh_p2wpkh_input(self):
txid = '6ab6432a6b7b04ecc335c6e8adccc45c25f46e33752478f0bcacaf3f1b61ad92'
srv = ServiceTest(cache_uri=DATABASEFILE_CACHE_UNITTESTS2)
srv = ServiceTest(cache_uri=DATABASE_CACHE_UNITTESTS2)
t = srv.gettransaction(txid)
self.assertEqual(t.size, 249)
self.assertEqual(srv.results_cache_n, 0)
Expand Down
16 changes: 2 additions & 14 deletions tests/test_tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,36 +29,24 @@ def database_init(dbname=DATABASE_NAME):
if os.getenv('UNITTEST_DATABASE') == 'postgresql':
con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True)
cur = con.cursor()
# try:
cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(
sql.Identifier(dbname))
)
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(dbname))
)
# except Exception:
# pass
# finally:
# cur.close()
# con.close()
cur.close()
con.close()
return 'postgresql://postgres:postgres@localhost:5432/' + dbname
elif os.getenv('UNITTEST_DATABASE') == 'mysql':
try:
con = mysql.connector.connect(user='user', host='localhost', password='password')
except:
try:
con = mysql.connector.connect(user='root', host='localhost', password='password')
except:
con = mysql.connector.connect(user='root', host='localhost')
con = mysql.connector.connect(user='user', host='localhost', password='password')
cur = con.cursor()
cur.execute("DROP DATABASE IF EXISTS {}".format(dbname))
cur.execute("CREATE DATABASE {}".format(dbname))
con.commit()
cur.close()
con.close()
return 'mysql://root@localhost:3306/' + dbname
return 'mysql://user:password@localhost:3306/' + dbname
else:
dburi = os.path.join(str(BCL_DATABASE_DIR), '%s.sqlite' % dbname)
if os.path.isfile(dburi):
Expand Down
51 changes: 10 additions & 41 deletions tests/test_wallets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,30 +43,13 @@
DATABASE_NAME = 'bitcoinlib_test'
DATABASE_NAME_2 = 'bitcoinlib2_test'

# db_uris = (
# ('sqlite', 'sqlite:///' + DATABASEFILE_UNITTESTS, 'sqlite:///' + DATABASEFILE_UNITTESTS_2),)

print("DATABASE USED: %s" % os.getenv('UNITTEST_DATABASE'))

# if UNITTESTS_FULL_DATABASE_TEST:
# db_uris += (
# ('mysql', 'mysql://root:root@localhost:3306/' + DATABASE_NAME,
# 'mysql://root:root@localhost:3306/' + DATABASE_NAME_2),
# ('postgresql', 'postgresql://postgres:postgres@localhost:5432/' + DATABASE_NAME,
# 'postgresql://postgres:postgres@localhost:5432/' + DATABASE_NAME_2),
# )
#
#
# params = (('SCHEMA', 'DATABASE_URI', 'DATABASE_URI_2'), (
# db_uris
# ))


def database_init(dbname=DATABASE_NAME):
session.close_all_sessions()
if os.getenv('UNITTEST_DATABASE') == 'postgresql':
# con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True)
# cur = con.cursor()
con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True)
cur = con.cursor()
# try:
# cur.execute(sql.SQL("ALTER DATABASE {} allow_connections = off").format(sql.Identifier(dbname)))
# cur.execute(sql.SQL("UPDATE pg_database SET datallowconn = 'false' WHERE datname = '{}'").format(
Expand All @@ -77,7 +60,6 @@ def database_init(dbname=DATABASE_NAME):
# except Exception as e:
# print(e)
# res = cur.execute(sql.SQL("SELECT sum(numbackends) FROM pg_stat_database"))
# print(res)
# res = cur.execute(sql.SQL("""
# DO $$ DECLARE
# r RECORD;
Expand All @@ -86,27 +68,14 @@ def database_init(dbname=DATABASE_NAME):
# EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE';
# END LOOP;
# END $$;"""))
# print(res)
# try:
# cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(dbname)))
# cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(dbname)))
# except:
# pass
# try:
# # drop all tables
# finally:
# cur.close()
# con.close()
# con = psycopg.connect(user='postgres', host='localhost', password='postgres', autocommit=True)
# cur = con.cursor()
# cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(dbname)))
# con.commit()
# cur.close()
# con.close()
# return 'postgresql://postgres:postgres@localhost:5432/' + dbname
# postgresql = testing.postgresql.Postgresql()
# return postgresql.url()
return 'testing.postgresql'
try:
cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(dbname)))
cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(dbname)))
except:
pass
cur.close()
con.close()
return 'postgresql://postgres:postgres@localhost:5432/' + dbname
elif os.getenv('UNITTEST_DATABASE') == 'mysql':
con = mysql.connector.connect(user='user', host='localhost', password='password')
cur = con.cursor()
Expand Down

0 comments on commit 1bce120

Please sign in to comment.