Skip to content

Commit

Permalink
create commitlog dir in fs::New (#2006)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Shubham8287
Shubham8287 authored Nov 21, 2024
1 parent 125ab58 commit f04d281
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 7 deletions.
8 changes: 4 additions & 4 deletions crates/commitlog/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ impl<T> Commitlog<T> {
/// free-standing functions in this module for how to traverse a read-only
/// commitlog.
pub fn open(root: CommitLogDir, opts: Options) -> io::Result<Self> {
let inner = commitlog::Generic::open(repo::Fs::new(root), opts)?;
let inner = commitlog::Generic::open(repo::Fs::new(root)?, opts)?;

Ok(Self {
inner: RwLock::new(inner),
Expand Down Expand Up @@ -444,7 +444,7 @@ pub fn commits_from(
root: CommitLogDir,
offset: u64,
) -> io::Result<impl Iterator<Item = Result<StoredCommit, error::Traversal>>> {
commitlog::commits_from(repo::Fs::new(root), DEFAULT_LOG_FORMAT_VERSION, offset)
commitlog::commits_from(repo::Fs::new(root)?, DEFAULT_LOG_FORMAT_VERSION, offset)
}

/// Obtain an iterator which traverses the commitlog located at the `root`
Expand Down Expand Up @@ -479,7 +479,7 @@ where
D::Error: From<error::Traversal>,
T: 'a,
{
commitlog::transactions_from(repo::Fs::new(root), DEFAULT_LOG_FORMAT_VERSION, offset, de)
commitlog::transactions_from(repo::Fs::new(root)?, DEFAULT_LOG_FORMAT_VERSION, offset, de)
}

/// Traverse the commitlog located at the `root` directory from the start and
Expand All @@ -505,5 +505,5 @@ where
D: Decoder,
D::Error: From<error::Traversal> + From<io::Error>,
{
commitlog::fold_transactions_from(repo::Fs::new(root), DEFAULT_LOG_FORMAT_VERSION, offset, de)
commitlog::fold_transactions_from(repo::Fs::new(root)?, DEFAULT_LOG_FORMAT_VERSION, offset, de)
}
5 changes: 3 additions & 2 deletions crates/commitlog/src/repo/fs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@ impl Fs {
/// Create a commitlog repository which stores segments in the directory `root`.
///
/// `root` must name an extant, accessible, writeable directory.
pub fn new(root: CommitLogDir) -> Self {
Self { root }
pub fn new(root: CommitLogDir) -> io::Result<Self> {
root.create()?;
Ok(Self { root })
}

/// Get the filename for a segment starting with `offset` within this
Expand Down
1 change: 0 additions & 1 deletion crates/core/src/db/relational_db.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1265,7 +1265,6 @@ pub type LocalDurability = Arc<durability::Local<ProductValue>>;
/// Note that this operation can be expensive, as it needs to traverse a suffix
/// of the commitlog.
pub async fn local_durability(commitlog_dir: CommitLogDir) -> io::Result<(LocalDurability, DiskSizeFn)> {
tokio::fs::create_dir_all(&commitlog_dir).await?;
let rt = tokio::runtime::Handle::current();
// TODO: Should this better be spawn_blocking?
let local = spawn_rayon(move || {
Expand Down

2 comments on commit f04d281

@github-actions
Copy link

@github-actions github-actions bot commented on f04d281 Nov 21, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmarking failed. Please check the workflow run for details.

@github-actions
Copy link

@github-actions github-actions bot commented on f04d281 Nov 21, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Callgrind benchmark results

Callgrind Benchmark Report

These benchmarks were run using callgrind,
an instruction-level profiler. They allow comparisons between sqlite (sqlite), SpacetimeDB running through a module (stdb_module), and the underlying SpacetimeDB data storage engine (stdb_raw). Callgrind emulates a CPU to collect the below estimates.

Measurement changes larger than five percent are in bold.

In-memory benchmarks

callgrind: empty transaction

db total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw 6426 6427 -0.02% 6544 6561 -0.26%
sqlite 5585 5579 0.11% 6113 6047 1.09%

callgrind: filter

db schema indices count preload _column data_type total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str no_index 64 128 1 u64 76579 76580 -0.00% 77007 77024 -0.02%
stdb_raw u32_u64_str no_index 64 128 2 string 118821 118822 -0.00% 119511 119528 -0.01%
stdb_raw u32_u64_str btree_each_column 64 128 2 string 25110 25118 -0.03% 25624 25624 0.00%
stdb_raw u32_u64_str btree_each_column 64 128 1 u64 24078 24079 -0.00% 24514 24515 -0.00%
sqlite u32_u64_str no_index 64 128 2 string 144695 144695 0.00% 146169 146101 0.05%
sqlite u32_u64_str no_index 64 128 1 u64 124044 124044 0.00% 125280 125268 0.01%
sqlite u32_u64_str btree_each_column 64 128 1 u64 131361 131361 0.00% 132813 132769 0.03%
sqlite u32_u64_str btree_each_column 64 128 2 string 134494 134494 0.00% 136140 136088 0.04%

callgrind: insert bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 64 128 878667 878606 0.01% 898069 903852 -0.64%
stdb_raw u32_u64_str btree_each_column 64 128 1032582 1029907 0.26% 1089996 1070563 1.82%
sqlite u32_u64_str unique_0 64 128 398320 398320 0.00% 417712 414636 0.74%
sqlite u32_u64_str btree_each_column 64 128 983637 983637 0.00% 1020543 1022767 -0.22%

callgrind: iterate

db schema indices count total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 153810 153811 -0.00% 153926 153927 -0.00%
stdb_raw u32_u64_str unique_0 64 16835 16836 -0.01% 16935 16936 -0.01%
sqlite u32_u64_str unique_0 1024 1068281 1068275 0.00% 1071651 1071649 0.00%
sqlite u32_u64_str unique_0 64 76267 76261 0.01% 77411 77281 0.17%

callgrind: serialize_product_value

count format total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
64 json 47528 47528 0.00% 50214 50214 0.00%
64 bsatn 25509 25509 0.00% 27753 27753 0.00%
16 bsatn 8200 8200 0.00% 9560 9560 0.00%
16 json 12188 12188 0.00% 14126 14126 0.00%

callgrind: update bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 1024 20559222 20320354 1.18% 21047298 20998040 0.23%
stdb_raw u32_u64_str unique_0 64 128 1288235 1289125 -0.07% 1354323 1367951 -1.00%
sqlite u32_u64_str unique_0 1024 1024 1802182 1802182 0.00% 1811352 1811698 -0.02%
sqlite u32_u64_str unique_0 64 128 128528 128528 0.00% 131244 131404 -0.12%
On-disk benchmarks

callgrind: empty transaction

db total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw 6431 6432 -0.02% 6557 6570 -0.20%
sqlite 5627 5621 0.11% 6211 6141 1.14%

callgrind: filter

db schema indices count preload _column data_type total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str no_index 64 128 1 u64 76584 76585 -0.00% 76992 77005 -0.02%
stdb_raw u32_u64_str no_index 64 128 2 string 118826 118848 -0.02% 119492 119590 -0.08%
stdb_raw u32_u64_str btree_each_column 64 128 2 string 25131 25333 -0.80% 25649 25855 -0.80%
stdb_raw u32_u64_str btree_each_column 64 128 1 u64 24083 24084 -0.00% 24503 24500 0.01%
sqlite u32_u64_str no_index 64 128 1 u64 125965 125965 0.00% 127565 127465 0.08%
sqlite u32_u64_str no_index 64 128 2 string 146616 146616 0.00% 148482 148378 0.07%
sqlite u32_u64_str btree_each_column 64 128 2 string 136616 136616 0.00% 138792 138708 0.06%
sqlite u32_u64_str btree_each_column 64 128 1 u64 133457 133475 -0.01% 135379 135329 0.04%

callgrind: insert bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 64 128 827485 827890 -0.05% 877135 852484 2.89%
stdb_raw u32_u64_str btree_each_column 64 128 976480 978621 -0.22% 1033682 1049691 -1.53%
sqlite u32_u64_str unique_0 64 128 415857 415857 0.00% 434571 431559 0.70%
sqlite u32_u64_str btree_each_column 64 128 1021908 1021908 0.00% 1057624 1060522 -0.27%

callgrind: iterate

db schema indices count total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 153815 153816 -0.00% 153915 153916 -0.00%
stdb_raw u32_u64_str unique_0 64 16840 16841 -0.01% 16940 16941 -0.01%
sqlite u32_u64_str unique_0 1024 1071349 1071343 0.00% 1075093 1075103 -0.00%
sqlite u32_u64_str unique_0 64 78039 78033 0.01% 79395 79297 0.12%

callgrind: serialize_product_value

count format total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
64 json 47528 47528 0.00% 50214 50214 0.00%
64 bsatn 25509 25509 0.00% 27753 27753 0.00%
16 bsatn 8200 8200 0.00% 9560 9560 0.00%
16 json 12188 12188 0.00% 14126 14126 0.00%

callgrind: update bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 1024 19046716 19054791 -0.04% 19568318 19803031 -1.19%
stdb_raw u32_u64_str unique_0 64 128 1240266 1240756 -0.04% 1303944 1317552 -1.03%
sqlite u32_u64_str unique_0 1024 1024 1809743 1809743 0.00% 1818405 1818483 -0.00%
sqlite u32_u64_str unique_0 64 128 132654 132654 0.00% 135586 135770 -0.14%

Please sign in to comment.