feat: add sst level config for benchmark

.github/workflows/ci.yml

@@ -70,17 +70,12 @@ jobs:
         run: |
           rustup component add clippy
           rustup component add rustfmt
-          cargo install cargo-sort
+          cargo install --git https://github.com/DevinR528/cargo-sort --rev 55ec890 --locked
       - name: Run Style Check
         run: |
           make clippy
           make fmt
           make check-cargo-toml
-      - name: Report Disk Usage
-        if: always()
-        run: |
-          du -sh ./target
-          df -h
 
   unit-test:
     name: unit-test

Cargo.toml

@@ -80,7 +80,7 @@ members = [
     "src/system_catalog",
     "src/table_engine",
     "src/tools",
-    "src/wal",
+    "src/wal"
 ]
 
 [workspace.dependencies]

src/benchmarks/bench.toml

@@ -40,6 +40,7 @@ runtime_thread_num = 1
 bench_measurement_time = "120s"
 max_projections = 5
 read_batch_row_num = 500
+sst_level = 0
 
 [merge_sst_bench.predicate]
 start_time_ms = 0

src/benchmarks/config/bench.toml

@@ -42,6 +42,7 @@ bench_measurement_time = "30s"
 bench_sample_size = 30
 max_projections = 5
 read_batch_row_num = 500
+sst_level = 0
 
 [merge_sst_bench.predicate]
 start_time_ms = 0

src/benchmarks/src/config.rs

@@ -85,6 +85,7 @@ pub struct MergeSstBenchConfig {
     pub max_projections: usize,
     pub num_rows_per_row_group: usize,
     pub predicate: BenchPredicate,
+    pub sst_level: u16,
 }
 
 #[derive(Deserialize)]

src/benchmarks/src/merge_sst_bench.rs

@@ -59,6 +59,7 @@ pub struct MergeSstBench {
     file_handles: Vec<FileHandle>,
     _receiver: UnboundedReceiver<Request>,
     dedup: bool,
+    sst_level: Level,
 }
 
 impl MergeSstBench {
@@ -121,6 +122,7 @@ impl MergeSstBench {
             file_handles,
             _receiver: rx,
             dedup: true,
+            sst_level: config.sst_level.into(),
         }
     }
 
@@ -168,7 +170,7 @@ impl MergeSstBench {
 
         builder
             // TODO: make level configurable
-            .mut_ssts_of_level(Level::MIN)
+            .mut_ssts_of_level(self.sst_level)
             .extend_from_slice(&self.file_handles);
 
         self.runtime.block_on(async {

src/benchmarks/src/util.rs

@@ -192,8 +192,8 @@ pub async fn file_handles_from_ssts(
 
         let file_meta = FileMeta {
             id: *file_id,
-            size: 0,
-            row_num: 0,
+            size: store.head(&path).await.unwrap().size as u64,
+            row_num: parquet_metadata.file_metadata().num_rows() as u64,
             time_range: sst_meta.time_range,
             max_seq: sst_meta.max_sequence,
             storage_format: StorageFormat::Columnar,

src/df_engine_extensions/Cargo.toml

@@ -38,7 +38,7 @@ catalog = { workspace = true, features = ["test"] }
 common_types = { workspace = true, features = ["test"] }
 datafusion = { workspace = true }
 datafusion-proto = { workspace = true }
-futures =   { workspace = true }
+futures = { workspace = true }
 generic_error = { workspace = true }
 horaedbproto = { workspace = true }
 lazy_static = { workspace = true }

src/horaedb/Cargo.toml

@@ -33,10 +33,7 @@ workspace = true
 [features]
 default = ["wal-rocksdb", "wal-table-kv", "wal-message-queue"]
 wal-table-kv = ["wal/wal-table-kv", "analytic_engine/wal-table-kv"]
-wal-message-queue = [
-    "wal/wal-message-queue",
-    "analytic_engine/wal-message-queue",
-]
+wal-message-queue = ["wal/wal-message-queue", "analytic_engine/wal-message-queue"]
 wal-rocksdb = ["wal/wal-rocksdb", "analytic_engine/wal-rocksdb"]
 
 [dependencies]

src/meta_client/Cargo.toml

@@ -43,7 +43,7 @@ reqwest = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 snafu = { workspace = true }
-table_engine =  { workspace = true }
+table_engine = { workspace = true }
 time_ext = { workspace = true }
 tokio = { workspace = true }
 tonic = { workspace = true }

src/wal/Cargo.toml

@@ -27,15 +27,15 @@ workspace = true
 [package.authors]
 workspace = true
 
+[package.edition]
+workspace = true
+
 [dependencies.rocksdb]
 git = "https://github.com/tikv/rust-rocksdb.git"
 rev = "f04f4dd8eacc30e67c24bc2529a6d9c6edb85f8f"
 features = ["portable"]
 optional = true
 
-[package.edition]
-workspace = true
-
 [features]
 wal-message-queue = ["dep:message_queue"]
 wal-table-kv = ["dep:table_kv"]