change to rocksdb

Cargo.lock

@@ -71,6 +71,30 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
+[[package]]
+name = "bindgen"
+version = "0.54.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66c0bb6167449588ff70803f4127f0684f9063097eca5016f37eb52b92c2cf36"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "cfg-if 0.1.10",
+ "clang-sys",
+ "clap",
+ "env_logger 0.7.1",
+ "lazy_static",
+ "lazycell",
+ "log",
+ "peeking_take_while",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "which",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -94,12 +118,6 @@ version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
 
-[[package]]
-name = "byteorder"
-version = "1.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
-
 [[package]]
 name = "bytes"
 version = "0.5.6"
@@ -115,6 +133,15 @@ dependencies = [
  "jobserver",
 ]
 
+[[package]]
+name = "cexpr"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
@@ -127,6 +154,17 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "clang-sys"
+version = "0.29.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe6837df1d5cba2397b835c8530f51723267e16abbf83892e9e5af4f0e5dd10a"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
 [[package]]
 name = "clap"
 version = "2.33.3"
@@ -142,15 +180,6 @@ dependencies = [
  "vec_map",
 ]
 
-[[package]]
-name = "cloudabi"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
-dependencies = [
- "bitflags",
-]
-
 [[package]]
 name = "confy"
 version = "0.4.1"
@@ -161,12 +190,6 @@ dependencies = [
  "toml",
 ]
 
-[[package]]
-name = "const_fn"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab"
-
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -189,29 +212,6 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
 
-[[package]]
-name = "crc32fast"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
-dependencies = [
- "cfg-if 1.0.0",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
-dependencies = [
- "cfg-if 1.0.0",
- "const_fn",
- "crossbeam-utils",
- "lazy_static",
- "memoffset",
- "scopeguard",
-]
-
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.1"
@@ -299,6 +299,19 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
+[[package]]
+name = "env_logger"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
+dependencies = [
+ "atty",
+ "humantime 1.3.0",
+ "log",
+ "regex",
+ "termcolor",
+]
+
 [[package]]
 name = "env_logger"
 version = "0.8.2"
@@ -306,7 +319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e"
 dependencies = [
  "atty",
- "humantime",
+ "humantime 2.0.1",
  "log",
  "regex",
  "termcolor",
@@ -344,16 +357,6 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
-[[package]]
-name = "fs2"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
-dependencies = [
- "libc",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "fuchsia-zircon"
 version = "0.3.3"
@@ -465,15 +468,6 @@ dependencies = [
  "slab",
 ]
 
-[[package]]
-name = "fxhash"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
-dependencies = [
- "byteorder",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.1.15"
@@ -509,7 +503,7 @@ dependencies = [
  "confy",
  "directories-next",
  "dirs-next",
- "env_logger",
+ "env_logger 0.8.2",
  "filetime",
  "futures",
  "futures-util",
@@ -540,11 +534,11 @@ dependencies = [
  "clap",
  "hbb_common",
  "lazy_static",
+ "rocksdb",
  "rust-ini",
  "serde",
  "serde_derive",
  "serde_json",
- "sled",
 ]
 
 [[package]]
@@ -558,18 +552,18 @@ dependencies = [
 
 [[package]]
 name = "humantime"
-version = "2.0.1"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a"
+checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
+dependencies = [
+ "quick-error",
+]
 
 [[package]]
-name = "instant"
-version = "0.1.9"
+name = "humantime"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
-dependencies = [
- "cfg-if 1.0.0",
-]
+checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a"
 
 [[package]]
 name = "iovec"
@@ -629,6 +623,12 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
 [[package]]
 name = "libc"
 version = "0.2.80"
@@ -636,23 +636,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
 
 [[package]]
-name = "libsodium-sys"
-version = "0.2.6"
+name = "libloading"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a685b64f837b339074115f2e7f7b431ac73681d08d75b389db7498b8892b8a58"
+checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
 dependencies = [
  "cc",
+ "winapi 0.3.9",
+]
+
+[[package]]
+name = "librocksdb-sys"
+version = "6.11.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb5b56f651c204634b936be2f92dbb42c36867e00ff7fe2405591f3b9fa66f09"
+dependencies = [
+ "bindgen",
+ "cc",
+ "glob",
  "libc",
- "pkg-config",
 ]
 
 [[package]]
-name = "lock_api"
-version = "0.4.2"
+name = "libsodium-sys"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
+checksum = "a685b64f837b339074115f2e7f7b431ac73681d08d75b389db7498b8892b8a58"
 dependencies = [
- "scopeguard",
+ "cc",
+ "libc",
+ "pkg-config",
 ]
 
 [[package]]
@@ -680,15 +693,6 @@ version = "2.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
 
-[[package]]
-name = "memoffset"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "mio"
 version = "0.6.22"
@@ -776,6 +780,16 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nom"
+version = "5.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
+dependencies = [
+ "memchr",
+ "version_check",
+]
+
 [[package]]
 name = "num_cpus"
 version = "1.13.0"
@@ -809,30 +823,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "parking_lot"
-version = "0.11.1"
+name = "peeking_take_while"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
-dependencies = [
- "instant",
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
-dependencies = [
- "cfg-if 0.1.10",
- "cloudabi",
- "instant",
- "libc",
- "redox_syscall",
- "smallvec",
- "winapi 0.3.9",
-]
+checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
 
 [[package]]
 name = "pin-project"
@@ -951,6 +945,12 @@ dependencies = [
  "protobuf-codegen",
 ]
 
+[[package]]
+name = "quick-error"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
+
 [[package]]
 name = "quinn"
 version = "0.6.1"
@@ -1087,6 +1087,16 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "rocksdb"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23d83c02c429044d58474eaf5ae31e062d0de894e21125b47437ec0edc1397e6"
+dependencies = [
+ "libc",
+ "librocksdb-sys",
+]
+
 [[package]]
 name = "rust-argon2"
 version = "0.8.3"
@@ -1109,6 +1119,12 @@ dependencies = [
  "ordered-multimap",
 ]
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustls"
 version = "0.17.0"
@@ -1156,12 +1172,6 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
-[[package]]
-name = "scopeguard"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-
 [[package]]
 name = "sct"
 version = "0.6.0"
@@ -1223,6 +1233,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "shlex"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.2.2"
@@ -1238,28 +1254,6 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
 
-[[package]]
-name = "sled"
-version = "0.34.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
-dependencies = [
- "crc32fast",
- "crossbeam-epoch",
- "crossbeam-utils",
- "fs2",
- "fxhash",
- "libc",
- "log",
- "parking_lot",
-]
-
-[[package]]
-name = "smallvec"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85"
-
 [[package]]
 name = "socket2"
 version = "0.3.17"
@@ -1546,6 +1540,15 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "which"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "winapi"
 version = "0.2.8"

Cargo.toml

@@ -13,7 +13,6 @@ path = "src/hbbr/main.rs"
 
 [dependencies]
 hbb_common = { path = "libs/hbb_common" }
-sled = "0.34"
 serde_derive = "1.0"
 serde = "1.0"
 serde_json = "1.0"
@@ -28,3 +27,7 @@ hbb_common = { path = "libs/hbb_common" }
 [workspace]
 members = ["libs/hbb_common"]
 
+[dependencies.rocksdb]
+default-features = false
+features = ["lz4"]
+version = "0.15"

libs/hbb_common

@@ -1 +1 @@
-Subproject commit 9b9677c6a438645fe1e77d308d08e5d908b501bc
+Subproject commit 3a1ef5db98239175384aa8034a27e387bbcb8816

src/rendezvous_server.rs

@@ -65,7 +65,7 @@ impl PeerMap {
     fn new() -> ResultType<Self> {
         Ok(Self {
             map: Default::default(),
-            db: super::SledAsync::new("./sled.db", true)?,
+            db: super::SledAsync::new("./hbbs.db", true)?,
         })
     }
 

src/sled_async.rs

@@ -3,49 +3,50 @@ use hbb_common::{
     tokio::{self, sync::mpsc},
     ResultType,
 };
+use rocksdb::DB;
 
 #[derive(Debug)]
 enum Action {
     Insert((String, Vec<u8>)),
-    Get((String, mpsc::Sender<Option<sled::IVec>>)),
+    Get((String, mpsc::Sender<Option<Vec<u8>>>)),
     _Close,
 }
 
 #[derive(Clone)]
 pub struct SledAsync {
-    db: sled::Db,
     tx: Option<mpsc::UnboundedSender<Action>>,
+    path: String,
 }
 
 impl SledAsync {
     pub fn new(path: &str, run: bool) -> ResultType<Self> {
         let mut res = Self {
-            db: sled::open(path)?,
             tx: None,
+            path: path.to_owned(),
         };
         if run {
-            res.run();
+            res.run()?;
         }
         Ok(res)
     }
 
-    pub fn run(&mut self) -> std::thread::JoinHandle<()> {
+    pub fn run(&mut self) -> ResultType<std::thread::JoinHandle<()>> {
         let (tx, rx) = mpsc::unbounded_channel::<Action>();
         self.tx = Some(tx);
-        let db = self.db.clone();
-        std::thread::spawn(move || {
+        let db = DB::open_default(&self.path)?;
+        Ok(std::thread::spawn(move || {
             Self::io_loop(db, rx);
             log::debug!("Exit SledAsync loop");
-        })
+        }))
     }
 
     #[tokio::main(basic_scheduler)]
-    async fn io_loop(db: sled::Db, rx: mpsc::UnboundedReceiver<Action>) {
+    async fn io_loop(db: DB, rx: mpsc::UnboundedReceiver<Action>) {
         let mut rx = rx;
         while let Some(x) = rx.recv().await {
             match x {
                 Action::Insert((key, value)) => {
-                    allow_err!(db.insert(key, value));
+                    allow_err!(db.put(&key, &value));
                 }
                 Action::Get((key, sender)) => {
                     let mut sender = sender;
@@ -67,9 +68,9 @@ impl SledAsync {
         allow_err!(j.join());
     }
 
-    pub async fn get(&mut self, key: String) -> Option<sled::IVec> {
+    pub async fn get(&mut self, key: String) -> Option<Vec<u8>> {
         if let Some(tx) = &self.tx {
-            let (tx_once, mut rx) = mpsc::channel::<Option<sled::IVec>>(1);
+            let (tx_once, mut rx) = mpsc::channel::<Option<Vec<u8>>>(1);
             allow_err!(tx.send(Action::Get((key, tx_once))));
             if let Some(v) = rx.recv().await {
                 return v;
@@ -79,7 +80,7 @@ impl SledAsync {
     }
 
     #[inline]
-    pub fn deserialize<'a, T: serde::Deserialize<'a>>(v: &'a Option<sled::IVec>) -> Option<T> {
+    pub fn deserialize<'a, T: serde::Deserialize<'a>>(v: &'a Option<Vec<u8>>) -> Option<T> {
         if let Some(v) = v {
             if let Ok(v) = std::str::from_utf8(v) {
                 if let Ok(v) = serde_json::from_str::<T>(&v) {