change to rocksdb

This commit is contained in:
open-trade 2021-03-19 11:32:28 +08:00
parent 06309ac068
commit 482b5e0a66
5 changed files with 167 additions and 160 deletions

291
Cargo.lock generated
View File

@ -71,6 +71,30 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bindgen"
version = "0.54.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66c0bb6167449588ff70803f4127f0684f9063097eca5016f37eb52b92c2cf36"
dependencies = [
"bitflags",
"cexpr",
"cfg-if 0.1.10",
"clang-sys",
"clap",
"env_logger 0.7.1",
"lazy_static",
"lazycell",
"log",
"peeking_take_while",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
"which",
]
[[package]]
name = "bitflags"
version = "1.2.1"
@ -94,12 +118,6 @@ version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
[[package]]
name = "byteorder"
version = "1.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
[[package]]
name = "bytes"
version = "0.5.6"
@ -115,6 +133,15 @@ dependencies = [
"jobserver",
]
[[package]]
name = "cexpr"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
dependencies = [
"nom",
]
[[package]]
name = "cfg-if"
version = "0.1.10"
@ -127,6 +154,17 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clang-sys"
version = "0.29.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe6837df1d5cba2397b835c8530f51723267e16abbf83892e9e5af4f0e5dd10a"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "clap"
version = "2.33.3"
@ -142,15 +180,6 @@ dependencies = [
"vec_map",
]
[[package]]
name = "cloudabi"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
dependencies = [
"bitflags",
]
[[package]]
name = "confy"
version = "0.4.1"
@ -161,12 +190,6 @@ dependencies = [
"toml",
]
[[package]]
name = "const_fn"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab"
[[package]]
name = "constant_time_eq"
version = "0.1.5"
@ -189,29 +212,6 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
[[package]]
name = "crc32fast"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
dependencies = [
"cfg-if 1.0.0",
"const_fn",
"crossbeam-utils",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.1"
@ -299,6 +299,19 @@ version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "env_logger"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
dependencies = [
"atty",
"humantime 1.3.0",
"log",
"regex",
"termcolor",
]
[[package]]
name = "env_logger"
version = "0.8.2"
@ -306,7 +319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e"
dependencies = [
"atty",
"humantime",
"humantime 2.0.1",
"log",
"regex",
"termcolor",
@ -344,16 +357,6 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi 0.3.9",
]
[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
@ -465,15 +468,6 @@ dependencies = [
"slab",
]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]]
name = "getrandom"
version = "0.1.15"
@ -509,7 +503,7 @@ dependencies = [
"confy",
"directories-next",
"dirs-next",
"env_logger",
"env_logger 0.8.2",
"filetime",
"futures",
"futures-util",
@ -540,11 +534,11 @@ dependencies = [
"clap",
"hbb_common",
"lazy_static",
"rocksdb",
"rust-ini",
"serde",
"serde_derive",
"serde_json",
"sled",
]
[[package]]
@ -556,21 +550,21 @@ dependencies = [
"libc",
]
[[package]]
name = "humantime"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
dependencies = [
"quick-error",
]
[[package]]
name = "humantime"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a"
[[package]]
name = "instant"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "iovec"
version = "0.1.4"
@ -629,12 +623,40 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
[[package]]
name = "libloading"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
dependencies = [
"cc",
"winapi 0.3.9",
]
[[package]]
name = "librocksdb-sys"
version = "6.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb5b56f651c204634b936be2f92dbb42c36867e00ff7fe2405591f3b9fa66f09"
dependencies = [
"bindgen",
"cc",
"glob",
"libc",
]
[[package]]
name = "libsodium-sys"
version = "0.2.6"
@ -646,15 +668,6 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "lock_api"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.11"
@ -680,15 +693,6 @@ version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memoffset"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
dependencies = [
"autocfg",
]
[[package]]
name = "mio"
version = "0.6.22"
@ -776,6 +780,16 @@ dependencies = [
"libc",
]
[[package]]
name = "nom"
version = "5.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
dependencies = [
"memchr",
"version_check",
]
[[package]]
name = "num_cpus"
version = "1.13.0"
@ -809,30 +823,10 @@ dependencies = [
]
[[package]]
name = "parking_lot"
version = "0.11.1"
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
dependencies = [
"cfg-if 0.1.10",
"cloudabi",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi 0.3.9",
]
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "pin-project"
@ -951,6 +945,12 @@ dependencies = [
"protobuf-codegen",
]
[[package]]
name = "quick-error"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quinn"
version = "0.6.1"
@ -1087,6 +1087,16 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "rocksdb"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23d83c02c429044d58474eaf5ae31e062d0de894e21125b47437ec0edc1397e6"
dependencies = [
"libc",
"librocksdb-sys",
]
[[package]]
name = "rust-argon2"
version = "0.8.3"
@ -1109,6 +1119,12 @@ dependencies = [
"ordered-multimap",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustls"
version = "0.17.0"
@ -1156,12 +1172,6 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sct"
version = "0.6.0"
@ -1223,6 +1233,12 @@ dependencies = [
"serde",
]
[[package]]
name = "shlex"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
[[package]]
name = "signal-hook-registry"
version = "1.2.2"
@ -1238,28 +1254,6 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
[[package]]
name = "sled"
version = "0.34.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
dependencies = [
"crc32fast",
"crossbeam-epoch",
"crossbeam-utils",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot",
]
[[package]]
name = "smallvec"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85"
[[package]]
name = "socket2"
version = "0.3.17"
@ -1546,6 +1540,15 @@ dependencies = [
"untrusted",
]
[[package]]
name = "which"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724"
dependencies = [
"libc",
]
[[package]]
name = "winapi"
version = "0.2.8"

View File

@ -13,7 +13,6 @@ path = "src/hbbr/main.rs"
[dependencies]
hbb_common = { path = "libs/hbb_common" }
sled = "0.34"
serde_derive = "1.0"
serde = "1.0"
serde_json = "1.0"
@ -28,3 +27,7 @@ hbb_common = { path = "libs/hbb_common" }
[workspace]
members = ["libs/hbb_common"]
[dependencies.rocksdb]
default-features = false
features = ["lz4"]
version = "0.15"

@ -1 +1 @@
Subproject commit 9b9677c6a438645fe1e77d308d08e5d908b501bc
Subproject commit 3a1ef5db98239175384aa8034a27e387bbcb8816

View File

@ -65,7 +65,7 @@ impl PeerMap {
fn new() -> ResultType<Self> {
Ok(Self {
map: Default::default(),
db: super::SledAsync::new("./sled.db", true)?,
db: super::SledAsync::new("./hbbs.db", true)?,
})
}

View File

@ -3,49 +3,50 @@ use hbb_common::{
tokio::{self, sync::mpsc},
ResultType,
};
use rocksdb::DB;
#[derive(Debug)]
enum Action {
Insert((String, Vec<u8>)),
Get((String, mpsc::Sender<Option<sled::IVec>>)),
Get((String, mpsc::Sender<Option<Vec<u8>>>)),
_Close,
}
#[derive(Clone)]
pub struct SledAsync {
db: sled::Db,
tx: Option<mpsc::UnboundedSender<Action>>,
path: String,
}
impl SledAsync {
pub fn new(path: &str, run: bool) -> ResultType<Self> {
let mut res = Self {
db: sled::open(path)?,
tx: None,
path: path.to_owned(),
};
if run {
res.run();
res.run()?;
}
Ok(res)
}
pub fn run(&mut self) -> std::thread::JoinHandle<()> {
pub fn run(&mut self) -> ResultType<std::thread::JoinHandle<()>> {
let (tx, rx) = mpsc::unbounded_channel::<Action>();
self.tx = Some(tx);
let db = self.db.clone();
std::thread::spawn(move || {
let db = DB::open_default(&self.path)?;
Ok(std::thread::spawn(move || {
Self::io_loop(db, rx);
log::debug!("Exit SledAsync loop");
})
}))
}
#[tokio::main(basic_scheduler)]
async fn io_loop(db: sled::Db, rx: mpsc::UnboundedReceiver<Action>) {
async fn io_loop(db: DB, rx: mpsc::UnboundedReceiver<Action>) {
let mut rx = rx;
while let Some(x) = rx.recv().await {
match x {
Action::Insert((key, value)) => {
allow_err!(db.insert(key, value));
allow_err!(db.put(&key, &value));
}
Action::Get((key, sender)) => {
let mut sender = sender;
@ -67,9 +68,9 @@ impl SledAsync {
allow_err!(j.join());
}
pub async fn get(&mut self, key: String) -> Option<sled::IVec> {
pub async fn get(&mut self, key: String) -> Option<Vec<u8>> {
if let Some(tx) = &self.tx {
let (tx_once, mut rx) = mpsc::channel::<Option<sled::IVec>>(1);
let (tx_once, mut rx) = mpsc::channel::<Option<Vec<u8>>>(1);
allow_err!(tx.send(Action::Get((key, tx_once))));
if let Some(v) = rx.recv().await {
return v;
@ -79,7 +80,7 @@ impl SledAsync {
}
#[inline]
pub fn deserialize<'a, T: serde::Deserialize<'a>>(v: &'a Option<sled::IVec>) -> Option<T> {
pub fn deserialize<'a, T: serde::Deserialize<'a>>(v: &'a Option<Vec<u8>>) -> Option<T> {
if let Some(v) = v {
if let Ok(v) = std::str::from_utf8(v) {
if let Ok(v) = serde_json::from_str::<T>(&v) {