Browse Source

Merge branch 'master' into 'next'

Integration with Persy

See merge request famedly/conduit!107
merge-requests/107/merge
tglman 4 years ago
parent
commit
5deb667e73
  1. 68
      Cargo.lock
  2. 5
      Cargo.toml
  3. 3
      src/database.rs
  4. 3
      src/database/abstraction.rs
  5. 535
      src/database/abstraction/persy.rs
  6. 12
      src/error.rs

68
Cargo.lock generated

@ -240,6 +240,7 @@ version = "0.2.0"
dependencies = [ dependencies = [
"base64 0.13.0", "base64 0.13.0",
"bytes", "bytes",
"chrono",
"crossbeam", "crossbeam",
"directories", "directories",
"heed", "heed",
@ -251,6 +252,7 @@ dependencies = [
"opentelemetry", "opentelemetry",
"opentelemetry-jaeger", "opentelemetry-jaeger",
"parking_lot", "parking_lot",
"persy",
"pretty_env_logger", "pretty_env_logger",
"rand 0.8.4", "rand 0.8.4",
"regex", "regex",
@ -269,6 +271,7 @@ dependencies = [
"thiserror", "thiserror",
"thread_local", "thread_local",
"threadpool", "threadpool",
"timer",
"tokio", "tokio",
"tracing", "tracing",
"tracing-flame", "tracing-flame",
@ -332,6 +335,21 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "crc"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10c2722795460108a7872e1cd933a85d6ec38abc4baecad51028f702da28889f"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403"
[[package]] [[package]]
name = "crc32fast" name = "crc32fast"
version = "1.2.1" version = "1.2.1"
@ -1562,6 +1580,32 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "persy"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de4cd9bda96e9bab3c961620ca512def7a7a880152780132632506abe4414458"
dependencies = [
"byteorder",
"crc",
"data-encoding",
"fs2",
"linked-hash-map",
"rand 0.8.4",
"thiserror",
"unsigned-varint",
"zigzag",
]
[[package]]
name = "pest"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
dependencies = [
"ucd-trie",
]
[[package]] [[package]]
name = "pin-project" name = "pin-project"
version = "1.0.8" version = "1.0.8"
@ -2829,6 +2873,15 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "timer"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31d42176308937165701f50638db1c31586f183f1aab416268216577aec7306b"
dependencies = [
"chrono",
]
[[package]] [[package]]
name = "tinyvec" name = "tinyvec"
version = "1.3.1" version = "1.3.1"
@ -3161,6 +3214,12 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7"
[[package]]
name = "unsigned-varint"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f8d425fafb8cd76bc3f22aace4af471d3156301d7508f2107e98fbeae10bc7f"
[[package]] [[package]]
name = "untrusted" name = "untrusted"
version = "0.7.1" version = "0.7.1"
@ -3414,6 +3473,15 @@ dependencies = [
"synstructure", "synstructure",
] ]
[[package]]
name = "zigzag"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70b40401a28d86ce16a330b863b86fd7dbee4d7c940587ab09ab8c019f9e3fdf"
dependencies = [
"num-traits",
]
[[package]] [[package]]
name = "zstd" name = "zstd"
version = "0.5.4+zstd.1.4.7" version = "0.5.4+zstd.1.4.7"

5
Cargo.toml

@ -28,6 +28,10 @@ tokio = "1.11.0"
# Used for storing data permanently # Used for storing data permanently
sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true } sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true }
#sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] } #sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] }
persy = { version = "1.0", optional = true }
# Used by the persy write cache for background flush
timer = "0.2"
chrono = "0.4"
# Used for the http request / response body type for Ruma endpoints used with reqwest # Used for the http request / response body type for Ruma endpoints used with reqwest
bytes = "1.1.0" bytes = "1.1.0"
@ -85,6 +89,7 @@ thread_local = "1.1.3"
[features] [features]
default = ["conduit_bin", "backend_sqlite"] default = ["conduit_bin", "backend_sqlite"]
backend_sled = ["sled"] backend_sled = ["sled"]
backend_persy = ["persy"]
backend_sqlite = ["sqlite"] backend_sqlite = ["sqlite"]
backend_heed = ["heed", "crossbeam"] backend_heed = ["heed", "crossbeam"]
sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"] sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"]

3
src/database.rs

@ -138,6 +138,9 @@ pub type Engine = abstraction::sqlite::Engine;
#[cfg(feature = "heed")] #[cfg(feature = "heed")]
pub type Engine = abstraction::heed::Engine; pub type Engine = abstraction::heed::Engine;
#[cfg(feature = "persy")]
pub type Engine = abstraction::persy::PersyEngine;
pub struct Database { pub struct Database {
_db: Arc<Engine>, _db: Arc<Engine>,
pub globals: globals::Globals, pub globals: globals::Globals,

3
src/database/abstraction.rs

@ -12,6 +12,9 @@ pub mod sqlite;
#[cfg(feature = "heed")] #[cfg(feature = "heed")]
pub mod heed; pub mod heed;
#[cfg(feature = "persy")]
pub mod persy;
pub trait DatabaseEngine: Sized { pub trait DatabaseEngine: Sized {
fn open(config: &Config) -> Result<Arc<Self>>; fn open(config: &Config) -> Result<Arc<Self>>;
fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>>; fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>>;

535
src/database/abstraction/persy.rs

@ -0,0 +1,535 @@
use crate::{
database::{
abstraction::{DatabaseEngine, Tree},
Config,
},
Result,
};
use persy::{ByteVec, OpenOptions, Persy, ValueMode};
use std::{
cmp::Ordering,
collections::{btree_map::Entry, BTreeMap, BTreeSet},
future::Future,
iter::Peekable,
pin::Pin,
sync::{Arc, RwLock},
time::{Duration, Instant},
};
use tracing::warn;
pub struct PersyEngine {
persy: Persy,
write_cache: Arc<RwLock<WriteCache>>,
}
impl DatabaseEngine for PersyEngine {
fn open(config: &Config) -> Result<Arc<Self>> {
let mut cfg = persy::Config::new();
cfg.change_cache_size((config.db_cache_capacity_mb * 1024.0 * 1024.0) as u64);
let persy = OpenOptions::new()
.create(true)
.config(cfg)
.open(&format!("{}/db.persy", config.database_path))?;
let write_cache_size = 1000;
let write_cache_window_millisecs = 1000;
let write_cache = Arc::new(RwLock::new(WriteCache::new(
&persy,
write_cache_size,
Duration::from_millis(write_cache_window_millisecs),
)));
/*
use timer::Timer;
let timer = Timer::new();
let timer_write_cache = write_cache.clone();
timer.schedule_repeating(
chrono::Duration::milliseconds((write_cache_window_millisecs / 4) as i64),
move || timer_write_cache.write().unwrap().flush_timed().unwrap(),
);
*/
Ok(Arc::new(PersyEngine { persy, write_cache }))
}
fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>> {
// Create if it doesn't exist
if !self.persy.exists_index(name)? {
let mut tx = self.persy.begin()?;
tx.create_index::<ByteVec, ByteVec>(name, ValueMode::Replace)?;
tx.prepare()?.commit()?;
}
Ok(Arc::new(PersyTree {
persy: self.persy.clone(),
name: name.to_owned(),
watchers: RwLock::new(BTreeMap::new()),
write_cache: self.write_cache.clone(),
}))
}
fn flush(self: &Arc<Self>) -> Result<()> {
self.write_cache.write().unwrap().flush_changes()?;
Ok(())
}
}
pub struct PersyTree {
persy: Persy,
name: String,
watchers: RwLock<BTreeMap<Vec<u8>, Vec<tokio::sync::oneshot::Sender<()>>>>,
write_cache: Arc<RwLock<WriteCache>>,
}
pub struct WriteCache {
add_cache: BTreeMap<String, BTreeMap<Vec<u8>, Vec<u8>>>,
remove_cache: BTreeMap<String, BTreeSet<Vec<u8>>>,
changes_count: u32,
last_flush: Instant,
persy: Persy,
max_size: u32,
max_time_window: Duration,
}
impl WriteCache {
fn new(persy: &Persy, max_size: u32, max_time_window: Duration) -> Self {
Self {
add_cache: Default::default(),
remove_cache: Default::default(),
changes_count: Default::default(),
last_flush: Instant::now(),
persy: persy.clone(),
max_size,
max_time_window,
}
}
}
impl WriteCache {
pub fn insert(&mut self, index: String, key: &[u8], value: &[u8]) -> Result<()> {
match self.add_cache.entry(index.clone()) {
Entry::Vacant(s) => {
let mut map = BTreeMap::new();
map.insert(key.to_owned(), value.to_owned());
s.insert(map);
}
Entry::Occupied(mut o) => {
o.get_mut().insert(key.to_owned(), value.to_owned());
}
}
self.remove_remove(index, key)?;
self.check_and_flush()?;
Ok(())
}
pub fn remove_remove(&mut self, index: String, key: &[u8]) -> Result<()> {
match self.remove_cache.entry(index) {
Entry::Vacant(_) => {}
Entry::Occupied(mut o) => {
o.get_mut().remove(key);
}
}
Ok(())
}
pub fn remove_insert(&mut self, index: String, key: &[u8]) -> Result<()> {
match self.add_cache.entry(index) {
Entry::Vacant(_) => {}
Entry::Occupied(mut o) => {
o.get_mut().remove(key);
}
}
Ok(())
}
pub fn remove(&mut self, index: String, key: &[u8]) -> Result<()> {
match self.remove_cache.entry(index.clone()) {
Entry::Vacant(s) => {
let mut map = BTreeSet::new();
map.insert(key.to_owned());
s.insert(map);
}
Entry::Occupied(mut o) => {
o.get_mut().insert(key.to_owned());
}
}
self.remove_insert(index, key)?;
self.check_and_flush()?;
Ok(())
}
pub fn check_and_flush(&mut self) -> Result<()> {
self.changes_count += 1;
if self.changes_count > self.max_size {
self.flush_changes()?;
self.changes_count = 0;
}
Ok(())
}
pub fn get(&self, index: &str, key: &[u8], value: Option<Vec<u8>>) -> Result<Option<Vec<u8>>> {
Ok(if let Some(changes) = self.add_cache.get(index) {
changes.get(key).map(|v| v.to_owned()).or(value)
} else if let Some(remove) = self.remove_cache.get(index) {
if remove.contains(key) {
None
} else {
value
}
} else {
value
})
}
fn flush_changes(&mut self) -> Result<()> {
let mut tx = self.persy.begin()?;
for (index, changes) in &self.add_cache {
for (key, value) in changes {
tx.put::<ByteVec, ByteVec>(
&index,
ByteVec::new(key.to_owned()),
ByteVec::new(value.to_owned()),
)?;
}
}
self.add_cache.clear();
for (index, changes) in &self.remove_cache {
for key in changes {
tx.remove::<ByteVec, ByteVec>(&index, ByteVec::new(key.to_owned()), None)?;
}
}
self.remove_cache.clear();
tx.prepare()?.commit()?;
self.last_flush = Instant::now();
Ok(())
}
pub fn iter<'a>(
&self,
index: &str,
mut iter: Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + Sync + 'a>,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + Sync + 'a> {
if let Some(adds) = self.add_cache.get(index) {
let added = adds.clone().into_iter().map(|(k, v)| (k.into(), v.into()));
iter = Box::new(UnionIter::new(iter, added, false))
}
if let Some(removes) = self.remove_cache.get(index) {
let to_filter = removes.clone();
iter = Box::new(iter.filter(move |x| to_filter.contains(&(*x.0).to_owned())))
}
iter
}
fn iter_from<'a>(
&self,
index: &str,
from: &[u8],
backwards: bool,
mut iter: Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a>,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> {
if let Some(adds) = self.add_cache.get(index) {
let range = if backwards {
adds.range(..from.to_owned())
} else {
adds.range(from.to_owned()..)
};
let added = range
.map(|(k, v)| (k.to_owned().into(), v.to_owned().into()))
.collect::<Vec<(Vec<u8>, Vec<u8>)>>();
let add_iter: Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send> = if backwards {
Box::new(added.into_iter().rev())
} else {
Box::new(added.into_iter())
};
iter = Box::new(UnionIter::new(iter, add_iter, backwards))
}
if let Some(removes) = self.remove_cache.get(index) {
let owned_from = from.to_owned();
let to_filter = removes.iter();
let to_filter = if backwards {
to_filter
.filter(|x| (..&owned_from).contains(x))
.cloned()
.collect::<Vec<Vec<u8>>>()
} else {
to_filter
.filter(|x| (&owned_from..).contains(x))
.cloned()
.collect::<Vec<Vec<u8>>>()
};
iter = Box::new(iter.filter(move |x| !to_filter.contains(&(*x.0).to_owned())))
}
iter
}
fn scan_prefix<'a>(
&self,
index: &str,
prefix: Vec<u8>,
mut iter: Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a>,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> {
if let Some(adds) = self.add_cache.get(index) {
let owned_prefix = prefix.to_owned();
let added = adds
.range(prefix.to_owned()..)
.take_while(move |(k, _)| k.starts_with(&owned_prefix))
.map(|(k, v)| (k.to_owned().into(), v.to_owned().into()))
.collect::<Vec<(Vec<u8>, Vec<u8>)>>();
iter = Box::new(UnionIter::new(iter, added.into_iter(), false))
}
if let Some(removes) = self.remove_cache.get(index) {
let to_filter = removes
.iter()
.filter(move |k| k.starts_with(&prefix))
.cloned()
.collect::<Vec<Vec<u8>>>();
iter = Box::new(iter.filter(move |x| !to_filter.contains(&(*x.0).to_owned())))
}
iter
}
#[allow(unused)]
pub fn flush_timed(&mut self) -> Result<()> {
if self.changes_count > 0 {
if Instant::now() - self.last_flush > self.max_time_window {
self.flush_changes()?;
}
}
Ok(())
}
}
struct UnionIter<T: Iterator<Item = I>, T1: Iterator<Item = I>, I> {
first: Peekable<T>,
second: Peekable<T1>,
backwards: bool,
}
impl<T: Iterator<Item = I>, T1: Iterator<Item = I>, I> UnionIter<T, T1, I> {
fn new(first: T, second: T1, backwards: bool) -> Self {
UnionIter {
first: first.peekable(),
second: second.peekable(),
backwards,
}
}
}
impl<K: Ord, V, T, T1> Iterator for UnionIter<T, T1, (K, V)>
where
T: Iterator<Item = (K, V)>,
T1: Iterator<Item = (K, V)>,
{
type Item = (K, V);
fn next(&mut self) -> Option<Self::Item> {
if let (Some(f), Some(s)) = (self.first.peek(), self.second.peek()) {
if self.backwards {
match f.0.cmp(&s.0) {
Ordering::Less => self.second.next(),
Ordering::Greater => self.first.next(),
Ordering::Equal => {
self.first.next();
self.second.next()
}
}
} else {
match f.0.cmp(&s.0) {
Ordering::Less => self.first.next(),
Ordering::Greater => self.second.next(),
Ordering::Equal => {
self.first.next();
self.second.next()
}
}
}
} else {
self.first.next().or_else(|| self.second.next())
}
}
}
impl Tree for PersyTree {
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
let result = self
.persy
.get::<ByteVec, ByteVec>(&self.name, &ByteVec::new(key.to_vec()))?
.next()
.map(|v| (*v).to_owned());
let result = self
.write_cache
.read()
.unwrap()
.get(&self.name, key, result)?;
Ok(result)
}
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
let watchers = self.watchers.read().unwrap();
let mut triggered = Vec::new();
for length in 0..=key.len() {
if watchers.contains_key(&key[..length]) {
triggered.push(&key[..length]);
}
}
drop(watchers);
if !triggered.is_empty() {
let mut watchers = self.watchers.write().unwrap();
for prefix in triggered {
if let Some(txs) = watchers.remove(prefix) {
for tx in txs {
let _ = tx.send(());
}
}
}
}
self.write_cache
.write()
.unwrap()
.insert(self.name.clone(), key, value)?;
Ok(())
}
fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
//TODO: evaluate if use instead a single big transaction
for (key, value) in iter {
self.insert(&key, &value)?;
}
Ok(())
}
fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()> {
for key in iter {
self.increment(&key)?;
}
Ok(())
}
fn remove(&self, key: &[u8]) -> Result<()> {
self.write_cache
.write()
.unwrap()
.remove(self.name.clone(), key)?;
Ok(())
}
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
let iter = self.persy.range::<ByteVec, ByteVec, _>(&self.name, ..);
match iter {
Ok(iter) => {
let result = Box::new(iter.filter_map(|(k, v)| {
v.into_iter()
.map(|val| ((*k).to_owned().into(), (*val).to_owned().into()))
.next()
}));
self.write_cache.read().unwrap().iter(&self.name, result)
}
Err(e) => {
warn!("error iterating {:?}", e);
Box::new(std::iter::empty())
}
}
}
fn iter_from<'a>(
&'a self,
from: &[u8],
backwards: bool,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
let range = if backwards {
self.persy
.range::<ByteVec, ByteVec, _>(&self.name, ..ByteVec::new(from.to_owned()))
} else {
self.persy
.range::<ByteVec, ByteVec, _>(&self.name, ByteVec::new(from.to_owned())..)
};
match range {
Ok(iter) => {
let map = iter.filter_map(|(k, v)| {
v.into_iter()
.map(|val| ((*k).to_owned().into(), (*val).to_owned().into()))
.next()
});
let result: Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> = if backwards
{
Box::new(map.rev())
} else {
Box::new(map)
};
self.write_cache
.read()
.unwrap()
.iter_from(&self.name, from, backwards, result)
}
Err(e) => {
warn!("error iterating with prefix {:?}", e);
Box::new(std::iter::empty())
}
}
}
fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
let old = self.get(key)?;
let new = crate::utils::increment(old.as_deref()).unwrap();
self.insert(key, &new)?;
Ok(new)
}
fn scan_prefix<'a>(
&'a self,
prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
let range_prefix = ByteVec::new(prefix.to_owned());
let range = self
.persy
.range::<ByteVec, ByteVec, _>(&self.name, range_prefix..);
match range {
Ok(iter) => {
let owned_prefix = prefix.clone();
let result = Box::new(
iter.take_while(move |(k, _)| (*k).starts_with(&owned_prefix))
.filter_map(|(k, v)| {
v.into_iter()
.map(|val| ((*k).to_owned().into(), (*val).to_owned().into()))
.next()
}),
);
self.write_cache
.read()
.unwrap()
.scan_prefix(&self.name, prefix, result)
}
Err(e) => {
warn!("error scanning prefix {:?}", e);
Box::new(std::iter::empty())
}
}
}
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
let (tx, rx) = tokio::sync::oneshot::channel();
self.watchers
.write()
.unwrap()
.entry(prefix.to_vec())
.or_default()
.push(tx);
Box::pin(async move {
// Tx is never destroyed
rx.await.unwrap();
})
}
}

12
src/error.rs

@ -1,3 +1,4 @@
use persy::PersyError;
use ruma::{ use ruma::{
api::client::{ api::client::{
error::{Error as RumaError, ErrorKind}, error::{Error as RumaError, ErrorKind},
@ -36,6 +37,9 @@ pub enum Error {
#[from] #[from]
source: rusqlite::Error, source: rusqlite::Error,
}, },
#[cfg(feature = "persy")]
#[error("There was a problem with the connection to the persy database.")]
PersyError { source: persy::PersyError },
#[cfg(feature = "heed")] #[cfg(feature = "heed")]
#[error("There was a problem with the connection to the heed database: {error}")] #[error("There was a problem with the connection to the heed database: {error}")]
HeedError { error: String }, HeedError { error: String },
@ -136,3 +140,11 @@ where
self.to_response().respond_to(r) self.to_response().respond_to(r)
} }
} }
impl<T: Into<PersyError>> From<persy::PE<T>> for Error {
fn from(err: persy::PE<T>) -> Self {
Error::PersyError {
source: err.error().into(),
}
}
}

Loading…
Cancel
Save