Jonathan de Jong
5 years ago
15 changed files with 772 additions and 314 deletions
@ -0,0 +1,32 @@
@@ -0,0 +1,32 @@
|
||||
# Conduit |
||||
version: '3' |
||||
|
||||
services: |
||||
homeserver: |
||||
image: ubuntu:21.04 |
||||
restart: unless-stopped |
||||
working_dir: "/srv/conduit" |
||||
entrypoint: /srv/conduit/conduit |
||||
ports: |
||||
- 8448:8000 |
||||
volumes: |
||||
- ../target/db:/srv/conduit/.local/share/conduit |
||||
- ../target/debug/conduit:/srv/conduit/conduit |
||||
- ./conduit.toml:/srv/conduit/conduit.toml:ro |
||||
environment: |
||||
# CONDUIT_SERVER_NAME: localhost:8000 # replace with your own name |
||||
# CONDUIT_TRUSTED_SERVERS: '["matrix.org"]' |
||||
### Uncomment and change values as desired |
||||
# CONDUIT_ADDRESS: 127.0.0.1 |
||||
# CONDUIT_PORT: 8000 |
||||
CONDUIT_CONFIG: '/srv/conduit/conduit.toml' |
||||
# Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging |
||||
# CONDUIT_LOG: debug # default is: "info,rocket=off,_=off,sled=off" |
||||
# CONDUIT_ALLOW_JAEGER: 'false' |
||||
# CONDUIT_ALLOW_REGISTRATION : 'false' |
||||
# CONDUIT_ALLOW_ENCRYPTION: 'false' |
||||
# CONDUIT_ALLOW_FEDERATION: 'false' |
||||
# CONDUIT_DATABASE_PATH: /srv/conduit/.local/share/conduit |
||||
# CONDUIT_WORKERS: 10 |
||||
# CONDUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB |
||||
|
||||
@ -0,0 +1,176 @@
@@ -0,0 +1,176 @@
|
||||
use super::super::Config; |
||||
use crate::{utils, Result}; |
||||
|
||||
use std::{future::Future, pin::Pin, sync::Arc}; |
||||
|
||||
use super::{DatabaseEngine, Tree}; |
||||
|
||||
use std::{collections::BTreeMap, sync::RwLock}; |
||||
|
||||
pub struct RocksDbEngine(rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>); |
||||
|
||||
pub struct RocksDbEngineTree<'a> { |
||||
db: Arc<RocksDbEngine>, |
||||
name: &'a str, |
||||
watchers: RwLock<BTreeMap<Vec<u8>, Vec<tokio::sync::oneshot::Sender<()>>>>, |
||||
} |
||||
|
||||
impl DatabaseEngine for RocksDbEngine { |
||||
fn open(config: &Config) -> Result<Arc<Self>> { |
||||
let mut db_opts = rocksdb::Options::default(); |
||||
db_opts.create_if_missing(true); |
||||
db_opts.set_max_open_files(16); |
||||
db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); |
||||
db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy); |
||||
db_opts.set_target_file_size_base(256 << 20); |
||||
db_opts.set_write_buffer_size(256 << 20); |
||||
|
||||
let mut block_based_options = rocksdb::BlockBasedOptions::default(); |
||||
block_based_options.set_block_size(512 << 10); |
||||
db_opts.set_block_based_table_factory(&block_based_options); |
||||
|
||||
let cfs = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::list_cf( |
||||
&db_opts, |
||||
&config.database_path, |
||||
) |
||||
.unwrap_or_default(); |
||||
|
||||
let mut options = rocksdb::Options::default(); |
||||
options.set_merge_operator_associative("increment", utils::increment_rocksdb); |
||||
|
||||
let db = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::open_cf_descriptors( |
||||
&db_opts, |
||||
&config.database_path, |
||||
cfs.iter() |
||||
.map(|name| rocksdb::ColumnFamilyDescriptor::new(name, options.clone())), |
||||
)?; |
||||
|
||||
Ok(Arc::new(RocksDbEngine(db))) |
||||
} |
||||
|
||||
fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>> { |
||||
let mut options = rocksdb::Options::default(); |
||||
options.set_merge_operator_associative("increment", utils::increment_rocksdb); |
||||
|
||||
// Create if it doesn't exist
|
||||
let _ = self.0.create_cf(name, &options); |
||||
|
||||
Ok(Arc::new(RocksDbEngineTree { |
||||
name, |
||||
db: Arc::clone(self), |
||||
watchers: RwLock::new(BTreeMap::new()), |
||||
})) |
||||
} |
||||
} |
||||
|
||||
impl RocksDbEngineTree<'_> { |
||||
fn cf(&self) -> rocksdb::BoundColumnFamily<'_> { |
||||
self.db.0.cf_handle(self.name).unwrap() |
||||
} |
||||
} |
||||
|
||||
impl Tree for RocksDbEngineTree<'_> { |
||||
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> { |
||||
Ok(self.db.0.get_cf(self.cf(), key)?) |
||||
} |
||||
|
||||
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { |
||||
let watchers = self.watchers.read().unwrap(); |
||||
let mut triggered = Vec::new(); |
||||
|
||||
for length in 0..=key.len() { |
||||
if watchers.contains_key(&key[..length]) { |
||||
triggered.push(&key[..length]); |
||||
} |
||||
} |
||||
|
||||
drop(watchers); |
||||
|
||||
if !triggered.is_empty() { |
||||
let mut watchers = self.watchers.write().unwrap(); |
||||
for prefix in triggered { |
||||
if let Some(txs) = watchers.remove(prefix) { |
||||
for tx in txs { |
||||
let _ = tx.send(()); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
Ok(self.db.0.put_cf(self.cf(), key, value)?) |
||||
} |
||||
|
||||
fn remove(&self, key: &[u8]) -> Result<()> { |
||||
Ok(self.db.0.delete_cf(self.cf(), key)?) |
||||
} |
||||
|
||||
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + Sync + 'a> { |
||||
Box::new( |
||||
self.db |
||||
.0 |
||||
.iterator_cf(self.cf(), rocksdb::IteratorMode::Start), |
||||
) |
||||
} |
||||
|
||||
fn iter_from<'a>( |
||||
&'a self, |
||||
from: &[u8], |
||||
backwards: bool, |
||||
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> { |
||||
Box::new(self.db.0.iterator_cf( |
||||
self.cf(), |
||||
rocksdb::IteratorMode::From( |
||||
from, |
||||
if backwards { |
||||
rocksdb::Direction::Reverse |
||||
} else { |
||||
rocksdb::Direction::Forward |
||||
}, |
||||
), |
||||
)) |
||||
} |
||||
|
||||
fn increment(&self, key: &[u8]) -> Result<Vec<u8>> { |
||||
let stats = rocksdb::perf::get_memory_usage_stats(Some(&[&self.db.0]), None).unwrap(); |
||||
dbg!(stats.mem_table_total); |
||||
dbg!(stats.mem_table_unflushed); |
||||
dbg!(stats.mem_table_readers_total); |
||||
dbg!(stats.cache_total); |
||||
// TODO: atomic?
|
||||
let old = self.get(key)?; |
||||
let new = utils::increment(old.as_deref()).unwrap(); |
||||
self.insert(key, &new)?; |
||||
Ok(new) |
||||
} |
||||
|
||||
fn scan_prefix<'a>( |
||||
&'a self, |
||||
prefix: Vec<u8>, |
||||
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> { |
||||
Box::new( |
||||
self.db |
||||
.0 |
||||
.iterator_cf( |
||||
self.cf(), |
||||
rocksdb::IteratorMode::From(&prefix, rocksdb::Direction::Forward), |
||||
) |
||||
.take_while(move |(k, _)| k.starts_with(&prefix)), |
||||
) |
||||
} |
||||
|
||||
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> { |
||||
let (tx, rx) = tokio::sync::oneshot::channel(); |
||||
|
||||
self.watchers |
||||
.write() |
||||
.unwrap() |
||||
.entry(prefix.to_vec()) |
||||
.or_default() |
||||
.push(tx); |
||||
|
||||
Box::pin(async move { |
||||
// Tx is never destroyed
|
||||
rx.await.unwrap(); |
||||
}) |
||||
} |
||||
} |
||||
@ -0,0 +1,115 @@
@@ -0,0 +1,115 @@
|
||||
use super::super::Config; |
||||
use crate::{utils, Result}; |
||||
use log::warn; |
||||
use std::{future::Future, pin::Pin, sync::Arc}; |
||||
|
||||
use super::{DatabaseEngine, Tree}; |
||||
|
||||
pub struct SledEngine(sled::Db); |
||||
|
||||
pub struct SledEngineTree(sled::Tree); |
||||
|
||||
impl DatabaseEngine for SledEngine { |
||||
fn open(config: &Config) -> Result<Arc<Self>> { |
||||
Ok(Arc::new(SledEngine( |
||||
sled::Config::default() |
||||
.path(&config.database_path) |
||||
.cache_capacity(config.cache_capacity as u64) |
||||
.use_compression(true) |
||||
.open()?, |
||||
))) |
||||
} |
||||
|
||||
fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>> { |
||||
Ok(Arc::new(SledEngineTree(self.0.open_tree(name)?))) |
||||
} |
||||
} |
||||
|
||||
impl Tree for SledEngineTree { |
||||
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> { |
||||
Ok(self.0.get(key)?.map(|v| v.to_vec())) |
||||
} |
||||
|
||||
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { |
||||
self.0.insert(key, value)?; |
||||
Ok(()) |
||||
} |
||||
|
||||
fn remove(&self, key: &[u8]) -> Result<()> { |
||||
self.0.remove(key)?; |
||||
Ok(()) |
||||
} |
||||
|
||||
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + Sync + 'a> { |
||||
Box::new( |
||||
self.0 |
||||
.iter() |
||||
.filter_map(|r| { |
||||
if let Err(e) = &r { |
||||
warn!("Error: {}", e); |
||||
} |
||||
r.ok() |
||||
}) |
||||
.map(|(k, v)| (k.to_vec().into(), v.to_vec().into())), |
||||
) |
||||
} |
||||
|
||||
fn iter_from( |
||||
&self, |
||||
from: &[u8], |
||||
backwards: bool, |
||||
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)>> { |
||||
let iter = if backwards { |
||||
self.0.range(..from) |
||||
} else { |
||||
self.0.range(from..) |
||||
}; |
||||
|
||||
let iter = iter |
||||
.filter_map(|r| { |
||||
if let Err(e) = &r { |
||||
warn!("Error: {}", e); |
||||
} |
||||
r.ok() |
||||
}) |
||||
.map(|(k, v)| (k.to_vec().into(), v.to_vec().into())); |
||||
|
||||
if backwards { |
||||
Box::new(iter.rev()) |
||||
} else { |
||||
Box::new(iter) |
||||
} |
||||
} |
||||
|
||||
fn increment(&self, key: &[u8]) -> Result<Vec<u8>> { |
||||
Ok(self |
||||
.0 |
||||
.update_and_fetch(key, utils::increment) |
||||
.map(|o| o.expect("increment always sets a value").to_vec())?) |
||||
} |
||||
|
||||
fn scan_prefix<'a>( |
||||
&'a self, |
||||
prefix: Vec<u8>, |
||||
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> { |
||||
let iter = self |
||||
.0 |
||||
.scan_prefix(prefix) |
||||
.filter_map(|r| { |
||||
if let Err(e) = &r { |
||||
warn!("Error: {}", e); |
||||
} |
||||
r.ok() |
||||
}) |
||||
.map(|(k, v)| (k.to_vec().into(), v.to_vec().into())); |
||||
|
||||
Box::new(iter) |
||||
} |
||||
|
||||
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> { |
||||
let prefix = prefix.to_vec(); |
||||
Box::pin(async move { |
||||
self.0.watch_prefix(prefix).await; |
||||
}) |
||||
} |
||||
} |
||||
@ -0,0 +1,273 @@
@@ -0,0 +1,273 @@
|
||||
use std::{future::Future, pin::Pin, sync::Arc, thread}; |
||||
|
||||
use crate::{database::Config, Result}; |
||||
|
||||
use super::{DatabaseEngine, Tree}; |
||||
|
||||
use std::{collections::BTreeMap, sync::RwLock}; |
||||
|
||||
use crossbeam::channel::{bounded, Sender as ChannelSender}; |
||||
use parking_lot::{Mutex, MutexGuard}; |
||||
use rusqlite::{params, Connection, OptionalExtension}; |
||||
|
||||
use tokio::sync::oneshot::Sender; |
||||
|
||||
type SqliteHandle = Arc<Mutex<Connection>>; |
||||
|
||||
// const SQL_CREATE_TABLE: &str =
|
||||
// "CREATE TABLE IF NOT EXISTS {} {{ \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL }}";
|
||||
// const SQL_SELECT: &str = "SELECT value FROM {} WHERE key = ?";
|
||||
// const SQL_INSERT: &str = "INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)";
|
||||
// const SQL_DELETE: &str = "DELETE FROM {} WHERE key = ?";
|
||||
// const SQL_SELECT_ITER: &str = "SELECT key, value FROM {}";
|
||||
// const SQL_SELECT_PREFIX: &str = "SELECT key, value FROM {} WHERE key LIKE ?||'%' ORDER BY key ASC";
|
||||
// const SQL_SELECT_ITER_FROM_FORWARDS: &str = "SELECT key, value FROM {} WHERE key >= ? ORDER BY ASC";
|
||||
// const SQL_SELECT_ITER_FROM_BACKWARDS: &str =
|
||||
// "SELECT key, value FROM {} WHERE key <= ? ORDER BY DESC";
|
||||
|
||||
pub struct SqliteEngine { |
||||
handle: SqliteHandle, |
||||
} |
||||
|
||||
impl DatabaseEngine for SqliteEngine { |
||||
fn open(config: &Config) -> Result<Arc<Self>> { |
||||
let conn = Connection::open(format!("{}/conduit.db", &config.database_path))?; |
||||
|
||||
conn.pragma_update(None, "journal_mode", &"WAL".to_owned())?; |
||||
|
||||
let handle = Arc::new(Mutex::new(conn)); |
||||
|
||||
Ok(Arc::new(SqliteEngine { handle })) |
||||
} |
||||
|
||||
fn open_tree(self: &Arc<Self>, name: &str) -> Result<Arc<dyn Tree>> { |
||||
self.handle.lock().execute(format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", name).as_str(), [])?; |
||||
|
||||
Ok(Arc::new(SqliteTable { |
||||
engine: Arc::clone(self), |
||||
name: name.to_owned(), |
||||
watchers: RwLock::new(BTreeMap::new()), |
||||
})) |
||||
} |
||||
} |
||||
|
||||
pub struct SqliteTable { |
||||
engine: Arc<SqliteEngine>, |
||||
name: String, |
||||
watchers: RwLock<BTreeMap<Vec<u8>, Vec<Sender<()>>>>, |
||||
} |
||||
|
||||
type TupleOfBytes = (Vec<u8>, Vec<u8>); |
||||
|
||||
impl SqliteTable { |
||||
fn get_with_guard( |
||||
&self, |
||||
guard: &MutexGuard<'_, Connection>, |
||||
key: &[u8], |
||||
) -> Result<Option<Vec<u8>>> { |
||||
Ok(guard |
||||
.prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())? |
||||
.query_row([key], |row| row.get(0)) |
||||
.optional()?) |
||||
} |
||||
|
||||
fn insert_with_guard( |
||||
&self, |
||||
guard: &MutexGuard<'_, Connection>, |
||||
key: &[u8], |
||||
value: &[u8], |
||||
) -> Result<()> { |
||||
guard.execute( |
||||
format!( |
||||
"INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)", |
||||
self.name |
||||
) |
||||
.as_str(), |
||||
[key, value], |
||||
)?; |
||||
Ok(()) |
||||
} |
||||
|
||||
fn _iter_from_thread<F>( |
||||
&self, |
||||
mutex: Arc<Mutex<Connection>>, |
||||
f: F, |
||||
) -> Box<dyn Iterator<Item = TupleOfBytes> + Send> |
||||
where |
||||
F: (FnOnce(MutexGuard<'_, Connection>, ChannelSender<TupleOfBytes>)) + Send + 'static, |
||||
{ |
||||
let (s, r) = bounded::<TupleOfBytes>(5); |
||||
|
||||
thread::spawn(move || { |
||||
let _ = f(mutex.lock(), s); |
||||
}); |
||||
|
||||
Box::new(r.into_iter()) |
||||
} |
||||
} |
||||
|
||||
macro_rules! iter_from_thread { |
||||
($self:expr, $sql:expr, $param:expr) => { |
||||
$self._iter_from_thread($self.engine.handle.clone(), move |guard, s| { |
||||
let _ = guard |
||||
.prepare($sql) |
||||
.unwrap() |
||||
.query_map($param, |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) |
||||
.unwrap() |
||||
.map(|r| r.unwrap()) |
||||
.try_for_each(|bob| s.send(bob)); |
||||
}) |
||||
}; |
||||
} |
||||
|
||||
impl Tree for SqliteTable { |
||||
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> { |
||||
self.get_with_guard(&self.engine.handle.lock(), key) |
||||
} |
||||
|
||||
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { |
||||
self.insert_with_guard(&self.engine.handle.lock(), key, value)?; |
||||
|
||||
let watchers = self.watchers.read().unwrap(); |
||||
let mut triggered = Vec::new(); |
||||
|
||||
for length in 0..=key.len() { |
||||
if watchers.contains_key(&key[..length]) { |
||||
triggered.push(&key[..length]); |
||||
} |
||||
} |
||||
|
||||
drop(watchers); |
||||
|
||||
if !triggered.is_empty() { |
||||
let mut watchers = self.watchers.write().unwrap(); |
||||
for prefix in triggered { |
||||
if let Some(txs) = watchers.remove(prefix) { |
||||
for tx in txs { |
||||
let _ = tx.send(()); |
||||
} |
||||
} |
||||
} |
||||
}; |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
fn remove(&self, key: &[u8]) -> Result<()> { |
||||
self.engine.handle.lock().execute( |
||||
format!("DELETE FROM {} WHERE key = ?", self.name).as_str(), |
||||
[key], |
||||
)?; |
||||
Ok(()) |
||||
} |
||||
|
||||
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = TupleOfBytes> + Send + 'a> { |
||||
let name = self.name.clone(); |
||||
iter_from_thread!( |
||||
self, |
||||
format!("SELECT key, value FROM {}", name).as_str(), |
||||
params![] |
||||
) |
||||
} |
||||
|
||||
fn iter_from<'a>( |
||||
&'a self, |
||||
from: &[u8], |
||||
backwards: bool, |
||||
) -> Box<dyn Iterator<Item = TupleOfBytes> + Send + 'a> { |
||||
let name = self.name.clone(); |
||||
let from = from.to_vec(); // TODO change interface?
|
||||
if backwards { |
||||
iter_from_thread!( |
||||
self, |
||||
format!( // TODO change to <= on rebase
|
||||
"SELECT key, value FROM {} WHERE key < ? ORDER BY key DESC", |
||||
name |
||||
) |
||||
.as_str(), |
||||
[from] |
||||
) |
||||
} else { |
||||
iter_from_thread!( |
||||
self, |
||||
format!( |
||||
"SELECT key, value FROM {} WHERE key >= ? ORDER BY key ASC", |
||||
name |
||||
) |
||||
.as_str(), |
||||
[from] |
||||
) |
||||
} |
||||
} |
||||
|
||||
fn increment(&self, key: &[u8]) -> Result<Vec<u8>> { |
||||
let guard = self.engine.handle.lock(); |
||||
|
||||
let old = self.get_with_guard(&guard, key)?; |
||||
|
||||
let new = |
||||
crate::utils::increment(old.as_deref()).expect("utils::increment always returns Some"); |
||||
|
||||
self.insert_with_guard(&guard, key, &new)?; |
||||
|
||||
Ok(new) |
||||
} |
||||
|
||||
// TODO: make this use take_while
|
||||
|
||||
fn scan_prefix<'a>( |
||||
&'a self, |
||||
prefix: Vec<u8>, |
||||
) -> Box<dyn Iterator<Item = TupleOfBytes> + Send + 'a> { |
||||
// let name = self.name.clone();
|
||||
// iter_from_thread!(
|
||||
// self,
|
||||
// format!(
|
||||
// "SELECT key, value FROM {} WHERE key BETWEEN ?1 AND ?1 || X'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF' ORDER BY key ASC",
|
||||
// name
|
||||
// )
|
||||
// .as_str(),
|
||||
// [prefix]
|
||||
// )
|
||||
Box::new(self.iter_from(&prefix, false).take_while(move |(key, _)| key.starts_with(&prefix))) |
||||
} |
||||
|
||||
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> { |
||||
let (tx, rx) = tokio::sync::oneshot::channel(); |
||||
|
||||
self.watchers |
||||
.write() |
||||
.unwrap() |
||||
.entry(prefix.to_vec()) |
||||
.or_default() |
||||
.push(tx); |
||||
|
||||
Box::pin(async move { |
||||
// Tx is never destroyed
|
||||
rx.await.unwrap(); |
||||
}) |
||||
} |
||||
|
||||
fn clear(&self) -> Result<()> { |
||||
self.engine.handle.lock().execute( |
||||
format!("DELETE FROM {}", self.name).as_str(), |
||||
[], |
||||
)?; |
||||
Ok(()) |
||||
} |
||||
} |
||||
|
||||
// TODO
|
||||
// struct Pool<const NUM_READERS: usize> {
|
||||
// writer: Mutex<Connection>,
|
||||
// readers: [Mutex<Connection>; NUM_READERS],
|
||||
// }
|
||||
|
||||
// // then, to pick a reader:
|
||||
// for r in &pool.readers {
|
||||
// if let Ok(reader) = r.try_lock() {
|
||||
// // use reader
|
||||
// }
|
||||
// }
|
||||
// // none unlocked, pick the next reader
|
||||
// pool.readers[pool.counter.fetch_add(1, Relaxed) % NUM_READERS].lock()
|
||||
Loading…
Reference in new issue