improvement: batched inserts for tokenids

src/database.rs

@@ -271,8 +271,8 @@ impl Database {
 
                 eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?,
                 referencedevents: builder.open_tree("referencedevents")?,
-                pdu_cache: Mutex::new(LruCache::new(100_000)),
+                pdu_cache: Mutex::new(LruCache::new(1_000_000)),
-                auth_chain_cache: Mutex::new(LruCache::new(100_000)),
+                auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
             },
             account_data: account_data::AccountData {
                 roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,

src/database/rooms.rs

@@ -905,18 +905,20 @@ impl Rooms {
             }
             EventType::RoomMessage => {
                 if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) {
-                    for word in body
+                    let mut batch = body
                         .split_terminator(|c: char| !c.is_alphanumeric())
                         .filter(|word| word.len() <= 50)
                         .map(str::to_lowercase)
-                    {
+                        .map(|word| {
-                        let mut key = pdu.room_id.as_bytes().to_vec();
+                            let mut key = pdu.room_id.as_bytes().to_vec();
-                        key.push(0xff);
+                            key.push(0xff);
-                        key.extend_from_slice(word.as_bytes());
+                            key.extend_from_slice(word.as_bytes());
-                        key.push(0xff);
+                            key.push(0xff);
-                        key.extend_from_slice(&pdu_id);
+                            key.extend_from_slice(&pdu_id);
-                        self.tokenids.insert(&key, &[])?;
+                            (key, Vec::new())
-                    }
+                        });
+
+                    self.tokenids.insert_batch(&mut batch)?;
 
                     if body.starts_with(&format!("@conduit:{}: ", db.globals.server_name()))
                         && self

src/server_server.rs

@@ -1,6 +1,6 @@
 use crate::{
     client_server::{self, claim_keys_helper, get_keys_helper},
-    database::DatabaseGuard,
+    database::{abstraction::sqlite::MILLI, DatabaseGuard},
     utils, ConduitResult, Database, Error, PduEvent, Result, Ruma,
 };
 use get_profile_information::v1::ProfileField;
@@ -1732,7 +1732,12 @@ fn get_auth_chain(starting_events: Vec<EventId>, db: &Database) -> Result<HashSe
             cached.clone()
         } else {
             drop(cache);
+            let start = Instant::now();
             let auth_chain = get_auth_chain_recursive(&event_id, HashSet::new(), db)?;
+            let elapsed = start.elapsed();
+            if elapsed > MILLI {
+                println!("auth chain for {} took {:?}", &event_id, elapsed)
+            }
 
             cache = db.rooms.auth_chain_cache();
 
@@ -1747,7 +1752,11 @@ fn get_auth_chain(starting_events: Vec<EventId>, db: &Database) -> Result<HashSe
     Ok(full_auth_chain)
 }
 
-fn get_auth_chain_recursive(event_id: &EventId, mut found: HashSet<EventId>, db: &Database) -> Result<HashSet<EventId>> {
+fn get_auth_chain_recursive(
+    event_id: &EventId,
+    mut found: HashSet<EventId>,
+    db: &Database,
+) -> Result<HashSet<EventId>> {
     if let Some(pdu) = db.rooms.get_pdu(&event_id)? {
         for auth_event in &pdu.auth_events {
             if !found.contains(auth_event) {