Rework sqlite db to implement nostr_database trait

Cargo.lock

@@ -384,14 +384,13 @@ dependencies = [
 
 [[package]]
 name = "deadpool"
-version = "0.9.5"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e"
+checksum = "fb84100978c1c7b37f09ed3ce3e5f843af02c2a2c431bae5b19230dad2c1b490"
 dependencies = [
  "async-trait",
  "deadpool-runtime",
  "num_cpus",
- "retain_mut",
  "tokio",
 ]
 
@@ -406,9 +405,9 @@ dependencies = [
 
 [[package]]
 name = "deadpool-sqlite"
-version = "0.5.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e026821eaacbce25ff0d54405e4421d71656fcae3e4a9323461280fcda6dbc7d"
+checksum = "b8010e36e12f3be22543a5e478b4af20aeead9a700dd69581a5e050a070fc22c"
 dependencies = [
  "deadpool",
  "deadpool-sync",
@@ -424,6 +423,15 @@ dependencies = [
  "deadpool-runtime",
 ]
 
+[[package]]
+name = "deranged"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
+dependencies = [
+ "powerfmt",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -479,9 +487,9 @@ dependencies = [
 
 [[package]]
 name = "fallible-iterator"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
 
 [[package]]
 name = "fallible-streaming-iterator"
@@ -501,16 +509,6 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "flatbuffers"
-version = "23.5.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640"
-dependencies = [
- "bitflags 1.3.2",
- "rustc_version",
-]
-
 [[package]]
 name = "flate2"
 version = "1.0.28"
@@ -988,9 +986,9 @@ checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
 
 [[package]]
 name = "libsqlite3-sys"
-version = "0.25.2"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa"
+checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
 dependencies = [
  "cc",
  "pkg-config",
@@ -1128,29 +1126,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aa0550256c8d4f0aaf74891ac986bd5ba46b2957c2c7e20f51838fa5819285f8"
 dependencies = [
  "async-trait",
- "flatbuffers",
  "nostr",
  "thiserror",
  "tokio",
  "tracing",
 ]
 
-[[package]]
-name = "nostr-sqlite"
-version = "0.27.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f0d3672bf705c0b1f1b1dc6682e397eafd8df4b379170136bbb52280ab76fd7"
-dependencies = [
- "async-trait",
- "deadpool-sqlite",
- "nostr",
- "nostr-database",
- "rusqlite",
- "thiserror",
- "tokio",
- "tracing",
-]
-
 [[package]]
 name = "nu-ansi-term"
 version = "0.49.0"
@@ -1318,6 +1299,12 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -1493,12 +1480,6 @@ dependencies = [
  "winreg",
 ]
 
-[[package]]
-name = "retain_mut"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0"
-
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -1516,16 +1497,30 @@ dependencies = [
 
 [[package]]
 name = "rusqlite"
-version = "0.28.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"
+checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.3.3",
+ "chrono",
  "fallible-iterator",
  "fallible-streaming-iterator",
  "hashlink",
  "libsqlite3-sys",
+ "serde_json",
  "smallvec",
+ "time",
+ "uuid",
+]
+
+[[package]]
+name = "rusqlite_migration"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4116d1697de2912db0b72069473dfb025f6c332b4a085ed041d121e8d745aea"
+dependencies = [
+ "log",
+ "rusqlite",
 ]
 
 [[package]]
@@ -1534,15 +1529,6 @@ version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
-[[package]]
-name = "rustc_version"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
-dependencies = [
- "semver",
-]
-
 [[package]]
 name = "rustix"
 version = "0.38.6"
@@ -1667,12 +1653,16 @@ dependencies = [
 
 [[package]]
 name = "sea-query"
-version = "0.30.4"
+version = "0.30.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41558fa9bb5f4d73952dac0b9d9c2ce23966493fc9ee0008037b01d709838a68"
+checksum = "4166a1e072292d46dc91f31617c2a1cdaf55a8be4b5c9f4bf2ba248e3ac4999b"
 dependencies = [
+ "chrono",
  "inherent",
  "sea-query-derive",
+ "serde_json",
+ "time",
+ "uuid",
 ]
 
 [[package]]
@@ -1688,6 +1678,16 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "sea-query-rusqlite"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e74efa64c7ba4d62ed221bfd6bf8a685b1bbc37e29c51b1039405fc6c33a2dd"
+dependencies = [
+ "rusqlite",
+ "sea-query",
+]
+
 [[package]]
 name = "secp256k1"
 version = "0.27.0"
@@ -1709,26 +1709,20 @@ dependencies = [
  "cc",
 ]
 
-[[package]]
-name = "semver"
-version = "1.0.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0"
-
 [[package]]
 name = "serde"
-version = "1.0.181"
+version = "1.0.193"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d3e73c93c3240c0bda063c239298e633114c69a888c3e37ca8bb33f343e9890"
+checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.181"
+version = "1.0.193"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be02f6cb0cd3a5ec20bbcfbcbd749f57daddb1a0882dc2e46a6c236c90b977ed"
+checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1827,17 +1821,21 @@ dependencies = [
  "argon2",
  "async-trait",
  "chrono",
+ "deadpool-sqlite",
  "flexbuffers",
  "flexi_logger",
  "futures-util",
  "lazy_static",
  "log",
  "nostr",
- "nostr-sqlite",
+ "nostr-database",
  "regex",
+ "rusqlite",
+ "rusqlite_migration",
  "rustls 0.21.6",
  "sailfish",
  "sea-query",
+ "sea-query-rusqlite",
  "serde",
  "serde_json",
  "sled",
@@ -1919,6 +1917,35 @@ dependencies = [
  "syn 2.0.28",
 ]
 
+[[package]]
+name = "time"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e"
+dependencies = [
+ "deranged",
+ "itoa",
+ "powerfmt",
+ "serde",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
+
+[[package]]
+name = "time-macros"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f"
+dependencies = [
+ "time-core",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.6.0"

Cargo.toml

@@ -17,16 +17,23 @@ futures-util = "0.3.28"
 rustls = "0.21"
 anyhow = "1.0"
 sled = "0.34.7"
-# sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-rustls", "sqlite", "migrate", "macros"] }
 flexi_logger = { version = "0.27.3", features = [ "async", "compress" ] }
 lazy_static = "1.4.0"
 log = "0.4"
+deadpool-sqlite = "0.7.0"
+rusqlite = { version = "0.30.0", features = [ "bundled", "vtab" ] }
+rusqlite_migration = "1.0.2"
 nostr = "0.27.0"
-nostr-sqlite = "0.27.0"
+nostr-database = "0.27.0"
 regex = "1.9.5"
 sailfish = "0.7.0"
 sea-query = { version = "0.30.4", features = ["backend-sqlite", "thread-safe"] }
-# sea-query-binder = { version = "0.5.0", features = ["sqlx-sqlite"] }
+sea-query-rusqlite = { version="0", features = [
+    "with-chrono",
+    "with-json",
+    "with-uuid",
+    "with-time",
+] }
 serde = "1.0"
 serde_json = "1.0"
 thiserror = "1.0.48"

src/noose/migrations/1697409647688_create_events.sql

@@ -21,3 +21,10 @@ CREATE TABLE tags (
 CREATE INDEX idx_tags_tag ON tags (tag);
 CREATE INDEX idx_tags_value ON tags (value);
 CREATE INDEX idx_tags_event_id ON tags (event_id);
+
+CREATE TABLE deleted_coordinates (
+    coordinate TEXT NOT NULL,
+    created_at INTEGER NOT NULL
+);
+
+CREATE INDEX idx_coordinates_coordinate ON coordinates (coordinate);

src/noose/migrations/1697410424624_pragma.sql

@@ -1,2 +1,6 @@
+PRAGMA encoding = "UTF-8";
+PRAGMA journal_mode=WAL;
 PRAGMA foreign_keys = ON;
 PRAGMA auto_vacuum = FULL;
+PRAGMA journal_size_limit=32768;
+PRAGMA mmap_size = 17179869184; -- cap mmap at 16GB

src/noose/migrations/1706115586021_event_seen_by_relays.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS event_seen_by_relays (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    event_id TEXT NOT NULL,
+    relay_url TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS event_seen_by_relays_index ON event_seen_by_relays(event_id,relay_url);

src/noose/migrations/mod.rs

@@ -0,0 +1,50 @@
+use rusqlite::Connection;
+use rusqlite_migration::{Migrations, M};
+
+pub struct MigrationRunner {}
+
+impl MigrationRunner {
+    pub fn up(connection: &mut Connection) -> bool {
+        let m_create_events = include_str!("./1697409647688_create_events.sql");
+        let m_event_seen_by_relays = include_str!("1706115586021_event_seen_by_relays.sql");
+        let m_add_realys = include_str!("./1697410161900_add_relays.sql");
+        let m_events_fts = include_str!("./1697410223576_events_fts.sql");
+        let m_users = include_str!("./1697410294265_users.sql");
+        let m_unattached_media = include_str!("./1697410480767_unattached_media.sql");
+        let m_pragma = include_str!("./1697410424624_pragma.sql");
+
+        let migrations = Migrations::new(vec![
+            M::up(m_create_events),
+            M::up(m_event_seen_by_relays),
+            M::up(m_add_realys),
+            M::up(m_events_fts),
+            M::up(m_users),
+            M::up(m_unattached_media),
+            M::up(m_pragma),
+        ]);
+
+        match migrations.to_latest(connection) {
+            Ok(()) => true,
+            Err(err) => {
+                log::error!("Migrations failed: {}", err.to_string());
+                false
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rusqlite::Connection;
+
+    use super::MigrationRunner;
+
+    #[test]
+    fn get_sql_path() {
+        let mut connection = Connection::open_in_memory().unwrap();
+
+        let runner = MigrationRunner::up(&mut connection);
+
+        assert!(runner);
+    }
+}

src/noose/mod.rs

@@ -3,11 +3,11 @@ use db::Noose;
 use pipeline::Pipeline;
 use tokio::runtime;
 pub mod db;
-mod nostr_db;
 pub mod pipeline;
 pub mod sled;
-// mod sqlite;
+mod sqlite;
 pub mod user;
+mod migrations;
 
 pub fn start(context: Context) {
     let rt = runtime::Runtime::new().unwrap();
@@ -30,17 +30,12 @@ pub fn start(context: Context) {
             sled_writer.start(sled_pubsub).await.unwrap();
         });
 
-        let nostr_db_writer_handle = tokio::task::spawn(async move {
+        let nostr_sqlite_writer_handle = tokio::task::spawn(async move {
-            let mut db_writer = nostr_db::NostrDb::new(db_config).await;
+            let mut db_writer = sqlite::NostrSqlite::new(db_config).await;
             db_writer.start(db_pubsub).await.unwrap();
         });
 
-        // let sqlite_writer_handle = tokio::task::spawn(async move {
+        nostr_sqlite_writer_handle.await.unwrap();
-        //     let mut db_writer = sqlite::SqliteDb::new().await;
-        //     db_writer.start(db_pubsub).await.unwrap();
-        // });
-
-        // sqlite_writer_handle.await.unwrap();
         pipeline_handle.await.unwrap();
     });
 }

src/noose/nostr_db.rs

@@ -1,111 +0,0 @@
-use crate::{
-    bussy::{channels, Command, Message, PubSub},
-    noose::Noose,
-    utils::{config::Config, error::Error, structs::Subscription},
-};
-use nostr_sqlite::{database::NostrDatabase, SQLiteDatabase};
-use std::sync::Arc;
-
-pub struct NostrDb {
-    db: SQLiteDatabase,
-}
-
-impl NostrDb {
-    pub async fn new(config: Arc<Config>) -> Self {
-        let db_path = config.get_db_path();
-        if let Ok(db) = SQLiteDatabase::open(db_path).await {
-            return Self { db };
-        }
-
-        panic!("[NostrDb] Failed to initialize database");
-    }
-}
-
-impl Noose for NostrDb {
-    async fn start(&mut self, pubsub: Arc<PubSub>) -> Result<(), Error> {
-        let mut subscriber = pubsub.subscribe(channels::MSG_NOOSE).await;
-
-        while let Ok(message) = subscriber.recv().await {
-            log::info!("[Noose] received message: {:?}", message);
-            let command = match message.content {
-                Command::DbReqWriteEvent(client_id, event) => match self.write_event(event).await {
-                    Ok(status) => Command::DbResOkWithStatus(client_id, status),
-                    Err(e) => Command::ServiceError(e),
-                },
-                Command::DbReqFindEvent(client_id, subscriptioin) => {
-                    match self.find_event(subscriptioin).await {
-                        Ok(relay_messages) => {
-                            Command::DbResRelayMessages(client_id, relay_messages)
-                        }
-                        Err(e) => Command::ServiceError(e),
-                    }
-                }
-                Command::DbReqEventCounts(client_id, subscriptioin) => {
-                    match self.counts(subscriptioin).await {
-                        Ok(relay_message) => Command::DbResEventCounts(client_id, relay_message),
-                        Err(e) => Command::ServiceError(e),
-                    }
-                }
-                _ => Command::Noop,
-            };
-            if command != Command::Noop {
-                let channel = message.source;
-                let message = Message {
-                    source: channels::MSG_NOOSE,
-                    content: command,
-                };
-
-                log::info!(
-                    "[Noose] publishing new message: {:?} to channel {}",
-                    message,
-                    channel
-                );
-
-                pubsub.publish(channel, message).await;
-            }
-        }
-
-        Ok(())
-    }
-
-    async fn write_event(&self, event: Box<nostr::Event>) -> Result<nostr::RelayMessage, Error> {
-        // TODO: Maybe do event validation and admin deletions here
-        match self.db.save_event(&event).await {
-            Ok(status) => {
-                let relay_message = nostr::RelayMessage::ok(event.id, status, "");
-                Ok(relay_message)
-            }
-            Err(err) => Err(Error::bad_request(err.to_string())),
-        }
-    }
-
-    async fn find_event(
-        &self,
-        subscription: Subscription,
-    ) -> Result<Vec<nostr::RelayMessage>, Error> {
-        match self
-            .db
-            .query(subscription.filters, nostr_sqlite::database::Order::Desc)
-            .await
-        {
-            Ok(events) => {
-                let relay_messages = events
-                    .into_iter()
-                    .map(|event| nostr::RelayMessage::event(subscription.id.clone(), event))
-                    .collect();
-                Ok(relay_messages)
-            }
-            Err(err) => Err(Error::bad_request(err.to_string())),
-        }
-    }
-
-    async fn counts(&self, subscription: Subscription) -> Result<nostr::RelayMessage, Error> {
-        match self.db.count(subscription.filters).await {
-            Ok(counts) => {
-                let relay_message = nostr::RelayMessage::count(subscription.id, counts);
-                Ok(relay_message)
-            }
-            Err(err) => Err(Error::internal_with_message(err.to_string())),
-        }
-    }
-}

src/utils/error.rs

@@ -4,6 +4,7 @@ use std::{
     convert::From,
     fmt::{self, Display},
 };
+use std::error::Error as StdError;
 use validator::ValidationErrors;
 use warp::{http::StatusCode, reject::Reject};
 
@@ -16,6 +17,16 @@ pub struct Error {
     pub sneedstr_version: Option<u16>,
 }
 
+impl StdError for Error {
+    fn source(&self) -> Option<&(dyn StdError + 'static)> {
+        None
+    }
+
+    fn cause(&self) -> Option<&dyn StdError> {
+        self.source()
+    }
+}
+
 impl Error {
     pub fn new(code: StatusCode, message: String) -> Self {
         Self {
@@ -74,12 +85,12 @@ impl Error {
 }
 
 impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}: {}", self.status_code(), &self.message)?;
+        write!(
-        if let Some(val) = &self.sneedstr_version {
+            f,
-            write!(f, "\ndiem ledger version: {}", val)?;
+            "Error {{ code: {}, message: '{}', sneedstr_version: {:?} }}",
-        }
+            self.code, self.message, self.sneedstr_version
-        Ok(())
+        )
     }
 }