1#![allow(clippy::await_holding_lock)]
6
7use std::{
8 borrow::Borrow,
9 collections::{BTreeMap, HashMap, VecDeque, btree_map::Iter},
10 marker::PhantomData,
11 ops::RangeBounds,
12 sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
13};
14
15use bincode::Options;
16use collectable::TryExtend;
17use ouroboros::self_referencing;
18use rand::distributions::{Alphanumeric, DistString};
19use rocksdb::Direction;
20use serde::{Serialize, de::DeserializeOwned};
21
22use crate::{
23 DbIterator, Map, TypedStoreError, be_fix_int_ser, rocks::errors::typed_store_err_from_bcs_err,
24};
25
26#[derive(Clone, Debug)]
29pub struct TestDB<K, V> {
30 pub rows: Arc<RwLock<BTreeMap<Vec<u8>, Vec<u8>>>>,
31 pub name: String,
32 _phantom: PhantomData<fn(K) -> V>,
33}
34
35impl<K, V> TestDB<K, V> {
36 pub fn open() -> Self {
37 TestDB {
38 rows: Arc::new(RwLock::new(BTreeMap::new())),
39 name: Alphanumeric.sample_string(&mut rand::thread_rng(), 16),
40 _phantom: PhantomData,
41 }
42 }
43 pub fn batch(&self) -> TestDBWriteBatch {
44 TestDBWriteBatch::default()
45 }
46}
47
48#[self_referencing(pub_extras)]
49pub struct TestDBIter<'a, K, V> {
50 pub rows: RwLockReadGuard<'a, BTreeMap<Vec<u8>, Vec<u8>>>,
51 #[borrows(mut rows)]
52 #[covariant]
53 pub iter: Iter<'this, Vec<u8>, Vec<u8>>,
54 phantom: PhantomData<(K, V)>,
55 pub direction: Direction,
56}
57
58#[self_referencing(pub_extras)]
59pub struct TestDBKeys<'a, K> {
60 rows: RwLockReadGuard<'a, BTreeMap<Vec<u8>, Vec<u8>>>,
61 #[borrows(mut rows)]
62 #[covariant]
63 pub iter: Iter<'this, Vec<u8>, Vec<u8>>,
64 phantom: PhantomData<K>,
65}
66
67#[self_referencing(pub_extras)]
68pub struct TestDBValues<'a, V> {
69 rows: RwLockReadGuard<'a, BTreeMap<Vec<u8>, Vec<u8>>>,
70 #[borrows(mut rows)]
71 #[covariant]
72 pub iter: Iter<'this, Vec<u8>, Vec<u8>>,
73 phantom: PhantomData<V>,
74}
75
76impl<K: DeserializeOwned, V: DeserializeOwned> Iterator for TestDBIter<'_, K, V> {
77 type Item = Result<(K, V), TypedStoreError>;
78
79 fn next(&mut self) -> Option<Self::Item> {
80 let mut out: Option<Self::Item> = None;
81 let config = bincode::DefaultOptions::new()
82 .with_big_endian()
83 .with_fixint_encoding();
84 self.with_mut(|fields| {
85 let resp = match fields.direction {
86 Direction::Forward => fields.iter.next(),
87 Direction::Reverse => panic!("Reverse iteration not supported in test db"),
88 };
89 if let Some((raw_key, raw_value)) = resp {
90 let key: K = config.deserialize(raw_key).ok().unwrap();
91 let value: V = bcs::from_bytes(raw_value).ok().unwrap();
92 out = Some(Ok((key, value)));
93 }
94 });
95 out
96 }
97}
98
99impl<'a, K: Serialize, V> TestDBIter<'a, K, V> {
100 pub fn skip_to(mut self, key: &K) -> Result<Self, TypedStoreError> {
104 self.with_mut(|fields| {
105 let serialized_key = be_fix_int_ser(key);
106 let mut peekable = fields.iter.peekable();
107 let mut peeked = peekable.peek();
108 while peeked.is_some() {
109 let serialized = be_fix_int_ser(peeked.unwrap());
110 if serialized >= serialized_key {
111 break;
112 } else {
113 peekable.next();
114 peeked = peekable.peek();
115 }
116 }
117 });
118 Ok(self)
119 }
120
121 pub fn skip_prior_to(mut self, key: &K) -> Result<Self, TypedStoreError> {
125 self.with_mut(|fields| {
126 let serialized_key = be_fix_int_ser(key);
127 let mut peekable = fields.iter.peekable();
128 let mut peeked = peekable.peek();
129 while peeked.is_some() {
130 let serialized = be_fix_int_ser(peeked.unwrap());
131 if serialized > serialized_key {
132 break;
133 } else {
134 peekable.next();
135 peeked = peekable.peek();
136 }
137 }
138 });
139 Ok(self)
140 }
141
142 pub fn skip_to_last(mut self) -> Self {
144 self.with_mut(|fields| {
145 fields.iter.last();
147 });
148 self
149 }
150
151 pub fn reverse(mut self) -> TestDBRevIter<'a, K, V> {
155 self.with_mut(|fields| {
156 *fields.direction = Direction::Reverse;
157 });
158 TestDBRevIter::new(self)
159 }
160}
161
162pub struct TestDBRevIter<'a, K, V> {
167 iter: TestDBIter<'a, K, V>,
168}
169
170impl<'a, K, V> TestDBRevIter<'a, K, V> {
171 fn new(iter: TestDBIter<'a, K, V>) -> Self {
172 Self { iter }
173 }
174}
175
176impl<K: DeserializeOwned, V: DeserializeOwned> Iterator for TestDBRevIter<'_, K, V> {
177 type Item = Result<(K, V), TypedStoreError>;
178
179 fn next(&mut self) -> Option<Self::Item> {
181 self.iter.next()
182 }
183}
184
185impl<K: DeserializeOwned> Iterator for TestDBKeys<'_, K> {
186 type Item = Result<K, TypedStoreError>;
187
188 fn next(&mut self) -> Option<Self::Item> {
189 let mut out: Option<Self::Item> = None;
190 self.with_mut(|fields| {
191 let config = bincode::DefaultOptions::new()
192 .with_big_endian()
193 .with_fixint_encoding();
194 if let Some((raw_key, _)) = fields.iter.next() {
195 let key: K = config.deserialize(raw_key).ok().unwrap();
196 out = Some(Ok(key));
197 }
198 });
199 out
200 }
201}
202
203impl<V: DeserializeOwned> Iterator for TestDBValues<'_, V> {
204 type Item = Result<V, TypedStoreError>;
205
206 fn next(&mut self) -> Option<Self::Item> {
207 let mut out: Option<Self::Item> = None;
208 self.with_mut(|fields| {
209 if let Some((_, raw_value)) = fields.iter.next() {
210 let value: V = bcs::from_bytes(raw_value).ok().unwrap();
211 out = Some(Ok(value));
212 }
213 });
214 out
215 }
216}
217
218impl<'a, K, V> Map<'a, K, V> for TestDB<K, V>
219where
220 K: Serialize + DeserializeOwned,
221 V: Serialize + DeserializeOwned,
222{
223 type Error = TypedStoreError;
224
225 fn contains_key(&self, key: &K) -> Result<bool, Self::Error> {
226 let raw_key = be_fix_int_ser(key);
227 let locked = self.rows.read().unwrap();
228 Ok(locked.contains_key(&raw_key))
229 }
230
231 fn get(&self, key: &K) -> Result<Option<V>, Self::Error> {
232 let raw_key = be_fix_int_ser(key);
233 let locked = self.rows.read().unwrap();
234 let res = locked.get(&raw_key);
235 Ok(res.map(|raw_value| bcs::from_bytes(raw_value).ok().unwrap()))
236 }
237
238 fn insert(&self, key: &K, value: &V) -> Result<(), Self::Error> {
239 let raw_key = be_fix_int_ser(key);
240 let raw_value = bcs::to_bytes(value).map_err(typed_store_err_from_bcs_err)?;
241 let mut locked = self.rows.write().unwrap();
242 locked.insert(raw_key, raw_value);
243 Ok(())
244 }
245
246 fn remove(&self, key: &K) -> Result<(), Self::Error> {
247 let raw_key = be_fix_int_ser(key);
248 let mut locked = self.rows.write().unwrap();
249 locked.remove(&raw_key);
250 Ok(())
251 }
252
253 fn schedule_delete_all(&self) -> Result<(), TypedStoreError> {
254 let mut locked = self.rows.write().unwrap();
255 locked.clear();
256 Ok(())
257 }
258
259 fn is_empty(&self) -> bool {
260 let locked = self.rows.read().unwrap();
261 locked.is_empty()
262 }
263
264 fn safe_iter(&'a self) -> DbIterator<'a, (K, V)> {
265 Box::new(
266 TestDBIterBuilder {
267 rows: self.rows.read().unwrap(),
268 iter_builder: |rows: &mut RwLockReadGuard<'a, BTreeMap<Vec<u8>, Vec<u8>>>| {
269 rows.iter()
270 },
271 phantom: PhantomData,
272 direction: Direction::Forward,
273 }
274 .build(),
275 )
276 }
277
278 fn safe_iter_with_bounds(
279 &'a self,
280 _lower_bound: Option<K>,
281 _upper_bound: Option<K>,
282 ) -> DbIterator<'a, (K, V)> {
283 unimplemented!("unimplemented API");
284 }
285
286 fn safe_range_iter(&'a self, _range: impl RangeBounds<K>) -> DbIterator<'a, (K, V)> {
287 unimplemented!("unimplemented API");
288 }
289
290 fn safe_range_iter_reversed(&'a self, _range: impl RangeBounds<K>) -> DbIterator<'a, (K, V)> {
291 unimplemented!("unimplemented API");
292 }
293
294 fn try_catch_up_with_primary(&self) -> Result<(), Self::Error> {
295 Ok(())
296 }
297}
298
299impl<J, K, U, V> TryExtend<(J, U)> for TestDB<K, V>
300where
301 J: Borrow<K>,
302 U: Borrow<V>,
303 K: Serialize,
304 V: Serialize,
305{
306 type Error = TypedStoreError;
307
308 fn try_extend<T>(&mut self, iter: &mut T) -> Result<(), Self::Error>
309 where
310 T: Iterator<Item = (J, U)>,
311 {
312 let mut wb = self.batch();
313 wb.insert_batch(self, iter)?;
314 wb.write()
315 }
316
317 fn try_extend_from_slice(&mut self, slice: &[(J, U)]) -> Result<(), Self::Error> {
318 let slice_of_refs = slice.iter().map(|(k, v)| (k.borrow(), v.borrow()));
319 let mut wb = self.batch();
320 wb.insert_batch(self, slice_of_refs)?;
321 wb.write()
322 }
323}
324
325pub type DeleteBatchPayload = (
326 Arc<RwLock<BTreeMap<Vec<u8>, Vec<u8>>>>,
327 String,
328 Vec<Vec<u8>>,
329);
330pub type DeleteRangePayload = (
331 Arc<RwLock<BTreeMap<Vec<u8>, Vec<u8>>>>,
332 String,
333 (Vec<u8>, Vec<u8>),
334);
335pub type InsertBatchPayload = (
336 Arc<RwLock<BTreeMap<Vec<u8>, Vec<u8>>>>,
337 String,
338 Vec<(Vec<u8>, Vec<u8>)>,
339);
340type DBAndName = (Arc<RwLock<BTreeMap<Vec<u8>, Vec<u8>>>>, String);
341
342pub enum WriteBatchOp {
343 DeleteBatch(DeleteBatchPayload),
344 DeleteRange(DeleteRangePayload),
345 InsertBatch(InsertBatchPayload),
346}
347
348#[derive(Default)]
349pub struct TestDBWriteBatch {
350 pub ops: VecDeque<WriteBatchOp>,
351}
352
353#[self_referencing]
354pub struct DBLocked {
355 db: Arc<RwLock<BTreeMap<Vec<u8>, Vec<u8>>>>,
356 #[borrows(db)]
357 #[covariant]
358 db_guard: RwLockWriteGuard<'this, BTreeMap<Vec<u8>, Vec<u8>>>,
359}
360
361impl TestDBWriteBatch {
362 pub fn write(self) -> Result<(), TypedStoreError> {
363 let mut dbs: Vec<DBAndName> = self
364 .ops
365 .iter()
366 .map(|op| match op {
367 WriteBatchOp::DeleteBatch((db, name, _)) => (db.clone(), name.clone()),
368 WriteBatchOp::DeleteRange((db, name, _)) => (db.clone(), name.clone()),
369 WriteBatchOp::InsertBatch((db, name, _)) => (db.clone(), name.clone()),
370 })
371 .collect();
372 dbs.sort_by_key(|(_k, v)| v.clone());
373 dbs.dedup_by_key(|(_k, v)| v.clone());
374 let mut db_locks = HashMap::new();
376 dbs.iter().for_each(|(db, name)| {
377 if !db_locks.contains_key(name) {
378 db_locks.insert(
379 name.clone(),
380 DBLockedBuilder {
381 db: db.clone(),
382 db_guard_builder: |db: &Arc<RwLock<BTreeMap<Vec<u8>, Vec<u8>>>>| {
383 db.write().unwrap()
384 },
385 }
386 .build(),
387 );
388 }
389 });
390 self.ops.iter().for_each(|op| match op {
391 WriteBatchOp::DeleteBatch((_, id, keys)) => {
392 let locked = db_locks.get_mut(id).unwrap();
393 locked.with_db_guard_mut(|db| {
394 keys.iter().for_each(|key| {
395 db.remove(key);
396 });
397 });
398 }
399 WriteBatchOp::DeleteRange((_, id, (from, to))) => {
400 let locked = db_locks.get_mut(id).unwrap();
401 locked.with_db_guard_mut(|db| {
402 db.retain(|k, _| k < from || k >= to);
403 });
404 }
405 WriteBatchOp::InsertBatch((_, id, key_values)) => {
406 let locked = db_locks.get_mut(id).unwrap();
407 locked.with_db_guard_mut(|db| {
408 key_values.iter().for_each(|(k, v)| {
409 db.insert(k.clone(), v.clone());
410 });
411 });
412 }
413 });
414 dbs.iter().rev().for_each(|(_db, id)| {
416 if db_locks.contains_key(id) {
417 db_locks.remove(id);
418 }
419 });
420 Ok(())
421 }
422 pub fn delete_batch<J: Borrow<K>, K: Serialize, V>(
424 &mut self,
425 db: &TestDB<K, V>,
426 purged_vals: impl IntoIterator<Item = J>,
427 ) -> Result<(), TypedStoreError> {
428 self.ops.push_back(WriteBatchOp::DeleteBatch((
429 db.rows.clone(),
430 db.name.clone(),
431 purged_vals
432 .into_iter()
433 .map(|key| be_fix_int_ser(&key.borrow()))
434 .collect(),
435 )));
436 Ok(())
437 }
438 pub fn delete_range<K: Serialize, V>(
441 &mut self,
442 db: &TestDB<K, V>,
443 from: &K,
444 to: &K,
445 ) -> Result<(), TypedStoreError> {
446 let raw_from = be_fix_int_ser(from);
447 let raw_to = be_fix_int_ser(to);
448 self.ops.push_back(WriteBatchOp::DeleteRange((
449 db.rows.clone(),
450 db.name.clone(),
451 (raw_from, raw_to),
452 )));
453 Ok(())
454 }
455 pub fn insert_batch<J: Borrow<K>, K: Serialize, U: Borrow<V>, V: Serialize>(
457 &mut self,
458 db: &TestDB<K, V>,
459 new_vals: impl IntoIterator<Item = (J, U)>,
460 ) -> Result<(), TypedStoreError> {
461 self.ops.push_back(WriteBatchOp::InsertBatch((
462 db.rows.clone(),
463 db.name.clone(),
464 new_vals
465 .into_iter()
466 .map(|(key, value)| {
467 (
468 be_fix_int_ser(&key.borrow()),
469 bcs::to_bytes(&value.borrow()).unwrap(),
470 )
471 })
472 .collect(),
473 )));
474 Ok(())
475 }
476}
477
478#[cfg(test)]
479mod test {
480 use crate::{Map, test_db::TestDB};
481
482 #[test]
483 fn test_contains_key() {
484 let db = TestDB::open();
485 db.insert(&123456789, &"123456789".to_string())
486 .expect("Failed to insert");
487 assert!(
488 db.contains_key(&123456789)
489 .expect("Failed to call contains key")
490 );
491 assert!(
492 !db.contains_key(&000000000)
493 .expect("Failed to call contains key")
494 );
495 }
496
497 #[test]
498 fn test_get() {
499 let db = TestDB::open();
500 db.insert(&123456789, &"123456789".to_string())
501 .expect("Failed to insert");
502 assert_eq!(
503 Some("123456789".to_string()),
504 db.get(&123456789).expect("Failed to get")
505 );
506 assert_eq!(None, db.get(&000000000).expect("Failed to get"));
507 }
508
509 #[test]
510 fn test_multi_get() {
511 let db = TestDB::open();
512 db.insert(&123, &"123".to_string())
513 .expect("Failed to insert");
514 db.insert(&456, &"456".to_string())
515 .expect("Failed to insert");
516
517 let result = db.multi_get([123, 456, 789]).expect("Failed to multi get");
518
519 assert_eq!(result.len(), 3);
520 assert_eq!(result[0], Some("123".to_string()));
521 assert_eq!(result[1], Some("456".to_string()));
522 assert_eq!(result[2], None);
523 }
524
525 #[test]
526 fn test_remove() {
527 let db = TestDB::open();
528 db.insert(&123456789, &"123456789".to_string())
529 .expect("Failed to insert");
530 assert!(db.get(&123456789).expect("Failed to get").is_some());
531
532 db.remove(&123456789).expect("Failed to remove");
533 assert!(db.get(&123456789).expect("Failed to get").is_none());
534 }
535
536 #[test]
537 fn test_iter() {
538 let db = TestDB::open();
539 db.insert(&123456789, &"123456789".to_string())
540 .expect("Failed to insert");
541
542 let mut iter = db.safe_iter();
543 assert_eq!(Some(Ok((123456789, "123456789".to_string()))), iter.next());
544 assert_eq!(None, iter.next());
545 }
546
547 #[test]
548 fn test_iter_reverse() {
549 let db = TestDB::open();
550 db.insert(&1, &"1".to_string()).expect("Failed to insert");
551 db.insert(&2, &"2".to_string()).expect("Failed to insert");
552 db.insert(&3, &"3".to_string()).expect("Failed to insert");
553 let mut iter = db.safe_iter();
554
555 assert_eq!(Some(Ok((1, "1".to_string()))), iter.next());
556 assert_eq!(Some(Ok((2, "2".to_string()))), iter.next());
557 assert_eq!(Some(Ok((3, "3".to_string()))), iter.next());
558 assert_eq!(None, iter.next());
559 }
560
561 #[test]
562 fn test_values() {
563 let db = TestDB::open();
564
565 db.insert(&123456789, &"123456789".to_string())
566 .expect("Failed to insert");
567 }
568
569 #[test]
570 fn test_insert_batch() {
571 let db = TestDB::open();
572 let keys_vals = (1..100).map(|i| (i, i.to_string()));
573 let mut wb = db.batch();
574 wb.insert_batch(&db, keys_vals.clone())
575 .expect("Failed to batch insert");
576 wb.write().expect("Failed to execute batch");
577 for (k, v) in keys_vals {
578 let val = db.get(&k).expect("Failed to get inserted key");
579 assert_eq!(Some(v), val);
580 }
581 }
582
583 #[test]
584 fn test_insert_batch_across_cf() {
585 let db_cf_1 = TestDB::open();
586 let keys_vals_1 = (1..100).map(|i| (i, i.to_string()));
587
588 let db_cf_2 = TestDB::open();
589 let keys_vals_2 = (1000..1100).map(|i| (i, i.to_string()));
590
591 let mut wb = db_cf_1.batch();
592 wb.insert_batch(&db_cf_1, keys_vals_1.clone())
593 .expect("Failed to batch insert");
594 wb.insert_batch(&db_cf_2, keys_vals_2.clone())
595 .expect("Failed to batch insert");
596 wb.write().expect("Failed to execute batch");
597 for (k, v) in keys_vals_1 {
598 let val = db_cf_1.get(&k).expect("Failed to get inserted key");
599 assert_eq!(Some(v), val);
600 }
601
602 for (k, v) in keys_vals_2 {
603 let val = db_cf_2.get(&k).expect("Failed to get inserted key");
604 assert_eq!(Some(v), val);
605 }
606 }
607
608 #[test]
609 fn test_delete_batch() {
610 let db: TestDB<i32, String> = TestDB::open();
611
612 let keys_vals = (1..100).map(|i| (i, i.to_string()));
613 let mut wb = db.batch();
614 wb.insert_batch(&db, keys_vals)
615 .expect("Failed to batch insert");
616
617 let deletion_keys = (1..100).step_by(2);
619 wb.delete_batch(&db, deletion_keys)
620 .expect("Failed to batch delete");
621
622 wb.write().expect("Failed to execute batch");
623
624 db.safe_iter().for_each(|item| {
625 assert!(item.unwrap().0 % 2 == 0);
626 });
627 }
628
629 #[test]
630 fn test_delete_range() {
631 let db: TestDB<i32, String> = TestDB::open();
632
633 let keys_vals = (0..101).map(|i| (i, i.to_string()));
635 let mut wb = db.batch();
636 wb.insert_batch(&db, keys_vals)
637 .expect("Failed to batch insert");
638
639 wb.delete_range(&db, &50, &100)
640 .expect("Failed to delete range");
641
642 wb.write().expect("Failed to execute batch");
643
644 for k in 0..50 {
645 assert!(db.contains_key(&k).expect("Failed to query legal key"),);
646 }
647 for k in 50..100 {
648 assert!(!db.contains_key(&k).expect("Failed to query legal key"));
649 }
650
651 assert!(db.contains_key(&100).expect("Failed to query legal key"));
653 }
654
655 #[test]
656 fn test_is_empty() {
657 let db: TestDB<i32, String> = TestDB::open();
658
659 assert!(db.is_empty());
661
662 let keys_vals = (0..101).map(|i| (i, i.to_string()));
663 let mut wb = db.batch();
664 wb.insert_batch(&db, keys_vals)
665 .expect("Failed to batch insert");
666
667 wb.write().expect("Failed to execute batch");
668
669 assert!(db.safe_iter().count() > 1);
671 assert!(!db.is_empty());
672 }
673
674 #[test]
675 fn test_multi_insert() {
676 let db: TestDB<i32, String> = TestDB::open();
678
679 let keys_vals = (0..101).map(|i| (i, i.to_string()));
681
682 db.multi_insert(keys_vals.clone())
683 .expect("Failed to multi-insert");
684
685 for (k, v) in keys_vals {
686 let val = db.get(&k).expect("Failed to get inserted key");
687 assert_eq!(Some(v), val);
688 }
689 }
690
691 #[test]
692 fn test_multi_remove() {
693 let db: TestDB<i32, String> = TestDB::open();
695
696 let keys_vals = (0..101).map(|i| (i, i.to_string()));
698
699 db.multi_insert(keys_vals.clone())
700 .expect("Failed to multi-insert");
701
702 for (k, v) in keys_vals.clone() {
704 let val = db.get(&k).expect("Failed to get inserted key");
705 assert_eq!(Some(v), val);
706 }
707
708 db.multi_remove(keys_vals.clone().map(|kv| kv.0).take(50))
710 .expect("Failed to multi-remove");
711 assert_eq!(db.safe_iter().count(), 101 - 50);
712
713 for (k, v) in keys_vals.skip(50) {
715 let val = db.get(&k).expect("Failed to get inserted key");
716 assert_eq!(Some(v), val);
717 }
718 }
719}