Merge remote-tracking branch 'origin/repository'

# Conflicts:
#	src/data/entities.rs

src/data.rs

@@ -1,4 +1,5 @@
 pub mod entities;
+pub mod repositories;
 
 use rusqlite::ToSql;
 use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ToSqlOutput, ValueRef};

src/data/entities.rs

@@ -3,10 +3,14 @@ use crate::core::srs::{FADE_PER_DAY, MAX_SCORE, OrderMode, WordOpenMode};
 use crate::data::{INVALID_ID, Id};
 use chrono::{DateTime, NaiveDate, Utc};
 use fsrs::MemoryState;
+use crate::notification::MaybeError;
+use chrono::{DateTime, Utc};
 use log::info;
 use rayon::prelude::*;
 use rhai::{Engine, Scope};
+use std::cmp::Ordering;
 use std::collections::HashMap;
+use std::marker::PhantomData;
 use std::time::Instant;
 
 #[derive(Clone, Debug)]
@@ -18,6 +22,17 @@ pub struct WordData {
     pub additional: HashMap<String, String>,
     pub group_id: Id,
 }
+
+impl EntityData for WordData {
+    fn id(&self) -> Id {
+        self.id
+    }
+
+    fn set_id(&mut self, id: Id) {
+        self.id = id;
+    }
+}
+
 #[derive(Clone)]
 pub struct WordGroup {
     pub id: Id,
@@ -168,3 +183,75 @@ impl CardStatistics {
         (self.score as f32 * multiplier).max(1.0)
     }
 }
+
+#[derive(Copy)]
+pub struct EntityHandle<E> {
+    pub(crate) id: Id,
+    index: i32,
+    pub(crate) _phantom: PhantomData<E>,
+}
+
+impl<T> Clone for EntityHandle<T> {
+    fn clone(&self) -> Self {
+        EntityHandle {
+            id: self.id,
+            index: self.index,
+            _phantom: Default::default(),
+        }
+    }
+}
+impl<T> EntityHandle<T> {
+    pub fn new(id: Id, index: usize, real: bool) -> Self {
+        Self {
+            id,
+            index: index as i32 | (!real as i32) << 31,
+            _phantom: Default::default(),
+        }
+    }
+    pub fn index(&self) -> usize {
+        (self.index << 1 >> 1) as usize
+    }
+
+    pub fn set_index(&mut self, index: usize) {
+        self.index = self.index >> 31 << 31 | index as i32;
+    }
+
+    pub fn real(&self) -> bool {
+        (self.index >> 31) == 0
+    }
+
+    pub fn set_real(&mut self, value: bool) {
+        self.index = self.index << 1 >> 1 | (!value as i32) << 31;
+    }
+}
+impl<T> Eq for EntityHandle<T> {}
+
+impl<T> PartialEq<Self> for EntityHandle<T> {
+    fn eq(&self, other: &Self) -> bool {
+        self.real() == other.real() && self.id == other.id
+    }
+}
+impl<T> PartialOrd<Self> for EntityHandle<T> {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl<T> Ord for EntityHandle<T> {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if self.real() && !other.real() {
+            return Ordering::Less;
+        }
+
+        if !self.real() && other.real() {
+            return Ordering::Greater;
+        }
+
+        self.id.cmp(&other.id)
+    }
+}
+
+pub trait EntityData {
+    fn id(&self) -> Id;
+    fn set_id(&mut self, id: Id);
+}

src/data/repositories.rs

@@ -0,0 +1,531 @@
+use crate::data::Id;
+use crate::data::entities::{EntityData, EntityHandle};
+use crate::helpers::into_result;
+use crate::notification::{MaybeError, NotificationData};
+use parking_lot::{Mutex, RwLock};
+use rayon::iter::IndexedParallelIterator;
+use rayon::iter::IntoParallelRefIterator;
+use rayon::iter::ParallelIterator;
+use rusqlite::Connection;
+use std::marker::PhantomData;
+use std::ops::Deref;
+use std::sync::Arc;
+
+pub trait Repository<T, D>
+where
+    D: DbLayer<T>,
+{
+    fn get(&self, handle: &mut EntityHandle<T>) -> Option<T>;
+    fn get_ref(&self, handle: &mut EntityHandle<T>) -> Option<&T>;
+    fn get_mut(&mut self, handle: &mut EntityHandle<T>) -> Option<&mut T>;
+    fn handle_by_id(&self, id: Id) -> EntityHandle<T>;
+    fn create(&mut self, new: T) -> EntityHandle<T>;
+    fn save(&mut self, entity: &mut EntityHandle<T>) -> MaybeError;
+    fn delete(&mut self, handle: EntityHandle<T>) -> MaybeError;
+    fn init(&mut self) -> MaybeError;
+    fn reload(&mut self) -> MaybeError;
+}
+
+struct QueryManger<T, R, D>
+where
+    R: Repository<T, D>,
+    D: DbLayer<T>,
+{
+    queries: Vec<Arc<Mutex<Query<T, R, D>>>>,
+}
+
+impl<T, R, D> QueryManger<T, R, D>
+where
+    R: Repository<T, D>,
+    D: DbLayer<T>,
+    T: EntityData,
+{
+    pub(crate) fn handle_save(&self, old: EntityHandle<T>, new: &mut EntityHandle<T>, data: &T) {
+        return;
+        if old.real() && new.real() {
+            return;
+        }
+
+        let affected = self.get_affected_queries(data);
+        for q in affected {
+            let mut query = self.queries[q].lock();
+            let index = query.cache.binary_search(&old).unwrap();
+            query.cache.remove(index);
+            let real_end = query.cache.iter().rposition(|r| r.real()).unwrap() + 1;
+            query.cache.insert(real_end, new.clone());
+        }
+    }
+    pub(crate) fn handle_delete(&self, handle: &EntityHandle<T>, data: &T) {
+        return;
+        let affected = self.get_affected_queries(data);
+        for q in affected {
+            let mut query = self.queries[q].lock();
+            let slice = query.cache.as_slice();
+            if let Ok(index) = slice.binary_search(&handle) {
+                query.cache.remove(index);
+            }
+        }
+    }
+
+    fn get_affected_queries(&self, data: &T) -> Vec<usize> {
+        self.queries
+            .iter()
+            .enumerate()
+            .filter(|(_, q)| {
+                let query = q.lock();
+                let rule = &query.rule;
+                return rule(data);
+
+                false
+            })
+            .map(|(i, _)| i)
+            .collect()
+    }
+
+    fn reload_queries(&mut self, data_repository: impl Repository<T, D>) {
+        for q in &self.queries {
+            let mut query = q.lock();
+            query.cache.clear();
+        }
+    }
+}
+pub struct Query<T, R, D>
+where
+    R: Repository<T, D>,
+    D: DbLayer<T>,
+{
+    cache: Vec<EntityHandle<T>>,
+    rule: Box<dyn Fn(&T) -> bool + Send + Sync>,
+    parallel: bool,
+    _marker: PhantomData<(R, D)>,
+    shift_stamp: u16,
+}
+
+impl<T: EntityData + Clone, R: Repository<T, D>, D: DbLayer<T>> Query<T, R, D> {
+    pub fn iter<'a>(&mut self, data_repository_inner: &'a R) -> impl Iterator<Item = &'a T>
+    where
+        T: 'a,
+    {
+        self.cache
+            .iter_mut()
+            .filter_map(move |e| data_repository_inner.get_ref(e))
+    }
+
+    pub fn for_each_mut<F>(&mut self, repo: &mut R, mut f: F)
+    where
+        F: FnMut(&mut T),
+    {
+        for handle in &mut self.cache {
+            if let Some(entity) = repo.get_mut(handle) {
+                f(entity);
+            }
+        }
+    }
+
+    pub fn load(&mut self, data_repository: &DataRepositoryInner<T, D>)
+    where
+        T: Send + Sync,
+    {
+        self.cache.clear();
+        if self.parallel {
+            self.cache = data_repository
+                .cache
+                .par_iter()
+                .enumerate()
+                .filter(|(_, entity)| (self.rule)(entity))
+                .map(|(i, entity)| EntityHandle::new(entity.id(), i, true))
+                .collect();
+        } else {
+            self.cache = data_repository
+                .cache
+                .iter()
+                .enumerate()
+                .filter(|(_, entity)| (self.rule)(entity))
+                .map(|(i, entity)| EntityHandle::new(entity.id(), i, true))
+                .collect();
+        }
+
+        self.cache.extend(
+            data_repository
+                .temporal
+                .iter()
+                .enumerate()
+                .filter(|(_, entity)| (self.rule)(entity))
+                .map(|(i, entity)| EntityHandle::new(entity.id(), i, false))
+                .collect::<Vec<_>>(),
+        );
+    }
+}
+
+pub trait BatchedRepository<T, D>
+where
+    D: BatchedDbLayer<T>,
+    T: EntityData,
+{
+    fn save_many(&mut self, handles: &mut [EntityHandle<T>]) -> MaybeError;
+    fn delete_many(&mut self, handles: &mut [EntityHandle<T>]) -> MaybeError;
+}
+
+pub trait DbLayer<T> {
+    fn write_entity(&self, entity: &T, connection: &Connection) -> MaybeError;
+    fn add_entity(&mut self, entity: &mut T, connection: &Connection) -> MaybeError;
+    fn delete_entity(&mut self, id: Id, connection: &Connection) -> MaybeError;
+    fn load_all(&self, storage: &mut Vec<T>, connection: &Connection) -> MaybeError;
+}
+
+pub trait BatchedDbLayer<T>
+where
+    T: EntityData,
+{
+    fn delete_many_entities(&mut self, ids: &[Id], connection: &Connection) -> MaybeError;
+    fn add_many_entities(&mut self, entity: &mut [T], connection: &Connection) -> MaybeError;
+}
+
+#[derive(Clone)]
+pub struct DataRepository<T: std::clone::Clone + EntityData, D: DbLayer<T>> {
+    inner: Arc<RwLock<DataRepositoryInner<T, D>>>,
+}
+
+pub struct DataRepositoryInner<T: std::clone::Clone + EntityData, D: DbLayer<T>> {
+    cache: Vec<T>,
+    temporal: Vec<T>,
+    next_temporal: u32,
+    connection: Arc<RwLock<Connection>>,
+    data_layer: D,
+    query_manger: QueryManger<T, Self, D>,
+    version: u16,
+    shifts: u16,
+}
+
+impl<T: std::clone::Clone + EntityData, D: DbLayer<T>> Deref for DataRepository<T, D> {
+    type Target = Arc<RwLock<DataRepositoryInner<T, D>>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl<T: EntityData + Clone, D: DbLayer<T>> DataRepositoryInner<T, D> {
+    pub fn query_par(
+        &self,
+        rule: Box<dyn Fn(&T) -> bool + Send + Sync>,
+    ) -> Query<T, DataRepositoryInner<T, D>, D>
+    where
+        T: Send + Sync,
+    {
+        let mut cache: Vec<EntityHandle<T>> = self
+            .cache
+            .par_iter()
+            .enumerate()
+            .filter(|(_, entity)| rule(entity))
+            .map(|(i, entity)| EntityHandle::new(entity.id(), i, true))
+            .collect();
+
+        cache.extend(
+            self.temporal
+                .iter()
+                .enumerate()
+                .filter(|(_, entity)| rule(entity))
+                .map(|(i, entity)| EntityHandle::new(entity.id(), i, false))
+                .collect::<Vec<_>>(),
+        );
+
+        Query {
+            cache,
+            rule,
+            parallel: true,
+            _marker: PhantomData,
+            shift_stamp: self.shifts,
+        }
+    }
+
+    pub fn query(
+        &self,
+        rule: Box<dyn Fn(&T) -> bool + Send + Sync>,
+    ) -> Query<T, DataRepositoryInner<T, D>, D>
+    where
+        T: Send + Sync,
+    {
+        let mut cache: Vec<EntityHandle<T>> = self
+            .cache
+            .iter()
+            .enumerate()
+            .filter(|(_, entity)| rule(entity))
+            .map(|(i, entity)| EntityHandle::new(entity.id(), i, true))
+            .collect();
+
+        cache.extend(
+            self.temporal
+                .iter()
+                .enumerate()
+                .filter(|(_, entity)| rule(entity))
+                .map(|(i, entity)| EntityHandle::new(entity.id(), i, false))
+                .collect::<Vec<_>>(),
+        );
+
+        Query {
+            cache,
+            rule,
+            parallel: false,
+            _marker: PhantomData,
+            shift_stamp: self.shifts,
+        }
+    }
+}
+
+impl<T: EntityData + Clone, D: DbLayer<T>> Repository<T, D> for DataRepositoryInner<T, D> {
+    fn get(&self, handle: &mut EntityHandle<T>) -> Option<T> {
+        let slice = if handle.real() {
+            self.cache.as_slice()
+        } else {
+            self.temporal.as_slice()
+        };
+        let index = fast_position_of(slice, handle)?;
+        Some(slice[index].clone())
+    }
+
+    fn get_ref(&self, handle: &mut EntityHandle<T>) -> Option<&T> {
+        let slice = if handle.real() {
+            self.cache.as_slice()
+        } else {
+            self.temporal.as_slice()
+        };
+        let index = fast_position_of(slice, handle)?;
+        Some(&slice[index])
+    }
+
+    fn get_mut(&mut self, handle: &mut EntityHandle<T>) -> Option<&mut T> {
+        let slice = if handle.real() {
+            self.cache.as_mut_slice()
+        } else {
+            self.temporal.as_mut_slice()
+        };
+        let index = fast_position_of(slice, handle)?;
+        Some(&mut slice[index])
+    }
+
+    fn handle_by_id(&self, id: Id) -> EntityHandle<T> {
+        EntityHandle::new(id, id.0 as usize, true)
+    }
+
+    fn create(&mut self, mut new: T) -> EntityHandle<T> {
+        let next_temporal = self.next_temporal;
+        self.next_temporal += 1;
+        let entity = EntityHandle::new(next_temporal.into(), self.temporal.len(), false);
+        new.set_id(entity.id);
+        self.temporal.push(new);
+        entity
+    }
+
+    fn save(&mut self, entity: &mut EntityHandle<T>) -> MaybeError {
+        let original_state = entity.clone();
+        if entity.real() {
+            let data = self.get_ref(entity).unwrap();
+            self.data_layer
+                .write_entity(data, &self.connection.read())?;
+        } else {
+            let index = fast_position_of(self.temporal.as_slice(), entity).unwrap();
+            let mut data = self.temporal.remove(index);
+            self.data_layer
+                .add_entity(&mut data, &self.connection.read())?;
+            entity.set_real(true);
+            entity.id = data.id();
+            entity.set_index(self.cache.len());
+            self.query_manger.handle_save(original_state, entity, &data);
+
+            self.cache.push(data);
+            self.shifts += 1;
+        }
+
+        Ok(())
+    }
+
+    fn delete(&mut self, mut handle: EntityHandle<T>) -> MaybeError {
+        let vec = if handle.real() {
+            self.data_layer
+                .delete_entity(handle.id, &self.connection.read())?;
+            &mut self.cache
+        } else {
+            &mut self.temporal
+        };
+
+        let index = into_result(
+            fast_position_of(vec.as_slice(), &mut handle),
+            NotificationData::DbSaveError,
+        )?;
+        let value = vec.remove(index);
+        self.query_manger.handle_delete(&handle, &value);
+        self.shifts += 1;
+        Ok(())
+    }
+
+    fn init(&mut self) -> MaybeError {
+        self.data_layer
+            .load_all(&mut self.cache, &self.connection.read())?;
+        self.version += 1;
+        Ok(())
+    }
+
+    fn reload(&mut self) -> MaybeError {
+        self.cache.clear();
+        self.temporal.clear();
+        self.init()?;
+        Ok(())
+    }
+}
+impl<T: EntityData + Clone, D: BatchedDbLayer<T> + DbLayer<T>> BatchedRepository<T, D>
+    for DataRepositoryInner<T, D>
+{
+    fn save_many(&mut self, handles: &mut [EntityHandle<T>]) -> MaybeError {
+        let mut temp = Vec::with_capacity(handles.len() / 2);
+        let mut cached = Vec::with_capacity(handles.len() / 2);
+        handles.sort_by_key(|a| a.id);
+        handles.iter_mut().for_each(|handle| {
+            if handle.real() {
+                cached.push(handle)
+            } else {
+                temp.push(handle)
+            }
+        });
+
+        let mut temp_values = Vec::with_capacity(temp.len());
+        let mut cell = 0;
+        let len = temp.len();
+        self.temporal.retain(|x| {
+            if cell == len {
+                return true;
+            }
+            let find = temp[cell].id;
+            let remove = x.id() == find;
+            if remove {
+                cell += 1;
+                temp_values.push(x.clone());
+            }
+            !remove
+        });
+
+        self.data_layer
+            .add_many_entities(&mut temp_values, &self.connection.read())?;
+
+        let start = self.cache.len();
+
+        for (i, (handle, data)) in temp.into_iter().zip(&mut temp_values).enumerate() {
+            let old = handle.clone();
+            handle.id = data.id();
+            handle.set_index(start + i);
+            handle.set_real(true);
+            self.query_manger.handle_save(old, handle, data);
+        }
+
+        self.cache.extend(temp_values);
+
+        for handle in cached.into_iter() {
+            let h = handle;
+            let idx = fast_position_of(self.cache.as_slice(), h).unwrap();
+            self.data_layer
+                .write_entity(&self.cache[idx], &self.connection.read())?;
+        }
+        self.shifts += 1;
+
+        Ok(())
+    }
+
+    fn delete_many(&mut self, handles: &mut [EntityHandle<T>]) -> MaybeError {
+        let (cached, temp) = split_temp_real(handles, |e| e.id);
+
+        let mut cell = 0;
+        let len = temp.len();
+        self.temporal.retain(|x| {
+            if cell == len {
+                return true;
+            }
+            let find = temp[cell];
+            let remove = x.id() == find;
+            if remove {
+                cell += 1;
+            }
+            !remove
+        });
+        self.data_layer
+            .delete_many_entities(cached.as_slice(), &self.connection.read())?;
+        let len = cached.len();
+        cell = 0;
+        self.cache.retain(|x| {
+            if cell == len {
+                return true;
+            }
+            let find = cached[cell];
+            let remove = x.id() == find;
+            if remove {
+                cell += 1;
+            }
+
+            !remove
+        });
+        self.shifts += 1;
+
+        Ok(())
+    }
+}
+
+fn split_temp_real<T, F, V>(handles: &mut [EntityHandle<T>], getter: F) -> (Vec<V>, Vec<V>)
+where
+    F: Fn(&EntityHandle<T>) -> V,
+{
+    let mut temp = Vec::with_capacity(handles.len() / 2);
+    let mut cached = Vec::with_capacity(handles.len() / 2);
+    handles.sort_by(|a, b| a.id.cmp(&b.id));
+    handles.iter().for_each(|handle| {
+        if handle.real() {
+            cached.push(getter(handle))
+        } else {
+            temp.push(getter(handle))
+        }
+    });
+    (cached, temp)
+}
+
+pub fn fast_position_of<E: EntityData>(
+    storage: &[E],
+    handle: &mut EntityHandle<E>,
+) -> Option<usize> {
+    if storage.is_empty() {
+        return None;
+    }
+
+    let max_idx = storage.len() - 1;
+    let upper = (handle.index()).min(max_idx);
+    let peek = storage[upper].id();
+
+    if peek == handle.id {
+        handle.set_index(upper);
+        return Some(upper);
+    }
+
+    if peek.0 < handle.id.0 {
+        return None;
+    }
+
+    let target_id_val = handle.id.0 as i64;
+    let peek_val = peek.0 as i64;
+    let delta = peek_val - target_id_val;
+
+    let new_index = (upper as i64 - delta).max(0) as usize;
+
+    if storage[new_index].id() == handle.id {
+        handle.set_index(new_index);
+        return Some(new_index);
+    }
+
+    if new_index == upper {
+        return None;
+    }
+
+    let index = storage[new_index..upper]
+        .binary_search_by(|e| e.id().cmp(&handle.id))
+        .ok()?
+        + new_index;
+
+    handle.set_index(index);
+    Some(index)
+}

src/main.rs

@@ -9,6 +9,7 @@ mod navigation;
 pub mod notification;
 mod pages;
 mod styling;
+pub mod tests;
 
 use crate::app::start;
 use mimalloc::MiMalloc;

src/tests.rs

@@ -0,0 +1,123 @@
+#[cfg(test)]
+mod mod_tests {
+    use crate::data::Id;
+    use crate::data::entities::{EntityData, EntityHandle};
+    use crate::data::repositories::fast_position_of;
+
+    struct ProtoEntity {
+        id: Id,
+        _data: i128,
+    }
+    impl EntityData for ProtoEntity {
+        fn id(&self) -> Id {
+            self.id
+        }
+
+        fn set_id(&mut self, id: Id) {
+            self.id = id;
+        }
+    }
+
+    impl ProtoEntity {
+        fn new(n: i128) -> Self {
+            Self {
+                id: (n as u32).into(),
+                _data: n,
+            }
+        }
+    }
+
+    #[test]
+    fn search_direct() {
+        let mut entity = EntityHandle::new(8.into(), 3, true);
+        assert_eq!(
+            fast_position_of(&[1, 2, 3, 8, 9, 12].map(ProtoEntity::new), &mut entity),
+            Some(3)
+        );
+    }
+
+    #[test]
+    fn search_0() {
+        let mut entity = EntityHandle::new(8.into(), 4, true);
+        assert_eq!(
+            fast_position_of(&[1, 2, 3, 8, 9, 12].map(ProtoEntity::new), &mut entity),
+            Some(3)
+        );
+    }
+
+    #[test]
+    fn search_1() {
+        let mut entity = EntityHandle::new(9.into(), 5, true);
+        assert_eq!(
+            fast_position_of(&[1, 2, 3, 8, 9, 12].map(ProtoEntity::new), &mut entity),
+            Some(4)
+        );
+    }
+
+    #[test]
+    fn search_2() {
+        let mut entity = EntityHandle::new(106.into(), 25, true);
+        assert_eq!(
+            fast_position_of(
+                &[
+                    1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 20, 21, 22, 23, 105, 106, 107,
+                    108, 109, 110
+                ]
+                .map(ProtoEntity::new),
+                &mut entity
+            ),
+            Some(18)
+        );
+    }
+
+    #[test]
+    fn id_0() {
+        let entity: EntityHandle<ProtoEntity> = EntityHandle::new(8.into(), 25, true);
+        assert_eq!(entity.real(), true)
+    }
+
+    #[test]
+    fn id_1() {
+        let entity: EntityHandle<ProtoEntity> = EntityHandle::new(8.into(), 25, false);
+        assert_eq!(entity.real(), false)
+    }
+
+    #[test]
+    fn id_3() {
+        let entity: EntityHandle<ProtoEntity> = EntityHandle::new(8.into(), 25, true);
+        assert_eq!(entity.index(), 25)
+    }
+
+    #[test]
+    fn id_4() {
+        let entity: EntityHandle<ProtoEntity> = EntityHandle::new(8.into(), 25, false);
+        assert_eq!(entity.index(), 25)
+    }
+    #[test]
+    fn id_5() {
+        let mut entity: EntityHandle<ProtoEntity> = EntityHandle::new(8.into(), 25, true);
+        entity.set_real(true);
+        assert_eq!(entity.real(), true)
+    }
+
+    #[test]
+    fn id_6() {
+        let mut entity: EntityHandle<ProtoEntity> = EntityHandle::new(8.into(), 25, false);
+        entity.set_real(true);
+        assert_eq!(entity.real(), true)
+    }
+
+    #[test]
+    fn id_7() {
+        let mut entity: EntityHandle<ProtoEntity> = EntityHandle::new(8.into(), 25, true);
+        entity.set_index(0);
+        assert_eq!(entity.index(), 0)
+    }
+
+    #[test]
+    fn id_8() {
+        let mut entity: EntityHandle<ProtoEntity> = EntityHandle::new(8.into(), 25, false);
+        entity.set_index(0);
+        assert_eq!(entity.index(), 0)
+    }
+}