jap_learn/src/lang.rs

use crate::data_provider::card_stats::load_stats_of_deck;
use crate::data_provider::card_stats::update_stat_score;
use crate::data_provider::history::{push_note, HistoryItem};
use crate::app::AppState;
use chrono::{DateTime, Utc};
use parking_lot::Mutex;
use rand::distr::weighted::WeightedIndex;
use rand::distr::Distribution;
use rand::prelude::SliceRandom;
use rand::rng;
use rand::rngs::ThreadRng;
use rayon::iter::IndexedParallelIterator;
use rayon::iter::IntoParallelRefIterator;
use rayon::iter::ParallelIterator;
use rhai::{Engine, Scope};
use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ToSqlOutput, ValueRef};
use rusqlite::ToSql;
use std::cmp::PartialEq;
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use std::num::ParseIntError;
use std::str::FromStr;
use std::sync::Arc;
use std::time::Instant;
use log::info;

const MAX_HISTORY_LEN: usize = 20;
const MAX_HISTORY_LEN_PART: f32 = 0.33;
const MAX_SCORE: u8 = 25;
const FADE_PER_DAY: f32 = 0.95;
#[derive(Clone, PartialEq, Eq, Debug, Hash, Copy, Ord, PartialOrd)]
pub struct Id(u32);
pub const INVALID_ID: Id = Id(0);
impl FromStr for Id {
    type Err = ParseIntError;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        Ok(Id(s.parse::<u32>()?))
    }
}

impl From<u32> for Id {
    fn from(value: u32) -> Self {
        Id(value)
    }
}

impl From<Id> for u32 {
    fn from(value: Id) -> Self {
        value.0
    }
}
impl Display for Id {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.0)
    }
}

impl FromSql for Id {
    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
        match value {
            ValueRef::Integer(i) => Ok(Id(i as u32)),
            _ => Err(FromSqlError::InvalidType),
        }
    }
}

impl ToSql for Id {
    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self.0 as i64))
    }
}

impl From<Id> for String {
    fn from(value: Id) -> Self {
        value.0.to_string()
    }
}

impl Id {
    pub(crate) fn is_valid(self) -> bool {
        self.0 != 0
    }
}


#[derive(Clone, Debug)]
pub struct WordData {
    pub id: Id,
    pub key: String,
    pub value: String,
    pub tags: String,
    pub additional: HashMap<String, String>,
    pub group_id: Id,
}
impl WordData {
    pub fn new() -> Self {
        Self {
            id: INVALID_ID,
            key: String::new(),
            value: String::new(),
            tags: String::new(),
            additional: Default::default(),
            group_id: 1.into(),
        }
    }
}
#[derive(Clone)]
pub struct WordGroup {
    pub id: Id,
    pub name: String,
}
#[derive(Clone, PartialEq)]
pub struct CardStatistics {
    pub id: Id,
    pub word_id: Id,
    pub set_id: Id,
    pub last_open: DateTime<Utc>,
    pub score: u8,
}
impl CardStatistics {
    pub fn update(&mut self, status: WordOpenMode) {
        match status {
            WordOpenMode::Easy => {
                self.score = (self.calculated_score() + 5.0).round() as u8;
            }
            WordOpenMode::Ok => self.score = (self.calculated_score() + 2.0).round() as u8,
            WordOpenMode::Hard => {
                self.score = (self.calculated_score() * 0.75).round() as u8;
            }
            WordOpenMode::None => {
                self.score = (self.calculated_score() * 0.4) as u8;
            }
        }

        self.score = self.score.clamp(1, MAX_SCORE);
        self.last_open = Utc::now();
    }

    pub fn calculated_score(&self) -> f32 {
        let time = Utc::now() - self.last_open;
        let days = time.num_days();
        let multiplier = FADE_PER_DAY.powi(days as i32);
        (self.score as f32 * multiplier).max(1.0)
    }
}
#[derive(Clone, Copy)]
pub enum WordOpenMode {
    Easy,
    Ok,
    Hard,
    None,
}
#[derive(Clone)]
pub struct DeckData {
    words: Vec<WordData>,
    set: Vec<CardStatistics>,
    current_word_index: Option<usize>,
    state: Arc<Mutex<AppState>>,
    order_module: OrderModule,
    settings: DeckSettings,
}
impl DeckData {
    pub fn new(settings: &DeckSettings, state: Arc<Mutex<AppState>>) -> Self {
        let state_for = state.clone();
        let state_locked = state.lock();

        let stats = load_stats_of_deck(settings, &state_locked.connection);
        let words = stats
            .iter()
            .map(|stat| {
                state_locked
                    .dictionary
                    .binary_search_by_key(&stat.word_id, |w| w.id)
                    .unwrap()
            })
            .map(|index| &state_locked.dictionary[index])
            .cloned()
            .collect();

        Self {
            set: stats,
            words,
            current_word_index: None,
            state: state_for,
            order_module: match settings.open_mode {
                OrderMode::Default => OrderModule::SemiRandomSRS(SemiRandomSRSModule::new()),
                OrderMode::TrainWorstFirst => {
                    OrderModule::WorstWordsSRS(WorstWordsSRSModule::new())
                }
                OrderMode::FullRandom => OrderModule::RandomSRS(RandomSRSModule::new()),
            },
            settings: settings.clone(),
        }
    }

    pub fn next(&mut self) -> (WordData, CardStatistics) {
        let index = match self.order_module.clone() {
            OrderModule::SemiRandomSRS(mut module) => {
                if !module.initialized {
                    module.init(self)
                }
                let index = module.next(self);
                self.order_module = OrderModule::SemiRandomSRS(module);
                index
            }
            OrderModule::RandomSRS(mut module) => {
                if !module.initialized {
                    module.init(self)
                }
                let index = module.next(self);
                self.order_module = OrderModule::RandomSRS(module);
                index
            }
            OrderModule::WorstWordsSRS(mut module) => {
                if !module.initialized {
                    module.init(self)
                }
                let index = module.next(self);
                self.order_module = OrderModule::WorstWordsSRS(module);
                index
            }
        };

        self.current_word_index = Some(index);
        (self.words[index].clone(), self.set[index].clone())
    }
    pub fn open(&mut self, status: WordOpenMode) {
        if self.current_word_index.is_none() {
            return;
        }
        let index = self.current_word_index.unwrap();

        let word : &mut CardStatistics = &mut self.set[index];
        let old_score = word.score;
        word.update(status);
        let new_score = word.score;

        match self.order_module {
            OrderModule::SemiRandomSRS(ref mut module) => {
                module.open(status, index, word);
            }
            OrderModule::RandomSRS(ref mut module) => {
                module.open(status, index, word);
            }
            OrderModule::WorstWordsSRS(ref mut module) => {
                module.open(status, index, word);
            }
        }
        update_stat_score(word, &self.state.lock().connection);
        let _ = push_note(
            self.settings.id,
            HistoryItem {
                timestamp: Utc::now(),
                word_id: word.word_id,
                mode: status,
                before: old_score,
                after: new_score,
            },
        );
    }
    pub fn len(&self) -> usize {
        self.set.len()
    }
}
#[derive(Clone, PartialEq, Copy, Eq)]
pub enum OrderMode {
    Default,
    TrainWorstFirst,
    FullRandom,
}
#[derive(Clone, Copy, Eq, PartialEq, Default)]
pub enum AppendMode {
    Full,
    #[default]
    Manual,
}
#[derive(Clone)]
enum OrderModule {
    SemiRandomSRS(SemiRandomSRSModule),
    RandomSRS(RandomSRSModule),
    WorstWordsSRS(WorstWordsSRSModule),
}
trait SRSModule {
    fn next(&mut self, set: &mut DeckData) -> usize;
    fn open(&mut self, status: WordOpenMode, index: usize, updated_word: &CardStatistics);
    fn init(&mut self, set: &mut DeckData);
}
#[derive(Clone)]
struct RandomSRSModule {
    basket: Vec<usize>,
    initialized: bool,
}
impl RandomSRSModule {
    fn new() -> RandomSRSModule {
        Self {
            basket: vec![],
            initialized: false,
        }
    }
}
impl SRSModule for RandomSRSModule {
    fn next(&mut self, set: &mut DeckData) -> usize {
        match self.basket.pop() {
            Some(index) => index,
            None => {
                self.basket = (0..set.words.len()).collect::<Vec<usize>>();
                self.basket.shuffle(&mut rand::rng());
                self.basket.pop().expect("empty basket")
            }
        }
    }

    fn open(&mut self, _: WordOpenMode, _: usize, _: &CardStatistics) {}

    fn init(&mut self, set: &mut DeckData) {
        self.initialized = true;
        self.basket = (0..set.set.len()).collect::<Vec<usize>>();
        self.basket.shuffle(&mut rand::rng())
    }
}
#[derive(Clone)]
struct SemiRandomSRSModule {
    history: Vec<usize>,
    last_weights: WeightedIndex<f32>,
    generator: ThreadRng,
    initialized: bool,
}
impl SemiRandomSRSModule {
    fn new() -> SemiRandomSRSModule {
        SemiRandomSRSModule {
            history: vec![],
            last_weights: WeightedIndex::new([1.0]).unwrap(),
            generator: rng(),
            initialized: false,
        }
    }
}
impl SRSModule for SemiRandomSRSModule {
    fn next(&mut self, set: &mut DeckData) -> usize {
        let index = self.last_weights.sample(&mut self.generator);

        if self.history.contains(&index) && self.history.len() != 1 {
            return self.next(set);
        }

        if self.history.len() == self.history_len(set) {
            self.history.remove(0);
        }
        self.history.push(index);

        index
    }
    fn open(&mut self, _: WordOpenMode, index: usize, word: &CardStatistics) {
        let new_weight = (100.0 / word.calculated_score()).powf(2.0);
        self.last_weights
            .update_weights(&[(index, &new_weight)])
            .expect("update weights error");
    }
    fn init(&mut self, set: &mut DeckData) {
        self.initialized = true;
        let weights = set
            .set
            .iter()
            .map(|s| (100.0 / s.calculated_score()).powf(2.0) * 2.0)
            .collect::<Vec<f32>>();
        self.last_weights = WeightedIndex::new(weights).unwrap();
    }
}
impl SemiRandomSRSModule {
    fn history_len(&self, set: &DeckData) -> usize {
        ((set.len() as f32 * MAX_HISTORY_LEN_PART) as usize).clamp(1, MAX_HISTORY_LEN)
    }
}
#[derive(Clone)]
struct WorstWordsSRSModule {
    initialized: bool,
    pool_size: usize,
    rounds_remaining: u8,
    pool: Vec<usize>,
    queue: Vec<usize>,
    rounds_count: u8,
}
impl SRSModule for WorstWordsSRSModule {
    fn next(&mut self, set: &mut DeckData) -> usize {
        if self.rounds_remaining == 0 {
            self.fill_pool(set);
            self.rounds_remaining = self.rounds_count;
        }

        match self.queue.pop() {
            Some(index) => index,
            None => {
                self.queue.append(&mut self.pool.clone());
                self.rounds_remaining -= 1;
                self.queue.pop().expect("empty queue")
            }
        }
    }

    fn open(&mut self, _: WordOpenMode, _: usize, _: &CardStatistics) {}

    fn init(&mut self, _: &mut DeckData) {
        self.initialized = true;
    }
}
impl WorstWordsSRSModule {
    fn new() -> WorstWordsSRSModule {
        WorstWordsSRSModule {
            initialized: false,
            pool_size: 15,
            rounds_count: 2,
            rounds_remaining: 0,
            pool: vec![],
            queue: vec![],
        }
    }
    fn fill_pool(&mut self, set: &DeckData) {
        let mut sorted = set
            .set
            .clone()
            .into_iter()
            .zip(0..set.set.len())
            .collect::<Vec<_>>();
        sorted.sort_by_key(|c| c.0.score);
        let mut worst = sorted[0..self.pool_size]
            .iter()
            .map(|(_, index)| *index)
            .collect::<Vec<usize>>();
        worst.shuffle(&mut rand::rng());
        self.pool = worst;
    }
}
#[derive(Clone)]
pub struct DeckSettings {
    pub id: Id,
    pub name: String,
    pub forward: String,
    pub backward: String,
    pub filter: String,
    pub count: Option<usize>,
    pub worst_words_list: Option<Vec<WordData>>,
    pub open_mode: OrderMode,
}
impl DeckSettings {
    pub(crate) fn with_name(name: String) -> DeckSettings {
        DeckSettings {
            id: 0.into(),
            name,
            forward: "".to_string(),
            backward: "".to_string(),
            filter: "true".to_string(),
            count: None,
            worst_words_list: None,
            open_mode: OrderMode::Default,
        }
    }
    pub fn check_filter(&self) -> bool {
        let time = Instant::now();
        let engine = Engine::new();
        let ast = engine.compile(&self.filter);
        info!("Compile time is {:?}", time.elapsed());
        ast.is_ok()
    }
    pub fn get_word_list(&self, state: &AppState) -> Vec<usize> {
        let time = Instant::now();
        let mut list = vec![];
        let engine = Engine::new();

        let ast = engine.compile(&self.filter);
        if let Ok(ast) = ast {
            let groups = &state.word_groups;

            list = state
                .dictionary
                .par_iter()
                .enumerate()
                .filter(|(_, word)| {
                    let mut more = rhai::Map::new();
                    for iced in &word.additional {
                        more.insert(iced.0.clone().into(), iced.1.clone().into());
                    }
                    let mut scope = Scope::new();
                    scope
                        .push_constant("id", word.id.0 as i64)
                        .push_constant("key", word.key.clone())
                        .push_constant("value", word.value.clone())
                        .push_constant("tags", word.tags.clone())
                        .push_constant("more", more)
                        .push_constant(
                            "group",
                            groups
                                .iter()
                                .find(|g| g.id == word.group_id)
                                .cloned()
                                .unwrap()
                                .name,
                        );

                    let result = engine.eval_ast_with_scope::<bool>(&mut scope, &ast);
                    result.is_ok() && result.unwrap()
                })
                .map(|(index, _)| index)
                .collect();

            info!("Collecting available words is {:?}", time.elapsed());

            list
        } else {
            list
        }
    }
    pub fn require_speech(&self) -> bool {
        self.forward == "speech" || self.backward == "speech"
    }
}