jap_learn/src/lang.rs

use crate::data_provider::card_stats::load_stats_of_deck;
use crate::data_provider::card_stats::update_stat_score;
use crate::data_provider::history::{push_note, HistoryItem};
use crate::AppState;
use chrono::{DateTime, Utc};
use parking_lot::Mutex;
use rand::distr::weighted::WeightedIndex;
use rand::distr::Distribution;
use rand::prelude::SliceRandom;
use rand::rng;
use rand::rngs::ThreadRng;
use rayon::iter::IndexedParallelIterator;
use rayon::iter::IntoParallelRefIterator;
use rayon::iter::ParallelIterator;
use rhai::{Engine, Scope};
use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ToSqlOutput, ValueRef};
use rusqlite::ToSql;
use std::cmp::PartialEq;
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use std::num::ParseIntError;
use std::str::FromStr;
use std::sync::Arc;
use std::time::Instant;

const MAX_HISTORY_LEN: usize = 20;
const MAX_HISTORY_LEN_PART: f32 = 0.33;
const MAX_SCORE: u8 = 25;
const FADE_PER_DAY: f32 = 0.95;
#[derive(Clone, PartialEq, Eq, Debug, Hash, Copy, Ord, PartialOrd)]
pub struct Id(u32);
pub const INVALID_ID: Id = Id(0);
impl FromStr for Id {
    type Err = ParseIntError;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        Ok(Id(s.parse::<u32>()?))
    }
}

impl From<u32> for Id {
    fn from(value: u32) -> Self {
        Id(value)
    }
}

impl From<Id> for u32 {
    fn from(value: Id) -> Self {
        value.0
    }
}
impl Display for Id {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.0)
    }
}

impl FromSql for Id {
    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
        match value {
            ValueRef::Integer(i) => Ok(Id(i as u32)),
            _ => Err(FromSqlError::InvalidType),
        }
    }
}

impl ToSql for Id {
    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self.0 as i64))
    }
}

impl From<Id> for String {
    fn from(value: Id) -> Self {
        value.0.to_string()
    }
}

impl Id {
    pub(crate) fn is_valid(self) -> bool {
        self.0 != 0
    }
}

#[derive(Clone, Debug)]
pub struct KanaSet {
    name: String,
    pub(crate) chars_type: KanaType,
    pub(crate) dictionary: Vec<Vec<(String, String)>>,
    pub(crate) include_map: [bool; 10],
}
#[derive(Clone, Debug)]
pub enum KanaType {
    Hiragana,
    Katakana,
}
impl KanaSet {
    pub fn hiragana() -> Self {
        Self {
            name: "Хирагана".to_string(),
            chars_type: KanaType::Hiragana,
            dictionary: vec![
                vec![
                    (String::from("ぁ"), String::from("a")),
                    (String::from("ぃ"), String::from("i")),
                    (String::from("ぅ"), String::from("u")),
                    (String::from("ぇ"), String::from("e")),
                    (String::from("ぉ"), String::from("o")),
                ],
                vec![
                    (String::from("さ"), String::from("sa")),
                    (String::from("し"), String::from("shi")),
                    (String::from("す"), String::from("su")),
                    (String::from("せ"), String::from("se")),
                    (String::from("そ"), String::from("so")),
                ],
                vec![
                    (String::from("か"), String::from("ka")),
                    (String::from("き"), String::from("ki")),
                    (String::from("く"), String::from("ku")),
                    (String::from("け"), String::from("ke")),
                    (String::from("こ"), String::from("ko")),
                ],
                // Ряд «та» (たちつてと)
                vec![
                    (String::from("た"), String::from("ta")),
                    (String::from("ち"), String::from("chi")),
                    (String::from("つ"), String::from("tsu")),
                    (String::from("て"), String::from("te")),
                    (String::from("と"), String::from("to")),
                ],
                // Ряд «на» (なにぬねの)
                vec![
                    (String::from("な"), String::from("na")),
                    (String::from("に"), String::from("ni")),
                    (String::from("ぬ"), String::from("nu")),
                    (String::from("ね"), String::from("ne")),
                    (String::from("の"), String::from("no")),
                ],
                // Ряд «ха» (はひふへほ)
                vec![
                    (String::from("は"), String::from("ha")),
                    (String::from("ひ"), String::from("hi")),
                    (String::from("ふ"), String::from("fu")),
                    (String::from("へ"), String::from("he")),
                    (String::from("ほ"), String::from("ho")),
                ],
                // Ряд «ма» (まみむめも)
                vec![
                    (String::from("ま"), String::from("ma")),
                    (String::from("み"), String::from("mi")),
                    (String::from("む"), String::from("mu")),
                    (String::from("め"), String::from("me")),
                    (String::from("も"), String::from("mo")),
                ],
                // Ряд «я» (やゆよ) — только 3 символа
                vec![
                    (String::from("や"), String::from("ya")),
                    (String::from("ゆ"), String::from("yu")),
                    (String::from("よ"), String::from("yo")),
                ],
                // Ряд «ра» (らりるれろ)
                vec![
                    (String::from("ら"), String::from("ra")),
                    (String::from("り"), String::from("ri")),
                    (String::from("る"), String::from("ru")),
                    (String::from("れ"), String::from("re")),
                    (String::from("ろ"), String::from("ro")),
                ],
                vec![
                    (String::from("わ"), String::from("wa")),
                    (String::from("を"), String::from("wo")),
                    (String::from("ん"), String::from("n")),
                ],
            ],
            include_map: [true; 10],
        }
    }

    pub fn katakana() -> Self {
        Self {
            name: "Катакана".to_string(),
            chars_type: KanaType::Katakana,
            dictionary: vec![
                vec![
                    (String::from("ァ"), String::from("a")),
                    (String::from("ィ"), String::from("i")),
                    (String::from("ゥ"), String::from("u")),
                    (String::from("ェ"), String::from("e")),
                    (String::from("ォ"), String::from("o")),
                ],
                vec![
                    (String::from("サ"), String::from("sa")),
                    (String::from("シ"), String::from("shi")),
                    (String::from("ス"), String::from("su")),
                    (String::from("セ"), String::from("se")),
                    (String::from("ソ"), String::from("so")),
                ],
                vec![
                    (String::from("カ"), String::from("ka")),
                    (String::from("キ"), String::from("ki")),
                    (String::from("ク"), String::from("ku")),
                    (String::from("ケ"), String::from("ke")),
                    (String::from("コ"), String::from("ko")),
                ],
                // Ряд «та» (タチツテト)
                vec![
                    (String::from("タ"), String::from("ta")),
                    (String::from("チ"), String::from("chi")),
                    (String::from("ツ"), String::from("tsu")),
                    (String::from("テ"), String::from("te")),
                    (String::from("ト"), String::from("to")),
                ],
                // Ряд «на» (ナニヌネノ)
                vec![
                    (String::from("ナ"), String::from("na")),
                    (String::from("ニ"), String::from("ni")),
                    (String::from("ヌ"), String::from("nu")),
                    (String::from("ネ"), String::from("ne")),
                    (String::from("ノ"), String::from("no")),
                ],
                // Ряд «ха» (ハヒフヘホ)
                vec![
                    (String::from("ハ"), String::from("ha")),
                    (String::from("ヒ"), String::from("hi")),
                    (String::from("フ"), String::from("fu")),
                    (String::from("ヘ"), String::from("he")),
                    (String::from("ホ"), String::from("ho")),
                ],
                // Ряд «ма» (マミムメモ)
                vec![
                    (String::from("マ"), String::from("ma")),
                    (String::from("ミ"), String::from("mi")),
                    (String::from("ム"), String::from("mu")),
                    (String::from("メ"), String::from("me")),
                    (String::from("モ"), String::from("mo")),
                ],
                // Ряд «я» (ヤユヨ) — только 3 символа
                vec![
                    (String::from("ヤ"), String::from("ya")),
                    (String::from("ユ"), String::from("yu")),
                    (String::from("ヨ"), String::from("yo")),
                ],
                // Ряд «ра» (ラリルレロ)
                vec![
                    (String::from("ラ"), String::from("ra")),
                    (String::from("リ"), String::from("ri")),
                    (String::from("ル"), String::from("ru")),
                    (String::from("レ"), String::from("re")),
                    (String::from("ロ"), String::from("ro")),
                ],
                vec![
                    (String::from("ワ"), String::from("wa")),
                    (String::from("ヲ"), String::from("wo")),
                    (String::from("ン"), String::from("n")),
                ],
            ],
            include_map: [true; 10],
        }
    }

    pub fn list(&self) -> Vec<(String, String)> {
        let mut current_set: Vec<(String, String)> = Vec::new();

        for i in 0..10 {
            if self.include_map[i] {
                self.dictionary[i].iter().for_each(|v| {
                    current_set.push(v.clone());
                });
            }
        }

        current_set
    }
}
impl Default for KanaSet {
    fn default() -> Self {
        KanaSet::hiragana()
    }
}
impl PartialEq<Self> for KanaSet {
    fn eq(&self, other: &Self) -> bool {
        self.name == other.name
    }
}
#[derive(Clone, Debug)]
pub struct WordData {
    pub id: Id,
    pub key: String,
    pub value: String,
    pub tags: String,
    pub additional: HashMap<String, String>,
    pub group_id: Id,
}
impl WordData {
    pub fn new() -> Self {
        Self {
            id: INVALID_ID,
            key: String::new(),
            value: String::new(),
            tags: String::new(),
            additional: Default::default(),
            group_id: 1.into(),
        }
    }
}
#[derive(Clone)]
pub struct WordGroup {
    pub id: Id,
    pub name: String,
}
#[derive(Clone, PartialEq)]
pub struct CardStatistics {
    pub id: Id,
    pub word_id: Id,
    pub set_id: Id,
    pub last_open: DateTime<Utc>,
    pub score: u8,
}
impl CardStatistics {
    pub fn update(&mut self, status: WordOpenMode) {
        match status {
            WordOpenMode::Easy => {
                self.score = (self.calculated_score() + 5.0).round() as u8;
            }
            WordOpenMode::Ok => self.score = (self.calculated_score() + 2.0).round() as u8,
            WordOpenMode::Hard => {
                self.score = (self.calculated_score() * 0.75).round() as u8;
            }
            WordOpenMode::None => {
                self.score = (self.calculated_score() * 0.4) as u8;
            }
        }

        self.score = self.score.clamp(1, MAX_SCORE);
        self.last_open = Utc::now();
    }

    pub fn calculated_score(&self) -> f32 {
        let time = Utc::now() - self.last_open;
        let days = time.num_days();
        let multiplier = FADE_PER_DAY.powi(days as i32);
        (self.score as f32 * multiplier).max(1.0)
    }
}
#[derive(Clone, Copy)]
pub enum WordOpenMode {
    Easy,
    Ok,
    Hard,
    None,
}
#[derive(Clone)]
pub struct DeckData {
    words: Vec<WordData>,
    set: Vec<CardStatistics>,
    current_word_index: Option<usize>,
    state: Arc<Mutex<AppState>>,
    order_module: OrderModule,
    settings: DeckSettings,
}
impl DeckData {
    pub fn new(settings: &DeckSettings, state: Arc<Mutex<AppState>>) -> Self {
        let state_for = state.clone();
        let state_locked = state.lock();

        let stats = load_stats_of_deck(settings, &state_locked.connection);
        let words = stats
            .iter()
            .map(|stat| {
                state_locked
                    .dictionary
                    .binary_search_by_key(&stat.word_id, |w| w.id)
                    .unwrap()
            })
            .map(|index| &state_locked.dictionary[index])
            .cloned()
            .collect();

        Self {
            set: stats,
            words,
            current_word_index: None,
            state: state_for,
            order_module: match settings.open_mode {
                OrderMode::Default => OrderModule::SemiRandomSRS(SemiRandomSRSModule::new()),
                OrderMode::TrainWorstFirst => {
                    OrderModule::WorstWordsSRS(WorstWordsSRSModule::new())
                }
                OrderMode::FullRandom => OrderModule::RandomSRS(RandomSRSModule::new()),
            },
            settings: settings.clone(),
        }
    }

    pub fn next(&mut self) -> (WordData, CardStatistics) {
        let index = match self.order_module.clone() {
            OrderModule::SemiRandomSRS(mut module) => {
                if !module.initialized {
                    module.init(self)
                }
                let index = module.next(self);
                self.order_module = OrderModule::SemiRandomSRS(module);
                index
            }
            OrderModule::RandomSRS(mut module) => {
                if !module.initialized {
                    module.init(self)
                }
                let index = module.next(self);
                self.order_module = OrderModule::RandomSRS(module);
                index
            }
            OrderModule::WorstWordsSRS(mut module) => {
                if !module.initialized {
                    module.init(self)
                }
                let index = module.next(self);
                self.order_module = OrderModule::WorstWordsSRS(module);
                index
            }
        };

        self.current_word_index = Some(index);
        (self.words[index].clone(), self.set[index].clone())
    }
    pub fn open(&mut self, status: WordOpenMode) {
        if self.current_word_index.is_none() {
            return;
        }
        let index = self.current_word_index.unwrap();

        let word = &mut self.set[index];
        let old_score = word.score;
        word.update(status);
        let new_score = word.score;

        match self.order_module.clone() {
            OrderModule::SemiRandomSRS(mut module) => {
                module.open(status, index, word.clone());
                self.order_module = OrderModule::SemiRandomSRS(module);
            }
            OrderModule::RandomSRS(mut module) => {
                module.open(status, index, word.clone());
                self.order_module = OrderModule::RandomSRS(module);
            }
            OrderModule::WorstWordsSRS(mut module) => {
                module.open(status, index, word.clone());
                self.order_module = OrderModule::WorstWordsSRS(module);
            }
        }
        update_stat_score(word, &self.state.lock().connection);
        push_note(
            self.settings.id,
            HistoryItem {
                timestamp: Utc::now(),
                word_id: word.word_id,
                mode: status,
                before: old_score,
                after: new_score,
            },
        )
    }
    pub fn len(&self) -> usize {
        self.set.len()
    }
}
#[derive(Clone, PartialEq, Copy, Eq)]
pub enum OrderMode {
    Default,
    TrainWorstFirst,
    FullRandom,
}
#[derive(Clone, Copy, Eq, PartialEq, Default)]
pub enum AppendMode {
    Full,
    #[default]
    Manual,
}
#[derive(Clone)]
enum OrderModule {
    SemiRandomSRS(SemiRandomSRSModule),
    RandomSRS(RandomSRSModule),
    WorstWordsSRS(WorstWordsSRSModule),
}
trait SRSModule {
    fn next(&mut self, set: &mut DeckData) -> usize;
    fn open(&mut self, status: WordOpenMode, index: usize, updated_word: CardStatistics);
    fn init(&mut self, set: &mut DeckData);
}
#[derive(Clone)]
struct RandomSRSModule {
    basket: Vec<usize>,
    initialized: bool,
}
impl RandomSRSModule {
    fn new() -> RandomSRSModule {
        Self {
            basket: vec![],
            initialized: false,
        }
    }
}
impl SRSModule for RandomSRSModule {
    fn next(&mut self, set: &mut DeckData) -> usize {
        match self.basket.pop() {
            Some(index) => index,
            None => {
                self.basket = (0..set.words.len()).collect::<Vec<usize>>();
                self.basket.shuffle(&mut rand::rng());
                self.basket.pop().expect("empty basket")
            }
        }
    }

    fn open(&mut self, _: WordOpenMode, _: usize, _: CardStatistics) {}

    fn init(&mut self, set: &mut DeckData) {
        self.initialized = true;
        self.basket = (0..set.set.len()).collect::<Vec<usize>>();
        self.basket.shuffle(&mut rand::rng())
    }
}
#[derive(Clone)]
struct SemiRandomSRSModule {
    history: Vec<usize>,
    last_weights: WeightedIndex<f32>,
    generator: ThreadRng,
    initialized: bool,
}
impl SemiRandomSRSModule {
    fn new() -> SemiRandomSRSModule {
        SemiRandomSRSModule {
            history: vec![],
            last_weights: WeightedIndex::new([1.0]).unwrap(),
            generator: rng(),
            initialized: false,
        }
    }
}
impl SRSModule for SemiRandomSRSModule {
    fn next(&mut self, set: &mut DeckData) -> usize {
        let index = self.last_weights.sample(&mut self.generator);

        if self.history.contains(&index) && self.history.len() != 1 {
            return self.next(set);
        }

        if self.history.len() == self.history_len(set) {
            self.history.remove(0);
        }
        self.history.push(index);

        index
    }
    fn open(&mut self, _: WordOpenMode, index: usize, word: CardStatistics) {
        let new_weight = (100.0 / word.calculated_score()).powf(2.0);
        self.last_weights
            .update_weights(&[(index, &new_weight)])
            .expect("update weights error");
    }
    fn init(&mut self, set: &mut DeckData) {
        self.initialized = true;
        let weights = set
            .set
            .iter()
            .map(|s| (100.0 / s.calculated_score()).powf(2.0) * 2.0)
            .collect::<Vec<f32>>();
        self.last_weights = WeightedIndex::new(weights).unwrap();
    }
}
impl SemiRandomSRSModule {
    fn history_len(&self, set: &DeckData) -> usize {
        ((set.len() as f32 * MAX_HISTORY_LEN_PART) as usize).clamp(1, MAX_HISTORY_LEN)
    }
}
#[derive(Clone)]
struct WorstWordsSRSModule {
    initialized: bool,
    pool_size: usize,
    rounds_remaining: u8,
    pool: Vec<usize>,
    queue: Vec<usize>,
    rounds_count: u8,
}
impl SRSModule for WorstWordsSRSModule {
    fn next(&mut self, set: &mut DeckData) -> usize {
        if self.rounds_remaining == 0 {
            self.fill_pool(set);
            self.rounds_remaining = self.rounds_count;
        }

        match self.queue.pop() {
            Some(index) => index,
            None => {
                self.queue.append(&mut self.pool.clone());
                self.rounds_remaining -= 1;
                self.queue.pop().expect("empty queue")
            }
        }
    }

    fn open(&mut self, _: WordOpenMode, _: usize, _: CardStatistics) {}

    fn init(&mut self, _: &mut DeckData) {
        self.initialized = true;
    }
}
impl WorstWordsSRSModule {
    fn new() -> WorstWordsSRSModule {
        WorstWordsSRSModule {
            initialized: false,
            pool_size: 15,
            rounds_count: 2,
            rounds_remaining: 0,
            pool: vec![],
            queue: vec![],
        }
    }
    fn fill_pool(&mut self, set: &DeckData) {
        let mut sorted = set
            .set
            .clone()
            .into_iter()
            .zip(0..set.set.len())
            .collect::<Vec<_>>();
        sorted.sort_by_key(|c| c.0.score);
        let mut worst = sorted[0..self.pool_size]
            .iter()
            .map(|(_, index)| *index)
            .collect::<Vec<usize>>();
        worst.shuffle(&mut rand::rng());
        self.pool = worst;
    }
}
#[derive(Clone)]
pub struct DeckSettings {
    pub id: Id,
    pub name: String,
    pub forward: String,
    pub backward: String,
    pub filter: String,
    pub count: Option<usize>,
    pub worst_words_list: Option<Vec<WordData>>,
    pub open_mode: OrderMode,
}
impl DeckSettings {
    pub(crate) fn with_name(name: String) -> DeckSettings {
        DeckSettings {
            id: 0.into(),
            name,
            forward: "".to_string(),
            backward: "".to_string(),
            filter: "true".to_string(),
            count: None,
            worst_words_list: None,
            open_mode: OrderMode::Default,
        }
    }
    pub fn check_filter(&self) -> bool {
        let time = Instant::now();
        let engine = Engine::new();
        let ast = engine.compile(&self.filter);
        println!("Compile time is {:?}", time.elapsed());
        ast.is_ok()
    }
    pub fn get_word_list(&self, state: &AppState) -> Vec<usize> {
        let time = Instant::now();
        let mut list = vec![];
        let engine = Engine::new();

        let ast = engine.compile(&self.filter);
        if let Ok(ast) = ast {
            let groups = &state.word_groups;

            list = state
                .dictionary
                .par_iter()
                .enumerate()
                .filter(|(_, word)| {
                    let mut more = rhai::Map::new();
                    for iced in &word.additional {
                        more.insert(iced.0.clone().into(), iced.1.clone().into());
                    }
                    let mut scope = Scope::new();
                    scope
                        .push_constant("id", word.id.0 as i64)
                        .push_constant("key", word.key.clone())
                        .push_constant("value", word.value.clone())
                        .push_constant("tags", word.tags.clone())
                        .push_constant("more", more)
                        .push_constant(
                            "group",
                            groups
                                .iter()
                                .find(|g| g.id == word.group_id)
                                .cloned()
                                .unwrap()
                                .name,
                        );

                    let result = engine.eval_ast_with_scope::<bool>(&mut scope, &ast);
                    result.is_ok() && result.unwrap()
                })
                .map(|(index, _)| index)
                .collect();

            println!("Collecting available words is {:?}", time.elapsed());

            list
        } else {
            list
        }
    }
    pub fn require_speech(&self) -> bool {
        self.forward == "speech" || self.backward == "speech"
    }
}