anki/rslib/src/notes.rs

// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html

use crate::backend_proto::note_is_duplicate_or_empty_out::State as DuplicateState;
use crate::{
    backend_proto as pb,
    collection::Collection,
    decks::DeckID,
    define_newtype,
    err::{AnkiError, Result},
    notetype::{CardGenContext, NoteField, NoteType, NoteTypeID},
    text::{ensure_string_in_nfc, strip_html_preserving_image_filenames},
    timestamp::TimestampSecs,
    types::Usn,
};
use itertools::Itertools;
use num_integer::Integer;
use regex::{Regex, Replacer};
use std::{
    borrow::Cow,
    collections::{HashMap, HashSet},
    convert::TryInto,
};

define_newtype!(NoteID, i64);

// fixme: ensure nulls and x1f not in field contents

#[derive(Default)]
pub(crate) struct TransformNoteOutput {
    pub changed: bool,
    pub generate_cards: bool,
    pub mark_modified: bool,
}

#[derive(Debug, PartialEq)]
pub struct Note {
    pub id: NoteID,
    pub guid: String,
    pub ntid: NoteTypeID,
    pub mtime: TimestampSecs,
    pub usn: Usn,
    pub tags: Vec<String>,
    pub(crate) fields: Vec<String>,
    pub(crate) sort_field: Option<String>,
    pub(crate) checksum: Option<u32>,
}

impl Note {
    pub(crate) fn new(notetype: &NoteType) -> Self {
        Note {
            id: NoteID(0),
            guid: guid(),
            ntid: notetype.id,
            mtime: TimestampSecs(0),
            usn: Usn(0),
            tags: vec![],
            fields: vec!["".to_string(); notetype.fields.len()],
            sort_field: None,
            checksum: None,
        }
    }

    pub fn fields(&self) -> &Vec<String> {
        &self.fields
    }

    pub fn set_field(&mut self, idx: usize, text: impl Into<String>) -> Result<()> {
        if idx >= self.fields.len() {
            return Err(AnkiError::invalid_input(
                "field idx out of range".to_string(),
            ));
        }

        self.fields[idx] = text.into();

        Ok(())
    }

    /// Prepare note for saving to the database. Does not mark it as modified.
    pub fn prepare_for_update(&mut self, nt: &NoteType, normalize_text: bool) -> Result<()> {
        assert!(nt.id == self.ntid);
        if nt.fields.len() != self.fields.len() {
            return Err(AnkiError::invalid_input(format!(
                "note has {} fields, expected {}",
                self.fields.len(),
                nt.fields.len()
            )));
        }

        if normalize_text {
            for field in &mut self.fields {
                ensure_string_in_nfc(field);
            }
        }

        let field1_nohtml = strip_html_preserving_image_filenames(&self.fields()[0]);
        let checksum = field_checksum(field1_nohtml.as_ref());
        let sort_field = if nt.config.sort_field_idx == 0 {
            field1_nohtml
        } else {
            strip_html_preserving_image_filenames(
                self.fields
                    .get(nt.config.sort_field_idx as usize)
                    .map(AsRef::as_ref)
                    .unwrap_or(""),
            )
        };
        self.sort_field = Some(sort_field.into());
        self.checksum = Some(checksum);
        Ok(())
    }

    pub(crate) fn set_modified(&mut self, usn: Usn) {
        self.mtime = TimestampSecs::now();
        self.usn = usn;
    }

    pub(crate) fn nonempty_fields<'a>(&self, fields: &'a [NoteField]) -> HashSet<&'a str> {
        self.fields
            .iter()
            .enumerate()
            .filter_map(|(ord, s)| {
                if s.trim().is_empty() {
                    None
                } else {
                    fields.get(ord).map(|f| f.name.as_str())
                }
            })
            .collect()
    }

    pub(crate) fn fields_map<'a>(
        &'a self,
        fields: &'a [NoteField],
    ) -> HashMap<&'a str, Cow<'a, str>> {
        self.fields
            .iter()
            .enumerate()
            .map(|(ord, field_content)| {
                (
                    fields.get(ord).map(|f| f.name.as_str()).unwrap_or(""),
                    field_content.as_str().into(),
                )
            })
            .collect()
    }

    pub(crate) fn replace_tags<T: Replacer>(&mut self, re: &Regex, mut repl: T) -> bool {
        let mut changed = false;
        for tag in &mut self.tags {
            if let Cow::Owned(rep) = re.replace_all(tag, repl.by_ref()) {
                *tag = rep;
                changed = true;
            }
        }
        changed
    }

    /// Pad or merge fields to match note type.
    pub(crate) fn fix_field_count(&mut self, nt: &NoteType) {
        while self.fields.len() < nt.fields.len() {
            self.fields.push("".into())
        }
        while self.fields.len() > nt.fields.len() && self.fields.len() > 1 {
            let last = self.fields.pop().unwrap();
            self.fields
                .last_mut()
                .unwrap()
                .push_str(&format!("; {}", last));
        }
    }
}

impl From<Note> for pb::Note {
    fn from(n: Note) -> Self {
        pb::Note {
            id: n.id.0,
            guid: n.guid,
            ntid: n.ntid.0,
            mtime_secs: n.mtime.0 as u32,
            usn: n.usn.0,
            tags: n.tags,
            fields: n.fields,
        }
    }
}

impl From<pb::Note> for Note {
    fn from(n: pb::Note) -> Self {
        Note {
            id: NoteID(n.id),
            guid: n.guid,
            ntid: NoteTypeID(n.ntid),
            mtime: TimestampSecs(n.mtime_secs as i64),
            usn: Usn(n.usn),
            tags: n.tags,
            fields: n.fields,
            sort_field: None,
            checksum: None,
        }
    }
}

/// Text must be passed to strip_html_preserving_image_filenames() by
/// caller prior to passing in here.
pub(crate) fn field_checksum(text: &str) -> u32 {
    let digest = sha1::Sha1::from(text).digest().bytes();
    u32::from_be_bytes(digest[..4].try_into().unwrap())
}

pub(crate) fn guid() -> String {
    anki_base91(rand::random())
}

fn anki_base91(mut n: u64) -> String {
    let table = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\
0123456789!#$%&()*+,-./:;<=>?@[]^_`{|}~";
    let mut buf = String::new();
    while n > 0 {
        let (q, r) = n.div_rem(&(table.len() as u64));
        buf.push(table[r as usize] as char);
        n = q;
    }

    buf.chars().rev().collect()
}

impl Collection {
    fn canonify_note_tags(&self, note: &mut Note, usn: Usn) -> Result<()> {
        if !note.tags.is_empty() {
            let tags = std::mem::replace(&mut note.tags, vec![]);
            note.tags = self.canonify_tags(tags, usn)?.0;
        }
        Ok(())
    }

    pub fn add_note(&mut self, note: &mut Note, did: DeckID) -> Result<()> {
        self.transact(None, |col| {
            let nt = col
                .get_notetype(note.ntid)?
                .ok_or_else(|| AnkiError::invalid_input("missing note type"))?;
            let ctx = CardGenContext::new(&nt, col.usn()?);
            let norm = col.normalize_note_text();
            col.add_note_inner(&ctx, note, did, norm)
        })
    }

    pub(crate) fn add_note_inner(
        &mut self,
        ctx: &CardGenContext,
        note: &mut Note,
        did: DeckID,
        normalize_text: bool,
    ) -> Result<()> {
        self.canonify_note_tags(note, ctx.usn)?;
        note.prepare_for_update(&ctx.notetype, normalize_text)?;
        note.set_modified(ctx.usn);
        self.storage.add_note(note)?;
        self.generate_cards_for_new_note(ctx, note, did)
    }

    pub fn update_note(&mut self, note: &mut Note) -> Result<()> {
        if let Some(existing_note) = self.storage.get_note(note.id)? {
            if &existing_note == note {
                // nothing to do
                return Ok(());
            }
        }

        self.transact(None, |col| {
            let nt = col
                .get_notetype(note.ntid)?
                .ok_or_else(|| AnkiError::invalid_input("missing note type"))?;
            let ctx = CardGenContext::new(&nt, col.usn()?);
            let norm = col.normalize_note_text();
            col.update_note_inner_generating_cards(&ctx, note, true, norm)
        })
    }

    pub(crate) fn update_note_inner_generating_cards(
        &mut self,
        ctx: &CardGenContext,
        note: &mut Note,
        mark_note_modified: bool,
        normalize_text: bool,
    ) -> Result<()> {
        self.update_note_inner_without_cards(
            note,
            ctx.notetype,
            ctx.usn,
            mark_note_modified,
            normalize_text,
        )?;
        self.generate_cards_for_existing_note(ctx, note)
    }

    pub(crate) fn update_note_inner_without_cards(
        &mut self,
        note: &mut Note,
        nt: &NoteType,
        usn: Usn,
        mark_note_modified: bool,
        normalize_text: bool,
    ) -> Result<()> {
        self.canonify_note_tags(note, usn)?;
        note.prepare_for_update(nt, normalize_text)?;
        if mark_note_modified {
            note.set_modified(usn);
        }
        self.storage.update_note(note)
    }

    /// Remove a note. Cards must already have been deleted.
    pub(crate) fn remove_note_only(&mut self, nid: NoteID, usn: Usn) -> Result<()> {
        if let Some(_note) = self.storage.get_note(nid)? {
            // fixme: undo
            self.storage.remove_note(nid)?;
            self.storage.add_note_grave(nid, usn)?;
        }
        Ok(())
    }

    /// Update cards and field cache after notes modified externally.
    /// If gencards is false, skip card generation.
    pub(crate) fn after_note_updates(
        &mut self,
        nids: &[NoteID],
        generate_cards: bool,
        mark_notes_modified: bool,
    ) -> Result<()> {
        self.transform_notes(nids, |_note, _nt| {
            Ok(TransformNoteOutput {
                changed: true,
                generate_cards,
                mark_modified: mark_notes_modified,
            })
        })
        .map(|_| ())
    }

    pub(crate) fn transform_notes<F>(
        &mut self,
        nids: &[NoteID],
        mut transformer: F,
    ) -> Result<usize>
    where
        F: FnMut(&mut Note, &NoteType) -> Result<TransformNoteOutput>,
    {
        let nids_by_notetype = self.storage.note_ids_by_notetype(nids)?;
        let norm = self.normalize_note_text();
        let mut changed_notes = 0;
        let usn = self.usn()?;

        for (ntid, group) in &nids_by_notetype.into_iter().group_by(|tup| tup.0) {
            let nt = self
                .get_notetype(ntid)?
                .ok_or_else(|| AnkiError::invalid_input("missing note type"))?;

            let mut genctx = None;
            for (_, nid) in group {
                // grab the note and transform it
                let mut note = self.storage.get_note(nid)?.unwrap();
                let out = transformer(&mut note, &nt)?;
                if !out.changed {
                    continue;
                }

                if out.generate_cards {
                    let ctx = genctx.get_or_insert_with(|| CardGenContext::new(&nt, usn));
                    self.update_note_inner_generating_cards(
                        &ctx,
                        &mut note,
                        out.mark_modified,
                        norm,
                    )?;
                } else {
                    self.update_note_inner_without_cards(
                        &mut note,
                        &nt,
                        usn,
                        out.mark_modified,
                        norm,
                    )?;
                }

                changed_notes += 1;
            }
        }

        Ok(changed_notes)
    }

    pub(crate) fn note_is_duplicate_or_empty(&self, note: &Note) -> Result<DuplicateState> {
        if let Some(field1) = note.fields.get(0) {
            let stripped = strip_html_preserving_image_filenames(field1);
            if stripped.trim().is_empty() {
                Ok(DuplicateState::Empty)
            } else {
                let csum = field_checksum(&stripped);
                for field in self
                    .storage
                    .note_fields_by_checksum(note.id, note.ntid, csum)?
                {
                    if strip_html_preserving_image_filenames(&field) == stripped {
                        return Ok(DuplicateState::Duplicate);
                    }
                }
                Ok(DuplicateState::Normal)
            }
        } else {
            Ok(DuplicateState::Empty)
        }
    }
}

#[cfg(test)]
mod test {
    use super::{anki_base91, field_checksum};
    use crate::{
        collection::open_test_collection, config::ConfigKey, decks::DeckID, err::Result,
        search::SortMode,
    };

    #[test]
    fn test_base91() {
        // match the python implementation for now
        assert_eq!(anki_base91(0), "");
        assert_eq!(anki_base91(1), "b");
        assert_eq!(anki_base91(u64::max_value()), "Rj&Z5m[>Zp");
        assert_eq!(anki_base91(1234567890), "saAKk");
    }

    #[test]
    fn test_field_checksum() {
        assert_eq!(field_checksum("test"), 2840236005);
        assert_eq!(field_checksum("今日"), 1464653051);
    }

    #[test]
    fn adding_cards() -> Result<()> {
        let mut col = open_test_collection();
        let nt = col
            .get_notetype_by_name("basic (and reversed card)")?
            .unwrap();

        let mut note = nt.new_note();
        // if no cards are generated, 1 card is added
        col.add_note(&mut note, DeckID(1)).unwrap();
        let existing = col.storage.existing_cards_for_note(note.id)?;
        assert_eq!(existing.len(), 1);
        assert_eq!(existing[0].ord, 0);

        // nothing changes if the first field is filled
        note.fields[0] = "test".into();
        col.update_note(&mut note).unwrap();
        let existing = col.storage.existing_cards_for_note(note.id)?;
        assert_eq!(existing.len(), 1);
        assert_eq!(existing[0].ord, 0);

        // second field causes another card to be generated
        note.fields[1] = "test".into();
        col.update_note(&mut note).unwrap();
        let existing = col.storage.existing_cards_for_note(note.id)?;
        assert_eq!(existing.len(), 2);
        assert_eq!(existing[1].ord, 1);

        // cloze cards also generate card 0 if no clozes are found
        let nt = col.get_notetype_by_name("cloze")?.unwrap();
        let mut note = nt.new_note();
        col.add_note(&mut note, DeckID(1)).unwrap();
        let existing = col.storage.existing_cards_for_note(note.id)?;
        assert_eq!(existing.len(), 1);
        assert_eq!(existing[0].ord, 0);
        assert_eq!(existing[0].original_deck_id, DeckID(1));

        // and generate cards for any cloze deletions
        note.fields[0] = "{{c1::foo}} {{c2::bar}} {{c3::baz}} {{c0::quux}} {{c501::over}}".into();
        col.update_note(&mut note)?;
        let existing = col.storage.existing_cards_for_note(note.id)?;
        let mut ords = existing.iter().map(|a| a.ord).collect::<Vec<_>>();
        ords.sort();
        assert_eq!(ords, vec![0, 1, 2, 499]);

        Ok(())
    }

    #[test]
    fn normalization() -> Result<()> {
        let mut col = open_test_collection();

        let nt = col.get_notetype_by_name("Basic")?.unwrap();
        let mut note = nt.new_note();
        note.fields[0] = "\u{fa47}".into();
        col.add_note(&mut note, DeckID(1))?;
        assert_eq!(note.fields[0], "\u{6f22}");
        // non-normalized searches should be converted
        assert_eq!(col.search_cards("\u{fa47}", SortMode::NoOrder)?.len(), 1);
        assert_eq!(
            col.search_cards("front:\u{fa47}", SortMode::NoOrder)?.len(),
            1
        );
        let cids = col.search_cards("", SortMode::NoOrder)?;
        col.remove_cards_inner(&cids)?;

        // if normalization turned off, note text is entered as-is
        let mut note = nt.new_note();
        note.fields[0] = "\u{fa47}".into();
        col.set_config(ConfigKey::NormalizeNoteText, &false)
            .unwrap();
        col.add_note(&mut note, DeckID(1))?;
        assert_eq!(note.fields[0], "\u{fa47}");
        // normalized searches won't match
        assert_eq!(col.search_cards("\u{6f22}", SortMode::NoOrder)?.len(), 0);
        // but original characters will
        assert_eq!(col.search_cards("\u{fa47}", SortMode::NoOrder)?.len(), 1);

        Ok(())
    }
}