From 8f9037cf0ff50720547d0202e42d6d601a84a966 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 2 Sep 2020 12:54:33 +1000 Subject: [PATCH 01/21] move filtered deck empty/fill to backend emptying of individual card ids still to be done --- proto/backend.proto | 2 + pylib/anki/sched.py | 72 +----------- pylib/anki/schedv2.py | 91 ++------------- rslib/src/backend/mod.rs | 8 ++ rslib/src/card.rs | 66 ++++++++--- rslib/src/decks/filtered.rs | 167 ++++++++++++++++++++++++++++ rslib/src/decks/mod.rs | 45 +------- rslib/src/sched/bury_and_suspend.rs | 7 +- rslib/src/search/cards.rs | 38 ++++--- rslib/src/stats/graphs.rs | 4 +- 10 files changed, 269 insertions(+), 231 deletions(-) create mode 100644 rslib/src/decks/filtered.rs diff --git a/proto/backend.proto b/proto/backend.proto index ac5340e32..bac9d8a9e 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -103,6 +103,8 @@ service BackendService { rpc RestoreBuriedAndSuspendedCards (CardIDs) returns (Empty); rpc UnburyCardsInCurrentDeck (UnburyCardsInCurrentDeckIn) returns (Empty); rpc BuryOrSuspendCards (BuryOrSuspendCardsIn) returns (Empty); + rpc EmptyFilteredDeck (DeckID) returns (Empty); + rpc RebuildFilteredDeck (DeckID) returns (UInt32); // stats diff --git a/pylib/anki/sched.py b/pylib/anki/sched.py index 924972b34..333c4d8e0 100644 --- a/pylib/anki/sched.py +++ b/pylib/anki/sched.py @@ -6,7 +6,7 @@ from __future__ import annotations import random import time from heapq import * -from typing import Any, List, Optional, Sequence, Tuple, Union +from typing import Any, List, Optional, Tuple, Union import anki from anki import hooks @@ -599,77 +599,9 @@ did = ? and queue = {QUEUE_TYPE_REV} and due <= ? limit ?""", idealIvl = self._fuzzedIvl(idealIvl) return idealIvl - # Dynamic deck handling + # Filtered deck handling ########################################################################## - def rebuildDyn(self, did: Optional[int] = None) -> Optional[Sequence[int]]: # type: ignore[override] - "Rebuild a dynamic deck." - did = did or self.col.decks.selected() - deck = self.col.decks.get(did) - assert deck["dyn"] - # move any existing cards back first, then fill - self.emptyDyn(did) - ids = self._fillDyn(deck) - if not ids: - return None - # and change to our new deck - self.col.decks.select(did) - return ids - - def _fillDyn(self, deck: Deck) -> Sequence[int]: # type: ignore[override] - search, limit, order = deck["terms"][0] - orderlimit = self._dynOrder(order, limit) - if search.strip(): - search = "(%s)" % search - search = "%s -is:suspended -is:buried -deck:filtered -is:learn" % search - try: - ids = self.col.findCards(search, order=orderlimit) - except: - ids = [] - return ids - # move the cards over - self.col.log(deck["id"], ids) - self._moveToDyn(deck["id"], ids) - return ids - - def emptyDyn(self, did: Optional[int], lim: Optional[str] = None) -> None: - if not lim: - lim = "did = %s" % did - self.col.log(self.col.db.list("select id from cards where %s" % lim)) - # move out of cram queue - self.col.db.execute( - f""" -update cards set did = odid, queue = (case when type = {CARD_TYPE_LRN} then {QUEUE_TYPE_NEW} -else type end), type = (case when type = {CARD_TYPE_LRN} then {CARD_TYPE_NEW} else type end), -due = odue, odue = 0, odid = 0, usn = ? where %s""" - % lim, - self.col.usn(), - ) - - def _moveToDyn(self, did: int, ids: Sequence[int]) -> None: # type: ignore[override] - deck = self.col.decks.get(did) - data = [] - t = intTime() - u = self.col.usn() - for c, id in enumerate(ids): - # start at -100000 so that reviews are all due - data.append((did, -100000 + c, u, id)) - # due reviews stay in the review queue. careful: can't use - # "odid or did", as sqlite converts to boolean - queue = f""" -(case when type={CARD_TYPE_REV} and (case when odue then odue <= %d else due <= %d end) - then {QUEUE_TYPE_REV} else {QUEUE_TYPE_NEW} end)""" - queue %= (self.today, self.today) - self.col.db.executemany( - """ -update cards set -odid = (case when odid then odid else did end), -odue = (case when odue then odue else due end), -did = ?, queue = %s, due = ?, usn = ? where id = ?""" - % queue, - data, - ) - def _dynIvlBoost(self, card: Card) -> int: assert card.odid and card.type == CARD_TYPE_REV assert card.factor diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index 7ca006b83..53ab369fb 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -25,7 +25,7 @@ import anki.backend_pb2 as pb from anki import hooks from anki.cards import Card from anki.consts import * -from anki.decks import Deck, DeckConfig, DeckManager, FilteredDeck, QueueConfig +from anki.decks import Deck, DeckConfig, DeckManager, QueueConfig from anki.lang import _ from anki.notes import Note from anki.rsbackend import ( @@ -1062,7 +1062,7 @@ select id from cards where did in %s and queue = {QUEUE_TYPE_REV} and due <= ? l return ivl - # Dynamic deck handling + # Filtered deck handling ########################################################################## _restoreQueueWhenEmptyingSnippet = f""" @@ -1076,41 +1076,19 @@ end) """ def rebuildDyn(self, did: Optional[int] = None) -> Optional[int]: - "Rebuild a dynamic deck." + "Rebuild a filtered deck." did = did or self.col.decks.selected() - deck = self.col.decks.get(did) - assert deck["dyn"] - # move any existing cards back first, then fill - self.emptyDyn(did) - cnt = self._fillDyn(deck) - if not cnt: + count = self.col.backend.rebuild_filtered_deck(did) or None + if not count: return None # and change to our new deck self.col.decks.select(did) - return cnt - - def _fillDyn(self, deck: FilteredDeck) -> int: - start = -100000 - total = 0 - for search, limit, order in deck["terms"]: - orderlimit = self._dynOrder(order, limit) - if search.strip(): - search = "(%s)" % search - search = "%s -is:suspended -is:buried -deck:filtered" % search - try: - ids = self.col.findCards(search, order=orderlimit) - except: - return total - # move the cards over - self.col.log(deck["id"], ids) - self._moveToDyn(deck["id"], ids, start=start + total) - total += len(ids) - return total + return count def emptyDyn(self, did: Optional[int], lim: Optional[str] = None) -> None: - if not lim: - lim = "did = %s" % did - self.col.log(self.col.db.list("select id from cards where %s" % lim)) + if lim is None: + self.col.backend.empty_filtered_deck(did) + return self.col.db.execute( """ @@ -1123,57 +1101,6 @@ due = (case when odue>0 then odue else due end), odue = 0, odid = 0, usn = ? whe def remFromDyn(self, cids: List[int]) -> None: self.emptyDyn(None, "id in %s and odid" % ids2str(cids)) - def _dynOrder(self, o: int, l: int) -> str: - if o == DYN_OLDEST: - t = "(select max(id) from revlog where cid=c.id)" - elif o == DYN_RANDOM: - t = "random()" - elif o == DYN_SMALLINT: - t = "ivl" - elif o == DYN_BIGINT: - t = "ivl desc" - elif o == DYN_LAPSES: - t = "lapses desc" - elif o == DYN_ADDED: - t = "n.id" - elif o == DYN_REVADDED: - t = "n.id desc" - elif o == DYN_DUEPRIORITY: - t = ( - f"(case when queue={QUEUE_TYPE_REV} and due <= %d then (ivl / cast(%d-due+0.001 as real)) else 100000+due end)" - % (self.today, self.today) - ) - else: # DYN_DUE or unknown - t = "c.due, c.ord" - return t + " limit %d" % l - - def _moveToDyn(self, did: int, ids: Sequence[int], start: int = -100000) -> None: - deck = self.col.decks.get(did) - data = [] - u = self.col.usn() - due = start - for id in ids: - data.append((did, due, u, id)) - due += 1 - - queue = "" - if not deck["resched"]: - queue = f",queue={QUEUE_TYPE_REV}" - - query = ( - """ -update cards set -odid = did, odue = due, -did = ?, -due = (case when due <= 0 then due else ? end), -usn = ? -%s -where id = ? -""" - % queue - ) - self.col.db.executemany(query, data) - def _removeFromFiltered(self, card: Card) -> None: if card.odid: card.did = card.odid diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 72be90733..977a1dce2 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -539,6 +539,14 @@ impl BackendService for Backend { }) } + fn empty_filtered_deck(&mut self, input: pb::DeckId) -> BackendResult { + self.with_col(|col| col.empty_filtered_deck(input.did.into()).map(Into::into)) + } + + fn rebuild_filtered_deck(&mut self, input: pb::DeckId) -> BackendResult { + self.with_col(|col| col.rebuild_filtered_deck(input.did.into()).map(Into::into)) + } + // statistics //----------------------------------------------- diff --git a/rslib/src/card.rs b/rslib/src/card.rs index 9baf69c8a..65ada96c5 100644 --- a/rslib/src/card.rs +++ b/rslib/src/card.rs @@ -1,7 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::decks::DeckID; +use crate::decks::{DeckFilterContext, DeckID}; use crate::define_newtype; use crate::err::{AnkiError, Result}; use crate::notes::NoteID; @@ -102,7 +102,42 @@ impl Card { self.usn = usn; } - pub(crate) fn return_home(&mut self, sched: SchedulerVersion) { + pub(crate) fn move_into_filtered_deck(&mut self, ctx: &DeckFilterContext, position: i32) { + // filtered and v1 learning cards are excluded, so odue should be guaranteed to be zero + if self.original_due != 0 { + println!("bug: odue was set"); + return; + } + + self.original_deck_id = self.deck_id; + self.deck_id = ctx.target_deck; + + self.original_due = self.due; + + if ctx.scheduler == SchedulerVersion::V1 { + if self.ctype == CardType::Review && self.due <= ctx.today as i32 { + // review cards that are due are left in the review queue + } else { + // new + non-due go into new queue + self.queue = CardQueue::New; + } + if self.due != 0 { + self.due = position; + } + } else { + // if rescheduling is disabled, all cards go in the review queue + if !ctx.config.reschedule { + self.queue = CardQueue::Review; + } + // fixme: can we unify this with v1 scheduler in the future? + // https://anki.tenderapp.com/discussions/ankidesktop/35978-rebuilding-filtered-deck-on-experimental-v2-empties-deck-and-reschedules-to-the-year-1745 + if self.due > 0 { + self.due = position; + } + } + } + + pub(crate) fn remove_from_filtered_deck(&mut self, sched: SchedulerVersion) { if self.original_deck_id.0 == 0 { // not in a filtered deck return; @@ -110,14 +145,11 @@ impl Card { self.deck_id = self.original_deck_id; self.original_deck_id.0 = 0; - if self.original_due > 0 { - self.due = self.original_due; - } - self.original_due = 0; - self.queue = match sched { + match sched { SchedulerVersion::V1 => { - match self.ctype { + self.due = self.original_due; + self.queue = match self.ctype { CardType::New => CardQueue::New, CardType::Learn => CardQueue::New, CardType::Review => CardQueue::Review, @@ -126,11 +158,19 @@ impl Card { println!("did not expect relearn type in v1 for card {}", self.id); CardQueue::New } + }; + if self.ctype == CardType::Learn { + self.ctype = CardType::New; } } SchedulerVersion::V2 => { + // original_due is cleared if card answered in filtered deck + if self.original_due > 0 { + self.due = self.original_due; + } + if (self.queue as i8) >= 0 { - match self.ctype { + self.queue = match self.ctype { CardType::Learn | CardType::Relearn => { if self.due > 1_000_000_000 { // unix timestamp @@ -143,15 +183,11 @@ impl Card { CardType::New => CardQueue::New, CardType::Review => CardQueue::Review, } - } else { - self.queue } } - }; - - if sched == SchedulerVersion::V1 && self.ctype == CardType::Learn { - self.ctype = CardType::New; } + + self.original_due = 0; } /// Remove the card from the (re)learning queue. diff --git a/rslib/src/decks/filtered.rs b/rslib/src/decks/filtered.rs new file mode 100644 index 000000000..c93de02ce --- /dev/null +++ b/rslib/src/decks/filtered.rs @@ -0,0 +1,167 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use super::{Deck, DeckID}; +pub use crate::backend_proto::{ + deck_kind::Kind as DeckKind, filtered_search_term::FilteredSearchOrder, Deck as DeckProto, + DeckCommon, DeckKind as DeckKindProto, FilteredDeck, FilteredSearchTerm, NormalDeck, +}; +use crate::{ + card::{CardID, CardQueue}, + collection::Collection, + config::SchedulerVersion, + err::Result, + prelude::AnkiError, + search::SortMode, + timestamp::TimestampSecs, + types::Usn, +}; + +impl Deck { + pub fn new_filtered() -> Deck { + let mut filt = FilteredDeck::default(); + filt.search_terms.push(FilteredSearchTerm { + search: "".into(), + limit: 100, + order: 0, + }); + filt.preview_delay = 10; + filt.reschedule = true; + Deck { + id: DeckID(0), + name: "".into(), + mtime_secs: TimestampSecs(0), + usn: Usn(0), + common: DeckCommon::default(), + kind: DeckKind::Filtered(filt), + } + } + + pub(crate) fn is_filtered(&self) -> bool { + matches!(self.kind, DeckKind::Filtered(_)) + } +} + +pub(crate) struct DeckFilterContext<'a> { + pub target_deck: DeckID, + pub config: &'a FilteredDeck, + pub scheduler: SchedulerVersion, + pub usn: Usn, + pub today: u32, +} + +impl Collection { + pub fn empty_filtered_deck(&mut self, did: DeckID) -> Result<()> { + self.transact(None, |col| col.return_all_cards_in_filtered_deck(did)) + } + pub(super) fn return_all_cards_in_filtered_deck(&mut self, did: DeckID) -> Result<()> { + let cids = self.storage.all_cards_in_single_deck(did)?; + self.return_cards_to_home_deck(&cids) + } + + // Unlike the old Python code, this also marks the cards as modified. + fn return_cards_to_home_deck(&mut self, cids: &[CardID]) -> Result<()> { + let sched = self.sched_ver(); + let usn = self.usn()?; + for cid in cids { + if let Some(mut card) = self.storage.get_card(*cid)? { + let original = card.clone(); + card.remove_from_filtered_deck(sched); + self.update_card(&mut card, &original, usn)?; + } + } + Ok(()) + } + + // Unlike the old Python code, this also marks the cards as modified. + pub fn rebuild_filtered_deck(&mut self, did: DeckID) -> Result { + let deck = self.get_deck(did)?.ok_or(AnkiError::NotFound)?; + let config = if let DeckKind::Filtered(kind) = &deck.kind { + kind + } else { + return Err(AnkiError::invalid_input("not filtered")); + }; + let ctx = DeckFilterContext { + target_deck: did, + config, + scheduler: self.sched_ver(), + usn: self.usn()?, + today: self.timing_today()?.days_elapsed, + }; + + self.transact(None, |col| { + col.return_all_cards_in_filtered_deck(did)?; + col.build_filtered_deck(ctx) + }) + } + + fn build_filtered_deck(&mut self, ctx: DeckFilterContext) -> Result { + let start = -100_000; + let mut position = start; + for term in &ctx.config.search_terms { + position = self.move_cards_matching_term(&ctx, term, position)?; + } + + Ok((position - start) as u32) + } + + /// Move matching cards into filtered deck. + /// Returns the new starting position. + fn move_cards_matching_term( + &mut self, + ctx: &DeckFilterContext, + term: &FilteredSearchTerm, + mut position: i32, + ) -> Result { + let search = format!( + "{} -is:suspended -is:buried -deck:filtered {}", + if term.search.trim().is_empty() { + "".to_string() + } else { + format!("({})", term.search) + }, + if ctx.scheduler == SchedulerVersion::V1 { + "-is:learn" + } else { + "" + } + ); + let order = order_and_limit_for_search(term, ctx.today); + + self.search_cards_into_table(&search, SortMode::Custom(order))?; + for mut card in self.storage.all_searched_cards()? { + let original = card.clone(); + card.move_into_filtered_deck(ctx, position); + self.update_card(&mut card, &original, ctx.usn)?; + position += 1; + } + + Ok(position) + } +} + +fn order_and_limit_for_search(term: &FilteredSearchTerm, today: u32) -> String { + let temp_string; + let order = match term.order() { + FilteredSearchOrder::OldestFirst => "(select max(id) from revlog where cid=c.id)", + FilteredSearchOrder::Random => "random()", + FilteredSearchOrder::IntervalsAscending => "ivl", + FilteredSearchOrder::IntervalsDescending => "ivl desc", + FilteredSearchOrder::Lapses => "lapses desc", + FilteredSearchOrder::Added => "n.id", + FilteredSearchOrder::ReverseAdded => "n.id desc", + FilteredSearchOrder::Due => "c.due, c.ord", + FilteredSearchOrder::DuePriority => { + temp_string = format!( + " +(case when queue={rev_queue} and due <= {today} +then (ivl / cast({today}-due+0.001 as real)) else 100000+due end)", + rev_queue = CardQueue::Review as i8, + today = today + ); + &temp_string + } + }; + + format!("{} limit {}", order, term.limit) +} diff --git a/rslib/src/decks/mod.rs b/rslib/src/decks/mod.rs index f1968bb4e..af7db1845 100644 --- a/rslib/src/decks/mod.rs +++ b/rslib/src/decks/mod.rs @@ -7,7 +7,6 @@ pub use crate::backend_proto::{ DeckCommon, DeckKind as DeckKindProto, FilteredDeck, FilteredSearchTerm, NormalDeck, }; use crate::{ - card::CardID, collection::Collection, deckconf::DeckConfID, define_newtype, @@ -18,9 +17,11 @@ use crate::{ types::Usn, }; mod counts; +mod filtered; mod schema11; mod tree; pub(crate) use counts::DueCounts; +pub(crate) use filtered::DeckFilterContext; pub use schema11::DeckSchema11; use std::{borrow::Cow, sync::Arc}; @@ -51,25 +52,6 @@ impl Deck { } } - pub fn new_filtered() -> Deck { - let mut filt = FilteredDeck::default(); - filt.search_terms.push(FilteredSearchTerm { - search: "".into(), - limit: 100, - order: 0, - }); - filt.preview_delay = 10; - filt.reschedule = true; - Deck { - id: DeckID(0), - name: "".into(), - mtime_secs: TimestampSecs(0), - usn: Usn(0), - common: DeckCommon::default(), - kind: DeckKind::Filtered(filt), - } - } - fn reset_stats_if_day_changed(&mut self, today: u32) { let c = &mut self.common; if c.last_day_studied != today { @@ -80,12 +62,6 @@ impl Deck { c.last_day_studied = today; } } -} - -impl Deck { - pub(crate) fn is_filtered(&self) -> bool { - matches!(self.kind, DeckKind::Filtered(_)) - } /// Returns deck config ID if deck is a normal deck. pub(crate) fn config_id(&self) -> Option { @@ -434,23 +410,6 @@ impl Collection { self.remove_cards_and_orphaned_notes(&cids) } - fn return_all_cards_in_filtered_deck(&mut self, did: DeckID) -> Result<()> { - let cids = self.storage.all_cards_in_single_deck(did)?; - self.return_cards_to_home_deck(&cids) - } - - fn return_cards_to_home_deck(&mut self, cids: &[CardID]) -> Result<()> { - let sched = self.sched_ver(); - for cid in cids { - if let Some(mut card) = self.storage.get_card(*cid)? { - // fixme: undo - card.return_home(sched); - self.storage.update_card(&card)?; - } - } - Ok(()) - } - pub fn get_all_deck_names(&self, skip_empty_default: bool) -> Result> { if skip_empty_default && self.default_deck_is_empty()? { Ok(self diff --git a/rslib/src/sched/bury_and_suspend.rs b/rslib/src/sched/bury_and_suspend.rs index fe445f580..a8201ead9 100644 --- a/rslib/src/sched/bury_and_suspend.rs +++ b/rslib/src/sched/bury_and_suspend.rs @@ -7,6 +7,7 @@ use crate::{ collection::Collection, config::SchedulerVersion, err::Result, + search::SortMode, }; use super::cutoff::SchedTimingToday; @@ -59,7 +60,7 @@ impl Collection { /// Unbury cards from the previous day. /// Done automatically, and does not mark the cards as modified. fn unbury_on_day_rollover(&mut self) -> Result<()> { - self.search_cards_into_table("is:buried")?; + self.search_cards_into_table("is:buried", SortMode::NoOrder)?; self.storage.for_each_card_in_search(|mut card| { card.restore_queue_after_bury_or_suspend(); self.storage.update_card(&card) @@ -94,7 +95,7 @@ impl Collection { UnburyDeckMode::SchedOnly => "is:buried-sibling", }; self.transact(None, |col| { - col.search_cards_into_table(&format!("deck:current {}", search))?; + col.search_cards_into_table(&format!("deck:current {}", search), SortMode::NoOrder)?; col.unsuspend_or_unbury_searched_cards() }) } @@ -125,7 +126,7 @@ impl Collection { }; if card.queue != desired_queue { if sched == SchedulerVersion::V1 { - card.return_home(sched); + card.remove_from_filtered_deck(sched); card.remove_from_learning(); } card.queue = desired_queue; diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index 4be6d3648..351abc89b 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -63,20 +63,7 @@ impl Collection { let writer = SqlWriter::new(self); let (mut sql, args) = writer.build_cards_query(&top_node, mode.required_table())?; - - match mode { - SortMode::NoOrder => (), - SortMode::FromConfig => unreachable!(), - SortMode::Builtin { kind, reverse } => { - prepare_sort(self, kind)?; - sql.push_str(" order by "); - write_order(&mut sql, kind, reverse)?; - } - SortMode::Custom(order_clause) => { - sql.push_str(" order by "); - sql.push_str(&order_clause); - } - } + self.add_order(&mut sql, mode)?; let mut stmt = self.storage.db.prepare(&sql)?; let ids: Vec<_> = stmt @@ -86,13 +73,32 @@ impl Collection { Ok(ids) } + fn add_order(&mut self, sql: &mut String, mode: SortMode) -> Result<()> { + match mode { + SortMode::NoOrder => (), + SortMode::FromConfig => unreachable!(), + SortMode::Builtin { kind, reverse } => { + prepare_sort(self, kind)?; + sql.push_str(" order by "); + write_order(sql, kind, reverse)?; + } + SortMode::Custom(order_clause) => { + sql.push_str(" order by "); + sql.push_str(&order_clause); + } + } + Ok(()) + } + /// Place the matched card ids into a temporary 'search_cids' table /// instead of returning them. Use clear_searched_cards() to remove it. - pub(crate) fn search_cards_into_table(&mut self, search: &str) -> Result<()> { + pub(crate) fn search_cards_into_table(&mut self, search: &str, mode: SortMode) -> Result<()> { let top_node = Node::Group(parse(search)?); let writer = SqlWriter::new(self); - let (sql, args) = writer.build_cards_query(&top_node, RequiredTable::Cards)?; + let (mut sql, args) = writer.build_cards_query(&top_node, mode.required_table())?; + self.add_order(&mut sql, mode)?; + self.storage .db .execute_batch(include_str!("search_cids_setup.sql"))?; diff --git a/rslib/src/stats/graphs.rs b/rslib/src/stats/graphs.rs index 601de79fe..8a2432315 100644 --- a/rslib/src/stats/graphs.rs +++ b/rslib/src/stats/graphs.rs @@ -1,7 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::{backend_proto as pb, prelude::*, revlog::RevlogEntry}; +use crate::{backend_proto as pb, prelude::*, revlog::RevlogEntry, search::SortMode}; impl Collection { pub(crate) fn graph_data_for_search( @@ -9,7 +9,7 @@ impl Collection { search: &str, days: u32, ) -> Result { - self.search_cards_into_table(search)?; + self.search_cards_into_table(search, SortMode::NoOrder)?; let all = search.trim().is_empty(); self.graph_data(all, days) } From 39212a38aad952b9c810f95110419cbef03e8d3f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 2 Sep 2020 14:41:46 +1000 Subject: [PATCH 02/21] move reschedCards() to backend --- proto/backend.proto | 7 ++++ pylib/anki/schedv2.py | 24 ++------------ rslib/src/backend/mod.rs | 12 +++++++ rslib/src/card.rs | 13 +++++++- rslib/src/decks/filtered.rs | 2 +- rslib/src/sched/bury_and_suspend.rs | 2 +- rslib/src/sched/mod.rs | 1 + rslib/src/sched/reviews.rs | 50 +++++++++++++++++++++++++++++ 8 files changed, 86 insertions(+), 25 deletions(-) create mode 100644 rslib/src/sched/reviews.rs diff --git a/proto/backend.proto b/proto/backend.proto index bac9d8a9e..b6fba6f84 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -105,6 +105,7 @@ service BackendService { rpc BuryOrSuspendCards (BuryOrSuspendCardsIn) returns (Empty); rpc EmptyFilteredDeck (DeckID) returns (Empty); rpc RebuildFilteredDeck (DeckID) returns (UInt32); + rpc ScheduleCardsAsReviews (ScheduleCardsAsReviewsIn) returns (Empty); // stats @@ -1056,3 +1057,9 @@ message BuryOrSuspendCardsIn { repeated int64 card_ids = 1; Mode mode = 2; } + +message ScheduleCardsAsReviewsIn { + repeated int64 card_ids = 1; + uint32 min_interval = 2; + uint32 max_interval = 3; +} diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index 53ab369fb..87b2e373d 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -1419,29 +1419,9 @@ and (queue={QUEUE_TYPE_NEW} or (queue={QUEUE_TYPE_REV} and due<=?))""", def reschedCards(self, ids: List[int], imin: int, imax: int) -> None: "Put cards in review queue with a new interval in days (min, max)." - d = [] - t = self.today - mod = intTime() - for id in ids: - r = random.randint(imin, imax) - d.append( - ( - max(1, r), - r + t, - self.col.usn(), - mod, - STARTING_FACTOR, - id, - ) - ) - self.remFromDyn(ids) - self.col.db.executemany( - f""" -update cards set type={CARD_TYPE_REV},queue={QUEUE_TYPE_REV},ivl=?,due=?,odue=0, -usn=?,mod=?,factor=? where id=?""", - d, + self.col.backend.schedule_cards_as_reviews( + card_ids=ids, min_interval=imin, max_interval=imax ) - self.col.log(ids) def resetCards(self, ids: List[int]) -> None: "Completely reset cards for export." diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 977a1dce2..755baacbf 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -547,6 +547,18 @@ impl BackendService for Backend { self.with_col(|col| col.rebuild_filtered_deck(input.did.into()).map(Into::into)) } + fn schedule_cards_as_reviews( + &mut self, + input: pb::ScheduleCardsAsReviewsIn, + ) -> BackendResult { + let cids: Vec<_> = input.card_ids.into_iter().map(CardID).collect(); + let (min, max) = (input.min_interval, input.max_interval); + self.with_col(|col| { + col.reschedule_cards_as_reviews(&cids, min, max) + .map(Into::into) + }) + } + // statistics //----------------------------------------------- diff --git a/rslib/src/card.rs b/rslib/src/card.rs index 65ada96c5..e50235ec3 100644 --- a/rslib/src/card.rs +++ b/rslib/src/card.rs @@ -137,7 +137,18 @@ impl Card { } } - pub(crate) fn remove_from_filtered_deck(&mut self, sched: SchedulerVersion) { + /// Restores to the original deck and clears original_due. + /// This does not update the queue or type, so should only be used as + /// part of an operation that adjusts those separately. + pub(crate) fn remove_from_filtered_deck_before_reschedule(&mut self) { + if self.original_deck_id.0 != 0 { + self.deck_id = self.original_deck_id; + self.original_deck_id.0 = 0; + self.original_due = 0; + } + } + + pub(crate) fn remove_from_filtered_deck_restoring_queue(&mut self, sched: SchedulerVersion) { if self.original_deck_id.0 == 0 { // not in a filtered deck return; diff --git a/rslib/src/decks/filtered.rs b/rslib/src/decks/filtered.rs index c93de02ce..0f9a278d8 100644 --- a/rslib/src/decks/filtered.rs +++ b/rslib/src/decks/filtered.rs @@ -66,7 +66,7 @@ impl Collection { for cid in cids { if let Some(mut card) = self.storage.get_card(*cid)? { let original = card.clone(); - card.remove_from_filtered_deck(sched); + card.remove_from_filtered_deck_restoring_queue(sched); self.update_card(&mut card, &original, usn)?; } } diff --git a/rslib/src/sched/bury_and_suspend.rs b/rslib/src/sched/bury_and_suspend.rs index a8201ead9..e14979328 100644 --- a/rslib/src/sched/bury_and_suspend.rs +++ b/rslib/src/sched/bury_and_suspend.rs @@ -126,7 +126,7 @@ impl Collection { }; if card.queue != desired_queue { if sched == SchedulerVersion::V1 { - card.remove_from_filtered_deck(sched); + card.remove_from_filtered_deck_restoring_queue(sched); card.remove_from_learning(); } card.queue = desired_queue; diff --git a/rslib/src/sched/mod.rs b/rslib/src/sched/mod.rs index f7c03a840..af0df2997 100644 --- a/rslib/src/sched/mod.rs +++ b/rslib/src/sched/mod.rs @@ -8,6 +8,7 @@ use crate::{ pub mod bury_and_suspend; pub(crate) mod congrats; pub mod cutoff; +mod reviews; pub mod timespan; use chrono::FixedOffset; diff --git a/rslib/src/sched/reviews.rs b/rslib/src/sched/reviews.rs new file mode 100644 index 000000000..0ad8062b3 --- /dev/null +++ b/rslib/src/sched/reviews.rs @@ -0,0 +1,50 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::{ + card::{Card, CardID, CardQueue, CardType}, + collection::Collection, + deckconf::INITIAL_EASE_FACTOR, + err::Result, +}; +use rand::distributions::{Distribution, Uniform}; + +impl Card { + fn schedule_as_review(&mut self, interval: u32, today: u32) { + self.remove_from_filtered_deck_before_reschedule(); + self.interval = interval.max(1); + self.due = (today + interval) as i32; + self.ctype = CardType::Review; + self.queue = CardQueue::Review; + if self.ease_factor == 0 { + // unlike the old Python code, we leave the ease factor alone + // if it's already set + self.ease_factor = INITIAL_EASE_FACTOR; + } + } +} + +impl Collection { + pub fn reschedule_cards_as_reviews( + &mut self, + cids: &[CardID], + min_days: u32, + max_days: u32, + ) -> Result<()> { + let usn = self.usn()?; + let today = self.timing_today()?.days_elapsed; + let mut rng = rand::thread_rng(); + let distribution = Uniform::from(min_days..=max_days); + self.transact(None, |col| { + col.set_search_table_to_card_ids(cids)?; + for mut card in col.storage.all_searched_cards()? { + let original = card.clone(); + let interval = distribution.sample(&mut rng); + card.schedule_as_review(interval, today); + col.update_card(&mut card, &original, usn)?; + } + col.clear_searched_cards()?; + Ok(()) + }) + } +} From ce49ca940167c4da181f124bf2cd0104559c2642 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 2 Sep 2020 17:18:29 +1000 Subject: [PATCH 03/21] log manual reschedule, but ignore the log entry in the stats --- proto/backend.proto | 6 ++- pylib/anki/collection.py | 3 ++ pylib/anki/stats.py | 4 +- qt/aqt/deckbrowser.py | 11 +--- rslib/ftl/card-stats.ftl | 2 + rslib/src/backend/mod.rs | 16 ++++-- rslib/src/card.rs | 30 +---------- rslib/src/revlog.rs | 38 ++++++++++++++ rslib/src/sched/learning.rs | 58 ++++++++++++++++++++++ rslib/src/sched/mod.rs | 1 + rslib/src/sched/reviews.rs | 1 + rslib/src/sched/timespan.rs | 44 ++++------------ rslib/src/stats/card.rs | 2 + rslib/src/stats/mod.rs | 3 ++ rslib/src/stats/today.rs | 45 +++++++++++++++++ rslib/src/storage/revlog/add.sql | 24 ++++++++- rslib/src/storage/revlog/mod.rs | 20 ++++++++ rslib/src/storage/revlog/studied_today.sql | 5 ++ rslib/src/timestamp.rs | 8 +++ ts/src/stats/reviews.ts | 4 ++ ts/src/stats/today.ts | 6 ++- 21 files changed, 249 insertions(+), 82 deletions(-) create mode 100644 rslib/src/sched/learning.rs create mode 100644 rslib/src/stats/today.rs create mode 100644 rslib/src/storage/revlog/studied_today.sql diff --git a/proto/backend.proto b/proto/backend.proto index b6fba6f84..95af6b8f1 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -95,7 +95,8 @@ service BackendService { rpc LocalMinutesWest (Int64) returns (Int32); rpc SetLocalMinutesWest (Int32) returns (Empty); rpc SchedTimingToday (Empty) returns (SchedTimingTodayOut); - rpc StudiedToday (StudiedTodayIn) returns (String); + rpc StudiedToday (Empty) returns (String); + rpc StudiedTodayMessage (StudiedTodayMessageIn) returns (String); rpc UpdateStats (UpdateStatsIn) returns (Empty); rpc ExtendLimits (ExtendLimitsIn) returns (Empty); rpc CountsForDeckToday (DeckID) returns (CountsForDeckTodayOut); @@ -685,7 +686,7 @@ message FormatTimespanIn { Context context = 2; } -message StudiedTodayIn { +message StudiedTodayMessageIn { uint32 cards = 1; double seconds = 2; } @@ -1016,6 +1017,7 @@ message RevlogEntry { REVIEW = 1; RELEARNING = 2; EARLY_REVIEW = 3; + MANUAL = 4; } int64 id = 1; int64 cid = 2; diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 320a6565b..d423f42a9 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -516,6 +516,9 @@ table.review-log {{ {revlog_style} }} return style + self.backend.card_stats(card_id) + def studied_today(self) -> str: + return self.backend.studied_today() + # legacy def cardStats(self, card: Card) -> str: diff --git a/pylib/anki/stats.py b/pylib/anki/stats.py index e3f34fe3a..aedf641ec 100644 --- a/pylib/anki/stats.py +++ b/pylib/anki/stats.py @@ -145,7 +145,9 @@ from revlog where id > ? """ return "" + str(s) + "" if cards: - b += self.col.backend.studied_today(cards=cards, seconds=float(thetime)) + b += self.col.backend.studied_today_message( + cards=cards, seconds=float(thetime) + ) # again/pass count b += "
" + _("Again count: %s") % bold(failed) if cards: diff --git a/qt/aqt/deckbrowser.py b/qt/aqt/deckbrowser.py index b08eff116..d79aa7c08 100644 --- a/qt/aqt/deckbrowser.py +++ b/qt/aqt/deckbrowser.py @@ -138,16 +138,7 @@ class DeckBrowser: self.web.eval("$(function() { window.scrollTo(0, %d, 'instant'); });" % offset) def _renderStats(self): - cards, thetime = self.mw.col.db.first( - """ -select count(), sum(time)/1000 from revlog -where id > ?""", - (self.mw.col.sched.dayCutoff - 86400) * 1000, - ) - cards = cards or 0 - thetime = thetime or 0 - buf = self.mw.col.backend.studied_today(cards=cards, seconds=float(thetime)) - return buf + return self.mw.col.studied_today() def _renderDeckTree(self, top: DeckTreeNode) -> str: buf = """ diff --git a/rslib/ftl/card-stats.ftl b/rslib/ftl/card-stats.ftl index 63d78df88..ddaf797d9 100644 --- a/rslib/ftl/card-stats.ftl +++ b/rslib/ftl/card-stats.ftl @@ -21,3 +21,5 @@ card-stats-review-log-type-learn = Learn card-stats-review-log-type-review = Review card-stats-review-log-type-relearn = Relearn card-stats-review-log-type-filtered = Filtered +card-stats-review-log-type-manual = Manual + diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 755baacbf..e157ba0fc 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -31,8 +31,9 @@ use crate::{ RenderCardOutput, }, sched::cutoff::local_minutes_west_for_stamp, - sched::timespan::{answer_button_time, studied_today, time_span}, + sched::timespan::{answer_button_time, time_span}, search::SortMode, + stats::studied_today, sync::{ get_remote_sync_meta, sync_abort, sync_login, FullSyncProgress, NormalSyncProgress, SyncActionRequired, SyncAuth, SyncMeta, SyncOutput, SyncStage, @@ -472,8 +473,17 @@ impl BackendService for Backend { }) } - fn studied_today(&mut self, input: pb::StudiedTodayIn) -> BackendResult { - Ok(studied_today(input.cards as usize, input.seconds as f32, &self.i18n).into()) + /// Fetch data from DB and return rendered string. + fn studied_today(&mut self, _input: pb::Empty) -> BackendResult { + self.with_col(|col| col.studied_today().map(Into::into)) + } + + /// Message rendering only, for old graphs. + fn studied_today_message( + &mut self, + input: pb::StudiedTodayMessageIn, + ) -> BackendResult { + Ok(studied_today(input.cards, input.seconds as f32, &self.i18n).into()) } fn update_stats(&mut self, input: pb::UpdateStatsIn) -> BackendResult { diff --git a/rslib/src/card.rs b/rslib/src/card.rs index e50235ec3..42404c21b 100644 --- a/rslib/src/card.rs +++ b/rslib/src/card.rs @@ -6,8 +6,8 @@ use crate::define_newtype; use crate::err::{AnkiError, Result}; use crate::notes::NoteID; use crate::{ - collection::Collection, config::SchedulerVersion, deckconf::INITIAL_EASE_FACTOR, - timestamp::TimestampSecs, types::Usn, undo::Undoable, + collection::Collection, config::SchedulerVersion, timestamp::TimestampSecs, types::Usn, + undo::Undoable, }; use num_enum::TryFromPrimitive; use serde_repr::{Deserialize_repr, Serialize_repr}; @@ -200,32 +200,6 @@ impl Card { self.original_due = 0; } - - /// Remove the card from the (re)learning queue. - /// This will reset cards in learning. - /// Only used in the V1 scheduler. - /// Unlike the legacy Python code, this sets the due# to 0 instead of - /// one past the previous max due number. - pub(crate) fn remove_from_learning(&mut self) { - if !matches!(self.queue, CardQueue::Learn | CardQueue::DayLearn) { - return; - } - - if self.ctype == CardType::Review { - // reviews are removed from relearning - self.due = self.original_due; - self.original_due = 0; - self.queue = CardQueue::Review; - } else { - // other cards are reset to new - self.ctype = CardType::New; - self.queue = CardQueue::New; - self.interval = 0; - self.due = 0; - self.original_due = 0; - self.ease_factor = INITIAL_EASE_FACTOR; - } - } } #[derive(Debug)] pub(crate) struct UpdateCardUndo(Card); diff --git a/rslib/src/revlog.rs b/rslib/src/revlog.rs index 1d4d9dd6c..71ab56a14 100644 --- a/rslib/src/revlog.rs +++ b/rslib/src/revlog.rs @@ -42,6 +42,7 @@ pub enum RevlogReviewKind { Review = 1, Relearning = 2, EarlyReview = 3, + Manual = 4, } impl Default for RevlogReviewKind { @@ -59,3 +60,40 @@ impl RevlogEntry { }) as u32 } } + +impl Card { + fn last_interval_for_revlog_todo(&self) -> i32 { + self.interval as i32 + + // fixme: need to pass in delays for (re)learning + // if let Some(delay) = self.current_learning_delay_seconds(&[]) { + // -(delay as i32) + // } else { + // self.interval as i32 + // } + } +} + +impl Collection { + pub(crate) fn log_manually_scheduled_review( + &mut self, + card: &Card, + usn: Usn, + next_interval: u32, + ) -> Result<()> { + println!("fixme: learning last_interval"); + // let deck = self.get_deck(card.deck_id)?.ok_or(AnkiError::NotFound)?; + let entry = RevlogEntry { + id: TimestampMillis::now(), + cid: card.id, + usn, + button_chosen: 0, + interval: next_interval as i32, + last_interval: card.last_interval_for_revlog_todo(), + ease_factor: card.ease_factor as u32, + taken_millis: 0, + review_kind: RevlogReviewKind::Manual, + }; + self.storage.add_revlog_entry(&entry) + } +} diff --git a/rslib/src/sched/learning.rs b/rslib/src/sched/learning.rs new file mode 100644 index 000000000..80f797bfa --- /dev/null +++ b/rslib/src/sched/learning.rs @@ -0,0 +1,58 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::{ + card::{Card, CardQueue, CardType}, + deckconf::INITIAL_EASE_FACTOR, +}; + +impl Card { + /// Remove the card from the (re)learning queue. + /// This will reset cards in learning. + /// Only used in the V1 scheduler. + /// Unlike the legacy Python code, this sets the due# to 0 instead of + /// one past the previous max due number. + pub(crate) fn remove_from_learning(&mut self) { + if !matches!(self.queue, CardQueue::Learn | CardQueue::DayLearn) { + return; + } + + if self.ctype == CardType::Review { + // reviews are removed from relearning + self.due = self.original_due; + self.original_due = 0; + self.queue = CardQueue::Review; + } else { + // other cards are reset to new + self.ctype = CardType::New; + self.queue = CardQueue::New; + self.interval = 0; + self.due = 0; + self.original_due = 0; + self.ease_factor = INITIAL_EASE_FACTOR; + } + } + + fn all_remaining_steps(&self) -> u32 { + self.remaining_steps % 1000 + } + + #[allow(dead_code)] + fn remaining_steps_today(&self) -> u32 { + self.remaining_steps / 1000 + } + + #[allow(dead_code)] + pub(crate) fn current_learning_delay_seconds(&self, delays: &[u32]) -> Option { + if self.queue == CardQueue::Learn { + let remaining = self.all_remaining_steps(); + delays + .iter() + .nth_back(remaining.saturating_sub(0) as usize) + .or(Some(&0)) + .map(|n| n * 60) + } else { + None + } + } +} diff --git a/rslib/src/sched/mod.rs b/rslib/src/sched/mod.rs index af0df2997..8cd709e81 100644 --- a/rslib/src/sched/mod.rs +++ b/rslib/src/sched/mod.rs @@ -8,6 +8,7 @@ use crate::{ pub mod bury_and_suspend; pub(crate) mod congrats; pub mod cutoff; +mod learning; mod reviews; pub mod timespan; diff --git a/rslib/src/sched/reviews.rs b/rslib/src/sched/reviews.rs index 0ad8062b3..e39b22585 100644 --- a/rslib/src/sched/reviews.rs +++ b/rslib/src/sched/reviews.rs @@ -40,6 +40,7 @@ impl Collection { for mut card in col.storage.all_searched_cards()? { let original = card.clone(); let interval = distribution.sample(&mut rng); + col.log_manually_scheduled_review(&card, usn, interval)?; card.schedule_as_review(interval, today); col.update_card(&mut card, &original, usn)?; } diff --git a/rslib/src/sched/timespan.rs b/rslib/src/sched/timespan.rs index ff74e186e..d8dfe15e4 100644 --- a/rslib/src/sched/timespan.rs +++ b/rslib/src/sched/timespan.rs @@ -41,21 +41,6 @@ pub fn time_span(seconds: f32, i18n: &I18n, precise: bool) -> String { i18n.trn(key, args) } -// fixme: this doesn't belong here -pub fn studied_today(cards: usize, secs: f32, i18n: &I18n) -> String { - let span = Timespan::from_secs(secs).natural_span(); - let amount = span.as_unit(); - let unit = span.unit().as_str(); - let secs_per = if cards > 0 { - secs / (cards as f32) - } else { - 0.0 - }; - let args = tr_args!["amount" => amount, "unit" => unit, - "cards" => cards, "secs-per-card" => secs_per]; - i18n.trn(TR::StatisticsStudiedToday, args) -} - const SECOND: f32 = 1.0; const MINUTE: f32 = 60.0 * SECOND; const HOUR: f32 = 60.0 * MINUTE; @@ -64,7 +49,7 @@ const MONTH: f32 = 30.0 * DAY; const YEAR: f32 = 12.0 * MONTH; #[derive(Clone, Copy)] -enum TimespanUnit { +pub(crate) enum TimespanUnit { Seconds, Minutes, Hours, @@ -74,7 +59,7 @@ enum TimespanUnit { } impl TimespanUnit { - fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { TimespanUnit::Seconds => "seconds", TimespanUnit::Minutes => "minutes", @@ -87,13 +72,13 @@ impl TimespanUnit { } #[derive(Clone, Copy)] -struct Timespan { +pub(crate) struct Timespan { seconds: f32, unit: TimespanUnit, } impl Timespan { - fn from_secs(seconds: f32) -> Self { + pub fn from_secs(seconds: f32) -> Self { Timespan { seconds, unit: TimespanUnit::Seconds, @@ -102,7 +87,7 @@ impl Timespan { /// Return the value as the configured unit, eg seconds=70/unit=Minutes /// returns 1.17 - fn as_unit(self) -> f32 { + pub fn as_unit(self) -> f32 { let s = self.seconds; match self.unit { TimespanUnit::Seconds => s, @@ -116,7 +101,7 @@ impl Timespan { /// Round seconds and days to integers, otherwise /// truncates to one decimal place. - fn as_rounded_unit(self) -> f32 { + pub fn as_rounded_unit(self) -> f32 { match self.unit { // seconds/days as integer TimespanUnit::Seconds | TimespanUnit::Days => self.as_unit().round(), @@ -125,13 +110,13 @@ impl Timespan { } } - fn unit(self) -> TimespanUnit { + pub fn unit(self) -> TimespanUnit { self.unit } /// Return a new timespan in the most appropriate unit, eg /// 70 secs -> timespan in minutes - fn natural_span(self) -> Timespan { + pub fn natural_span(self) -> Timespan { let secs = self.seconds.abs(); let unit = if secs < MINUTE { TimespanUnit::Seconds @@ -158,7 +143,7 @@ impl Timespan { mod test { use crate::i18n::I18n; use crate::log; - use crate::sched::timespan::{answer_button_time, studied_today, time_span, MONTH}; + use crate::sched::timespan::{answer_button_time, time_span, MONTH}; #[test] fn answer_buttons() { @@ -180,15 +165,4 @@ mod test { assert_eq!(time_span(45.0 * 86_400.0, &i18n, false), "1.5 months"); assert_eq!(time_span(365.0 * 86_400.0 * 1.5, &i18n, false), "1.5 years"); } - - #[test] - fn combo() { - // temporary test of fluent term handling - let log = log::terminal(); - let i18n = I18n::new(&["zz"], "", log); - assert_eq!( - &studied_today(3, 13.0, &i18n).replace("\n", " "), - "Studied 3 cards in 13 seconds today (4.33s/card)" - ); - } } diff --git a/rslib/src/stats/card.rs b/rslib/src/stats/card.rs index c5a07bb12..863dafd61 100644 --- a/rslib/src/stats/card.rs +++ b/rslib/src/stats/card.rs @@ -229,12 +229,14 @@ fn revlog_to_text(e: RevlogEntry, i18n: &I18n, offset: FixedOffset) -> RevlogTex RevlogReviewKind::Review => i18n.tr(TR::CardStatsReviewLogTypeReview).into(), RevlogReviewKind::Relearning => i18n.tr(TR::CardStatsReviewLogTypeRelearn).into(), RevlogReviewKind::EarlyReview => i18n.tr(TR::CardStatsReviewLogTypeFiltered).into(), + RevlogReviewKind::Manual => i18n.tr(TR::CardStatsReviewLogTypeManual).into(), }; let kind_class = match e.review_kind { RevlogReviewKind::Learning => String::from("revlog-learn"), RevlogReviewKind::Review => String::from("revlog-review"), RevlogReviewKind::Relearning => String::from("revlog-relearn"), RevlogReviewKind::EarlyReview => String::from("revlog-filtered"), + RevlogReviewKind::Manual => String::from("revlog-manual"), }; let rating = e.button_chosen.to_string(); let interval = if e.interval == 0 { diff --git a/rslib/src/stats/mod.rs b/rslib/src/stats/mod.rs index 33b8dd19b..b855d8ce5 100644 --- a/rslib/src/stats/mod.rs +++ b/rslib/src/stats/mod.rs @@ -3,3 +3,6 @@ mod card; mod graphs; +mod today; + +pub use today::studied_today; diff --git a/rslib/src/stats/today.rs b/rslib/src/stats/today.rs new file mode 100644 index 000000000..27b1a0b4e --- /dev/null +++ b/rslib/src/stats/today.rs @@ -0,0 +1,45 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::{i18n::I18n, prelude::*, sched::timespan::Timespan}; + +pub fn studied_today(cards: u32, secs: f32, i18n: &I18n) -> String { + let span = Timespan::from_secs(secs).natural_span(); + let amount = span.as_unit(); + let unit = span.unit().as_str(); + let secs_per = if cards > 0 { + secs / (cards as f32) + } else { + 0.0 + }; + let args = tr_args!["amount" => amount, "unit" => unit, + "cards" => cards, "secs-per-card" => secs_per]; + i18n.trn(TR::StatisticsStudiedToday, args) +} + +impl Collection { + pub fn studied_today(&self) -> Result { + let today = self + .storage + .studied_today(self.timing_today()?.next_day_at)?; + Ok(studied_today(today.cards, today.seconds as f32, &self.i18n)) + } +} + +#[cfg(test)] +mod test { + use super::studied_today; + use crate::i18n::I18n; + use crate::log; + + #[test] + fn today() { + // temporary test of fluent term handling + let log = log::terminal(); + let i18n = I18n::new(&["zz"], "", log); + assert_eq!( + &studied_today(3, 13.0, &i18n).replace("\n", " "), + "Studied 3 cards in 13 seconds today (4.33s/card)" + ); + } +} diff --git a/rslib/src/storage/revlog/add.sql b/rslib/src/storage/revlog/add.sql index a3155a942..b63301200 100644 --- a/rslib/src/storage/revlog/add.sql +++ b/rslib/src/storage/revlog/add.sql @@ -10,5 +10,25 @@ insert time, type ) -values - (?, ?, ?, ?, ?, ?, ?, ?, ?) \ No newline at end of file +values ( + ( + case + when ?1 in ( + select id + from revlog + ) then ( + select max(id) + 1 + from revlog + ) + else ?1 + end + ), + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ? + ) \ No newline at end of file diff --git a/rslib/src/storage/revlog/mod.rs b/rslib/src/storage/revlog/mod.rs index fd9434ea2..991f16929 100644 --- a/rslib/src/storage/revlog/mod.rs +++ b/rslib/src/storage/revlog/mod.rs @@ -15,6 +15,11 @@ use rusqlite::{ }; use std::convert::TryFrom; +pub(crate) struct StudiedToday { + pub cards: u32, + pub seconds: f64, +} + impl FromSql for RevlogReviewKind { fn column_result(value: ValueRef<'_>) -> std::result::Result { if let ValueRef::Integer(i) = value { @@ -113,4 +118,19 @@ impl SqliteStorage { })? .collect() } + + pub(crate) fn studied_today(&self, day_cutoff: i64) -> Result { + let start = (day_cutoff - 86_400) * 1_000; + self.db + .prepare_cached(include_str!("studied_today.sql"))? + .query_map(&[start, RevlogReviewKind::Manual as i64], |row| { + Ok(StudiedToday { + cards: row.get(0)?, + seconds: row.get(1)?, + }) + })? + .next() + .unwrap() + .map_err(Into::into) + } } diff --git a/rslib/src/storage/revlog/studied_today.sql b/rslib/src/storage/revlog/studied_today.sql new file mode 100644 index 000000000..73f8cfba3 --- /dev/null +++ b/rslib/src/storage/revlog/studied_today.sql @@ -0,0 +1,5 @@ +select count(), + coalesce(sum(time) / 1000.0, 0.0) +from revlog +where id > ? + and type != ? \ No newline at end of file diff --git a/rslib/src/timestamp.rs b/rslib/src/timestamp.rs index 1645ac17d..0189c718a 100644 --- a/rslib/src/timestamp.rs +++ b/rslib/src/timestamp.rs @@ -15,6 +15,10 @@ impl TimestampSecs { Self(elapsed().as_secs() as i64) } + pub fn zero() -> Self { + Self(0) + } + pub fn elapsed_secs(self) -> u64 { (Self::now().0 - self.0).max(0) as u64 } @@ -30,6 +34,10 @@ impl TimestampMillis { Self(elapsed().as_millis() as i64) } + pub fn zero() -> Self { + Self(0) + } + pub fn as_secs(self) -> TimestampSecs { TimestampSecs(self.0 / 1000) } diff --git a/ts/src/stats/reviews.ts b/ts/src/stats/reviews.ts index b6a41d9df..a33b878a8 100644 --- a/ts/src/stats/reviews.ts +++ b/ts/src/stats/reviews.ts @@ -47,6 +47,10 @@ export function gatherData(data: pb.BackendProto.GraphsOut): GraphData { const empty = { mature: 0, young: 0, learn: 0, relearn: 0, early: 0 }; for (const review of data.revlog as pb.BackendProto.RevlogEntry[]) { + if (review.reviewKind == ReviewKind.MANUAL) { + // don't count days with only manual scheduling + continue; + } const day = Math.ceil( ((review.id as number) / 1000 - data.nextDayAtSecs) / 86400 ); diff --git a/ts/src/stats/today.ts b/ts/src/stats/today.ts index 1c861d4c8..db77073bc 100644 --- a/ts/src/stats/today.ts +++ b/ts/src/stats/today.ts @@ -1,7 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -import pb from "../backend/proto"; +import pb, { BackendProto } from "../backend/proto"; import { studiedToday } from "../time"; import { I18n } from "../i18n"; @@ -30,6 +30,10 @@ export function gatherData(data: pb.BackendProto.GraphsOut, i18n: I18n): TodayDa continue; } + if (review.reviewKind == ReviewKind.MANUAL) { + continue; + } + // total answerCount += 1; answerMillis += review.takenMillis; From e56f83be8479c7fa81684eebfd3d73971e74fd4b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 3 Sep 2020 12:43:18 +1000 Subject: [PATCH 04/21] set local offset when saving preferences --- rslib/src/preferences.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rslib/src/preferences.rs b/rslib/src/preferences.rs index 93231c41f..337a61e4c 100644 --- a/rslib/src/preferences.rs +++ b/rslib/src/preferences.rs @@ -9,6 +9,7 @@ use crate::{ collection::Collection, err::Result, sched::cutoff::local_minutes_west_for_stamp, + timestamp::TimestampSecs, }; impl Collection { @@ -79,6 +80,10 @@ impl Collection { self.set_creation_mins_west(None)?; } + if s.scheduler_version != 1 { + self.set_local_mins_west(local_minutes_west_for_stamp(TimestampSecs::now().0))?; + } + // fixme: currently scheduler change unhandled Ok(()) } From b65174a026916cfb2638270fc0101b81571f8389 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 3 Sep 2020 15:44:14 +1000 Subject: [PATCH 05/21] move card sorting and resetting to backend --- proto/backend.proto | 16 ++ pylib/anki/schedv2.py | 89 +++-------- rslib/src/backend/mod.rs | 28 ++++ rslib/src/config.rs | 4 + rslib/src/sched/bury_and_suspend.rs | 6 +- rslib/src/sched/mod.rs | 1 + rslib/src/sched/new.rs | 143 ++++++++++++++++++ rslib/src/sched/reviews.rs | 2 +- rslib/src/search/cards.rs | 16 +- rslib/src/stats/graphs.rs | 2 +- .../src/storage/card/at_or_above_position.sql | 5 + rslib/src/storage/card/mod.rs | 20 +++ .../card}/search_cids_setup.sql | 0 13 files changed, 245 insertions(+), 87 deletions(-) create mode 100644 rslib/src/sched/new.rs create mode 100644 rslib/src/storage/card/at_or_above_position.sql rename rslib/src/{search => storage/card}/search_cids_setup.sql (100%) diff --git a/proto/backend.proto b/proto/backend.proto index 95af6b8f1..483f69fc1 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -107,6 +107,9 @@ service BackendService { rpc EmptyFilteredDeck (DeckID) returns (Empty); rpc RebuildFilteredDeck (DeckID) returns (UInt32); rpc ScheduleCardsAsReviews (ScheduleCardsAsReviewsIn) returns (Empty); + rpc ScheduleCardsAsNew (CardIDs) returns (Empty); + rpc SortCards (SortCardsIn) returns (Empty); + rpc SortDeck (SortDeckIn) returns (Empty); // stats @@ -1065,3 +1068,16 @@ message ScheduleCardsAsReviewsIn { uint32 min_interval = 2; uint32 max_interval = 3; } + +message SortCardsIn { + repeated int64 card_ids = 1; + uint32 starting_from = 2; + uint32 step_size = 3; + bool randomize = 4; + bool shift_existing = 5; +} + +message SortDeckIn { + int64 deck_id = 1; + bool randomize = 2; +} diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index 87b2e373d..b91aed151 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -15,7 +15,6 @@ from typing import ( List, Optional, Sequence, - Set, Tuple, Union, ) @@ -1401,26 +1400,16 @@ and (queue={QUEUE_TYPE_NEW} or (queue={QUEUE_TYPE_REV} and due<=?))""", # Resetting ########################################################################## - def forgetCards(self, ids: List[int]) -> None: + def schedule_cards_as_new(self, card_ids: List[int]) -> None: "Put cards at the end of the new queue." - self.remFromDyn(ids) - self.col.db.execute( - f"update cards set type={CARD_TYPE_NEW},queue={QUEUE_TYPE_NEW},ivl=0,due=0,odue=0,factor=?" - " where id in " + ids2str(ids), - STARTING_FACTOR, - ) - pmax = ( - self.col.db.scalar(f"select max(due) from cards where type={CARD_TYPE_NEW}") - or 0 - ) - # takes care of mod + usn - self.sortCards(ids, start=pmax + 1) - self.col.log(ids) + self.col.backend.schedule_cards_as_new(card_ids) - def reschedCards(self, ids: List[int], imin: int, imax: int) -> None: - "Put cards in review queue with a new interval in days (min, max)." + def schedule_cards_as_reviews( + self, card_ids: List[int], min_interval: int, max_interval: int + ) -> None: + "Make cards review cards, with a new interval randomly selected from range." self.col.backend.schedule_cards_as_reviews( - card_ids=ids, min_interval=imin, max_interval=imax + card_ids=card_ids, min_interval=min_interval, max_interval=max_interval ) def resetCards(self, ids: List[int]) -> None: @@ -1440,6 +1429,11 @@ and (queue={QUEUE_TYPE_NEW} or (queue={QUEUE_TYPE_REV} and due<=?))""", self.forgetCards(nonNew) self.col.log(ids) + # legacy + + forgetCards = schedule_cards_as_new + reschedCards = schedule_cards_as_reviews + # Repositioning new cards ########################################################################## @@ -1451,60 +1445,19 @@ and (queue={QUEUE_TYPE_NEW} or (queue={QUEUE_TYPE_REV} and due<=?))""", shuffle: bool = False, shift: bool = False, ) -> None: - scids = ids2str(cids) - now = intTime() - nids = [] - nidsSet: Set[int] = set() - for id in cids: - nid = self.col.db.scalar("select nid from cards where id = ?", id) - if nid not in nidsSet: - nids.append(nid) - nidsSet.add(nid) - if not nids: - # no new cards - return - # determine nid ordering - due = {} - if shuffle: - random.shuffle(nids) - for c, nid in enumerate(nids): - due[nid] = start + c * step - # pylint: disable=undefined-loop-variable - high = start + c * step - # shift? - if shift: - low = self.col.db.scalar( - f"select min(due) from cards where due >= ? and type = {CARD_TYPE_NEW} " - "and id not in %s" % scids, - start, - ) - if low is not None: - shiftby = high - low + 1 - self.col.db.execute( - f""" -update cards set mod=?, usn=?, due=due+? where id not in %s -and due >= ? and queue = {QUEUE_TYPE_NEW}""" - % scids, - now, - self.col.usn(), - shiftby, - low, - ) - # reorder cards - d = [] - for id, nid in self.col.db.execute( - f"select id, nid from cards where type = {CARD_TYPE_NEW} and id in " + scids - ): - d.append((due[nid], now, self.col.usn(), id)) - self.col.db.executemany("update cards set due=?,mod=?,usn=? where id = ?", d) + self.col.backend.sort_cards( + card_ids=cids, + starting_from=start, + step_size=step, + randomize=shuffle, + shift_existing=shift, + ) def randomizeCards(self, did: int) -> None: - cids = self.col.db.list("select id from cards where did = ?", did) - self.sortCards(cids, shuffle=True) + self.col.backend.sort_deck(deck_id=did, randomize=True) def orderCards(self, did: int) -> None: - cids = self.col.db.list("select id from cards where did = ? order by nid", did) - self.sortCards(cids) + self.col.backend.sort_deck(deck_id=did, randomize=False) def resortConf(self, conf) -> None: for did in self.col.decks.didsForConf(conf): diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index e157ba0fc..281f37bbe 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -569,6 +569,34 @@ impl BackendService for Backend { }) } + fn schedule_cards_as_new(&mut self, input: pb::CardIDs) -> BackendResult { + self.with_col(|col| { + col.reschedule_cards_as_new(&input.into_native()) + .map(Into::into) + }) + } + + fn sort_cards(&mut self, input: pb::SortCardsIn) -> BackendResult { + let cids: Vec<_> = input.card_ids.into_iter().map(CardID).collect(); + let (start, step, random, shift) = ( + input.starting_from, + input.step_size, + input.randomize, + input.shift_existing, + ); + self.with_col(|col| { + col.sort_cards(&cids, start, step, random, shift) + .map(Into::into) + }) + } + + fn sort_deck(&mut self, input: pb::SortDeckIn) -> BackendResult { + self.with_col(|col| { + col.sort_deck(input.deck_id.into(), input.randomize) + .map(Into::into) + }) + } + // statistics //----------------------------------------------- diff --git a/rslib/src/config.rs b/rslib/src/config.rs index b3c2e845e..70294ef1f 100644 --- a/rslib/src/config.rs +++ b/rslib/src/config.rs @@ -179,6 +179,10 @@ impl Collection { self.set_config(ConfigKey::CurrentNoteTypeID, &id) } + pub(crate) fn get_next_card_position(&self) -> u32 { + self.get_config_default(ConfigKey::NextNewCardPosition) + } + pub(crate) fn get_and_update_next_card_position(&self) -> Result { let pos: u32 = self .get_config_optional(ConfigKey::NextNewCardPosition) diff --git a/rslib/src/sched/bury_and_suspend.rs b/rslib/src/sched/bury_and_suspend.rs index e14979328..a51c3e416 100644 --- a/rslib/src/sched/bury_and_suspend.rs +++ b/rslib/src/sched/bury_and_suspend.rs @@ -65,7 +65,7 @@ impl Collection { card.restore_queue_after_bury_or_suspend(); self.storage.update_card(&card) })?; - self.clear_searched_cards() + self.storage.clear_searched_cards_table() } /// Unsuspend/unbury cards in search table, and clear it. @@ -78,7 +78,7 @@ impl Collection { self.update_card(&mut card, &original, usn)?; } } - self.clear_searched_cards() + self.storage.clear_searched_cards_table() } pub fn unbury_or_unsuspend_cards(&mut self, cids: &[CardID]) -> Result<()> { @@ -134,7 +134,7 @@ impl Collection { } } - self.clear_searched_cards() + self.storage.clear_searched_cards_table() } pub fn bury_or_suspend_cards( diff --git a/rslib/src/sched/mod.rs b/rslib/src/sched/mod.rs index 8cd709e81..1a90ac43e 100644 --- a/rslib/src/sched/mod.rs +++ b/rslib/src/sched/mod.rs @@ -9,6 +9,7 @@ pub mod bury_and_suspend; pub(crate) mod congrats; pub mod cutoff; mod learning; +pub mod new; mod reviews; pub mod timespan; diff --git a/rslib/src/sched/new.rs b/rslib/src/sched/new.rs new file mode 100644 index 000000000..7ae5ac197 --- /dev/null +++ b/rslib/src/sched/new.rs @@ -0,0 +1,143 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::{ + card::{Card, CardID, CardQueue, CardType}, + collection::Collection, + deckconf::INITIAL_EASE_FACTOR, + decks::DeckID, + err::Result, + notes::NoteID, + search::SortMode, + types::Usn, +}; +use rand::seq::SliceRandom; +use std::collections::{HashMap, HashSet}; + +impl Card { + fn schedule_as_new(&mut self, position: u32) { + self.remove_from_filtered_deck_before_reschedule(); + self.due = position as i32; + self.ctype = CardType::New; + self.queue = CardQueue::New; + self.interval = 0; + if self.ease_factor == 0 { + // unlike the old Python code, we leave the ease factor alone + // if it's already set + self.ease_factor = INITIAL_EASE_FACTOR; + } + } + + /// If the card is new, change its position. + fn set_new_position(&mut self, position: u32) { + if self.queue != CardQueue::New || self.ctype != CardType::New { + return; + } + self.due = position as i32; + } +} +pub(crate) struct NewCardSorter { + position: HashMap, +} + +impl NewCardSorter { + pub(crate) fn new(cards: &[Card], starting_from: u32, step: u32, random: bool) -> Self { + let nids: HashSet<_> = cards.iter().map(|c| c.note_id).collect(); + let mut nids: Vec<_> = nids.into_iter().collect(); + if random { + nids.shuffle(&mut rand::thread_rng()); + } else { + nids.sort_unstable(); + } + + NewCardSorter { + position: nids + .into_iter() + .enumerate() + .map(|(i, nid)| (nid, ((i as u32) * step) + starting_from)) + .collect(), + } + } + + pub(crate) fn position(&self, card: &Card) -> u32 { + self.position + .get(&card.note_id) + .cloned() + .unwrap_or_default() + } +} + +impl Collection { + pub fn reschedule_cards_as_new(&mut self, cids: &[CardID]) -> Result<()> { + let usn = self.usn()?; + let mut position = self.get_next_card_position(); + self.transact(None, |col| { + col.set_search_table_to_card_ids(cids)?; + let cards = col.storage.all_searched_cards()?; + for mut card in cards { + let original = card.clone(); + col.log_manually_scheduled_review(&card, usn, 0)?; + card.schedule_as_new(position); + col.update_card(&mut card, &original, usn)?; + position += 1; + } + col.set_next_card_position(position)?; + col.storage.clear_searched_cards_table()?; + Ok(()) + }) + } + + pub fn sort_cards( + &mut self, + cids: &[CardID], + starting_from: u32, + step: u32, + random: bool, + shift: bool, + ) -> Result<()> { + let usn = self.usn()?; + self.transact(None, |col| { + col.sort_cards_inner(cids, starting_from, step, random, shift, usn) + }) + } + + fn sort_cards_inner( + &mut self, + cids: &[CardID], + starting_from: u32, + step: u32, + random: bool, + shift: bool, + usn: Usn, + ) -> Result<()> { + if shift { + self.shift_existing_cards(starting_from, step * cids.len() as u32, usn)?; + } + self.set_search_table_to_card_ids(cids)?; + let cards = self.storage.all_searched_cards()?; + let sorter = NewCardSorter::new(&cards, starting_from, step, random); + for mut card in cards { + let original = card.clone(); + card.set_new_position(sorter.position(&card)); + self.update_card(&mut card, &original, usn)?; + } + self.storage.clear_searched_cards_table() + } + + /// This creates a transaction - we probably want to split it out + /// in the future if calling it as part of a deck options update. + pub fn sort_deck(&mut self, deck: DeckID, random: bool) -> Result<()> { + let cids = self.search_cards(&format!("did:{}", deck), SortMode::NoOrder)?; + self.sort_cards(&cids, 1, 1, random, false) + } + + fn shift_existing_cards(&mut self, start: u32, by: u32, usn: Usn) -> Result<()> { + self.storage.search_cards_at_or_above_position(start)?; + for mut card in self.storage.all_searched_cards()? { + let original = card.clone(); + card.set_new_position(card.due as u32 + by); + self.update_card(&mut card, &original, usn)?; + } + Ok(()) + } +} diff --git a/rslib/src/sched/reviews.rs b/rslib/src/sched/reviews.rs index e39b22585..1d5b4c945 100644 --- a/rslib/src/sched/reviews.rs +++ b/rslib/src/sched/reviews.rs @@ -44,7 +44,7 @@ impl Collection { card.schedule_as_review(interval, today); col.update_card(&mut card, &original, usn)?; } - col.clear_searched_cards()?; + col.storage.clear_searched_cards_table()?; Ok(()) }) } diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index 351abc89b..5f9f5fbd4 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -9,7 +9,6 @@ use crate::{ card::CardID, card::CardType, collection::Collection, config::SortKind, err::Result, search::parser::parse, }; -use rusqlite::NO_PARAMS; #[derive(Debug, PartialEq, Clone)] pub enum SortMode { @@ -99,9 +98,7 @@ impl Collection { let (mut sql, args) = writer.build_cards_query(&top_node, mode.required_table())?; self.add_order(&mut sql, mode)?; - self.storage - .db - .execute_batch(include_str!("search_cids_setup.sql"))?; + self.storage.setup_searched_cards_table()?; let sql = format!("insert into search_cids {}", sql); self.storage.db.prepare(&sql)?.execute(&args)?; @@ -113,9 +110,7 @@ impl Collection { /// when ids have arrived outside of a search. /// Clear with clear_searched_cards(). pub(crate) fn set_search_table_to_card_ids(&mut self, cards: &[CardID]) -> Result<()> { - self.storage - .db - .execute_batch(include_str!("search_cids_setup.sql"))?; + self.storage.setup_searched_cards_table()?; let mut stmt = self .storage .db @@ -127,13 +122,6 @@ impl Collection { Ok(()) } - pub(crate) fn clear_searched_cards(&self) -> Result<()> { - self.storage - .db - .execute("drop table if exists search_cids", NO_PARAMS)?; - Ok(()) - } - /// If the sort mode is based on a config setting, look it up. fn resolve_config_sort(&self, mode: &mut SortMode) { if mode == &SortMode::FromConfig { diff --git a/rslib/src/stats/graphs.rs b/rslib/src/stats/graphs.rs index 8a2432315..d2afcbee0 100644 --- a/rslib/src/stats/graphs.rs +++ b/rslib/src/stats/graphs.rs @@ -33,7 +33,7 @@ impl Collection { .get_revlog_entries_for_searched_cards(revlog_start)? }; - self.clear_searched_cards()?; + self.storage.clear_searched_cards_table()?; Ok(pb::GraphsOut { cards: cards.into_iter().map(Into::into).collect(), diff --git a/rslib/src/storage/card/at_or_above_position.sql b/rslib/src/storage/card/at_or_above_position.sql new file mode 100644 index 000000000..2621aa55a --- /dev/null +++ b/rslib/src/storage/card/at_or_above_position.sql @@ -0,0 +1,5 @@ +insert into search_cids +select id +from cards +where due >= ? + and type = ? \ No newline at end of file diff --git a/rslib/src/storage/card/mod.rs b/rslib/src/storage/card/mod.rs index 521b5ed81..de7640678 100644 --- a/rslib/src/storage/card/mod.rs +++ b/rslib/src/storage/card/mod.rs @@ -315,6 +315,26 @@ impl super::SqliteStorage { .next() .unwrap() } + + pub(crate) fn search_cards_at_or_above_position(&self, start: u32) -> Result<()> { + self.setup_searched_cards_table()?; + self.db + .prepare(include_str!("at_or_above_position.sql"))? + .execute(&[start, CardType::New as u32])?; + Ok(()) + } + + pub(crate) fn setup_searched_cards_table(&self) -> Result<()> { + self.db + .execute_batch(include_str!("search_cids_setup.sql"))?; + Ok(()) + } + + pub(crate) fn clear_searched_cards_table(&self) -> Result<()> { + self.db + .execute("drop table if exists search_cids", NO_PARAMS)?; + Ok(()) + } } #[cfg(test)] diff --git a/rslib/src/search/search_cids_setup.sql b/rslib/src/storage/card/search_cids_setup.sql similarity index 100% rename from rslib/src/search/search_cids_setup.sql rename to rslib/src/storage/card/search_cids_setup.sql From 97778bec4315277d44fceb5cc9c6b7f725b8feee Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 3 Sep 2020 16:00:06 +1000 Subject: [PATCH 06/21] log 0 day interval as 1 day Allows distinguishing it from resetting to new, and is treated as a one day delay when answering anyway. --- rslib/src/sched/reviews.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rslib/src/sched/reviews.rs b/rslib/src/sched/reviews.rs index 1d5b4c945..160a23faa 100644 --- a/rslib/src/sched/reviews.rs +++ b/rslib/src/sched/reviews.rs @@ -40,7 +40,7 @@ impl Collection { for mut card in col.storage.all_searched_cards()? { let original = card.clone(); let interval = distribution.sample(&mut rng); - col.log_manually_scheduled_review(&card, usn, interval)?; + col.log_manually_scheduled_review(&card, usn, interval.max(1))?; card.schedule_as_review(interval, today); col.update_card(&mut card, &original, usn)?; } From 9214c4a700bbcd2606d7af5f0d671fba76ce0752 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 3 Sep 2020 16:43:22 +1000 Subject: [PATCH 07/21] support did:[deck id] search --- rslib/src/search/parser.rs | 3 +++ rslib/src/search/sqlwriter.rs | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 70e33f08d..6d3cd038c 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -2,6 +2,7 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html use crate::{ + decks::DeckID, err::{AnkiError, Result}, notetype::NoteTypeID, }; @@ -64,6 +65,7 @@ pub(super) enum SearchNode<'a> { EditedInDays(u32), CardTemplate(TemplateKind), Deck(Cow<'a, str>), + DeckID(DeckID), NoteTypeID(NoteTypeID), NoteType(Cow<'a, str>), Rated { @@ -283,6 +285,7 @@ fn search_node_for_text_with_argument<'a>( "mid" => SearchNode::NoteTypeID(val.parse()?), "nid" => SearchNode::NoteIDs(check_id_list(val)?), "cid" => SearchNode::CardIDs(check_id_list(val)?), + "did" => SearchNode::DeckID(val.parse()?), "card" => parse_template(val.as_ref()), "is" => parse_state(val.as_ref())?, "flag" => parse_flag(val.as_ref())?, diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 385fe71f8..c57b55081 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -140,6 +140,9 @@ impl SqlWriter<'_> { SearchNode::NoteTypeID(ntid) => { write!(self.sql, "n.mid = {}", ntid).unwrap(); } + SearchNode::DeckID(did) => { + write!(self.sql, "c.did = {}", did).unwrap(); + } SearchNode::NoteType(notetype) => self.write_note_type(&norm(notetype))?, SearchNode::Rated { days, ease } => self.write_rated(*days, *ease)?, SearchNode::Tag(tag) => self.write_tag(&norm(tag))?, @@ -505,6 +508,7 @@ impl SearchNode<'_> { match self { SearchNode::AddedInDays(_) => RequiredTable::Cards, SearchNode::Deck(_) => RequiredTable::Cards, + SearchNode::DeckID(_) => RequiredTable::Cards, SearchNode::Rated { .. } => RequiredTable::Cards, SearchNode::State(_) => RequiredTable::Cards, SearchNode::Flag(_) => RequiredTable::Cards, From 56ceb6ba76a44d8e0aeeb07d8c3256819e59efae Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 3 Sep 2020 17:42:46 +1000 Subject: [PATCH 08/21] set_deck() --- proto/backend.proto | 6 ++ pylib/anki/collection.py | 3 + qt/aqt/browser.py | 18 +--- rslib/src/backend/mod.rs | 6 ++ rslib/src/card.rs | 124 ++++++---------------------- rslib/src/decks/mod.rs | 2 - rslib/src/{decks => }/filtered.rs | 105 ++++++++++++++++++++++- rslib/src/lib.rs | 1 + rslib/src/sched/bury_and_suspend.rs | 4 +- rslib/src/sched/new.rs | 4 +- rslib/src/sched/reviews.rs | 2 +- rslib/src/search/cards.rs | 16 ---- rslib/src/storage/card/mod.rs | 15 ++++ 13 files changed, 167 insertions(+), 139 deletions(-) rename rslib/src/{decks => }/filtered.rs (58%) diff --git a/proto/backend.proto b/proto/backend.proto index 483f69fc1..b438cca38 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -150,6 +150,7 @@ service BackendService { rpc UpdateCard (Card) returns (Empty); rpc AddCard (Card) returns (CardID); rpc RemoveCards (RemoveCardsIn) returns (Empty); + rpc SetDeck (SetDeckIn) returns (Empty); // notes @@ -1081,3 +1082,8 @@ message SortDeckIn { int64 deck_id = 1; bool randomize = 2; } + +message SetDeckIn { + repeated int64 card_ids = 1; + int64 deck_id = 2; +} diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index d423f42a9..cf39340e8 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -384,6 +384,9 @@ class Collection: "You probably want .remove_notes_by_card() instead." self.backend.remove_cards(card_ids=card_ids) + def set_deck(self, card_ids: List[int], deck_id: int) -> None: + self.backend.set_deck(card_ids=card_ids, deck_id=deck_id) + # legacy def remCards(self, ids: List[int], notes: bool = True) -> None: diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index 816b85f42..d36909606 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -22,7 +22,7 @@ from anki.models import NoteType from anki.notes import Note from anki.rsbackend import TR, DeckTreeNode, InvalidInput from anki.stats import CardStats -from anki.utils import htmlToTextLine, ids2str, intTime, isMac, isWin +from anki.utils import htmlToTextLine, ids2str, isMac, isWin from aqt import AnkiQt, gui_hooks from aqt.editor import Editor from aqt.exporting import ExportDialog @@ -1601,21 +1601,7 @@ where id in %s""" return self.model.beginReset() self.mw.checkpoint(_("Change Deck")) - mod = intTime() - usn = self.col.usn() - # normal cards - scids = ids2str(cids) - # remove any cards from filtered deck first - self.col.sched.remFromDyn(cids) - # then move into new deck - self.col.db.execute( - """ -update cards set usn=?, mod=?, did=? where id in """ - + scids, - usn, - mod, - did, - ) + self.col.set_deck(cids, did) self.model.endReset() self.mw.requireReset(reason=ResetReason.BrowserSetDeck, context=self) diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 281f37bbe..3741bf93c 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -817,6 +817,12 @@ impl BackendService for Backend { }) } + fn set_deck(&mut self, input: pb::SetDeckIn) -> BackendResult { + let cids: Vec<_> = input.card_ids.into_iter().map(CardID).collect(); + let deck_id = input.deck_id.into(); + self.with_col(|col| col.set_deck(&cids, deck_id).map(Into::into)) + } + // notes //------------------------------------------------------------------- diff --git a/rslib/src/card.rs b/rslib/src/card.rs index 42404c21b..4841adeb2 100644 --- a/rslib/src/card.rs +++ b/rslib/src/card.rs @@ -1,7 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::decks::{DeckFilterContext, DeckID}; +use crate::decks::DeckID; use crate::define_newtype; use crate::err::{AnkiError, Result}; use crate::notes::NoteID; @@ -102,103 +102,10 @@ impl Card { self.usn = usn; } - pub(crate) fn move_into_filtered_deck(&mut self, ctx: &DeckFilterContext, position: i32) { - // filtered and v1 learning cards are excluded, so odue should be guaranteed to be zero - if self.original_due != 0 { - println!("bug: odue was set"); - return; - } - - self.original_deck_id = self.deck_id; - self.deck_id = ctx.target_deck; - - self.original_due = self.due; - - if ctx.scheduler == SchedulerVersion::V1 { - if self.ctype == CardType::Review && self.due <= ctx.today as i32 { - // review cards that are due are left in the review queue - } else { - // new + non-due go into new queue - self.queue = CardQueue::New; - } - if self.due != 0 { - self.due = position; - } - } else { - // if rescheduling is disabled, all cards go in the review queue - if !ctx.config.reschedule { - self.queue = CardQueue::Review; - } - // fixme: can we unify this with v1 scheduler in the future? - // https://anki.tenderapp.com/discussions/ankidesktop/35978-rebuilding-filtered-deck-on-experimental-v2-empties-deck-and-reschedules-to-the-year-1745 - if self.due > 0 { - self.due = position; - } - } - } - - /// Restores to the original deck and clears original_due. - /// This does not update the queue or type, so should only be used as - /// part of an operation that adjusts those separately. - pub(crate) fn remove_from_filtered_deck_before_reschedule(&mut self) { - if self.original_deck_id.0 != 0 { - self.deck_id = self.original_deck_id; - self.original_deck_id.0 = 0; - self.original_due = 0; - } - } - - pub(crate) fn remove_from_filtered_deck_restoring_queue(&mut self, sched: SchedulerVersion) { - if self.original_deck_id.0 == 0 { - // not in a filtered deck - return; - } - - self.deck_id = self.original_deck_id; - self.original_deck_id.0 = 0; - - match sched { - SchedulerVersion::V1 => { - self.due = self.original_due; - self.queue = match self.ctype { - CardType::New => CardQueue::New, - CardType::Learn => CardQueue::New, - CardType::Review => CardQueue::Review, - // not applicable in v1, should not happen - CardType::Relearn => { - println!("did not expect relearn type in v1 for card {}", self.id); - CardQueue::New - } - }; - if self.ctype == CardType::Learn { - self.ctype = CardType::New; - } - } - SchedulerVersion::V2 => { - // original_due is cleared if card answered in filtered deck - if self.original_due > 0 { - self.due = self.original_due; - } - - if (self.queue as i8) >= 0 { - self.queue = match self.ctype { - CardType::Learn | CardType::Relearn => { - if self.due > 1_000_000_000 { - // unix timestamp - CardQueue::Learn - } else { - // day number - CardQueue::DayLearn - } - } - CardType::New => CardQueue::New, - CardType::Review => CardQueue::Review, - } - } - } - } - - self.original_due = 0; + /// Caller must ensure provided deck exists and is not filtered. + fn set_deck(&mut self, deck: DeckID, sched: SchedulerVersion) { + self.remove_from_filtered_deck_restoring_queue(sched); + self.deck_id = deck; } } #[derive(Debug)] @@ -289,6 +196,27 @@ impl Collection { Ok(()) } + + pub fn set_deck(&mut self, cards: &[CardID], deck_id: DeckID) -> Result<()> { + let deck = self.get_deck(deck_id)?.ok_or(AnkiError::NotFound)?; + if deck.is_filtered() { + return Err(AnkiError::DeckIsFiltered); + } + self.storage.set_search_table_to_card_ids(cards)?; + let sched = self.sched_ver(); + let usn = self.usn()?; + self.transact(None, |col| { + for mut card in col.storage.all_searched_cards()? { + if card.deck_id == deck_id { + continue; + } + let original = card.clone(); + card.set_deck(deck_id, sched); + col.update_card(&mut card, &original, usn)?; + } + Ok(()) + }) + } } #[cfg(test)] diff --git a/rslib/src/decks/mod.rs b/rslib/src/decks/mod.rs index af7db1845..a0ea612a3 100644 --- a/rslib/src/decks/mod.rs +++ b/rslib/src/decks/mod.rs @@ -17,11 +17,9 @@ use crate::{ types::Usn, }; mod counts; -mod filtered; mod schema11; mod tree; pub(crate) use counts::DueCounts; -pub(crate) use filtered::DeckFilterContext; pub use schema11::DeckSchema11; use std::{borrow::Cow, sync::Arc}; diff --git a/rslib/src/decks/filtered.rs b/rslib/src/filtered.rs similarity index 58% rename from rslib/src/decks/filtered.rs rename to rslib/src/filtered.rs index 0f9a278d8..fca4422e9 100644 --- a/rslib/src/decks/filtered.rs +++ b/rslib/src/filtered.rs @@ -1,13 +1,13 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use super::{Deck, DeckID}; pub use crate::backend_proto::{ deck_kind::Kind as DeckKind, filtered_search_term::FilteredSearchOrder, Deck as DeckProto, DeckCommon, DeckKind as DeckKindProto, FilteredDeck, FilteredSearchTerm, NormalDeck, }; +use crate::decks::{Deck, DeckID}; use crate::{ - card::{CardID, CardQueue}, + card::{Card, CardID, CardQueue, CardType}, collection::Collection, config::SchedulerVersion, err::Result, @@ -17,6 +17,107 @@ use crate::{ types::Usn, }; +impl Card { + pub(crate) fn move_into_filtered_deck(&mut self, ctx: &DeckFilterContext, position: i32) { + // filtered and v1 learning cards are excluded, so odue should be guaranteed to be zero + if self.original_due != 0 { + println!("bug: odue was set"); + return; + } + + self.original_deck_id = self.deck_id; + self.deck_id = ctx.target_deck; + + self.original_due = self.due; + + if ctx.scheduler == SchedulerVersion::V1 { + if self.ctype == CardType::Review && self.due <= ctx.today as i32 { + // review cards that are due are left in the review queue + } else { + // new + non-due go into new queue + self.queue = CardQueue::New; + } + if self.due != 0 { + self.due = position; + } + } else { + // if rescheduling is disabled, all cards go in the review queue + if !ctx.config.reschedule { + self.queue = CardQueue::Review; + } + // fixme: can we unify this with v1 scheduler in the future? + // https://anki.tenderapp.com/discussions/ankidesktop/35978-rebuilding-filtered-deck-on-experimental-v2-empties-deck-and-reschedules-to-the-year-1745 + if self.due > 0 { + self.due = position; + } + } + } + + /// Restores to the original deck and clears original_due. + /// This does not update the queue or type, so should only be used as + /// part of an operation that adjusts those separately. + pub(crate) fn remove_from_filtered_deck_before_reschedule(&mut self) { + if self.original_deck_id.0 != 0 { + self.deck_id = self.original_deck_id; + self.original_deck_id.0 = 0; + self.original_due = 0; + } + } + + pub(crate) fn remove_from_filtered_deck_restoring_queue(&mut self, sched: SchedulerVersion) { + if self.original_deck_id.0 == 0 { + // not in a filtered deck + return; + } + + self.deck_id = self.original_deck_id; + self.original_deck_id.0 = 0; + + match sched { + SchedulerVersion::V1 => { + self.due = self.original_due; + self.queue = match self.ctype { + CardType::New => CardQueue::New, + CardType::Learn => CardQueue::New, + CardType::Review => CardQueue::Review, + // not applicable in v1, should not happen + CardType::Relearn => { + println!("did not expect relearn type in v1 for card {}", self.id); + CardQueue::New + } + }; + if self.ctype == CardType::Learn { + self.ctype = CardType::New; + } + } + SchedulerVersion::V2 => { + // original_due is cleared if card answered in filtered deck + if self.original_due > 0 { + self.due = self.original_due; + } + + if (self.queue as i8) >= 0 { + self.queue = match self.ctype { + CardType::Learn | CardType::Relearn => { + if self.due > 1_000_000_000 { + // unix timestamp + CardQueue::Learn + } else { + // day number + CardQueue::DayLearn + } + } + CardType::New => CardQueue::New, + CardType::Review => CardQueue::Review, + } + } + } + } + + self.original_due = 0; + } +} + impl Deck { pub fn new_filtered() -> Deck { let mut filt = FilteredDeck::default(); diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index 574569f58..1cca25c30 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -13,6 +13,7 @@ pub mod dbcheck; pub mod deckconf; pub mod decks; pub mod err; +pub mod filtered; pub mod findreplace; pub mod i18n; pub mod latex; diff --git a/rslib/src/sched/bury_and_suspend.rs b/rslib/src/sched/bury_and_suspend.rs index a51c3e416..01fdd0ad0 100644 --- a/rslib/src/sched/bury_and_suspend.rs +++ b/rslib/src/sched/bury_and_suspend.rs @@ -83,7 +83,7 @@ impl Collection { pub fn unbury_or_unsuspend_cards(&mut self, cids: &[CardID]) -> Result<()> { self.transact(None, |col| { - col.set_search_table_to_card_ids(cids)?; + col.storage.set_search_table_to_card_ids(cids)?; col.unsuspend_or_unbury_searched_cards() }) } @@ -143,7 +143,7 @@ impl Collection { mode: pb::bury_or_suspend_cards_in::Mode, ) -> Result<()> { self.transact(None, |col| { - col.set_search_table_to_card_ids(cids)?; + col.storage.set_search_table_to_card_ids(cids)?; col.bury_or_suspend_searched_cards(mode) }) } diff --git a/rslib/src/sched/new.rs b/rslib/src/sched/new.rs index 7ae5ac197..dc7bb9e68 100644 --- a/rslib/src/sched/new.rs +++ b/rslib/src/sched/new.rs @@ -72,7 +72,7 @@ impl Collection { let usn = self.usn()?; let mut position = self.get_next_card_position(); self.transact(None, |col| { - col.set_search_table_to_card_ids(cids)?; + col.storage.set_search_table_to_card_ids(cids)?; let cards = col.storage.all_searched_cards()?; for mut card in cards { let original = card.clone(); @@ -113,7 +113,7 @@ impl Collection { if shift { self.shift_existing_cards(starting_from, step * cids.len() as u32, usn)?; } - self.set_search_table_to_card_ids(cids)?; + self.storage.set_search_table_to_card_ids(cids)?; let cards = self.storage.all_searched_cards()?; let sorter = NewCardSorter::new(&cards, starting_from, step, random); for mut card in cards { diff --git a/rslib/src/sched/reviews.rs b/rslib/src/sched/reviews.rs index 160a23faa..182cbe08e 100644 --- a/rslib/src/sched/reviews.rs +++ b/rslib/src/sched/reviews.rs @@ -36,7 +36,7 @@ impl Collection { let mut rng = rand::thread_rng(); let distribution = Uniform::from(min_days..=max_days); self.transact(None, |col| { - col.set_search_table_to_card_ids(cids)?; + col.storage.set_search_table_to_card_ids(cids)?; for mut card in col.storage.all_searched_cards()? { let original = card.clone(); let interval = distribution.sample(&mut rng); diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index 5f9f5fbd4..fba1a752b 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -106,22 +106,6 @@ impl Collection { Ok(()) } - /// Injects the provided card IDs into the search_cids table, for - /// when ids have arrived outside of a search. - /// Clear with clear_searched_cards(). - pub(crate) fn set_search_table_to_card_ids(&mut self, cards: &[CardID]) -> Result<()> { - self.storage.setup_searched_cards_table()?; - let mut stmt = self - .storage - .db - .prepare_cached("insert into search_cids values (?)")?; - for cid in cards { - stmt.execute(&[cid])?; - } - - Ok(()) - } - /// If the sort mode is based on a config setting, look it up. fn resolve_config_sort(&self, mode: &mut SortMode) { if mode == &SortMode::FromConfig { diff --git a/rslib/src/storage/card/mod.rs b/rslib/src/storage/card/mod.rs index de7640678..f001e1f3b 100644 --- a/rslib/src/storage/card/mod.rs +++ b/rslib/src/storage/card/mod.rs @@ -335,6 +335,21 @@ impl super::SqliteStorage { .execute("drop table if exists search_cids", NO_PARAMS)?; Ok(()) } + + /// Injects the provided card IDs into the search_cids table, for + /// when ids have arrived outside of a search. + /// Clear with clear_searched_cards(). + pub(crate) fn set_search_table_to_card_ids(&mut self, cards: &[CardID]) -> Result<()> { + self.setup_searched_cards_table()?; + let mut stmt = self + .db + .prepare_cached("insert into search_cids values (?)")?; + for cid in cards { + stmt.execute(&[cid])?; + } + + Ok(()) + } } #[cfg(test)] From f87fa762beb6cc7b85518d52144b604b473b17b1 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 3 Sep 2020 17:43:07 +1000 Subject: [PATCH 09/21] empty_filtered_deck() --- pylib/anki/schedv2.py | 60 ++++++++++++++++++++----------------- pylib/tests/test_schedv1.py | 10 +++---- pylib/tests/test_schedv2.py | 6 ++-- qt/aqt/customstudy.py | 2 +- qt/aqt/overview.py | 4 +-- 5 files changed, 44 insertions(+), 38 deletions(-) diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index b91aed151..dbec53d3d 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -1064,42 +1064,21 @@ select id from cards where did in %s and queue = {QUEUE_TYPE_REV} and due <= ? l # Filtered deck handling ########################################################################## - _restoreQueueWhenEmptyingSnippet = f""" -queue = (case when queue < 0 then queue - when type in (1,{CARD_TYPE_RELEARNING}) then - (case when (case when odue then odue else due end) > 1000000000 then 1 else - {QUEUE_TYPE_DAY_LEARN_RELEARN} end) -else - type -end) -""" + def rebuild_filtered_deck(self, deck_id: int) -> int: + return self.col.backend.rebuild_filtered_deck(deck_id) + + def empty_filtered_deck(self, deck_id: int) -> None: + self.col.backend.empty_filtered_deck(deck_id) def rebuildDyn(self, did: Optional[int] = None) -> Optional[int]: - "Rebuild a filtered deck." did = did or self.col.decks.selected() - count = self.col.backend.rebuild_filtered_deck(did) or None + count = self.rebuild_filtered_deck(did) or None if not count: return None # and change to our new deck self.col.decks.select(did) return count - def emptyDyn(self, did: Optional[int], lim: Optional[str] = None) -> None: - if lim is None: - self.col.backend.empty_filtered_deck(did) - return - - self.col.db.execute( - """ -update cards set did = odid, %s, -due = (case when odue>0 then odue else due end), odue = 0, odid = 0, usn = ? where %s""" - % (self._restoreQueueWhenEmptyingSnippet, lim), - self.col.usn(), - ) - - def remFromDyn(self, cids: List[int]) -> None: - self.emptyDyn(None, "id in %s and odid" % ids2str(cids)) - def _removeFromFiltered(self, card: Card) -> None: if card.odid: card.did = card.odid @@ -1121,6 +1100,33 @@ due = (case when odue>0 then odue else due end), odue = 0, odid = 0, usn = ? whe else: card.queue = card.type + # legacy + + def emptyDyn(self, did: Optional[int], lim: Optional[str] = None) -> None: + if lim is None: + self.empty_filtered_deck(did) + return + + queue = f""" +queue = (case when queue < 0 then queue + when type in (1,{CARD_TYPE_RELEARNING}) then + (case when (case when odue then odue else due end) > 1000000000 then 1 else + {QUEUE_TYPE_DAY_LEARN_RELEARN} end) +else + type +end) +""" + self.col.db.execute( + """ +update cards set did = odid, %s, +due = (case when odue>0 then odue else due end), odue = 0, odid = 0, usn = ? where %s""" + % (queue, lim), + self.col.usn(), + ) + + def remFromDyn(self, cids: List[int]) -> None: + self.emptyDyn(None, "id in %s and odid" % ids2str(cids)) + # Leeches ########################################################################## diff --git a/pylib/tests/test_schedv1.py b/pylib/tests/test_schedv1.py index 025ec55de..407eddd12 100644 --- a/pylib/tests/test_schedv1.py +++ b/pylib/tests/test_schedv1.py @@ -682,7 +682,7 @@ def test_cram_rem(): assert c.type == CARD_TYPE_LRN and c.queue == QUEUE_TYPE_LRN assert c.due != oldDue # if we terminate cramming prematurely it should be set back to new - col.sched.emptyDyn(did) + col.sched.empty_filtered_deck(did) c.load() assert c.type == CARD_TYPE_NEW and c.queue == QUEUE_TYPE_NEW assert c.due == oldDue @@ -734,7 +734,7 @@ def test_cram_resched(): col.reset() c = col.sched.getCard() col.sched.answerCard(c, 1) - col.sched.emptyDyn(did) + col.sched.empty_filtered_deck(did) c.load() assert c.ivl == 100 assert c.due == col.sched.today + 25 @@ -746,7 +746,7 @@ def test_cram_resched(): c = col.sched.getCard() col.sched.answerCard(c, 1) col.sched.answerCard(c, 3) - col.sched.emptyDyn(did) + col.sched.empty_filtered_deck(did) c.load() assert c.ivl == 100 assert c.due == col.sched.today + 25 @@ -758,7 +758,7 @@ def test_cram_resched(): col.reset() c = col.sched.getCard() col.sched.answerCard(c, 3) - col.sched.emptyDyn(did) + col.sched.empty_filtered_deck(did) c.load() assert c.ivl == 100 assert c.due == -25 @@ -770,7 +770,7 @@ def test_cram_resched(): col.reset() c = col.sched.getCard() col.sched.answerCard(c, 1) - col.sched.emptyDyn(did) + col.sched.empty_filtered_deck(did) c.load() assert c.ivl == 100 assert c.due == -25 diff --git a/pylib/tests/test_schedv2.py b/pylib/tests/test_schedv2.py index 6e928ae7b..fdd906d88 100644 --- a/pylib/tests/test_schedv2.py +++ b/pylib/tests/test_schedv2.py @@ -773,7 +773,7 @@ def test_filt_keep_lrn_state(): assert c.due - intTime() > 60 * 60 # emptying the deck preserves learning state - col.sched.emptyDyn(did) + col.sched.empty_filtered_deck(did) c.load() assert c.type == CARD_TYPE_LRN and c.queue == QUEUE_TYPE_LRN assert c.left == 1001 @@ -823,7 +823,7 @@ def test_preview(): assert c.id == orig.id # emptying the filtered deck should restore card - col.sched.emptyDyn(did) + col.sched.empty_filtered_deck(did) c.load() assert c.queue == QUEUE_TYPE_NEW assert c.reps == 0 @@ -1255,7 +1255,7 @@ def test_negativeDueFilter(): # into and out of filtered deck did = col.decks.newDyn("Cram") col.sched.rebuildDyn(did) - col.sched.emptyDyn(did) + col.sched.empty_filtered_deck(did) col.reset() c.load() diff --git a/qt/aqt/customstudy.py b/qt/aqt/customstudy.py index 1b5debc8e..6a36479d7 100644 --- a/qt/aqt/customstudy.py +++ b/qt/aqt/customstudy.py @@ -145,7 +145,7 @@ class CustomStudy(QDialog): return QDialog.accept(self) else: # safe to empty - self.mw.col.sched.emptyDyn(cur["id"]) + self.mw.col.sched.empty_filtered_deck(cur["id"]) # reuse; don't delete as it may have children dyn = cur self.mw.col.decks.select(cur["id"]) diff --git a/qt/aqt/overview.py b/qt/aqt/overview.py index 58f49c005..430eefd2b 100644 --- a/qt/aqt/overview.py +++ b/qt/aqt/overview.py @@ -79,7 +79,7 @@ class Overview: self.mw.col.sched.rebuildDyn() self.mw.reset() elif url == "empty": - self.mw.col.sched.emptyDyn(self.mw.col.decks.selected()) + self.mw.col.sched.empty_filtered_deck(self.mw.col.decks.selected()) self.mw.reset() elif url == "decks": self.mw.moveToState("deckBrowser") @@ -112,7 +112,7 @@ class Overview: def onEmptyKey(self): if self._filteredDeck(): - self.mw.col.sched.emptyDyn(self.mw.col.decks.selected()) + self.mw.col.sched.empty_filtered_deck(self.mw.col.decks.selected()) self.mw.reset() def onCustomStudyKey(self): From ade7f438ce81bc9eae38bf28f3b62d2389661567 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 3 Sep 2020 18:02:47 +1000 Subject: [PATCH 10/21] rebuild_filtered_deck() and new_filtered() --- pylib/anki/decks.py | 6 +++++- pylib/anki/schedv2.py | 18 +++++++++--------- pylib/tests/test_decks.py | 2 +- pylib/tests/test_schedv1.py | 36 ++++++++++++++++++------------------ pylib/tests/test_schedv2.py | 22 +++++++++++----------- qt/aqt/customstudy.py | 4 ++-- qt/aqt/dyndeckconf.py | 2 +- qt/aqt/main.py | 2 +- qt/aqt/overview.py | 4 ++-- 9 files changed, 50 insertions(+), 46 deletions(-) diff --git a/pylib/anki/decks.py b/pylib/anki/decks.py index 2da3cbe9c..a6284950e 100644 --- a/pylib/anki/decks.py +++ b/pylib/anki/decks.py @@ -556,7 +556,7 @@ class DeckManager: # Dynamic decks ########################################################################## - def newDyn(self, name: str) -> int: + def new_filtered(self, name: str) -> int: "Return a new dynamic deck and set it as the current deck." did = self.id(name, type=1) self.select(did) @@ -565,3 +565,7 @@ class DeckManager: # 1 for dyn, 0 for standard def isDyn(self, did: Union[int, str]) -> int: return self.get(did)["dyn"] + + # legacy + + newDyn = new_filtered diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index dbec53d3d..1b3660a7d 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -1070,15 +1070,6 @@ select id from cards where did in %s and queue = {QUEUE_TYPE_REV} and due <= ? l def empty_filtered_deck(self, deck_id: int) -> None: self.col.backend.empty_filtered_deck(deck_id) - def rebuildDyn(self, did: Optional[int] = None) -> Optional[int]: - did = did or self.col.decks.selected() - count = self.rebuild_filtered_deck(did) or None - if not count: - return None - # and change to our new deck - self.col.decks.select(did) - return count - def _removeFromFiltered(self, card: Card) -> None: if card.odid: card.did = card.odid @@ -1102,6 +1093,15 @@ select id from cards where did in %s and queue = {QUEUE_TYPE_REV} and due <= ? l # legacy + def rebuildDyn(self, did: Optional[int] = None) -> Optional[int]: + did = did or self.col.decks.selected() + count = self.rebuild_filtered_deck(did) or None + if not count: + return None + # and change to our new deck + self.col.decks.select(did) + return count + def emptyDyn(self, did: Optional[int], lim: Optional[str] = None) -> None: if lim is None: self.empty_filtered_deck(did) diff --git a/pylib/tests/test_decks.py b/pylib/tests/test_decks.py index 4dbe61551..59fb8e130 100644 --- a/pylib/tests/test_decks.py +++ b/pylib/tests/test_decks.py @@ -84,7 +84,7 @@ def test_rename(): for n in "yo", "yo::two", "yo::two::three": assert n in names # over filtered - filteredId = col.decks.newDyn("filtered") + filteredId = col.decks.new_filtered("filtered") filtered = col.decks.get(filteredId) childId = col.decks.id("child") child = col.decks.get(childId) diff --git a/pylib/tests/test_schedv1.py b/pylib/tests/test_schedv1.py index 407eddd12..fe5452cb1 100644 --- a/pylib/tests/test_schedv1.py +++ b/pylib/tests/test_schedv1.py @@ -545,8 +545,8 @@ def test_suspend(): # should cope with cards in cram decks c.due = 1 c.flush() - col.decks.newDyn("tmp") - col.sched.rebuildDyn() + did = col.decks.new_filtered("tmp") + col.sched.rebuild_filtered_deck(did) c.load() assert c.due != 1 assert c.did != 1 @@ -575,8 +575,8 @@ def test_cram(): assert col.sched.counts() == (0, 0, 0) cardcopy = copy.copy(c) # create a dynamic deck and refresh it - did = col.decks.newDyn("Cram") - col.sched.rebuildDyn(did) + did = col.decks.new_filtered("Cram") + col.sched.rebuild_filtered_deck(did) col.reset() # should appear as new in the deck list assert sorted(col.sched.deck_due_tree().children)[0].new_count == 1 @@ -616,7 +616,7 @@ def test_cram(): # and it will have moved back to the previous deck assert c.did == 1 # cram the deck again - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() c = col.sched.getCard() # check ivls again - passing should be idempotent @@ -646,8 +646,8 @@ def test_cram(): col.reset() assert col.sched.counts() == (0, 0, 1) # cram again - did = col.decks.newDyn("Cram") - col.sched.rebuildDyn(did) + did = col.decks.new_filtered("Cram") + col.sched.rebuild_filtered_deck(did) col.reset() assert col.sched.counts() == (0, 0, 1) c.load() @@ -673,8 +673,8 @@ def test_cram_rem(): note["Front"] = "one" col.addNote(note) oldDue = note.cards()[0].due - did = col.decks.newDyn("Cram") - col.sched.rebuildDyn(did) + did = col.decks.new_filtered("Cram") + col.sched.rebuild_filtered_deck(did) col.reset() c = col.sched.getCard() col.sched.answerCard(c, 2) @@ -695,11 +695,11 @@ def test_cram_resched(): note["Front"] = "one" col.addNote(note) # cram deck - did = col.decks.newDyn("Cram") + did = col.decks.new_filtered("Cram") cram = col.decks.get(did) cram["resched"] = False col.decks.save(cram) - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() # graduate should return it to new c = col.sched.getCard() @@ -718,7 +718,7 @@ def test_cram_resched(): c.factor = STARTING_FACTOR c.flush() cardcopy = copy.copy(c) - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() c = col.sched.getCard() assert ni(c, 1) == 600 @@ -730,7 +730,7 @@ def test_cram_resched(): # check failure too c = cardcopy c.flush() - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() c = col.sched.getCard() col.sched.answerCard(c, 1) @@ -741,7 +741,7 @@ def test_cram_resched(): # fail+grad early c = cardcopy c.flush() - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() c = col.sched.getCard() col.sched.answerCard(c, 1) @@ -754,7 +754,7 @@ def test_cram_resched(): c = cardcopy c.due = -25 c.flush() - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() c = col.sched.getCard() col.sched.answerCard(c, 3) @@ -766,7 +766,7 @@ def test_cram_resched(): c = cardcopy c.due = -25 c.flush() - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() c = col.sched.getCard() col.sched.answerCard(c, 1) @@ -778,7 +778,7 @@ def test_cram_resched(): c = cardcopy c.due = -25 c.flush() - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() c = col.sched.getCard() col.sched.answerCard(c, 1) @@ -789,7 +789,7 @@ def test_cram_resched(): # lapsed card pulled into cram # col.sched._cardConf(c)['lapse']['mult']=0.5 # col.sched.answerCard(c, 1) - # col.sched.rebuildDyn(did) + # col.sched.rebuild_filtered_deck(did) # col.reset() # c = col.sched.getCard() # col.sched.answerCard(c, 2) diff --git a/pylib/tests/test_schedv2.py b/pylib/tests/test_schedv2.py index fdd906d88..f09a63094 100644 --- a/pylib/tests/test_schedv2.py +++ b/pylib/tests/test_schedv2.py @@ -668,8 +668,8 @@ def test_suspend(): # should cope with cards in cram decks c.due = 1 c.flush() - col.decks.newDyn("tmp") - col.sched.rebuildDyn() + did = col.decks.new_filtered("tmp") + col.sched.rebuild_filtered_deck(did) c.load() assert c.due != 1 assert c.did != 1 @@ -698,8 +698,8 @@ def test_filt_reviewing_early_normal(): col.reset() assert col.sched.counts() == (0, 0, 0) # create a dynamic deck and refresh it - did = col.decks.newDyn("Cram") - col.sched.rebuildDyn(did) + did = col.decks.new_filtered("Cram") + col.sched.rebuild_filtered_deck(did) col.reset() # should appear as normal in the deck list assert sorted(col.sched.deck_due_tree().children)[0].review_count == 1 @@ -727,7 +727,7 @@ def test_filt_reviewing_early_normal(): c.ivl = 100 c.due = col.sched.today + 75 c.flush() - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() c = col.sched.getCard() @@ -758,8 +758,8 @@ def test_filt_keep_lrn_state(): assert c.type == CARD_TYPE_LRN and c.queue == QUEUE_TYPE_LRN # create a dynamic deck and refresh it - did = col.decks.newDyn("Cram") - col.sched.rebuildDyn(did) + did = col.decks.new_filtered("Cram") + col.sched.rebuild_filtered_deck(did) col.reset() # card should still be in learning state @@ -792,11 +792,11 @@ def test_preview(): note2["Front"] = "two" col.addNote(note2) # cram deck - did = col.decks.newDyn("Cram") + did = col.decks.new_filtered("Cram") cram = col.decks.get(did) cram["resched"] = False col.decks.save(cram) - col.sched.rebuildDyn(did) + col.sched.rebuild_filtered_deck(did) col.reset() # grab the first card c = col.sched.getCard() @@ -1253,8 +1253,8 @@ def test_negativeDueFilter(): c.flush() # into and out of filtered deck - did = col.decks.newDyn("Cram") - col.sched.rebuildDyn(did) + did = col.decks.new_filtered("Cram") + col.sched.rebuild_filtered_deck(did) col.sched.empty_filtered_deck(did) col.reset() diff --git a/qt/aqt/customstudy.py b/qt/aqt/customstudy.py index 6a36479d7..bf049575e 100644 --- a/qt/aqt/customstudy.py +++ b/qt/aqt/customstudy.py @@ -150,7 +150,7 @@ class CustomStudy(QDialog): dyn = cur self.mw.col.decks.select(cur["id"]) else: - did = self.mw.col.decks.newDyn(_("Custom Study Session")) + did = self.mw.col.decks.new_filtered(_("Custom Study Session")) dyn = self.mw.col.decks.get(did) # and then set various options if i == RADIO_FORGOT: @@ -186,7 +186,7 @@ class CustomStudy(QDialog): self.mw.col.decks.save(dyn) # generate cards self.created_custom_study = True - if not self.mw.col.sched.rebuildDyn(): + if not self.mw.col.sched.rebuild_filtered_deck(dyn["id"]): return showWarning(_("No cards matched the criteria you provided.")) self.mw.moveToState("overview") QDialog.accept(self) diff --git a/qt/aqt/dyndeckconf.py b/qt/aqt/dyndeckconf.py index 91c5d89e6..b1c018193 100644 --- a/qt/aqt/dyndeckconf.py +++ b/qt/aqt/dyndeckconf.py @@ -122,7 +122,7 @@ class DeckConf(QDialog): def accept(self): if not self.saveConf(): return - if not self.mw.col.sched.rebuildDyn(): + if not self.mw.col.sched.rebuild_filtered_deck(self.deck["id"]): if askUser( _( """\ diff --git a/qt/aqt/main.py b/qt/aqt/main.py index f9b80ef3a..25be4c4ac 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -1159,7 +1159,7 @@ title="%s" %s>%s""" % ( while self.col.decks.id_for_name(_("Filtered Deck %d") % n): n += 1 name = _("Filtered Deck %d") % n - did = self.col.decks.newDyn(name) + did = self.col.decks.new_filtered(name) diag = aqt.dyndeckconf.DeckConf(self, first=True, search=search) if not diag.ok: # user cancelled first config diff --git a/qt/aqt/overview.py b/qt/aqt/overview.py index 430eefd2b..bc02b4e85 100644 --- a/qt/aqt/overview.py +++ b/qt/aqt/overview.py @@ -76,7 +76,7 @@ class Overview: deck = self.mw.col.decks.current() self.mw.onCram("'deck:%s'" % deck["name"]) elif url == "refresh": - self.mw.col.sched.rebuildDyn() + self.mw.col.sched.rebuild_filtered_deck(self.mw.col.decks.selected()) self.mw.reset() elif url == "empty": self.mw.col.sched.empty_filtered_deck(self.mw.col.decks.selected()) @@ -107,7 +107,7 @@ class Overview: def onRebuildKey(self): if self._filteredDeck(): - self.mw.col.sched.rebuildDyn() + self.mw.col.sched.rebuild_filtered_deck(self.mw.col.decks.selected()) self.mw.reset() def onEmptyKey(self): From a43135f3d5a798462d2bf82bb535c9bef317d872 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 4 Sep 2020 08:31:33 +1000 Subject: [PATCH 11/21] fix custom study link being shown in filtered deck https://forums.ankiweb.net/t/crashing-after-custom-study-learn-forgotten-cards/3013 --- ts/src/sched/CongratsPage.svelte | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ts/src/sched/CongratsPage.svelte b/ts/src/sched/CongratsPage.svelte index 273d0af57..dd24aa439 100644 --- a/ts/src/sched/CongratsPage.svelte +++ b/ts/src/sched/CongratsPage.svelte @@ -59,9 +59,11 @@

{/if} -

- {@html customStudyMsg} -

+ {#if !info.isFilteredDeck} +

+ {@html customStudyMsg} +

+ {/if} {/if} From dc1f489d3e8fb65e09936a0a46113d039dd688ba Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 4 Sep 2020 08:34:26 +1000 Subject: [PATCH 12/21] Revert "Merge pull request #749 from ANH25/editor-media-autoplay" This reverts commit 9aa3d7867e184f3c8d91e9e5686d2881bc9b2cba, reversing changes made to c3a1acb524433611b66a58f693ef56324104673f. https://github.com/ankitects/anki/pull/749#issuecomment-686385112 --- qt/aqt/editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qt/aqt/editor.py b/qt/aqt/editor.py index 2a980d037..eb587f00a 100644 --- a/qt/aqt/editor.py +++ b/qt/aqt/editor.py @@ -743,7 +743,6 @@ to a cloze type first, via 'Notes>Change Note Type'""" ) return if file: - av_player.play_file(file) self.addMedia(file) # Media downloads @@ -761,6 +760,7 @@ to a cloze type first, via 'Notes>Change Note Type'""" name = urllib.parse.quote(fname.encode("utf8")) return '' % name else: + av_player.play_file(fname) return "[sound:%s]" % fname def urlToFile(self, url: str) -> Optional[str]: From c82a084edff075aa78180baeb9234b681e26a64f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 4 Sep 2020 09:26:21 +1000 Subject: [PATCH 13/21] handle quoted html chars in media check https://forums.ankiweb.net/t/unable-to-play-longer-audio-on-cards/1313/30 --- rslib/src/media/check.rs | 21 ++++++++++++++++++--- rslib/src/text.rs | 22 +++++++++++++++------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index de4dbe773..a6018c1dd 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -455,7 +455,7 @@ fn normalize_and_maybe_rename_files<'a>( } // normalize fname into NFC - let mut fname = normalize_to_nfc(media_ref.fname); + let mut fname = normalize_to_nfc(&media_ref.fname_decoded); // and look it up to see if it's been renamed if let Some(new_name) = renamed.get(fname.as_ref()) { fname = new_name.to_owned().into(); @@ -486,7 +486,13 @@ fn normalize_and_maybe_rename_files<'a>( } fn rename_media_ref_in_field(field: &str, media_ref: &MediaRef, new_name: &str) -> String { - let updated_tag = media_ref.full_ref.replace(media_ref.fname, new_name); + let new_name = if matches!(media_ref.fname_decoded, Cow::Owned(_)) { + // filename had quoted characters like & - need to re-encode + htmlescape::encode_minimal(new_name) + } else { + new_name.into() + }; + let updated_tag = media_ref.full_ref.replace(media_ref.fname, &new_name); field.replace(media_ref.full_ref, &updated_tag) } @@ -522,6 +528,7 @@ pub(crate) mod test { pub(crate) const MEDIACHECK_ANKI2: &[u8] = include_bytes!("../../tests/support/mediacheck.anki2"); + use super::normalize_and_maybe_rename_files; use crate::collection::{open_collection, Collection}; use crate::err::Result; use crate::i18n::I18n; @@ -530,7 +537,7 @@ pub(crate) mod test { use crate::media::files::trash_folder; use crate::media::MediaManager; use std::path::Path; - use std::{fs, io}; + use std::{collections::HashMap, fs, io}; use tempfile::{tempdir, TempDir}; fn common_setup() -> Result<(TempDir, MediaManager, Collection)> { @@ -730,4 +737,12 @@ Unused: unused.jpg Ok(()) } + + #[test] + fn html_encoding() { + let field = "[sound:a & b.mp3]"; + let mut seen = Default::default(); + normalize_and_maybe_rename_files(field, &HashMap::new(), &mut seen, Path::new("/tmp")); + assert!(seen.contains("a & b.mp3")); + } } diff --git a/rslib/src/text.rs b/rslib/src/text.rs index 17cf98ec7..0793281c0 100644 --- a/rslib/src/text.rs +++ b/rslib/src/text.rs @@ -142,28 +142,36 @@ pub fn extract_av_tags<'a>(text: &'a str, question_side: bool) -> (Cow<'a, str>, pub(crate) struct MediaRef<'a> { pub full_ref: &'a str, pub fname: &'a str, + /// audio files may have things like & that need decoding + pub fname_decoded: Cow<'a, str>, } pub(crate) fn extract_media_refs(text: &str) -> Vec { let mut out = vec![]; for caps in IMG_TAG.captures_iter(text) { + let fname = caps + .get(1) + .or_else(|| caps.get(2)) + .or_else(|| caps.get(3)) + .unwrap() + .as_str(); + let fname_decoded = fname.into(); out.push(MediaRef { full_ref: caps.get(0).unwrap().as_str(), - fname: caps - .get(1) - .or_else(|| caps.get(2)) - .or_else(|| caps.get(3)) - .unwrap() - .as_str(), + fname, + fname_decoded, }); } for caps in AV_TAGS.captures_iter(text) { if let Some(m) = caps.get(1) { + let fname = m.as_str(); + let fname_decoded = decode_entities(fname); out.push(MediaRef { full_ref: caps.get(0).unwrap().as_str(), - fname: m.as_str(), + fname, + fname_decoded, }); } } From 21749a3b0cafc989bcd36978d7512c1d1520f5c7 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 4 Sep 2020 09:36:18 +1000 Subject: [PATCH 14/21] sound tags need filename html escaped https://forums.ankiweb.net/t/unable-to-play-longer-audio-on-cards/1313/30 --- qt/aqt/editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qt/aqt/editor.py b/qt/aqt/editor.py index eb587f00a..2735a0b05 100644 --- a/qt/aqt/editor.py +++ b/qt/aqt/editor.py @@ -761,7 +761,7 @@ to a cloze type first, via 'Notes>Change Note Type'""" return '' % name else: av_player.play_file(fname) - return "[sound:%s]" % fname + return "[sound:%s]" % html.escape(fname, quote=False) def urlToFile(self, url: str) -> Optional[str]: l = url.lower() From 0455e760c6e94e383a9d3b52973f2241f9920996 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 4 Sep 2020 10:13:30 +1000 Subject: [PATCH 15/21] another showDupes fix from ANH https://forums.ankiweb.net/t/show-duplicates-not-working/2883/10 --- qt/aqt/editor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qt/aqt/editor.py b/qt/aqt/editor.py index 2735a0b05..05d57d507 100644 --- a/qt/aqt/editor.py +++ b/qt/aqt/editor.py @@ -493,7 +493,9 @@ class Editor: self.web.eval("setBackgrounds(%s);" % json.dumps(cols)) def showDupes(self): - contents = html.escape(stripHTMLMedia(self.note.fields[0])) + contents = html.escape( + stripHTMLMedia(self.note.fields[0]), quote=False + ).replace('"', r"\"") browser = aqt.dialogs.open("Browser", self.mw) browser.form.searchEdit.lineEdit().setText( '"dupe:%s,%s"' % (self.note.model()["id"], contents) From 70339e07f9ad3d7e38e026a30790525a3aa90870 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Thu, 3 Sep 2020 14:45:18 +1000 Subject: [PATCH 16/21] rslib: auto-format protobuf code prost-build doesn't generate rustfmt-safe code, so we had to add it to the ignore list for rustfmt on "make check". However, the ignore list isn't supported by stable rustfmt, so we have to work around this some other way -- in this case, just do "rustfmt" on the generated file in "build.rs" (this way, formatting errors in checked-in source code are still caught but generated code doesn't cause spurrious errors). Signed-off-by: Aleksa Sarai --- rslib/build.rs | 14 ++++++++++---- rslib/rustfmt.toml | 1 - 2 files changed, 10 insertions(+), 5 deletions(-) delete mode 100644 rslib/rustfmt.toml diff --git a/rslib/build.rs b/rslib/build.rs index 9722afc41..5c044fcc6 100644 --- a/rslib/build.rs +++ b/rslib/build.rs @@ -1,6 +1,7 @@ use std::fmt::Write; use std::fs; use std::path::Path; +use std::process::Command; use fluent_syntax::ast::{Entry::Message, ResourceEntry}; use fluent_syntax::parser::parse; @@ -200,15 +201,20 @@ fn main() -> std::io::Result<()> { fs::write(rust_string_path, rust_string_vec(&idents))?; // output protobuf generated code - // we avoid default OUT_DIR for now, as it breaks code completion - std::env::set_var("OUT_DIR", "src"); println!("cargo:rerun-if-changed=../proto/backend.proto"); - let mut config = prost_build::Config::new(); - config.service_generator(service_generator()); config + // we avoid default OUT_DIR for now, as it breaks code completion + .out_dir("src") + .service_generator(service_generator()) .compile_protos(&["../proto/backend.proto"], &["../proto"]) .unwrap(); + // rustfmt the protobuf code + let rustfmt = Command::new("rustfmt") + .arg(Path::new("src/backend_proto.rs")) + .status() + .unwrap(); + assert!(rustfmt.success(), "rustfmt backend_proto.rs failed"); // write the other language ftl files let mut ftl_lang_dirs = vec!["./ftl/repo/core".to_string()]; diff --git a/rslib/rustfmt.toml b/rslib/rustfmt.toml deleted file mode 100644 index bd0ab67a8..000000000 --- a/rslib/rustfmt.toml +++ /dev/null @@ -1 +0,0 @@ -ignore = ["backend_proto.rs"] From 312393e82560f845dfd361922a103f988b88cfbf Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Fri, 4 Sep 2020 16:20:39 +1000 Subject: [PATCH 17/21] rs: remove most &mut references in BackendService The previous implementation had some slightly questionable memory safety properties (older versions of PyO3 didn't uphold the Rust aliasing rules and would thus create multiple &mut references to #[pyclass] objects). This explains why Backend has internal Mutexs even though all of its methods took &mut self. The solution is to simply make all methods take &self, which luckily doesn't pose too make issues -- most of the code inside Backend already has sufficient locking. The only two things which needed to be explicitly handled where: 1. "self.runtime" which was fairly easy to handle. All usages of the Runtime only require an immutable reference to create a new Handle, so we could switch to OnceCell which provides lazy-initialisation semantics without needing a more heavy-handed Mutex. 2. "self.sync_abort" was simply wrapped in a Mutex<>, though some of the odd semantics of sync_abort (not being able to handle multiple processes synchronising at the same time) become pretty obvious with this change (for now we just log a warning in that case). In addition, switch to an RAII-style guard to make sure we don't forget to clear the abort_handle. As a result, we now no longer break Rust's aliasing rules and we can build with newer versions of PyO3 which have runtime checks for these things (and build on stable Rust). Signed-off-by: Aleksa Sarai --- rslib/Cargo.toml | 2 + rslib/build.rs | 4 +- rslib/src/backend/mod.rs | 311 +++++++++++++++++++-------------------- rspy/src/lib.rs | 8 +- 4 files changed, 163 insertions(+), 162 deletions(-) diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index fc3a3f437..4519dac2a 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -52,6 +52,8 @@ pin-project = "0.4.22" async-compression = { version = "0.3.5", features = ["stream", "gzip"] } askama = "0.10.1" hyper = "0.13.7" +once_cell = "1.4.1" +scopeguard = "1.1.0" [target.'cfg(target_vendor="apple")'.dependencies.rusqlite] version = "0.23.1" diff --git a/rslib/build.rs b/rslib/build.rs index 5c044fcc6..acc1ed09a 100644 --- a/rslib/build.rs +++ b/rslib/build.rs @@ -116,7 +116,7 @@ fn write_method_trait(buf: &mut String, service: &prost_build::Service) { use prost::Message; pub type BackendResult = std::result::Result; pub trait BackendService { - fn run_command_bytes2_inner(&mut self, method: u32, input: &[u8]) -> std::result::Result, crate::err::AnkiError> { + fn run_command_bytes2_inner(&self, method: u32, input: &[u8]) -> std::result::Result, crate::err::AnkiError> { match method { "#, ); @@ -146,7 +146,7 @@ pub trait BackendService { write!( buf, concat!( - " fn {method_name}(&mut self, input: {input_type}) -> ", + " fn {method_name}(&self, input: {input_type}) -> ", "BackendResult<{output_type}>;\n" ), method_name = method.name, diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 3741bf93c..698d06135 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -44,11 +44,13 @@ use crate::{ types::Usn, }; use fluent::FluentValue; -use futures::future::{AbortHandle, Abortable}; +use futures::future::{AbortHandle, AbortRegistration, Abortable}; use log::error; +use once_cell::sync::OnceCell; use pb::{sync_status_out, BackendService}; use prost::Message; use serde_json::Value as JsonValue; +use slog::warn; use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; use std::{ @@ -85,13 +87,16 @@ struct ProgressState { last_progress: Option, } +// fixme: this should support multiple abort handles. +type AbortHandleSlot = Arc>>; + pub struct Backend { col: Arc>>, i18n: I18n, server: bool, - sync_abort: Option, + sync_abort: AbortHandleSlot, progress_state: Arc>, - runtime: Option, + runtime: OnceCell, state: Arc>, } @@ -267,12 +272,12 @@ impl From for DeckConfID { } impl BackendService for Backend { - fn latest_progress(&mut self, _input: Empty) -> BackendResult { + fn latest_progress(&self, _input: Empty) -> BackendResult { let progress = self.progress_state.lock().unwrap().last_progress; Ok(progress_to_proto(progress, &self.i18n)) } - fn set_wants_abort(&mut self, _input: Empty) -> BackendResult { + fn set_wants_abort(&self, _input: Empty) -> BackendResult { self.progress_state.lock().unwrap().want_abort = true; Ok(().into()) } @@ -280,7 +285,7 @@ impl BackendService for Backend { // card rendering fn render_existing_card( - &mut self, + &self, input: pb::RenderExistingCardIn, ) -> BackendResult { self.with_col(|col| { @@ -290,7 +295,7 @@ impl BackendService for Backend { } fn render_uncommitted_card( - &mut self, + &self, input: pb::RenderUncommittedCardIn, ) -> BackendResult { let schema11: CardTemplateSchema11 = serde_json::from_slice(&input.template)?; @@ -307,7 +312,7 @@ impl BackendService for Backend { }) } - fn get_empty_cards(&mut self, _input: pb::Empty) -> Result { + fn get_empty_cards(&self, _input: pb::Empty) -> Result { self.with_col(|col| { let mut empty = col.empty_cards()?; let report = col.empty_cards_report(&mut empty)?; @@ -327,16 +332,13 @@ impl BackendService for Backend { }) } - fn strip_av_tags(&mut self, input: pb::String) -> BackendResult { + fn strip_av_tags(&self, input: pb::String) -> BackendResult { Ok(pb::String { val: strip_av_tags(&input.val).into(), }) } - fn extract_av_tags( - &mut self, - input: pb::ExtractAvTagsIn, - ) -> BackendResult { + fn extract_av_tags(&self, input: pb::ExtractAvTagsIn) -> BackendResult { let (text, tags) = extract_av_tags(&input.text, input.question_side); let pt_tags = tags .into_iter() @@ -368,7 +370,7 @@ impl BackendService for Backend { }) } - fn extract_latex(&mut self, input: pb::ExtractLatexIn) -> BackendResult { + fn extract_latex(&self, input: pb::ExtractLatexIn) -> BackendResult { let func = if input.expand_clozes { extract_latex_expanding_clozes } else { @@ -391,7 +393,7 @@ impl BackendService for Backend { // searching //----------------------------------------------- - fn search_cards(&mut self, input: pb::SearchCardsIn) -> Result { + fn search_cards(&self, input: pb::SearchCardsIn) -> Result { self.with_col(|col| { let order = if let Some(order) = input.order { use pb::sort_order::Value as V; @@ -415,7 +417,7 @@ impl BackendService for Backend { }) } - fn search_notes(&mut self, input: pb::SearchNotesIn) -> Result { + fn search_notes(&self, input: pb::SearchNotesIn) -> Result { self.with_col(|col| { let nids = col.search_notes(&input.search)?; Ok(pb::SearchNotesOut { @@ -424,7 +426,7 @@ impl BackendService for Backend { }) } - fn find_and_replace(&mut self, input: pb::FindAndReplaceIn) -> BackendResult { + fn find_and_replace(&self, input: pb::FindAndReplaceIn) -> BackendResult { let mut search = if input.regex { input.search } else { @@ -451,7 +453,7 @@ impl BackendService for Backend { /// This behaves like _updateCutoff() in older code - it also unburies at the start of /// a new day. - fn sched_timing_today(&mut self, _input: pb::Empty) -> Result { + fn sched_timing_today(&self, _input: pb::Empty) -> Result { self.with_col(|col| { let timing = col.timing_today()?; col.unbury_if_day_rolled_over(timing)?; @@ -459,13 +461,13 @@ impl BackendService for Backend { }) } - fn local_minutes_west(&mut self, input: pb::Int64) -> BackendResult { + fn local_minutes_west(&self, input: pb::Int64) -> BackendResult { Ok(pb::Int32 { val: local_minutes_west_for_stamp(input.val), }) } - fn set_local_minutes_west(&mut self, input: pb::Int32) -> BackendResult { + fn set_local_minutes_west(&self, input: pb::Int32) -> BackendResult { self.with_col(|col| { col.transact(None, |col| { col.set_local_mins_west(input.val).map(Into::into) @@ -474,19 +476,16 @@ impl BackendService for Backend { } /// Fetch data from DB and return rendered string. - fn studied_today(&mut self, _input: pb::Empty) -> BackendResult { + fn studied_today(&self, _input: pb::Empty) -> BackendResult { self.with_col(|col| col.studied_today().map(Into::into)) } /// Message rendering only, for old graphs. - fn studied_today_message( - &mut self, - input: pb::StudiedTodayMessageIn, - ) -> BackendResult { + fn studied_today_message(&self, input: pb::StudiedTodayMessageIn) -> BackendResult { Ok(studied_today(input.cards, input.seconds as f32, &self.i18n).into()) } - fn update_stats(&mut self, input: pb::UpdateStatsIn) -> BackendResult { + fn update_stats(&self, input: pb::UpdateStatsIn) -> BackendResult { self.with_col(|col| { col.transact(None, |col| { let today = col.current_due_day(0)?; @@ -496,7 +495,7 @@ impl BackendService for Backend { }) } - fn extend_limits(&mut self, input: pb::ExtendLimitsIn) -> BackendResult { + fn extend_limits(&self, input: pb::ExtendLimitsIn) -> BackendResult { self.with_col(|col| { col.transact(None, |col| { let today = col.current_due_day(0)?; @@ -513,18 +512,15 @@ impl BackendService for Backend { }) } - fn counts_for_deck_today( - &mut self, - input: pb::DeckId, - ) -> BackendResult { + fn counts_for_deck_today(&self, input: pb::DeckId) -> BackendResult { self.with_col(|col| col.counts_for_deck_today(input.did.into())) } - fn congrats_info(&mut self, _input: Empty) -> BackendResult { + fn congrats_info(&self, _input: Empty) -> BackendResult { self.with_col(|col| col.congrats_info()) } - fn restore_buried_and_suspended_cards(&mut self, input: pb::CardIDs) -> BackendResult { + fn restore_buried_and_suspended_cards(&self, input: pb::CardIDs) -> BackendResult { self.with_col(|col| { col.unbury_or_unsuspend_cards(&input.into_native()) .map(Into::into) @@ -532,7 +528,7 @@ impl BackendService for Backend { } fn unbury_cards_in_current_deck( - &mut self, + &self, input: pb::UnburyCardsInCurrentDeckIn, ) -> BackendResult { self.with_col(|col| { @@ -541,7 +537,7 @@ impl BackendService for Backend { }) } - fn bury_or_suspend_cards(&mut self, input: pb::BuryOrSuspendCardsIn) -> BackendResult { + fn bury_or_suspend_cards(&self, input: pb::BuryOrSuspendCardsIn) -> BackendResult { self.with_col(|col| { let mode = input.mode(); let cids: Vec<_> = input.card_ids.into_iter().map(CardID).collect(); @@ -549,16 +545,16 @@ impl BackendService for Backend { }) } - fn empty_filtered_deck(&mut self, input: pb::DeckId) -> BackendResult { + fn empty_filtered_deck(&self, input: pb::DeckId) -> BackendResult { self.with_col(|col| col.empty_filtered_deck(input.did.into()).map(Into::into)) } - fn rebuild_filtered_deck(&mut self, input: pb::DeckId) -> BackendResult { + fn rebuild_filtered_deck(&self, input: pb::DeckId) -> BackendResult { self.with_col(|col| col.rebuild_filtered_deck(input.did.into()).map(Into::into)) } fn schedule_cards_as_reviews( - &mut self, + &self, input: pb::ScheduleCardsAsReviewsIn, ) -> BackendResult { let cids: Vec<_> = input.card_ids.into_iter().map(CardID).collect(); @@ -569,14 +565,14 @@ impl BackendService for Backend { }) } - fn schedule_cards_as_new(&mut self, input: pb::CardIDs) -> BackendResult { + fn schedule_cards_as_new(&self, input: pb::CardIDs) -> BackendResult { self.with_col(|col| { col.reschedule_cards_as_new(&input.into_native()) .map(Into::into) }) } - fn sort_cards(&mut self, input: pb::SortCardsIn) -> BackendResult { + fn sort_cards(&self, input: pb::SortCardsIn) -> BackendResult { let cids: Vec<_> = input.card_ids.into_iter().map(CardID).collect(); let (start, step, random, shift) = ( input.starting_from, @@ -590,7 +586,7 @@ impl BackendService for Backend { }) } - fn sort_deck(&mut self, input: pb::SortDeckIn) -> BackendResult { + fn sort_deck(&self, input: pb::SortDeckIn) -> BackendResult { self.with_col(|col| { col.sort_deck(input.deck_id.into(), input.randomize) .map(Into::into) @@ -600,19 +596,19 @@ impl BackendService for Backend { // statistics //----------------------------------------------- - fn card_stats(&mut self, input: pb::CardId) -> BackendResult { + fn card_stats(&self, input: pb::CardId) -> BackendResult { self.with_col(|col| col.card_stats(input.into())) .map(Into::into) } - fn graphs(&mut self, input: pb::GraphsIn) -> BackendResult { + fn graphs(&self, input: pb::GraphsIn) -> BackendResult { self.with_col(|col| col.graph_data_for_search(&input.search, input.days)) } // decks //----------------------------------------------- - fn deck_tree(&mut self, input: pb::DeckTreeIn) -> Result { + fn deck_tree(&self, input: pb::DeckTreeIn) -> Result { let lim = if input.top_deck_id > 0 { Some(DeckID(input.top_deck_id)) } else { @@ -628,7 +624,7 @@ impl BackendService for Backend { }) } - fn deck_tree_legacy(&mut self, _input: pb::Empty) -> BackendResult { + fn deck_tree_legacy(&self, _input: pb::Empty) -> BackendResult { self.with_col(|col| { let tree = col.legacy_deck_tree()?; serde_json::to_vec(&tree) @@ -637,7 +633,7 @@ impl BackendService for Backend { }) } - fn get_deck_legacy(&mut self, input: pb::DeckId) -> Result { + fn get_deck_legacy(&self, input: pb::DeckId) -> Result { self.with_col(|col| { let deck: DeckSchema11 = col .storage @@ -650,7 +646,7 @@ impl BackendService for Backend { }) } - fn get_deck_id_by_name(&mut self, input: pb::String) -> Result { + fn get_deck_id_by_name(&self, input: pb::String) -> Result { self.with_col(|col| { col.get_deck_id(&input.val).and_then(|d| { d.ok_or(AnkiError::NotFound) @@ -659,7 +655,7 @@ impl BackendService for Backend { }) } - fn get_all_decks_legacy(&mut self, _input: Empty) -> BackendResult { + fn get_all_decks_legacy(&self, _input: Empty) -> BackendResult { self.with_col(|col| { let decks = col.storage.get_all_decks_as_schema11()?; serde_json::to_vec(&decks).map_err(Into::into) @@ -667,7 +663,7 @@ impl BackendService for Backend { .map(Into::into) } - fn get_deck_names(&mut self, input: pb::GetDeckNamesIn) -> Result { + fn get_deck_names(&self, input: pb::GetDeckNamesIn) -> Result { self.with_col(|col| { let names = if input.include_filtered { col.get_all_deck_names(input.skip_empty_default)? @@ -683,10 +679,7 @@ impl BackendService for Backend { }) } - fn add_or_update_deck_legacy( - &mut self, - input: pb::AddOrUpdateDeckLegacyIn, - ) -> Result { + fn add_or_update_deck_legacy(&self, input: pb::AddOrUpdateDeckLegacyIn) -> Result { self.with_col(|col| { let schema11: DeckSchema11 = serde_json::from_slice(&input.deck)?; let mut deck: Deck = schema11.into(); @@ -702,7 +695,7 @@ impl BackendService for Backend { }) } - fn new_deck_legacy(&mut self, input: pb::Bool) -> BackendResult { + fn new_deck_legacy(&self, input: pb::Bool) -> BackendResult { let deck = if input.val { Deck::new_filtered() } else { @@ -714,7 +707,7 @@ impl BackendService for Backend { .map(Into::into) } - fn remove_deck(&mut self, input: pb::DeckId) -> BackendResult { + fn remove_deck(&self, input: pb::DeckId) -> BackendResult { self.with_col(|col| col.remove_deck_and_child_decks(input.into())) .map(Into::into) } @@ -723,7 +716,7 @@ impl BackendService for Backend { //---------------------------------------------------- fn add_or_update_deck_config_legacy( - &mut self, + &self, input: AddOrUpdateDeckConfigLegacyIn, ) -> BackendResult { let conf: DeckConfSchema11 = serde_json::from_slice(&input.config)?; @@ -737,7 +730,7 @@ impl BackendService for Backend { .map(Into::into) } - fn all_deck_config_legacy(&mut self, _input: Empty) -> BackendResult { + fn all_deck_config_legacy(&self, _input: Empty) -> BackendResult { self.with_col(|col| { let conf: Vec = col .storage @@ -750,18 +743,18 @@ impl BackendService for Backend { .map(Into::into) } - fn new_deck_config_legacy(&mut self, _input: Empty) -> BackendResult { + fn new_deck_config_legacy(&self, _input: Empty) -> BackendResult { serde_json::to_vec(&DeckConfSchema11::default()) .map_err(Into::into) .map(Into::into) } - fn remove_deck_config(&mut self, input: pb::DeckConfigId) -> BackendResult { + fn remove_deck_config(&self, input: pb::DeckConfigId) -> BackendResult { self.with_col(|col| col.transact(None, |col| col.remove_deck_config(input.into()))) .map(Into::into) } - fn get_deck_config_legacy(&mut self, input: pb::DeckConfigId) -> BackendResult { + fn get_deck_config_legacy(&self, input: pb::DeckConfigId) -> BackendResult { self.with_col(|col| { let conf = col.get_deck_config(input.into(), true)?.unwrap(); let conf: DeckConfSchema11 = conf.into(); @@ -773,7 +766,7 @@ impl BackendService for Backend { // cards //------------------------------------------------------------------- - fn get_card(&mut self, input: pb::CardId) -> BackendResult { + fn get_card(&self, input: pb::CardId) -> BackendResult { self.with_col(|col| { col.storage .get_card(input.into()) @@ -782,7 +775,7 @@ impl BackendService for Backend { }) } - fn update_card(&mut self, input: pb::Card) -> BackendResult { + fn update_card(&self, input: pb::Card) -> BackendResult { let mut card = pbcard_to_native(input)?; self.with_col(|col| { col.transact(None, |ctx| { @@ -796,13 +789,13 @@ impl BackendService for Backend { .map(Into::into) } - fn add_card(&mut self, input: pb::Card) -> BackendResult { + fn add_card(&self, input: pb::Card) -> BackendResult { let mut card = pbcard_to_native(input)?; self.with_col(|col| col.transact(None, |ctx| ctx.add_card(&mut card)))?; Ok(pb::CardId { cid: card.id.0 }) } - fn remove_cards(&mut self, input: pb::RemoveCardsIn) -> BackendResult { + fn remove_cards(&self, input: pb::RemoveCardsIn) -> BackendResult { self.with_col(|col| { col.transact(None, |col| { col.remove_cards_and_orphaned_notes( @@ -817,7 +810,7 @@ impl BackendService for Backend { }) } - fn set_deck(&mut self, input: pb::SetDeckIn) -> BackendResult { + fn set_deck(&self, input: pb::SetDeckIn) -> BackendResult { let cids: Vec<_> = input.card_ids.into_iter().map(CardID).collect(); let deck_id = input.deck_id.into(); self.with_col(|col| col.set_deck(&cids, deck_id).map(Into::into)) @@ -826,14 +819,14 @@ impl BackendService for Backend { // notes //------------------------------------------------------------------- - fn new_note(&mut self, input: pb::NoteTypeId) -> BackendResult { + fn new_note(&self, input: pb::NoteTypeId) -> BackendResult { self.with_col(|col| { let nt = col.get_notetype(input.into())?.ok_or(AnkiError::NotFound)?; Ok(nt.new_note().into()) }) } - fn add_note(&mut self, input: pb::AddNoteIn) -> BackendResult { + fn add_note(&self, input: pb::AddNoteIn) -> BackendResult { self.with_col(|col| { let mut note: Note = input.note.ok_or(AnkiError::NotFound)?.into(); col.add_note(&mut note, DeckID(input.deck_id)) @@ -841,7 +834,7 @@ impl BackendService for Backend { }) } - fn update_note(&mut self, input: pb::Note) -> BackendResult { + fn update_note(&self, input: pb::Note) -> BackendResult { self.with_col(|col| { let mut note: Note = input.into(); col.update_note(&mut note) @@ -849,7 +842,7 @@ impl BackendService for Backend { .map(Into::into) } - fn get_note(&mut self, input: pb::NoteId) -> BackendResult { + fn get_note(&self, input: pb::NoteId) -> BackendResult { self.with_col(|col| { col.storage .get_note(input.into())? @@ -858,7 +851,7 @@ impl BackendService for Backend { }) } - fn remove_notes(&mut self, input: pb::RemoveNotesIn) -> BackendResult { + fn remove_notes(&self, input: pb::RemoveNotesIn) -> BackendResult { self.with_col(|col| { if !input.note_ids.is_empty() { col.remove_notes( @@ -883,7 +876,7 @@ impl BackendService for Backend { }) } - fn add_note_tags(&mut self, input: pb::AddNoteTagsIn) -> BackendResult { + fn add_note_tags(&self, input: pb::AddNoteTagsIn) -> BackendResult { self.with_col(|col| { col.add_tags_for_notes(&to_nids(input.nids), &input.tags) .map(|n| n as u32) @@ -891,7 +884,7 @@ impl BackendService for Backend { .map(Into::into) } - fn update_note_tags(&mut self, input: pb::UpdateNoteTagsIn) -> BackendResult { + fn update_note_tags(&self, input: pb::UpdateNoteTagsIn) -> BackendResult { self.with_col(|col| { col.replace_tags_for_notes( &to_nids(input.nids), @@ -903,10 +896,7 @@ impl BackendService for Backend { }) } - fn cloze_numbers_in_note( - &mut self, - note: pb::Note, - ) -> BackendResult { + fn cloze_numbers_in_note(&self, note: pb::Note) -> BackendResult { let mut set = HashSet::with_capacity(4); for field in ¬e.fields { add_cloze_numbers_in_string(field, &mut set); @@ -917,7 +907,7 @@ impl BackendService for Backend { } fn field_names_for_notes( - &mut self, + &self, input: pb::FieldNamesForNotesIn, ) -> BackendResult { self.with_col(|col| { @@ -928,7 +918,7 @@ impl BackendService for Backend { }) } - fn after_note_updates(&mut self, input: pb::AfterNoteUpdatesIn) -> BackendResult { + fn after_note_updates(&self, input: pb::AfterNoteUpdatesIn) -> BackendResult { self.with_col(|col| { col.transact(None, |col| { col.after_note_updates( @@ -942,7 +932,7 @@ impl BackendService for Backend { } fn note_is_duplicate_or_empty( - &mut self, + &self, input: pb::Note, ) -> BackendResult { let note: Note = input.into(); @@ -952,7 +942,7 @@ impl BackendService for Backend { }) } - fn cards_of_note(&mut self, input: pb::NoteId) -> BackendResult { + fn cards_of_note(&self, input: pb::NoteId) -> BackendResult { self.with_col(|col| { col.storage .all_card_ids_of_note(NoteID(input.nid)) @@ -965,10 +955,7 @@ impl BackendService for Backend { // notetypes //------------------------------------------------------------------- - fn get_stock_notetype_legacy( - &mut self, - input: pb::GetStockNotetypeIn, - ) -> BackendResult { + fn get_stock_notetype_legacy(&self, input: pb::GetStockNotetypeIn) -> BackendResult { // fixme: use individual functions instead of full vec let mut all = all_stock_notetypes(&self.i18n); let idx = (input.kind as usize).min(all.len() - 1); @@ -979,7 +966,7 @@ impl BackendService for Backend { .map(Into::into) } - fn get_notetype_names(&mut self, _input: Empty) -> BackendResult { + fn get_notetype_names(&self, _input: Empty) -> BackendResult { self.with_col(|col| { let entries: Vec<_> = col .storage @@ -991,10 +978,7 @@ impl BackendService for Backend { }) } - fn get_notetype_names_and_counts( - &mut self, - _input: Empty, - ) -> BackendResult { + fn get_notetype_names_and_counts(&self, _input: Empty) -> BackendResult { self.with_col(|col| { let entries: Vec<_> = col .storage @@ -1010,7 +994,7 @@ impl BackendService for Backend { }) } - fn get_notetype_legacy(&mut self, input: pb::NoteTypeId) -> BackendResult { + fn get_notetype_legacy(&self, input: pb::NoteTypeId) -> BackendResult { self.with_col(|col| { let schema11: NoteTypeSchema11 = col .storage @@ -1021,7 +1005,7 @@ impl BackendService for Backend { }) } - fn get_notetype_id_by_name(&mut self, input: pb::String) -> BackendResult { + fn get_notetype_id_by_name(&self, input: pb::String) -> BackendResult { self.with_col(|col| { col.storage .get_notetype_id(&input.val) @@ -1031,7 +1015,7 @@ impl BackendService for Backend { } fn add_or_update_notetype( - &mut self, + &self, input: pb::AddOrUpdateNotetypeIn, ) -> BackendResult { self.with_col(|col| { @@ -1046,7 +1030,7 @@ impl BackendService for Backend { }) } - fn remove_notetype(&mut self, input: pb::NoteTypeId) -> BackendResult { + fn remove_notetype(&self, input: pb::NoteTypeId) -> BackendResult { self.with_col(|col| col.remove_notetype(input.into())) .map(Into::into) } @@ -1054,7 +1038,7 @@ impl BackendService for Backend { // media //------------------------------------------------------------------- - fn add_media_file(&mut self, input: pb::AddMediaFileIn) -> BackendResult { + fn add_media_file(&self, input: pb::AddMediaFileIn) -> BackendResult { self.with_col(|col| { let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; let mut ctx = mgr.dbctx(); @@ -1065,7 +1049,7 @@ impl BackendService for Backend { }) } - fn empty_trash(&mut self, _input: Empty) -> BackendResult { + fn empty_trash(&self, _input: Empty) -> BackendResult { let mut handler = self.new_progress_handler(); let progress_fn = move |progress| handler.update(Progress::MediaCheck(progress as u32), true); @@ -1081,7 +1065,7 @@ impl BackendService for Backend { .map(Into::into) } - fn restore_trash(&mut self, _input: Empty) -> BackendResult { + fn restore_trash(&self, _input: Empty) -> BackendResult { let mut handler = self.new_progress_handler(); let progress_fn = move |progress| handler.update(Progress::MediaCheck(progress as u32), true); @@ -1097,7 +1081,7 @@ impl BackendService for Backend { .map(Into::into) } - fn trash_media_files(&mut self, input: pb::TrashMediaFilesIn) -> BackendResult { + fn trash_media_files(&self, input: pb::TrashMediaFilesIn) -> BackendResult { self.with_col(|col| { let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; let mut ctx = mgr.dbctx(); @@ -1106,7 +1090,7 @@ impl BackendService for Backend { .map(Into::into) } - fn check_media(&mut self, _input: pb::Empty) -> Result { + fn check_media(&self, _input: pb::Empty) -> Result { let mut handler = self.new_progress_handler(); let progress_fn = move |progress| handler.update(Progress::MediaCheck(progress as u32), true); @@ -1131,7 +1115,7 @@ impl BackendService for Backend { // collection //------------------------------------------------------------------- - fn check_database(&mut self, _input: pb::Empty) -> BackendResult { + fn check_database(&self, _input: pb::Empty) -> BackendResult { let mut handler = self.new_progress_handler(); let progress_fn = move |progress, throttle| { handler.update(Progress::DatabaseCheck(progress), throttle); @@ -1144,7 +1128,7 @@ impl BackendService for Backend { }) } - fn open_collection(&mut self, input: pb::OpenCollectionIn) -> BackendResult { + fn open_collection(&self, input: pb::OpenCollectionIn) -> BackendResult { let mut col = self.col.lock().unwrap(); if col.is_some() { return Err(AnkiError::CollectionAlreadyOpen); @@ -1173,7 +1157,7 @@ impl BackendService for Backend { Ok(().into()) } - fn close_collection(&mut self, input: pb::CloseCollectionIn) -> BackendResult { + fn close_collection(&self, input: pb::CloseCollectionIn) -> BackendResult { self.abort_media_sync_and_wait(); let mut col = self.col.lock().unwrap(); @@ -1195,41 +1179,41 @@ impl BackendService for Backend { // sync //------------------------------------------------------------------- - fn sync_login(&mut self, input: pb::SyncLoginIn) -> BackendResult { + fn sync_login(&self, input: pb::SyncLoginIn) -> BackendResult { self.sync_login_inner(input) } - fn sync_status(&mut self, input: pb::SyncAuth) -> BackendResult { + fn sync_status(&self, input: pb::SyncAuth) -> BackendResult { self.sync_status_inner(input) } - fn sync_collection(&mut self, input: pb::SyncAuth) -> BackendResult { + fn sync_collection(&self, input: pb::SyncAuth) -> BackendResult { self.sync_collection_inner(input) } - fn full_upload(&mut self, input: pb::SyncAuth) -> BackendResult { + fn full_upload(&self, input: pb::SyncAuth) -> BackendResult { self.full_sync_inner(input, true)?; Ok(().into()) } - fn full_download(&mut self, input: pb::SyncAuth) -> BackendResult { + fn full_download(&self, input: pb::SyncAuth) -> BackendResult { self.full_sync_inner(input, false)?; Ok(().into()) } - fn sync_media(&mut self, input: pb::SyncAuth) -> BackendResult { + fn sync_media(&self, input: pb::SyncAuth) -> BackendResult { self.sync_media_inner(input).map(Into::into) } - fn abort_sync(&mut self, _input: Empty) -> BackendResult { - if let Some(handle) = self.sync_abort.take() { + fn abort_sync(&self, _input: Empty) -> BackendResult { + if let Some(handle) = self.sync_abort.lock().unwrap().take() { handle.abort(); } Ok(().into()) } /// Abort the media sync. Does not wait for completion. - fn abort_media_sync(&mut self, _input: Empty) -> BackendResult { + fn abort_media_sync(&self, _input: Empty) -> BackendResult { let guard = self.state.lock().unwrap(); if let Some(handle) = &guard.media_sync_abort { handle.abort(); @@ -1237,14 +1221,14 @@ impl BackendService for Backend { Ok(().into()) } - fn before_upload(&mut self, _input: Empty) -> BackendResult { + fn before_upload(&self, _input: Empty) -> BackendResult { self.with_col(|col| col.before_upload().map(Into::into)) } // i18n/messages //------------------------------------------------------------------- - fn translate_string(&mut self, input: pb::TranslateStringIn) -> BackendResult { + fn translate_string(&self, input: pb::TranslateStringIn) -> BackendResult { let key = match pb::FluentString::from_i32(input.key) { Some(key) => key, None => return Ok("invalid key".to_string().into()), @@ -1259,7 +1243,7 @@ impl BackendService for Backend { Ok(self.i18n.trn(key, map).into()) } - fn format_timespan(&mut self, input: pb::FormatTimespanIn) -> BackendResult { + fn format_timespan(&self, input: pb::FormatTimespanIn) -> BackendResult { let context = match pb::format_timespan_in::Context::from_i32(input.context) { Some(context) => context, None => return Ok("".to_string().into()), @@ -1276,7 +1260,7 @@ impl BackendService for Backend { .into()) } - fn i18n_resources(&mut self, _input: Empty) -> BackendResult { + fn i18n_resources(&self, _input: Empty) -> BackendResult { serde_json::to_vec(&self.i18n.resources_for_js()) .map(Into::into) .map_err(Into::into) @@ -1285,7 +1269,7 @@ impl BackendService for Backend { // tags //------------------------------------------------------------------- - fn all_tags(&mut self, _input: Empty) -> BackendResult { + fn all_tags(&self, _input: Empty) -> BackendResult { let tags = self.with_col(|col| col.storage.all_tags())?; let tags: Vec<_> = tags .into_iter() @@ -1294,7 +1278,7 @@ impl BackendService for Backend { Ok(pb::AllTagsOut { tags }) } - fn register_tags(&mut self, input: pb::RegisterTagsIn) -> BackendResult { + fn register_tags(&self, input: pb::RegisterTagsIn) -> BackendResult { self.with_col(|col| { col.transact(None, |col| { let usn = if input.preserve_usn { @@ -1311,7 +1295,7 @@ impl BackendService for Backend { // config/preferences //------------------------------------------------------------------- - fn get_config_json(&mut self, input: pb::String) -> BackendResult { + fn get_config_json(&self, input: pb::String) -> BackendResult { self.with_col(|col| { let val: Option = col.get_config_optional(input.val.as_str()); val.ok_or(AnkiError::NotFound) @@ -1320,7 +1304,7 @@ impl BackendService for Backend { }) } - fn set_config_json(&mut self, input: pb::SetConfigJsonIn) -> BackendResult { + fn set_config_json(&self, input: pb::SetConfigJsonIn) -> BackendResult { self.with_col(|col| { col.transact(None, |col| { // ensure it's a well-formed object @@ -1331,12 +1315,12 @@ impl BackendService for Backend { .map(Into::into) } - fn remove_config(&mut self, input: pb::String) -> BackendResult { + fn remove_config(&self, input: pb::String) -> BackendResult { self.with_col(|col| col.transact(None, |col| col.remove_config(input.val.as_str()))) .map(Into::into) } - fn set_all_config(&mut self, input: pb::Json) -> BackendResult { + fn set_all_config(&self, input: pb::Json) -> BackendResult { let val: HashMap = serde_json::from_slice(&input.json)?; self.with_col(|col| { col.transact(None, |col| { @@ -1347,7 +1331,7 @@ impl BackendService for Backend { .map(Into::into) } - fn get_all_config(&mut self, _input: Empty) -> BackendResult { + fn get_all_config(&self, _input: Empty) -> BackendResult { self.with_col(|col| { let conf = col.storage.get_all_config()?; serde_json::to_vec(&conf).map_err(Into::into) @@ -1355,11 +1339,11 @@ impl BackendService for Backend { .map(Into::into) } - fn get_preferences(&mut self, _input: Empty) -> BackendResult { + fn get_preferences(&self, _input: Empty) -> BackendResult { self.with_col(|col| col.get_preferences()) } - fn set_preferences(&mut self, input: pb::Preferences) -> BackendResult { + fn set_preferences(&self, input: pb::Preferences) -> BackendResult { self.with_col(|col| col.transact(None, |col| col.set_preferences(input))) .map(Into::into) } @@ -1371,12 +1355,12 @@ impl Backend { col: Arc::new(Mutex::new(None)), i18n, server, - sync_abort: None, + sync_abort: Arc::new(Mutex::new(None)), progress_state: Arc::new(Mutex::new(ProgressState { want_abort: false, last_progress: None, })), - runtime: None, + runtime: OnceCell::new(), state: Arc::new(Mutex::new(BackendState::default())), } } @@ -1385,11 +1369,7 @@ impl Backend { &self.i18n } - pub fn run_command_bytes( - &mut self, - method: u32, - input: &[u8], - ) -> result::Result, Vec> { + pub fn run_command_bytes(&self, method: u32, input: &[u8]) -> result::Result, Vec> { self.run_command_bytes2_inner(method, input).map_err(|err| { let backend_err = anki_error_to_proto_error(err, &self.i18n); let mut bytes = Vec::new(); @@ -1421,26 +1401,54 @@ impl Backend { guard.last_progress = None; } ThrottlingProgressHandler { - state: self.progress_state.clone(), + state: Arc::clone(&self.progress_state), last_update: coarsetime::Instant::now(), } } - fn runtime_handle(&mut self) -> runtime::Handle { - if self.runtime.is_none() { - self.runtime = Some( + fn runtime_handle(&self) -> runtime::Handle { + self.runtime + .get_or_init(|| { runtime::Builder::new() .threaded_scheduler() .core_threads(1) .enable_all() .build() - .unwrap(), - ) - } - self.runtime.as_ref().unwrap().handle().clone() + .unwrap() + }) + .handle() + .clone() } - fn sync_media_inner(&mut self, input: pb::SyncAuth) -> Result<()> { + fn sync_abort_handle( + &self, + ) -> BackendResult<( + scopeguard::ScopeGuard, + AbortRegistration, + )> { + let (abort_handle, abort_reg) = AbortHandle::new_pair(); + + // Register the new abort_handle. + let old_handle = self.sync_abort.lock().unwrap().replace(abort_handle); + if old_handle.is_some() { + // NOTE: In the future we would ideally be able to handle multiple + // abort handles by just iterating over them all in + // abort_sync). But for now, just log a warning if there was + // already one present -- but don't abort it either. + let log = self.with_col(|col| Ok(col.log.clone()))?; + warn!( + log, + "new sync_abort handle registered, but old one was still present (old sync job might not be cancelled on abort)" + ); + } + // Clear the abort handle after the caller is done and drops the guard. + let guard = scopeguard::guard(Arc::clone(&self.sync_abort), |sync_abort| { + sync_abort.lock().unwrap().take(); + }); + Ok((guard, abort_reg)) + } + + fn sync_media_inner(&self, input: pb::SyncAuth) -> Result<()> { // mark media sync as active let (abort_handle, abort_reg) = AbortHandle::new_pair(); { @@ -1485,7 +1493,7 @@ impl Backend { } /// Abort the media sync. Won't return until aborted. - fn abort_media_sync_and_wait(&mut self) { + fn abort_media_sync_and_wait(&self) { let guard = self.state.lock().unwrap(); if let Some(handle) = &guard.media_sync_abort { handle.abort(); @@ -1500,9 +1508,8 @@ impl Backend { } } - fn sync_login_inner(&mut self, input: pb::SyncLoginIn) -> BackendResult { - let (abort_handle, abort_reg) = AbortHandle::new_pair(); - self.sync_abort = Some(abort_handle); + fn sync_login_inner(&self, input: pb::SyncLoginIn) -> BackendResult { + let (_guard, abort_reg) = self.sync_abort_handle()?; let rt = self.runtime_handle(); let sync_fut = sync_login(&input.username, &input.password); @@ -1511,14 +1518,13 @@ impl Backend { Ok(sync_result) => sync_result, Err(_) => Err(AnkiError::Interrupted), }; - self.sync_abort = None; ret.map(|a| pb::SyncAuth { hkey: a.hkey, host_number: a.host_number, }) } - fn sync_status_inner(&mut self, input: pb::SyncAuth) -> BackendResult { + fn sync_status_inner(&self, input: pb::SyncAuth) -> BackendResult { // any local changes mean we can skip the network round-trip let req = self.with_col(|col| col.get_local_sync_status())?; if req != pb::sync_status_out::Required::NoChanges { @@ -1547,12 +1553,8 @@ impl Backend { Ok(response.into()) } - fn sync_collection_inner( - &mut self, - input: pb::SyncAuth, - ) -> BackendResult { - let (abort_handle, abort_reg) = AbortHandle::new_pair(); - self.sync_abort = Some(abort_handle); + fn sync_collection_inner(&self, input: pb::SyncAuth) -> BackendResult { + let (_guard, abort_reg) = self.sync_abort_handle()?; let rt = self.runtime_handle(); let input_copy = input.clone(); @@ -1580,7 +1582,6 @@ impl Backend { } } }); - self.sync_abort = None; let output: SyncOutput = ret?; self.state @@ -1591,7 +1592,7 @@ impl Backend { Ok(output.into()) } - fn full_sync_inner(&mut self, input: pb::SyncAuth, upload: bool) -> Result<()> { + fn full_sync_inner(&self, input: pb::SyncAuth, upload: bool) -> Result<()> { self.abort_media_sync_and_wait(); let rt = self.runtime_handle(); @@ -1603,8 +1604,7 @@ impl Backend { let col_inner = col.take().unwrap(); - let (abort_handle, abort_reg) = AbortHandle::new_pair(); - self.sync_abort = Some(abort_handle); + let (_guard, abort_reg) = self.sync_abort_handle()?; let col_path = col_inner.col_path.clone(); let media_folder_path = col_inner.media_folder.clone(); @@ -1625,7 +1625,6 @@ impl Backend { let abortable_sync = Abortable::new(sync_fut, abort_reg); rt.block_on(abortable_sync) }; - self.sync_abort = None; // ensure re-opened regardless of outcome col.replace(open_collection( diff --git a/rspy/src/lib.rs b/rspy/src/lib.rs index ca501182d..0986ead04 100644 --- a/rspy/src/lib.rs +++ b/rspy/src/lib.rs @@ -61,10 +61,10 @@ fn want_release_gil(method: u32) -> bool { #[pymethods] impl Backend { - fn command(&mut self, py: Python, method: u32, input: &PyBytes) -> PyResult { + fn command(&self, py: Python, method: u32, input: &PyBytes) -> PyResult { let in_bytes = input.as_bytes(); if want_release_gil(method) { - py.allow_threads(move || self.backend.run_command_bytes(method, in_bytes)) + py.allow_threads(|| self.backend.run_command_bytes(method, in_bytes)) } else { self.backend.run_command_bytes(method, in_bytes) } @@ -77,9 +77,9 @@ impl Backend { /// This takes and returns JSON, due to Python's slow protobuf /// encoding/decoding. - fn db_command(&mut self, py: Python, input: &PyBytes) -> PyResult { + fn db_command(&self, py: Python, input: &PyBytes) -> PyResult { let in_bytes = input.as_bytes(); - let out_res = py.allow_threads(move || { + let out_res = py.allow_threads(|| { self.backend .run_db_command_bytes(in_bytes) .map_err(BackendError::py_err) From d2ada5a1ee1d9e13fe8c526b4ba89adc08c1782d Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Wed, 2 Sep 2020 17:02:10 +1000 Subject: [PATCH 18/21] rslib: disable bzip2 feature We can't link against libzip2.so.1 when building manylinux wheels, but the only dependency which required this was "zip" and we don't appear to make use of bzip2 in Anki. This fixes the following "make build" error on Linux: x maturin failed Caused by: Failed to ensure manylinux compliance Caused by: Your library is not manylinux compliant because it links the following forbidden libraries: ["libbz2.so.1"] Signed-off-by: Aleksa Sarai --- rslib/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index 4519dac2a..8f4d873c1 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -28,7 +28,7 @@ serde = "1.0.114" serde_json = "1.0.56" tokio = { version = "0.2.21", features = ["fs", "rt-threaded"] } serde_derive = "1.0.114" -zip = "0.5.6" +zip = { version = "0.5.6", default-features = false, features = ["deflate", "time"] } serde_tuple = "0.5.0" coarsetime = { git = "https://github.com/ankitects/rust-coarsetime.git", branch="old-mac-compat" } utime = "0.3.1" From 16864e616febdab77ba9a8af4950696dd651b5d0 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Wed, 2 Sep 2020 16:03:41 +1000 Subject: [PATCH 19/21] rust: switch to stable compilers The only dependency which required nightly compilers was pyo3, which supports stable Rust (1.39.0 and later) since version 0.11.0. Supporting stable Rust makes it easier to package Anki for distributions. No other code changes were required. Signed-off-by: Aleksa Sarai --- CONTRIBUTORS | 3 ++- rslib/rust-toolchain | 2 +- rspy/Cargo.toml | 2 +- rspy/rust-toolchain | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 81b10b43a..8e7a3ff32 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -46,8 +46,9 @@ Matt Krump Alexander Presnyakov abdo aplaice -phwoo +phwoo Soren Bjornstad +Aleksa Sarai ******************** diff --git a/rslib/rust-toolchain b/rslib/rust-toolchain index 22bde8163..2bf5ad044 100644 --- a/rslib/rust-toolchain +++ b/rslib/rust-toolchain @@ -1 +1 @@ -nightly-2020-06-25 +stable diff --git a/rspy/Cargo.toml b/rspy/Cargo.toml index 0b8e4941c..71d6c812c 100644 --- a/rspy/Cargo.toml +++ b/rspy/Cargo.toml @@ -11,7 +11,7 @@ readme = "README.md" anki = { path = "../rslib" } [dependencies.pyo3] -version = "0.8.0" +version = "0.11.0" features = ["extension-module"] [lib] diff --git a/rspy/rust-toolchain b/rspy/rust-toolchain index 22bde8163..2bf5ad044 100644 --- a/rspy/rust-toolchain +++ b/rspy/rust-toolchain @@ -1 +1 @@ -nightly-2020-06-25 +stable From d4edbd5b11c2425aa8bbc6893fd5052d7507da2b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 7 Sep 2020 16:03:21 +1000 Subject: [PATCH 20/21] doc updates thanks to Robert: https://github.com/ankitects/anki/pull/756 closes #756 --- CONTRIBUTORS | 10 ++++++++++ README.development | 18 +++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 8e7a3ff32..401e24833 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -13,6 +13,16 @@ licensed under the BSD 3 clause license. If any pull request you make contains code that you don't own the copyright to, you agree to make that clear when submitting the request. +When submitting a pull request, GitHub Actions will check that the Git email you +are submitting from matches the one you used to edit this file. A common issue +is adding yourself to this file using the username on your computer, but then +using GitHub to rebase or edit a pull request online. This will result in your +Git email becoming something like user@noreply.github.com. To prevent the +automatic check from failing, you can edit this file again using GitHub's online +editor, making a trivial edit like adding a space after your name, and then pull +requests will work regardless of whether you create them using your computer or +GitHub's online interface. + For users who previously confirmed the license of their contributions on the support site, it would be great if you could add your name below as well. diff --git a/README.development b/README.development index b38a515e3..de41bf3fb 100644 --- a/README.development +++ b/README.development @@ -26,7 +26,8 @@ $ pyenv/bin/python -c 'import aqt; aqt.run()' Building from source -------------------- -To start, make sure you have the following installed: +You will need the following dependencies. Some OS-specific tips on installing +them are further down this document. - Python 3.7+ - portaudio @@ -44,14 +45,14 @@ To start, make sure you have the following installed: - git - curl -The build scripts assume a UNIX-like environment, so on Windows you will -need to use WSL or Cygwin to use them. - Once you've installed the above components, execute ./run in this repo, which will build the subcomponents, and start Anki. Any arguments included on the command line will be passed on to Anki. The first run will take quite a while to download and build everything - please be patient. +Don't name the Git checkout ~/Anki or ~/Documents/Anki, as those folders +were used on old Anki versions and will be automatically moved. + Before contributing code, please read README.contributing. If you'd like to contribute translations, please see the translations section @@ -67,6 +68,10 @@ Subcomponents - proto contains the interface used to communicate between different languages. +The pyenv folder is created when running make for the first time. +It is a Python virtual environment that contains Anki's libraries +and all the required depenencies. + Makefile -------------- @@ -99,7 +104,7 @@ Install Python 3.7+ if it's not installed. Install other dependencies: ``` -sudo apt install portaudio19-dev mpv lame npm rsync gcc gettext git curl +sudo apt install portaudio19-dev mpv lame npm rsync gcc gettext git curl python3-dev python3-venv libxcb-xinerama0 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh source $HOME/.cargo/env wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip @@ -120,6 +125,9 @@ $ brew link gettext --force Windows users (using Visual Studio) ---------- +The build scripts assume a UNIX-like environment, so on Windows you will +need to use WSL or Cygwin to use them. + User-contributed instructions for building using Cygwin: 1. Download and install Cygwin and put its `/bin/` directory on your system path (This PC > Properties > Advancded system settings > Environment Variables > double-click Path > New). From 4faaeb492fb303c9da16416aa8effd81320c687f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 7 Sep 2020 16:09:52 +1000 Subject: [PATCH 21/21] typo --- README.development | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.development b/README.development index de41bf3fb..76083c605 100644 --- a/README.development +++ b/README.development @@ -70,7 +70,7 @@ Subcomponents The pyenv folder is created when running make for the first time. It is a Python virtual environment that contains Anki's libraries -and all the required depenencies. +and all the required dependencies. Makefile --------------