2020-02-10 08:58:54 +01:00
|
|
|
# Copyright: Ankitects Pty Ltd and contributors
|
|
|
|
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
2020-02-11 08:30:10 +01:00
|
|
|
import itertools
|
2020-02-10 08:58:54 +01:00
|
|
|
import time
|
|
|
|
from concurrent.futures import Future
|
2021-10-03 10:59:42 +02:00
|
|
|
from typing import Iterable, Sequence, TypeVar
|
2020-02-10 08:58:54 +01:00
|
|
|
|
|
|
|
import aqt
|
2022-02-13 04:40:47 +01:00
|
|
|
import aqt.progress
|
2021-12-03 23:45:26 +01:00
|
|
|
from anki.collection import Collection, SearchNode
|
2021-01-31 06:55:08 +01:00
|
|
|
from anki.errors import Interrupted
|
2021-06-20 07:49:20 +02:00
|
|
|
from anki.media import CheckMediaResponse
|
2023-02-20 09:48:09 +01:00
|
|
|
from anki.notes import NoteId
|
2022-05-27 08:25:34 +02:00
|
|
|
from aqt import gui_hooks
|
2021-12-03 23:45:26 +01:00
|
|
|
from aqt.operations import QueryOp
|
2023-02-20 09:48:09 +01:00
|
|
|
from aqt.operations.tag import add_tags_to_notes
|
2020-02-10 08:58:54 +01:00
|
|
|
from aqt.qt import *
|
2021-01-07 05:46:55 +01:00
|
|
|
from aqt.utils import (
|
|
|
|
askUser,
|
|
|
|
disable_help_button,
|
|
|
|
restoreGeom,
|
|
|
|
saveGeom,
|
|
|
|
showText,
|
|
|
|
tooltip,
|
|
|
|
tr,
|
|
|
|
)
|
2020-02-10 08:58:54 +01:00
|
|
|
|
2020-02-11 08:30:10 +01:00
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
|
|
|
2021-10-03 10:59:42 +02:00
|
|
|
def chunked_list(l: Iterable[T], n: int) -> Iterable[list[T]]:
|
2020-02-11 08:30:10 +01:00
|
|
|
l = iter(l)
|
|
|
|
while True:
|
|
|
|
res = list(itertools.islice(l, n))
|
|
|
|
if not res:
|
|
|
|
return
|
|
|
|
yield res
|
|
|
|
|
2020-02-10 08:58:54 +01:00
|
|
|
|
|
|
|
def check_media_db(mw: aqt.AnkiQt) -> None:
|
|
|
|
c = MediaChecker(mw)
|
|
|
|
c.check()
|
|
|
|
|
|
|
|
|
|
|
|
class MediaChecker:
|
2021-10-03 10:59:42 +02:00
|
|
|
progress_dialog: aqt.progress.ProgressDialog | None
|
2020-02-10 08:58:54 +01:00
|
|
|
|
|
|
|
def __init__(self, mw: aqt.AnkiQt) -> None:
|
|
|
|
self.mw = mw
|
2021-10-03 10:59:42 +02:00
|
|
|
self._progress_timer: QTimer | None = None
|
2020-02-10 08:58:54 +01:00
|
|
|
|
|
|
|
def check(self) -> None:
|
|
|
|
self.progress_dialog = self.mw.progress.start()
|
2020-05-29 11:59:50 +02:00
|
|
|
self._set_progress_enabled(True)
|
2020-02-10 08:58:54 +01:00
|
|
|
self.mw.taskman.run_in_background(self._check, self._on_finished)
|
|
|
|
|
2020-05-29 11:59:50 +02:00
|
|
|
def _set_progress_enabled(self, enabled: bool) -> None:
|
|
|
|
if self._progress_timer:
|
Refactor progress handling (#2549)
Previously it was Backend's responsibility to store the last progress,
and when calling routines in Collection, one had to construct and pass
in a Fn, which wasn't the most ergonomic. This PR adds the last progress
state to the collection, so that the routines no longer need a separate
progress arg, and makes some other tweaks to improve ergonomics.
ThrottlingProgressHandler has been tweaked so that it now stores the
current state, so that callers don't need to store it separately. When
a long-running routine starts, it calls col.new_progress_handler(),
which automatically initializes the data to defaults, and updates the
shared UI state, so we no longer need to manually update the state at
the start of an operation.
The backend shares the Arc<Mutex<>> with the collection, so it can get
at the current state, and so we can update the state when importing a
backup.
Other tweaks:
- The current Incrementor was awkward to use in the media check, which
uses a single incrementing value across multiple method calls, so I've
added a simpler alternative for such cases. The old incrementor method
has been kept, but implemented directly on ThrottlingProgressHandler.
- The full sync code was passing the progress handler in a complicated
way that may once have been required, but no longer is.
- On the Qt side, timers are now stopped before deletion, or they keep
running for a few seconds.
- I left the ChangeTracker using a closure, as it's used for both importing
and syncing.
2023-06-19 05:48:32 +02:00
|
|
|
self._progress_timer.stop()
|
Backups (#1685)
* Add zstd dep
* Implement backend backup with zstd
* Implement backup thinning
* Write backup meta
* Use new file ending anki21b
* Asynchronously backup on collection close in Rust
* Revert "Add zstd dep"
This reverts commit 3fcb2141d2be15f907269d13275c41971431385c.
* Add zstd again
* Take backup col path from col struct
* Fix formatting
* Implement backup restoring on backend
* Normalize restored media file names
* Refactor `extract_legacy_data()`
A bit cumbersome due to borrowing rules.
* Refactor
* Make thinning calendar-based and gradual
* Consider last kept backups of previous stages
* Import full apkgs and colpkgs with backend
* Expose new backup settings
* Test `BackupThinner` and make it deterministic
* Mark backup_path when closing optional
* Delete leaky timer
* Add progress updates for restoring media
* Write restored collection to tempfile first
* Do collection compression in the background thread
This has us currently storing an uncompressed and compressed copy of
the collection in memory (not ideal), but means the collection can be
closed without waiting for compression to complete. On a large collection,
this takes a close and reopen from about 0.55s to about 0.07s. The old
backup code for comparison: about 0.35s for compression off, about
8.5s for zip compression.
* Use multithreading in zstd compression
On my system, this reduces the compression time of a large collection
from about 0.55s to 0.08s.
* Stream compressed collection data into zip file
* Tweak backup explanation
+ Fix incorrect tab order for ignore accents option
* Decouple restoring backup and full import
In the first case, no profile is opened, unless the new collection
succeeds to load.
In the second case, either the old collection is reloaded or the new one
is loaded.
* Fix number gap in Progress message
* Don't revert backup when media fails but report it
* Tweak error flow
* Remove native BackupLimits enum
* Fix type annotation
* Add thinning test for whole year
* Satisfy linter
* Await async backup to finish
* Move restart disclaimer out of backup tab
Should be visible regardless of the current tab.
* Write restored collection in chunks
* Refactor
* Write media in chunks and refactor
* Log error if removing file fails
* join_backup_task -> await_backup_completion
* Refactor backup.rs
* Refactor backup meta and collection extraction
* Fix wrong error being returned
* Call sync_all() on new collection
* Add ImportError
* Store logger in Backend, instead of creating one on demand
init_backend() accepts a Logger rather than a log file, to allow other
callers to customize the logger if they wish.
In the future we may want to explore using the tracing crate as an
alternative; it's a bit more ergonomic, as a logger doesn't need to be
passed around, and it plays more nicely with async code.
* Sync file contents prior to rename; sync folder after rename.
* Limit backup creation to once per 30 min
* Use zstd::stream::copy_decode
* Make importing abortable
* Don't revert if backup media is aborted
* Set throttle implicitly
* Change force flag to minimum_backup_interval
* Don't attempt to open folders on Windows
* Join last backup thread before starting new one
Also refactor.
* Disable auto sync and backup when restoring again
* Force backup on full download
* Include the reason why a media file import failed, and the file path
- Introduce a FileIoError that contains a string representation of
the underlying I/O error, and an associated path. There are a few
places in the code where we're currently manually including the filename
in a custom error message, and this is a step towards a more consistent
approach (but we may be better served with a more general approach in
the future similar to Anyhow's .context())
- Move the error message into importing.ftl, as it's a bit neater
when error messages live in the same file as the rest of the messages
associated with some functionality.
* Fix importing of media files
* Minor wording tweaks
* Save an allocation
I18n strings with replacements are already strings, so we can skip the
extra allocation. Not that it matters here at all.
* Terminate import if file missing from archive
If a third-party tool is creating invalid archives, the user should know
about it. This should be rare, so I did not attempt to make it
translatable.
* Skip multithreaded compression on small collections
Co-authored-by: Damien Elmes <gpg@ankiweb.net>
2022-03-07 06:11:31 +01:00
|
|
|
self._progress_timer.deleteLater()
|
2020-05-29 11:59:50 +02:00
|
|
|
self._progress_timer = None
|
|
|
|
if enabled:
|
2021-02-08 07:46:57 +01:00
|
|
|
self._progress_timer = timer = QTimer()
|
|
|
|
timer.setSingleShot(False)
|
|
|
|
timer.setInterval(100)
|
|
|
|
qconnect(timer.timeout, self._on_progress)
|
|
|
|
timer.start()
|
2020-05-29 11:59:50 +02:00
|
|
|
|
|
|
|
def _on_progress(self) -> None:
|
2021-02-08 07:46:57 +01:00
|
|
|
if not self.mw.col:
|
|
|
|
return
|
2020-05-29 11:59:50 +02:00
|
|
|
progress = self.mw.col.latest_progress()
|
2021-02-08 07:40:27 +01:00
|
|
|
if not progress.HasField("media_check"):
|
2020-05-29 11:59:50 +02:00
|
|
|
return
|
2021-02-08 07:40:27 +01:00
|
|
|
label = progress.media_check
|
2020-05-29 11:59:50 +02:00
|
|
|
|
|
|
|
try:
|
|
|
|
if self.progress_dialog.wantCancel:
|
2021-01-31 09:46:43 +01:00
|
|
|
self.mw.col.set_wants_abort()
|
2020-05-29 11:59:50 +02:00
|
|
|
except AttributeError:
|
|
|
|
# dialog may not be active
|
|
|
|
pass
|
2020-02-10 08:58:54 +01:00
|
|
|
|
2021-02-08 07:40:27 +01:00
|
|
|
self.mw.taskman.run_on_main(lambda: self.mw.progress.update(label=label))
|
2020-02-10 08:58:54 +01:00
|
|
|
|
2021-06-20 07:49:20 +02:00
|
|
|
def _check(self) -> CheckMediaResponse:
|
2020-02-10 08:58:54 +01:00
|
|
|
"Run the check on a background thread."
|
|
|
|
return self.mw.col.media.check()
|
|
|
|
|
2020-02-14 07:15:18 +01:00
|
|
|
def _on_finished(self, future: Future) -> None:
|
2020-05-29 11:59:50 +02:00
|
|
|
self._set_progress_enabled(False)
|
2020-02-10 08:58:54 +01:00
|
|
|
self.mw.progress.finish()
|
|
|
|
self.progress_dialog = None
|
|
|
|
|
|
|
|
exc = future.exception()
|
|
|
|
if isinstance(exc, Interrupted):
|
|
|
|
return
|
|
|
|
|
2021-06-20 07:49:20 +02:00
|
|
|
output: CheckMediaResponse = future.result()
|
2022-05-27 08:25:34 +02:00
|
|
|
gui_hooks.media_check_did_finish(output)
|
2020-02-14 07:15:18 +01:00
|
|
|
report = output.report
|
2020-02-10 08:58:54 +01:00
|
|
|
|
|
|
|
# show report and offer to delete
|
|
|
|
diag = QDialog(self.mw)
|
2021-03-26 04:48:26 +01:00
|
|
|
diag.setWindowTitle(tr.media_check_window_title())
|
2021-01-07 05:46:55 +01:00
|
|
|
disable_help_button(diag)
|
2020-02-10 08:58:54 +01:00
|
|
|
layout = QVBoxLayout(diag)
|
|
|
|
diag.setLayout(layout)
|
2021-02-08 07:42:21 +01:00
|
|
|
text = QPlainTextEdit()
|
2020-02-10 08:58:54 +01:00
|
|
|
text.setReadOnly(True)
|
|
|
|
text.setPlainText(report)
|
2021-10-05 05:53:01 +02:00
|
|
|
text.setWordWrapMode(QTextOption.WrapMode.NoWrap)
|
2020-02-10 08:58:54 +01:00
|
|
|
layout.addWidget(text)
|
2021-10-05 05:53:01 +02:00
|
|
|
box = QDialogButtonBox(QDialogButtonBox.StandardButton.Close)
|
2020-02-10 08:58:54 +01:00
|
|
|
layout.addWidget(box)
|
2020-02-11 08:30:10 +01:00
|
|
|
|
2020-02-10 08:58:54 +01:00
|
|
|
if output.unused:
|
2021-03-26 04:48:26 +01:00
|
|
|
b = QPushButton(tr.media_check_delete_unused())
|
2020-02-10 08:58:54 +01:00
|
|
|
b.setAutoDefault(False)
|
2021-10-05 05:53:01 +02:00
|
|
|
box.addButton(b, QDialogButtonBox.ButtonRole.RejectRole)
|
2020-05-04 05:23:08 +02:00
|
|
|
qconnect(b.clicked, lambda c: self._on_trash_files(output.unused))
|
2020-02-10 08:58:54 +01:00
|
|
|
|
2020-02-11 06:09:33 +01:00
|
|
|
if output.missing:
|
2023-02-20 09:48:09 +01:00
|
|
|
b = QPushButton(tr.media_check_add_tag())
|
|
|
|
b.setAutoDefault(False)
|
|
|
|
box.addButton(b, QDialogButtonBox.ButtonRole.RejectRole)
|
|
|
|
qconnect(
|
|
|
|
b.clicked,
|
|
|
|
lambda: add_missing_media_tag(self.mw, output.missing_media_notes),
|
|
|
|
)
|
|
|
|
|
2020-02-11 06:09:33 +01:00
|
|
|
if any(map(lambda x: x.startswith("latex-"), output.missing)):
|
2021-03-26 04:48:26 +01:00
|
|
|
b = QPushButton(tr.media_check_render_latex())
|
2020-02-11 06:09:33 +01:00
|
|
|
b.setAutoDefault(False)
|
2021-10-05 05:53:01 +02:00
|
|
|
box.addButton(b, QDialogButtonBox.ButtonRole.RejectRole)
|
2020-05-04 05:23:08 +02:00
|
|
|
qconnect(b.clicked, self._on_render_latex)
|
2020-02-11 06:09:33 +01:00
|
|
|
|
2020-03-10 03:49:40 +01:00
|
|
|
if output.have_trash:
|
2021-03-26 04:48:26 +01:00
|
|
|
b = QPushButton(tr.media_check_empty_trash())
|
2020-03-10 03:49:40 +01:00
|
|
|
b.setAutoDefault(False)
|
2021-10-05 05:53:01 +02:00
|
|
|
box.addButton(b, QDialogButtonBox.ButtonRole.RejectRole)
|
2020-05-04 05:23:08 +02:00
|
|
|
qconnect(b.clicked, lambda c: self._on_empty_trash())
|
2020-03-10 04:35:09 +01:00
|
|
|
|
2021-03-26 04:48:26 +01:00
|
|
|
b = QPushButton(tr.media_check_restore_trash())
|
2020-03-10 04:35:09 +01:00
|
|
|
b.setAutoDefault(False)
|
2021-10-05 05:53:01 +02:00
|
|
|
box.addButton(b, QDialogButtonBox.ButtonRole.RejectRole)
|
2020-05-04 05:23:08 +02:00
|
|
|
qconnect(b.clicked, lambda c: self._on_restore_trash())
|
2020-03-10 04:35:09 +01:00
|
|
|
|
2020-05-04 05:23:08 +02:00
|
|
|
qconnect(box.rejected, diag.reject)
|
2020-02-10 08:58:54 +01:00
|
|
|
diag.setMinimumHeight(400)
|
|
|
|
diag.setMinimumWidth(500)
|
2023-07-03 15:57:56 +02:00
|
|
|
restoreGeom(diag, "checkmediadb", default_size=(800, 800))
|
2021-10-05 02:01:45 +02:00
|
|
|
diag.exec()
|
2020-02-10 08:58:54 +01:00
|
|
|
saveGeom(diag, "checkmediadb")
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _on_render_latex(self) -> None:
|
2020-02-11 06:09:33 +01:00
|
|
|
self.progress_dialog = self.mw.progress.start()
|
|
|
|
try:
|
2020-02-11 06:36:05 +01:00
|
|
|
out = self.mw.col.media.render_all_latex(self._on_render_latex_progress)
|
2020-02-11 07:46:57 +01:00
|
|
|
if self.progress_dialog.wantCancel:
|
|
|
|
return
|
2020-02-11 06:09:33 +01:00
|
|
|
finally:
|
|
|
|
self.mw.progress.finish()
|
2020-02-11 07:46:57 +01:00
|
|
|
self.progress_dialog = None
|
2020-02-11 06:36:05 +01:00
|
|
|
|
|
|
|
if out is not None:
|
|
|
|
nid, err = out
|
2021-02-11 10:57:19 +01:00
|
|
|
aqt.dialogs.open("Browser", self.mw, search=(SearchNode(nid=nid),))
|
2020-02-11 06:36:05 +01:00
|
|
|
showText(err, type="html")
|
|
|
|
else:
|
2021-03-26 04:48:26 +01:00
|
|
|
tooltip(tr.media_check_all_latex_rendered())
|
2020-02-11 06:09:33 +01:00
|
|
|
|
|
|
|
def _on_render_latex_progress(self, count: int) -> bool:
|
|
|
|
if self.progress_dialog.wantCancel:
|
|
|
|
return False
|
|
|
|
|
2021-03-26 05:21:04 +01:00
|
|
|
self.mw.progress.update(tr.media_check_checked(count=count))
|
2020-02-11 06:09:33 +01:00
|
|
|
return True
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _on_trash_files(self, fnames: Sequence[str]) -> None:
|
2021-03-26 04:48:26 +01:00
|
|
|
if not askUser(tr.media_check_delete_unused_confirm()):
|
2020-02-11 08:30:10 +01:00
|
|
|
return
|
|
|
|
|
2020-02-17 01:18:20 +01:00
|
|
|
total = len(fnames)
|
2021-12-03 23:45:26 +01:00
|
|
|
|
|
|
|
def trash(col: Collection) -> None:
|
2021-12-04 00:09:46 +01:00
|
|
|
last_progress = 0.0
|
|
|
|
remaining = total
|
2021-12-03 23:45:26 +01:00
|
|
|
|
2020-02-11 08:30:10 +01:00
|
|
|
for chunk in chunked_list(fnames, 25):
|
2021-12-03 23:45:26 +01:00
|
|
|
col.media.trash_files(chunk)
|
2020-02-11 08:30:10 +01:00
|
|
|
remaining -= len(chunk)
|
2021-12-04 00:09:46 +01:00
|
|
|
if time.time() - last_progress >= 0.1:
|
2021-12-03 23:45:26 +01:00
|
|
|
self.mw.taskman.run_on_main(
|
|
|
|
lambda: self.mw.progress.update(
|
|
|
|
label=tr.media_check_files_remaining(count=remaining),
|
|
|
|
value=total - remaining,
|
|
|
|
max=total,
|
|
|
|
)
|
2020-02-11 08:30:10 +01:00
|
|
|
)
|
2021-12-04 00:09:46 +01:00
|
|
|
last_progress = time.time()
|
2020-02-11 08:30:10 +01:00
|
|
|
|
2021-12-03 23:45:26 +01:00
|
|
|
QueryOp(
|
|
|
|
parent=aqt.mw,
|
|
|
|
op=trash,
|
|
|
|
success=lambda _: tooltip(
|
|
|
|
tr.media_check_delete_unused_complete(count=total)
|
|
|
|
),
|
|
|
|
).with_progress().run_in_background()
|
2020-03-10 03:49:40 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _on_empty_trash(self) -> None:
|
2020-03-10 03:49:40 +01:00
|
|
|
self.progress_dialog = self.mw.progress.start()
|
2020-05-29 11:59:50 +02:00
|
|
|
self._set_progress_enabled(True)
|
2020-03-10 03:49:40 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def empty_trash() -> None:
|
2021-01-31 09:46:43 +01:00
|
|
|
self.mw.col.media.empty_trash()
|
2020-03-10 03:49:40 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def on_done(fut: Future) -> None:
|
2020-03-10 03:49:40 +01:00
|
|
|
self.mw.progress.finish()
|
2020-05-29 11:59:50 +02:00
|
|
|
self._set_progress_enabled(False)
|
2020-03-10 03:49:40 +01:00
|
|
|
# check for errors
|
|
|
|
fut.result()
|
|
|
|
|
2021-03-26 04:48:26 +01:00
|
|
|
tooltip(tr.media_check_trash_emptied())
|
2020-03-10 03:49:40 +01:00
|
|
|
|
|
|
|
self.mw.taskman.run_in_background(empty_trash, on_done)
|
2020-03-10 04:35:09 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _on_restore_trash(self) -> None:
|
2020-03-10 04:35:09 +01:00
|
|
|
self.progress_dialog = self.mw.progress.start()
|
2020-05-29 11:59:50 +02:00
|
|
|
self._set_progress_enabled(True)
|
2020-03-10 04:35:09 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def restore_trash() -> None:
|
2021-01-31 09:46:43 +01:00
|
|
|
self.mw.col.media.restore_trash()
|
2020-03-10 04:35:09 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def on_done(fut: Future) -> None:
|
2020-03-10 04:35:09 +01:00
|
|
|
self.mw.progress.finish()
|
2020-05-29 11:59:50 +02:00
|
|
|
self._set_progress_enabled(False)
|
2020-03-10 04:35:09 +01:00
|
|
|
# check for errors
|
|
|
|
fut.result()
|
|
|
|
|
2021-03-26 04:48:26 +01:00
|
|
|
tooltip(tr.media_check_trash_restored())
|
2020-03-10 04:35:09 +01:00
|
|
|
|
|
|
|
self.mw.taskman.run_in_background(restore_trash, on_done)
|
2023-02-20 09:48:09 +01:00
|
|
|
|
|
|
|
|
|
|
|
def add_missing_media_tag(parent: QWidget, missing_media_notes: Sequence[int]) -> None:
|
|
|
|
add_tags_to_notes(
|
|
|
|
parent=parent,
|
|
|
|
note_ids=list(map(NoteId, missing_media_notes)),
|
|
|
|
space_separated_tags=tr.media_check_missing_media_tag(),
|
|
|
|
).run_in_background()
|