anki/qt/aqt/mediasync.py

191 lines
6.1 KiB
Python
Raw Normal View History

# Copyright: Ankitects Pty Ltd and contributors
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
from __future__ import annotations
import time
from concurrent.futures import Future
from datetime import datetime
from typing import Any, Callable
import aqt
import aqt.forms
import aqt.main
from anki.collection import Collection
Rework syncing code, and replace local sync server (#2329) This PR replaces the existing Python-driven sync server with a new one in Rust. The new server supports both collection and media syncing, and is compatible with both the new protocol mentioned below, and older clients. A setting has been added to the preferences screen to point Anki to a local server, and a similar setting is likely to come to AnkiMobile soon. Documentation is available here: <https://docs.ankiweb.net/sync-server.html> In addition to the new server and refactoring, this PR also makes changes to the sync protocol. The existing sync protocol places payloads and metadata inside a multipart POST body, which causes a few headaches: - Legacy clients build the request in a non-deterministic order, meaning the entire request needs to be scanned to extract the metadata. - Reqwest's multipart API directly writes the multipart body, without exposing the resulting stream to us, making it harder to track the progress of the transfer. We've been relying on a patched version of reqwest for timeouts, which is a pain to keep up to date. To address these issues, the metadata is now sent in a HTTP header, with the data payload sent directly in the body. Instead of the slower gzip, we now use zstd. The old timeout handling code has been replaced with a new implementation that wraps the request and response body streams to track progress, allowing us to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout. The main other change to the protocol is that one-way syncs no longer need to downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
from anki.errors import Interrupted
PEP8 for rest of pylib (#1451) * PEP8 dbproxy.py * PEP8 errors.py * PEP8 httpclient.py * PEP8 lang.py * PEP8 latex.py * Add decorator to deprectate key words * Make replacement for deprecated attribute optional * Use new helper `_print_replacement_warning()` * PEP8 media.py * PEP8 rsbackend.py * PEP8 sound.py * PEP8 stdmodels.py * PEP8 storage.py * PEP8 sync.py * PEP8 tags.py * PEP8 template.py * PEP8 types.py * Fix DeprecatedNamesMixinForModule The class methods need to be overridden with instance methods, so every module has its own dicts. * Use `# pylint: disable=invalid-name` instead of id * PEP8 utils.py * Only decorate `__getattr__` with `@no_type_check` * Fix mypy issue with snakecase Importing it from `anki._vendor` raises attribute errors. * Format * Remove inheritance of DeprecatedNamesMixin There's almost no shared code now and overriding classmethods with instance methods raises mypy issues. * Fix traceback frames of deprecation warnings * remove fn/TimedLog (dae) Neither Anki nor add-ons appear to have been using it * fix some issues with stringcase use (dae) - the wheel was depending on the PyPI version instead of our vendored version - _vendor:stringcase should not have been listed in the anki py_library. We already include the sources in py_srcs, and need to refer to them directly. By listing _vendor:stringcase as well, we were making a top-level stringcase library available, which would have only worked for distributing because the wheel definition was also incorrect. - mypy errors are what caused me to mistakenly add the above - they were because the type: ignore at the top of stringcase.py was causing mypy to completely ignore the file, so it was not aware of any attributes it contained.
2021-10-25 06:50:13 +02:00
from anki.utils import int_time
from aqt import gui_hooks
from aqt.operations import QueryOp
from aqt.qt import QDialog, QDialogButtonBox, QPushButton, Qt, QTimer, qconnect
from aqt.utils import disable_help_button, show_info, tr
class MediaSyncer:
2021-02-01 14:28:21 +01:00
def __init__(self, mw: aqt.main.AnkiQt) -> None:
self.mw = mw
self._syncing: bool = False
self.last_progress = ""
self._last_progress_at = 0
gui_hooks.media_sync_did_start_or_stop.append(self._on_start_stop)
def start(self) -> None:
"Start media syncing in the background, if it's not already running."
if not self.mw.pm.media_syncing_enabled() or not (
auth := self.mw.pm.sync_auth()
):
return
def run(col: Collection) -> None:
col.sync_media(auth)
# this will exit after the thread is spawned, but may block if there's an existing
# backend lock
QueryOp(parent=aqt.mw, op=run, success=lambda _: 1).run_in_background()
self.start_monitoring()
2020-05-30 04:28:22 +02:00
def start_monitoring(self) -> None:
if self._syncing:
return
self._syncing = True
gui_hooks.media_sync_did_start_or_stop(True)
self._update_progress(tr.sync_media_starting())
def monitor() -> None:
while True:
resp = self.mw.col.media_sync_status()
if not resp.active:
return
if p := resp.progress:
self._update_progress(f"{p.added}, {p.removed}, {p.checked}")
time.sleep(0.25)
self.mw.taskman.run_in_background(
monitor, self._on_finished, uses_collection=False
)
def _update_progress(self, progress: str) -> None:
self.last_progress = progress
self.mw.taskman.run_on_main(lambda: gui_hooks.media_sync_did_progress(progress))
def _on_finished(self, future: Future) -> None:
self._syncing = False
self._last_progress_at = int_time()
gui_hooks.media_sync_did_start_or_stop(False)
exc = future.exception()
if exc is not None:
self._handle_sync_error(exc)
else:
self._update_progress(tr.sync_media_complete())
2021-02-01 14:28:21 +01:00
def _handle_sync_error(self, exc: BaseException) -> None:
2020-02-04 06:16:11 +01:00
if isinstance(exc, Interrupted):
self._update_progress(tr.sync_media_aborted())
2020-02-04 06:16:11 +01:00
return
Rework syncing code, and replace local sync server (#2329) This PR replaces the existing Python-driven sync server with a new one in Rust. The new server supports both collection and media syncing, and is compatible with both the new protocol mentioned below, and older clients. A setting has been added to the preferences screen to point Anki to a local server, and a similar setting is likely to come to AnkiMobile soon. Documentation is available here: <https://docs.ankiweb.net/sync-server.html> In addition to the new server and refactoring, this PR also makes changes to the sync protocol. The existing sync protocol places payloads and metadata inside a multipart POST body, which causes a few headaches: - Legacy clients build the request in a non-deterministic order, meaning the entire request needs to be scanned to extract the metadata. - Reqwest's multipart API directly writes the multipart body, without exposing the resulting stream to us, making it harder to track the progress of the transfer. We've been relying on a patched version of reqwest for timeouts, which is a pain to keep up to date. To address these issues, the metadata is now sent in a HTTP header, with the data payload sent directly in the body. Instead of the slower gzip, we now use zstd. The old timeout handling code has been replaced with a new implementation that wraps the request and response body streams to track progress, allowing us to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout. The main other change to the protocol is that one-way syncs no longer need to downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
else:
show_info(str(exc), modality=Qt.WindowModality.NonModal)
return
2020-02-04 06:16:11 +01:00
def abort(self) -> None:
if not self.is_syncing():
return
self.mw.col.set_wants_abort()
self.mw.col.abort_media_sync()
self._update_progress(tr.sync_media_aborting())
def is_syncing(self) -> bool:
return self._syncing
2020-02-27 04:27:58 +01:00
def _on_start_stop(self, running: bool) -> None:
self.mw.toolbar.set_sync_active(running)
2021-02-01 14:28:21 +01:00
def show_sync_log(self) -> None:
aqt.dialogs.open("sync_log", self.mw, self)
2021-02-01 14:28:21 +01:00
def show_diag_until_finished(self, on_finished: Callable[[], None]) -> None:
2020-02-05 03:23:15 +01:00
# nothing to do if not syncing
if not self.is_syncing():
return on_finished()
2020-02-05 03:23:15 +01:00
diag: MediaSyncDialog = aqt.dialogs.open("sync_log", self.mw, self, True)
diag.show()
timer: QTimer | None = None
2021-02-01 14:28:21 +01:00
def check_finished() -> None:
if not self.is_syncing():
timer.deleteLater()
on_finished()
timer = self.mw.progress.timer(150, check_finished, True, False, parent=self.mw)
2020-02-05 03:23:15 +01:00
2020-02-05 03:38:36 +01:00
def seconds_since_last_sync(self) -> int:
if self.is_syncing():
return 0
return int_time() - self._last_progress_at
2020-02-05 03:38:36 +01:00
class MediaSyncDialog(QDialog):
silentlyClose = True
2020-02-05 03:23:15 +01:00
def __init__(
self, mw: aqt.main.AnkiQt, syncer: MediaSyncer, close_when_done: bool = False
) -> None:
super().__init__(mw)
self.mw = mw
self._syncer = syncer
2020-02-05 03:23:15 +01:00
self._close_when_done = close_when_done
self.form = aqt.forms.synclog.Ui_Dialog()
self.form.setupUi(self)
2021-03-26 04:48:26 +01:00
self.setWindowTitle(tr.sync_media_log_title())
disable_help_button(self)
2021-03-26 04:48:26 +01:00
self.abort_button = QPushButton(tr.sync_abort_button())
qconnect(self.abort_button.clicked, self._on_abort)
self.abort_button.setAutoDefault(False)
self.form.buttonBox.addButton(
self.abort_button, QDialogButtonBox.ButtonRole.ActionRole
)
self.abort_button.setHidden(not self._syncer.is_syncing())
gui_hooks.media_sync_did_progress.append(self._on_log_entry)
2020-02-05 03:23:15 +01:00
gui_hooks.media_sync_did_start_or_stop.append(self._on_start_stop)
self._on_log_entry(syncer.last_progress)
self.show()
2020-02-05 03:23:15 +01:00
def reject(self) -> None:
if self._close_when_done and self._syncer.is_syncing():
# closing while syncing on close starts an abort
self._on_abort()
return
aqt.dialogs.markClosed("sync_log")
2020-02-05 03:23:15 +01:00
QDialog.reject(self)
def reopen(
self, mw: aqt.AnkiQt, syncer: Any, close_when_done: bool = False
) -> None:
2020-02-05 03:23:15 +01:00
self._close_when_done = close_when_done
self.show()
def _on_abort(self, *_args: Any) -> None:
self._syncer.abort()
self.abort_button.setHidden(True)
def _on_log_entry(self, entry: str) -> None:
dt = datetime.fromtimestamp(int_time())
time = dt.strftime("%H:%M:%S")
text = f"{time}: {entry}"
self.form.log_label.setText(text)
if not self._syncer.is_syncing():
self.abort_button.setHidden(True)
2020-02-05 03:23:15 +01:00
def _on_start_stop(self, running: bool) -> None:
if not running and self._close_when_done:
aqt.dialogs.markClosed("sync_log")
self._close_when_done = False
self.close()