2020-02-04 00:07:15 +01:00
|
|
|
# Copyright: Ankitects Pty Ltd and contributors
|
|
|
|
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
|
|
|
|
2020-02-04 02:41:20 +01:00
|
|
|
from __future__ import annotations
|
|
|
|
|
2020-02-04 00:07:15 +01:00
|
|
|
import time
|
|
|
|
from concurrent.futures import Future
|
2023-09-10 05:22:20 +02:00
|
|
|
from datetime import datetime
|
|
|
|
from typing import Any, Callable
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
import aqt
|
2022-02-13 04:40:47 +01:00
|
|
|
import aqt.forms
|
|
|
|
import aqt.main
|
2023-09-10 05:22:20 +02:00
|
|
|
from anki.collection import Collection
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
from anki.errors import Interrupted
|
2021-10-25 06:50:13 +02:00
|
|
|
from anki.utils import int_time
|
2020-02-04 00:07:15 +01:00
|
|
|
from aqt import gui_hooks
|
2023-09-10 05:22:20 +02:00
|
|
|
from aqt.operations import QueryOp
|
|
|
|
from aqt.qt import QDialog, QDialogButtonBox, QPushButton, Qt, QTimer, qconnect
|
|
|
|
from aqt.utils import disable_help_button, show_info, tr
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
|
|
|
|
class MediaSyncer:
|
2021-02-01 14:28:21 +01:00
|
|
|
def __init__(self, mw: aqt.main.AnkiQt) -> None:
|
2020-02-04 03:26:10 +01:00
|
|
|
self.mw = mw
|
2020-02-06 09:16:39 +01:00
|
|
|
self._syncing: bool = False
|
2023-09-10 05:22:20 +02:00
|
|
|
self.last_progress = ""
|
|
|
|
self._last_progress_at = 0
|
2020-02-05 02:55:14 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop.append(self._on_start_stop)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2020-02-04 03:26:10 +01:00
|
|
|
def start(self) -> None:
|
2020-02-04 00:07:15 +01:00
|
|
|
"Start media syncing in the background, if it's not already running."
|
2023-09-10 05:22:20 +02:00
|
|
|
if not self.mw.pm.media_syncing_enabled() or not (
|
|
|
|
auth := self.mw.pm.sync_auth()
|
|
|
|
):
|
2020-02-04 00:07:15 +01:00
|
|
|
return
|
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
def run(col: Collection) -> None:
|
|
|
|
col.sync_media(auth)
|
2020-02-04 03:26:10 +01:00
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
# this will exit after the thread is spawned, but may block if there's an existing
|
|
|
|
# backend lock
|
|
|
|
QueryOp(parent=aqt.mw, op=run, success=lambda _: 1).run_in_background()
|
|
|
|
|
|
|
|
self.start_monitoring()
|
2020-05-30 04:28:22 +02:00
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
def start_monitoring(self) -> None:
|
|
|
|
if self._syncing:
|
|
|
|
return
|
2020-02-06 09:16:39 +01:00
|
|
|
self._syncing = True
|
2020-02-05 02:55:14 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop(True)
|
2023-09-10 05:22:20 +02:00
|
|
|
self._update_progress(tr.sync_media_starting())
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
def monitor() -> None:
|
|
|
|
while True:
|
|
|
|
resp = self.mw.col.media_sync_status()
|
|
|
|
if not resp.active:
|
|
|
|
return
|
|
|
|
if p := resp.progress:
|
|
|
|
self._update_progress(f"{p.added}, {p.removed}, {p.checked}")
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
time.sleep(0.25)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2023-10-17 01:47:50 +02:00
|
|
|
self.mw.taskman.run_in_background(
|
|
|
|
monitor, self._on_finished, uses_collection=False
|
|
|
|
)
|
2023-09-10 05:22:20 +02:00
|
|
|
|
|
|
|
def _update_progress(self, progress: str) -> None:
|
|
|
|
self.last_progress = progress
|
|
|
|
self.mw.taskman.run_on_main(lambda: gui_hooks.media_sync_did_progress(progress))
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
def _on_finished(self, future: Future) -> None:
|
2020-02-06 09:16:39 +01:00
|
|
|
self._syncing = False
|
2023-09-10 05:22:20 +02:00
|
|
|
self._last_progress_at = int_time()
|
2020-02-05 02:55:14 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop(False)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
exc = future.exception()
|
|
|
|
if exc is not None:
|
2020-02-04 03:26:10 +01:00
|
|
|
self._handle_sync_error(exc)
|
|
|
|
else:
|
2023-09-10 05:22:20 +02:00
|
|
|
self._update_progress(tr.sync_media_complete())
|
2020-02-04 03:26:10 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _handle_sync_error(self, exc: BaseException) -> None:
|
2020-02-04 06:16:11 +01:00
|
|
|
if isinstance(exc, Interrupted):
|
2023-09-10 05:22:20 +02:00
|
|
|
self._update_progress(tr.sync_media_aborted())
|
2020-02-04 06:16:11 +01:00
|
|
|
return
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
else:
|
2023-09-10 05:22:20 +02:00
|
|
|
show_info(str(exc), modality=Qt.WindowModality.NonModal)
|
2020-03-23 10:06:13 +01:00
|
|
|
return
|
2020-02-04 06:16:11 +01:00
|
|
|
|
2020-02-04 00:07:15 +01:00
|
|
|
def abort(self) -> None:
|
2020-02-04 03:26:10 +01:00
|
|
|
if not self.is_syncing():
|
|
|
|
return
|
2021-01-31 09:46:43 +01:00
|
|
|
self.mw.col.set_wants_abort()
|
|
|
|
self.mw.col.abort_media_sync()
|
2023-09-10 05:22:20 +02:00
|
|
|
self._update_progress(tr.sync_media_aborting())
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2020-02-04 02:41:20 +01:00
|
|
|
def is_syncing(self) -> bool:
|
2020-02-06 09:16:39 +01:00
|
|
|
return self._syncing
|
2020-02-04 02:41:20 +01:00
|
|
|
|
2020-02-27 04:27:58 +01:00
|
|
|
def _on_start_stop(self, running: bool) -> None:
|
|
|
|
self.mw.toolbar.set_sync_active(running)
|
2020-02-04 03:26:10 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def show_sync_log(self) -> None:
|
2020-02-04 03:26:10 +01:00
|
|
|
aqt.dialogs.open("sync_log", self.mw, self)
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def show_diag_until_finished(self, on_finished: Callable[[], None]) -> None:
|
2020-02-05 03:23:15 +01:00
|
|
|
# nothing to do if not syncing
|
|
|
|
if not self.is_syncing():
|
2020-05-31 10:51:05 +02:00
|
|
|
return on_finished()
|
2020-02-05 03:23:15 +01:00
|
|
|
|
|
|
|
diag: MediaSyncDialog = aqt.dialogs.open("sync_log", self.mw, self, True)
|
2020-05-31 10:51:05 +02:00
|
|
|
diag.show()
|
|
|
|
|
2021-10-03 10:59:42 +02:00
|
|
|
timer: QTimer | None = None
|
2020-05-31 10:51:05 +02:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def check_finished() -> None:
|
2020-05-31 10:51:05 +02:00
|
|
|
if not self.is_syncing():
|
2022-02-24 12:15:56 +01:00
|
|
|
timer.deleteLater()
|
2020-05-31 10:51:05 +02:00
|
|
|
on_finished()
|
|
|
|
|
2022-02-22 11:09:43 +01:00
|
|
|
timer = self.mw.progress.timer(150, check_finished, True, False, parent=self.mw)
|
2020-02-05 03:23:15 +01:00
|
|
|
|
2020-02-05 03:38:36 +01:00
|
|
|
def seconds_since_last_sync(self) -> int:
|
|
|
|
if self.is_syncing():
|
|
|
|
return 0
|
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
return int_time() - self._last_progress_at
|
2020-02-05 03:38:36 +01:00
|
|
|
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
class MediaSyncDialog(QDialog):
|
2020-02-04 02:41:20 +01:00
|
|
|
silentlyClose = True
|
|
|
|
|
2020-02-05 03:23:15 +01:00
|
|
|
def __init__(
|
|
|
|
self, mw: aqt.main.AnkiQt, syncer: MediaSyncer, close_when_done: bool = False
|
|
|
|
) -> None:
|
2020-02-04 02:41:20 +01:00
|
|
|
super().__init__(mw)
|
|
|
|
self.mw = mw
|
2020-02-04 00:07:15 +01:00
|
|
|
self._syncer = syncer
|
2020-02-05 03:23:15 +01:00
|
|
|
self._close_when_done = close_when_done
|
2020-02-04 00:07:15 +01:00
|
|
|
self.form = aqt.forms.synclog.Ui_Dialog()
|
|
|
|
self.form.setupUi(self)
|
2021-03-26 04:48:26 +01:00
|
|
|
self.setWindowTitle(tr.sync_media_log_title())
|
2021-01-07 05:24:49 +01:00
|
|
|
disable_help_button(self)
|
2021-03-26 04:48:26 +01:00
|
|
|
self.abort_button = QPushButton(tr.sync_abort_button())
|
2020-05-04 05:23:08 +02:00
|
|
|
qconnect(self.abort_button.clicked, self._on_abort)
|
2020-02-04 03:26:10 +01:00
|
|
|
self.abort_button.setAutoDefault(False)
|
2021-10-05 05:53:01 +02:00
|
|
|
self.form.buttonBox.addButton(
|
|
|
|
self.abort_button, QDialogButtonBox.ButtonRole.ActionRole
|
|
|
|
)
|
2020-02-27 03:22:24 +01:00
|
|
|
self.abort_button.setHidden(not self._syncer.is_syncing())
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
gui_hooks.media_sync_did_progress.append(self._on_log_entry)
|
2020-02-05 03:23:15 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop.append(self._on_start_stop)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
self._on_log_entry(syncer.last_progress)
|
2020-02-04 02:41:20 +01:00
|
|
|
self.show()
|
|
|
|
|
2020-02-05 03:23:15 +01:00
|
|
|
def reject(self) -> None:
|
|
|
|
if self._close_when_done and self._syncer.is_syncing():
|
|
|
|
# closing while syncing on close starts an abort
|
|
|
|
self._on_abort()
|
|
|
|
return
|
2020-02-04 02:41:20 +01:00
|
|
|
|
|
|
|
aqt.dialogs.markClosed("sync_log")
|
2020-02-05 03:23:15 +01:00
|
|
|
QDialog.reject(self)
|
2020-02-04 02:41:20 +01:00
|
|
|
|
2021-02-02 15:00:29 +01:00
|
|
|
def reopen(
|
|
|
|
self, mw: aqt.AnkiQt, syncer: Any, close_when_done: bool = False
|
|
|
|
) -> None:
|
2020-02-05 03:23:15 +01:00
|
|
|
self._close_when_done = close_when_done
|
2020-02-04 02:41:20 +01:00
|
|
|
self.show()
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2021-02-02 15:00:29 +01:00
|
|
|
def _on_abort(self, *_args: Any) -> None:
|
2020-02-04 00:07:15 +01:00
|
|
|
self._syncer.abort()
|
|
|
|
self.abort_button.setHidden(True)
|
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
def _on_log_entry(self, entry: str) -> None:
|
|
|
|
dt = datetime.fromtimestamp(int_time())
|
|
|
|
time = dt.strftime("%H:%M:%S")
|
|
|
|
text = f"{time}: {entry}"
|
|
|
|
self.form.log_label.setText(text)
|
2020-02-04 03:26:10 +01:00
|
|
|
if not self._syncer.is_syncing():
|
|
|
|
self.abort_button.setHidden(True)
|
2020-02-05 03:23:15 +01:00
|
|
|
|
|
|
|
def _on_start_stop(self, running: bool) -> None:
|
|
|
|
if not running and self._close_when_done:
|
|
|
|
aqt.dialogs.markClosed("sync_log")
|
|
|
|
self._close_when_done = False
|
|
|
|
self.close()
|