2020-02-04 00:07:15 +01:00
|
|
|
# Copyright: Ankitects Pty Ltd and contributors
|
|
|
|
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
|
|
|
|
2020-02-04 02:41:20 +01:00
|
|
|
from __future__ import annotations
|
|
|
|
|
2020-02-04 00:07:15 +01:00
|
|
|
import time
|
|
|
|
from concurrent.futures import Future
|
|
|
|
from dataclasses import dataclass
|
2021-10-03 10:59:42 +02:00
|
|
|
from typing import Any, Callable, Union
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
import aqt
|
2022-02-13 04:40:47 +01:00
|
|
|
import aqt.forms
|
|
|
|
import aqt.main
|
2021-02-08 07:40:27 +01:00
|
|
|
from anki.collection import Progress
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
from anki.errors import Interrupted
|
2020-12-18 07:50:55 +01:00
|
|
|
from anki.types import assert_exhaustive
|
2021-10-25 06:50:13 +02:00
|
|
|
from anki.utils import int_time
|
2020-02-04 00:07:15 +01:00
|
|
|
from aqt import gui_hooks
|
2020-06-12 18:45:04 +02:00
|
|
|
from aqt.qt import QDialog, QDialogButtonBox, QPushButton, QTextCursor, QTimer, qconnect
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
from aqt.utils import disable_help_button, tr
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2021-02-08 07:40:27 +01:00
|
|
|
LogEntry = Union[Progress.MediaSync, str]
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
class LogEntryWithTime:
|
|
|
|
time: int
|
|
|
|
entry: LogEntry
|
|
|
|
|
|
|
|
|
|
|
|
class MediaSyncer:
|
2021-02-01 14:28:21 +01:00
|
|
|
def __init__(self, mw: aqt.main.AnkiQt) -> None:
|
2020-02-04 03:26:10 +01:00
|
|
|
self.mw = mw
|
2020-02-06 09:16:39 +01:00
|
|
|
self._syncing: bool = False
|
2021-10-03 10:59:42 +02:00
|
|
|
self._log: list[LogEntryWithTime] = []
|
|
|
|
self._progress_timer: QTimer | None = None
|
2020-02-05 02:55:14 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop.append(self._on_start_stop)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _on_progress(self) -> None:
|
2020-05-29 11:59:50 +02:00
|
|
|
progress = self.mw.col.latest_progress()
|
2021-02-08 07:40:27 +01:00
|
|
|
if not progress.HasField("media_sync"):
|
2020-05-29 11:59:50 +02:00
|
|
|
return
|
2021-02-08 07:40:27 +01:00
|
|
|
sync_progress = progress.media_sync
|
|
|
|
self._log_and_notify(sync_progress)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2020-02-04 03:26:10 +01:00
|
|
|
def start(self) -> None:
|
2020-02-04 00:07:15 +01:00
|
|
|
"Start media syncing in the background, if it's not already running."
|
2020-02-06 09:16:39 +01:00
|
|
|
if self._syncing:
|
2020-02-04 00:07:15 +01:00
|
|
|
return
|
|
|
|
|
2020-02-04 03:26:10 +01:00
|
|
|
if not self.mw.pm.media_syncing_enabled():
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_disabled())
|
2020-02-04 03:26:10 +01:00
|
|
|
return
|
|
|
|
|
2020-05-30 04:28:22 +02:00
|
|
|
auth = self.mw.pm.sync_auth()
|
|
|
|
if auth is None:
|
|
|
|
return
|
|
|
|
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_starting())
|
2020-02-06 09:16:39 +01:00
|
|
|
self._syncing = True
|
2020-05-31 10:51:05 +02:00
|
|
|
self._progress_timer = self.mw.progress.timer(
|
2022-02-22 11:09:43 +01:00
|
|
|
1000, self._on_progress, True, True, parent=self.mw
|
2020-05-31 10:51:05 +02:00
|
|
|
)
|
2020-02-05 02:55:14 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop(True)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
def run() -> None:
|
2021-01-31 09:46:43 +01:00
|
|
|
self.mw.col.sync_media(auth)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2020-02-04 03:26:10 +01:00
|
|
|
self.mw.taskman.run_in_background(run, self._on_finished)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
def _log_and_notify(self, entry: LogEntry) -> None:
|
2021-10-25 06:50:13 +02:00
|
|
|
entry_with_time = LogEntryWithTime(time=int_time(), entry=entry)
|
2020-02-04 00:07:15 +01:00
|
|
|
self._log.append(entry_with_time)
|
2020-02-04 03:26:10 +01:00
|
|
|
self.mw.taskman.run_on_main(
|
2020-02-04 00:07:15 +01:00
|
|
|
lambda: gui_hooks.media_sync_did_progress(entry_with_time)
|
|
|
|
)
|
|
|
|
|
|
|
|
def _on_finished(self, future: Future) -> None:
|
2020-02-06 09:16:39 +01:00
|
|
|
self._syncing = False
|
2020-05-29 11:59:50 +02:00
|
|
|
if self._progress_timer:
|
2022-02-24 12:15:56 +01:00
|
|
|
self._progress_timer.deleteLater()
|
2020-05-29 11:59:50 +02:00
|
|
|
self._progress_timer = None
|
2020-02-05 02:55:14 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop(False)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
exc = future.exception()
|
|
|
|
if exc is not None:
|
2020-02-04 03:26:10 +01:00
|
|
|
self._handle_sync_error(exc)
|
|
|
|
else:
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_complete())
|
2020-02-04 03:26:10 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _handle_sync_error(self, exc: BaseException) -> None:
|
2020-02-04 06:16:11 +01:00
|
|
|
if isinstance(exc, Interrupted):
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_aborted())
|
2020-02-04 06:16:11 +01:00
|
|
|
return
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
else:
|
|
|
|
# Avoid popups for errors; they can cause a deadlock if
|
|
|
|
# a modal window happens to be active, or a duplicate auth
|
|
|
|
# failed message if the password is changed.
|
2020-03-23 10:06:13 +01:00
|
|
|
self._log_and_notify(str(exc))
|
|
|
|
return
|
2020-02-04 06:16:11 +01:00
|
|
|
|
2021-10-03 10:59:42 +02:00
|
|
|
def entries(self) -> list[LogEntryWithTime]:
|
2020-02-04 00:07:15 +01:00
|
|
|
return self._log
|
|
|
|
|
|
|
|
def abort(self) -> None:
|
2020-02-04 03:26:10 +01:00
|
|
|
if not self.is_syncing():
|
|
|
|
return
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_aborting())
|
2021-01-31 09:46:43 +01:00
|
|
|
self.mw.col.set_wants_abort()
|
|
|
|
self.mw.col.abort_media_sync()
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2020-02-04 02:41:20 +01:00
|
|
|
def is_syncing(self) -> bool:
|
2020-02-06 09:16:39 +01:00
|
|
|
return self._syncing
|
2020-02-04 02:41:20 +01:00
|
|
|
|
2020-02-27 04:27:58 +01:00
|
|
|
def _on_start_stop(self, running: bool) -> None:
|
|
|
|
self.mw.toolbar.set_sync_active(running)
|
2020-02-04 03:26:10 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def show_sync_log(self) -> None:
|
2020-02-04 03:26:10 +01:00
|
|
|
aqt.dialogs.open("sync_log", self.mw, self)
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def show_diag_until_finished(self, on_finished: Callable[[], None]) -> None:
|
2020-02-05 03:23:15 +01:00
|
|
|
# nothing to do if not syncing
|
|
|
|
if not self.is_syncing():
|
2020-05-31 10:51:05 +02:00
|
|
|
return on_finished()
|
2020-02-05 03:23:15 +01:00
|
|
|
|
|
|
|
diag: MediaSyncDialog = aqt.dialogs.open("sync_log", self.mw, self, True)
|
2020-05-31 10:51:05 +02:00
|
|
|
diag.show()
|
|
|
|
|
2021-10-03 10:59:42 +02:00
|
|
|
timer: QTimer | None = None
|
2020-05-31 10:51:05 +02:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def check_finished() -> None:
|
2020-05-31 10:51:05 +02:00
|
|
|
if not self.is_syncing():
|
2022-02-24 12:15:56 +01:00
|
|
|
timer.deleteLater()
|
2020-05-31 10:51:05 +02:00
|
|
|
on_finished()
|
|
|
|
|
2022-02-22 11:09:43 +01:00
|
|
|
timer = self.mw.progress.timer(150, check_finished, True, False, parent=self.mw)
|
2020-02-05 03:23:15 +01:00
|
|
|
|
2020-02-05 03:38:36 +01:00
|
|
|
def seconds_since_last_sync(self) -> int:
|
|
|
|
if self.is_syncing():
|
|
|
|
return 0
|
|
|
|
|
|
|
|
if self._log:
|
|
|
|
last = self._log[-1].time
|
|
|
|
else:
|
|
|
|
last = 0
|
2021-10-25 06:50:13 +02:00
|
|
|
return int_time() - last
|
2020-02-05 03:38:36 +01:00
|
|
|
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
class MediaSyncDialog(QDialog):
|
2020-02-04 02:41:20 +01:00
|
|
|
silentlyClose = True
|
|
|
|
|
2020-02-05 03:23:15 +01:00
|
|
|
def __init__(
|
|
|
|
self, mw: aqt.main.AnkiQt, syncer: MediaSyncer, close_when_done: bool = False
|
|
|
|
) -> None:
|
2020-02-04 02:41:20 +01:00
|
|
|
super().__init__(mw)
|
|
|
|
self.mw = mw
|
2020-02-04 00:07:15 +01:00
|
|
|
self._syncer = syncer
|
2020-02-05 03:23:15 +01:00
|
|
|
self._close_when_done = close_when_done
|
2020-02-04 00:07:15 +01:00
|
|
|
self.form = aqt.forms.synclog.Ui_Dialog()
|
|
|
|
self.form.setupUi(self)
|
2021-03-26 04:48:26 +01:00
|
|
|
self.setWindowTitle(tr.sync_media_log_title())
|
2021-01-07 05:24:49 +01:00
|
|
|
disable_help_button(self)
|
2021-03-26 04:48:26 +01:00
|
|
|
self.abort_button = QPushButton(tr.sync_abort_button())
|
2020-05-04 05:23:08 +02:00
|
|
|
qconnect(self.abort_button.clicked, self._on_abort)
|
2020-02-04 03:26:10 +01:00
|
|
|
self.abort_button.setAutoDefault(False)
|
2021-10-05 05:53:01 +02:00
|
|
|
self.form.buttonBox.addButton(
|
|
|
|
self.abort_button, QDialogButtonBox.ButtonRole.ActionRole
|
|
|
|
)
|
2020-02-27 03:22:24 +01:00
|
|
|
self.abort_button.setHidden(not self._syncer.is_syncing())
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
gui_hooks.media_sync_did_progress.append(self._on_log_entry)
|
2020-02-05 03:23:15 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop.append(self._on_start_stop)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
self.form.plainTextEdit.setPlainText(
|
|
|
|
"\n".join(self._entry_to_text(x) for x in syncer.entries())
|
|
|
|
)
|
2021-10-05 05:53:01 +02:00
|
|
|
self.form.plainTextEdit.moveCursor(QTextCursor.MoveOperation.End)
|
2020-02-04 02:41:20 +01:00
|
|
|
self.show()
|
|
|
|
|
2020-02-05 03:23:15 +01:00
|
|
|
def reject(self) -> None:
|
|
|
|
if self._close_when_done and self._syncer.is_syncing():
|
|
|
|
# closing while syncing on close starts an abort
|
|
|
|
self._on_abort()
|
|
|
|
return
|
2020-02-04 02:41:20 +01:00
|
|
|
|
|
|
|
aqt.dialogs.markClosed("sync_log")
|
2020-02-05 03:23:15 +01:00
|
|
|
QDialog.reject(self)
|
2020-02-04 02:41:20 +01:00
|
|
|
|
2021-02-02 15:00:29 +01:00
|
|
|
def reopen(
|
|
|
|
self, mw: aqt.AnkiQt, syncer: Any, close_when_done: bool = False
|
|
|
|
) -> None:
|
2020-02-05 03:23:15 +01:00
|
|
|
self._close_when_done = close_when_done
|
2020-02-04 02:41:20 +01:00
|
|
|
self.show()
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2021-02-02 15:00:29 +01:00
|
|
|
def _on_abort(self, *_args: Any) -> None:
|
2020-02-04 00:07:15 +01:00
|
|
|
self._syncer.abort()
|
|
|
|
self.abort_button.setHidden(True)
|
|
|
|
|
|
|
|
def _time_and_text(self, stamp: int, text: str) -> str:
|
|
|
|
asctime = time.asctime(time.localtime(stamp))
|
|
|
|
return f"{asctime}: {text}"
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _entry_to_text(self, entry: LogEntryWithTime) -> str:
|
2020-02-04 03:26:10 +01:00
|
|
|
if isinstance(entry.entry, str):
|
|
|
|
txt = entry.entry
|
2021-02-08 07:40:27 +01:00
|
|
|
elif isinstance(entry.entry, Progress.MediaSync):
|
2020-02-04 00:07:15 +01:00
|
|
|
txt = self._logentry_to_text(entry.entry)
|
|
|
|
else:
|
2020-12-18 07:50:55 +01:00
|
|
|
assert_exhaustive(entry.entry)
|
2020-02-04 00:07:15 +01:00
|
|
|
return self._time_and_text(entry.time, txt)
|
|
|
|
|
2021-02-08 07:40:27 +01:00
|
|
|
def _logentry_to_text(self, e: Progress.MediaSync) -> str:
|
2020-02-16 09:46:51 +01:00
|
|
|
return f"{e.added}, {e.removed}, {e.checked}"
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _on_log_entry(self, entry: LogEntryWithTime) -> None:
|
2020-02-04 00:07:15 +01:00
|
|
|
self.form.plainTextEdit.appendPlainText(self._entry_to_text(entry))
|
2020-02-04 03:26:10 +01:00
|
|
|
if not self._syncer.is_syncing():
|
|
|
|
self.abort_button.setHidden(True)
|
2020-02-05 03:23:15 +01:00
|
|
|
|
|
|
|
def _on_start_stop(self, running: bool) -> None:
|
|
|
|
if not running and self._close_when_done:
|
|
|
|
aqt.dialogs.markClosed("sync_log")
|
|
|
|
self._close_when_done = False
|
|
|
|
self.close()
|