2020-02-04 00:07:15 +01:00
|
|
|
# Copyright: Ankitects Pty Ltd and contributors
|
|
|
|
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
|
|
|
|
2020-02-04 02:41:20 +01:00
|
|
|
from __future__ import annotations
|
|
|
|
|
2020-02-04 00:07:15 +01:00
|
|
|
import time
|
|
|
|
from concurrent.futures import Future
|
|
|
|
from dataclasses import dataclass
|
2021-10-03 10:59:42 +02:00
|
|
|
from typing import Any, Callable, Union
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
import aqt
|
2022-02-13 04:40:47 +01:00
|
|
|
import aqt.forms
|
|
|
|
import aqt.main
|
2021-02-08 07:40:27 +01:00
|
|
|
from anki.collection import Progress
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
from anki.errors import Interrupted
|
2020-12-18 07:50:55 +01:00
|
|
|
from anki.types import assert_exhaustive
|
2021-10-25 06:50:13 +02:00
|
|
|
from anki.utils import int_time
|
2020-02-04 00:07:15 +01:00
|
|
|
from aqt import gui_hooks
|
2020-06-12 18:45:04 +02:00
|
|
|
from aqt.qt import QDialog, QDialogButtonBox, QPushButton, QTextCursor, QTimer, qconnect
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
from aqt.utils import disable_help_button, tr
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2021-02-08 07:40:27 +01:00
|
|
|
LogEntry = Union[Progress.MediaSync, str]
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
class LogEntryWithTime:
|
|
|
|
time: int
|
|
|
|
entry: LogEntry
|
|
|
|
|
|
|
|
|
|
|
|
class MediaSyncer:
|
2021-02-01 14:28:21 +01:00
|
|
|
def __init__(self, mw: aqt.main.AnkiQt) -> None:
|
2020-02-04 03:26:10 +01:00
|
|
|
self.mw = mw
|
2020-02-06 09:16:39 +01:00
|
|
|
self._syncing: bool = False
|
2021-10-03 10:59:42 +02:00
|
|
|
self._log: list[LogEntryWithTime] = []
|
|
|
|
self._progress_timer: QTimer | None = None
|
2020-02-05 02:55:14 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop.append(self._on_start_stop)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _on_progress(self) -> None:
|
2020-05-29 11:59:50 +02:00
|
|
|
progress = self.mw.col.latest_progress()
|
2021-02-08 07:40:27 +01:00
|
|
|
if not progress.HasField("media_sync"):
|
2020-05-29 11:59:50 +02:00
|
|
|
return
|
2021-02-08 07:40:27 +01:00
|
|
|
sync_progress = progress.media_sync
|
|
|
|
self._log_and_notify(sync_progress)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2020-02-04 03:26:10 +01:00
|
|
|
def start(self) -> None:
|
2020-02-04 00:07:15 +01:00
|
|
|
"Start media syncing in the background, if it's not already running."
|
2020-02-06 09:16:39 +01:00
|
|
|
if self._syncing:
|
2020-02-04 00:07:15 +01:00
|
|
|
return
|
|
|
|
|
2020-02-04 03:26:10 +01:00
|
|
|
if not self.mw.pm.media_syncing_enabled():
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_disabled())
|
2020-02-04 03:26:10 +01:00
|
|
|
return
|
|
|
|
|
2020-05-30 04:28:22 +02:00
|
|
|
auth = self.mw.pm.sync_auth()
|
|
|
|
if auth is None:
|
|
|
|
return
|
|
|
|
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_starting())
|
2020-02-06 09:16:39 +01:00
|
|
|
self._syncing = True
|
2020-05-31 10:51:05 +02:00
|
|
|
self._progress_timer = self.mw.progress.timer(
|
2022-02-22 11:09:43 +01:00
|
|
|
1000, self._on_progress, True, True, parent=self.mw
|
2020-05-31 10:51:05 +02:00
|
|
|
)
|
2020-02-05 02:55:14 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop(True)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
def run() -> None:
|
2021-01-31 09:46:43 +01:00
|
|
|
self.mw.col.sync_media(auth)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2020-02-04 03:26:10 +01:00
|
|
|
self.mw.taskman.run_in_background(run, self._on_finished)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
def _log_and_notify(self, entry: LogEntry) -> None:
|
2021-10-25 06:50:13 +02:00
|
|
|
entry_with_time = LogEntryWithTime(time=int_time(), entry=entry)
|
2020-02-04 00:07:15 +01:00
|
|
|
self._log.append(entry_with_time)
|
2020-02-04 03:26:10 +01:00
|
|
|
self.mw.taskman.run_on_main(
|
2020-02-04 00:07:15 +01:00
|
|
|
lambda: gui_hooks.media_sync_did_progress(entry_with_time)
|
|
|
|
)
|
|
|
|
|
|
|
|
def _on_finished(self, future: Future) -> None:
|
2020-02-06 09:16:39 +01:00
|
|
|
self._syncing = False
|
2020-05-29 11:59:50 +02:00
|
|
|
if self._progress_timer:
|
Refactor progress handling (#2549)
Previously it was Backend's responsibility to store the last progress,
and when calling routines in Collection, one had to construct and pass
in a Fn, which wasn't the most ergonomic. This PR adds the last progress
state to the collection, so that the routines no longer need a separate
progress arg, and makes some other tweaks to improve ergonomics.
ThrottlingProgressHandler has been tweaked so that it now stores the
current state, so that callers don't need to store it separately. When
a long-running routine starts, it calls col.new_progress_handler(),
which automatically initializes the data to defaults, and updates the
shared UI state, so we no longer need to manually update the state at
the start of an operation.
The backend shares the Arc<Mutex<>> with the collection, so it can get
at the current state, and so we can update the state when importing a
backup.
Other tweaks:
- The current Incrementor was awkward to use in the media check, which
uses a single incrementing value across multiple method calls, so I've
added a simpler alternative for such cases. The old incrementor method
has been kept, but implemented directly on ThrottlingProgressHandler.
- The full sync code was passing the progress handler in a complicated
way that may once have been required, but no longer is.
- On the Qt side, timers are now stopped before deletion, or they keep
running for a few seconds.
- I left the ChangeTracker using a closure, as it's used for both importing
and syncing.
2023-06-19 05:48:32 +02:00
|
|
|
self._progress_timer.stop()
|
2022-02-24 12:15:56 +01:00
|
|
|
self._progress_timer.deleteLater()
|
2020-05-29 11:59:50 +02:00
|
|
|
self._progress_timer = None
|
2020-02-05 02:55:14 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop(False)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
exc = future.exception()
|
|
|
|
if exc is not None:
|
2020-02-04 03:26:10 +01:00
|
|
|
self._handle_sync_error(exc)
|
|
|
|
else:
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_complete())
|
2020-02-04 03:26:10 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _handle_sync_error(self, exc: BaseException) -> None:
|
2020-02-04 06:16:11 +01:00
|
|
|
if isinstance(exc, Interrupted):
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_aborted())
|
2020-02-04 06:16:11 +01:00
|
|
|
return
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
else:
|
|
|
|
# Avoid popups for errors; they can cause a deadlock if
|
|
|
|
# a modal window happens to be active, or a duplicate auth
|
|
|
|
# failed message if the password is changed.
|
2020-03-23 10:06:13 +01:00
|
|
|
self._log_and_notify(str(exc))
|
|
|
|
return
|
2020-02-04 06:16:11 +01:00
|
|
|
|
2021-10-03 10:59:42 +02:00
|
|
|
def entries(self) -> list[LogEntryWithTime]:
|
2020-02-04 00:07:15 +01:00
|
|
|
return self._log
|
|
|
|
|
|
|
|
def abort(self) -> None:
|
2020-02-04 03:26:10 +01:00
|
|
|
if not self.is_syncing():
|
|
|
|
return
|
2021-03-26 04:48:26 +01:00
|
|
|
self._log_and_notify(tr.sync_media_aborting())
|
2021-01-31 09:46:43 +01:00
|
|
|
self.mw.col.set_wants_abort()
|
|
|
|
self.mw.col.abort_media_sync()
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2020-02-04 02:41:20 +01:00
|
|
|
def is_syncing(self) -> bool:
|
2020-02-06 09:16:39 +01:00
|
|
|
return self._syncing
|
2020-02-04 02:41:20 +01:00
|
|
|
|
2020-02-27 04:27:58 +01:00
|
|
|
def _on_start_stop(self, running: bool) -> None:
|
|
|
|
self.mw.toolbar.set_sync_active(running)
|
2020-02-04 03:26:10 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def show_sync_log(self) -> None:
|
2020-02-04 03:26:10 +01:00
|
|
|
aqt.dialogs.open("sync_log", self.mw, self)
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def show_diag_until_finished(self, on_finished: Callable[[], None]) -> None:
|
2020-02-05 03:23:15 +01:00
|
|
|
# nothing to do if not syncing
|
|
|
|
if not self.is_syncing():
|
2020-05-31 10:51:05 +02:00
|
|
|
return on_finished()
|
2020-02-05 03:23:15 +01:00
|
|
|
|
|
|
|
diag: MediaSyncDialog = aqt.dialogs.open("sync_log", self.mw, self, True)
|
2020-05-31 10:51:05 +02:00
|
|
|
diag.show()
|
|
|
|
|
2021-10-03 10:59:42 +02:00
|
|
|
timer: QTimer | None = None
|
2020-05-31 10:51:05 +02:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def check_finished() -> None:
|
2020-05-31 10:51:05 +02:00
|
|
|
if not self.is_syncing():
|
2022-02-24 12:15:56 +01:00
|
|
|
timer.deleteLater()
|
2020-05-31 10:51:05 +02:00
|
|
|
on_finished()
|
|
|
|
|
2022-02-22 11:09:43 +01:00
|
|
|
timer = self.mw.progress.timer(150, check_finished, True, False, parent=self.mw)
|
2020-02-05 03:23:15 +01:00
|
|
|
|
2020-02-05 03:38:36 +01:00
|
|
|
def seconds_since_last_sync(self) -> int:
|
|
|
|
if self.is_syncing():
|
|
|
|
return 0
|
|
|
|
|
|
|
|
if self._log:
|
|
|
|
last = self._log[-1].time
|
|
|
|
else:
|
|
|
|
last = 0
|
2021-10-25 06:50:13 +02:00
|
|
|
return int_time() - last
|
2020-02-05 03:38:36 +01:00
|
|
|
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
class MediaSyncDialog(QDialog):
|
2020-02-04 02:41:20 +01:00
|
|
|
silentlyClose = True
|
|
|
|
|
2020-02-05 03:23:15 +01:00
|
|
|
def __init__(
|
|
|
|
self, mw: aqt.main.AnkiQt, syncer: MediaSyncer, close_when_done: bool = False
|
|
|
|
) -> None:
|
2020-02-04 02:41:20 +01:00
|
|
|
super().__init__(mw)
|
|
|
|
self.mw = mw
|
2020-02-04 00:07:15 +01:00
|
|
|
self._syncer = syncer
|
2020-02-05 03:23:15 +01:00
|
|
|
self._close_when_done = close_when_done
|
2020-02-04 00:07:15 +01:00
|
|
|
self.form = aqt.forms.synclog.Ui_Dialog()
|
|
|
|
self.form.setupUi(self)
|
2021-03-26 04:48:26 +01:00
|
|
|
self.setWindowTitle(tr.sync_media_log_title())
|
2021-01-07 05:24:49 +01:00
|
|
|
disable_help_button(self)
|
2021-03-26 04:48:26 +01:00
|
|
|
self.abort_button = QPushButton(tr.sync_abort_button())
|
2020-05-04 05:23:08 +02:00
|
|
|
qconnect(self.abort_button.clicked, self._on_abort)
|
2020-02-04 03:26:10 +01:00
|
|
|
self.abort_button.setAutoDefault(False)
|
2021-10-05 05:53:01 +02:00
|
|
|
self.form.buttonBox.addButton(
|
|
|
|
self.abort_button, QDialogButtonBox.ButtonRole.ActionRole
|
|
|
|
)
|
2020-02-27 03:22:24 +01:00
|
|
|
self.abort_button.setHidden(not self._syncer.is_syncing())
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
gui_hooks.media_sync_did_progress.append(self._on_log_entry)
|
2020-02-05 03:23:15 +01:00
|
|
|
gui_hooks.media_sync_did_start_or_stop.append(self._on_start_stop)
|
2020-02-04 00:07:15 +01:00
|
|
|
|
|
|
|
self.form.plainTextEdit.setPlainText(
|
|
|
|
"\n".join(self._entry_to_text(x) for x in syncer.entries())
|
|
|
|
)
|
2021-10-05 05:53:01 +02:00
|
|
|
self.form.plainTextEdit.moveCursor(QTextCursor.MoveOperation.End)
|
2020-02-04 02:41:20 +01:00
|
|
|
self.show()
|
|
|
|
|
2020-02-05 03:23:15 +01:00
|
|
|
def reject(self) -> None:
|
|
|
|
if self._close_when_done and self._syncer.is_syncing():
|
|
|
|
# closing while syncing on close starts an abort
|
|
|
|
self._on_abort()
|
|
|
|
return
|
2020-02-04 02:41:20 +01:00
|
|
|
|
|
|
|
aqt.dialogs.markClosed("sync_log")
|
2020-02-05 03:23:15 +01:00
|
|
|
QDialog.reject(self)
|
2020-02-04 02:41:20 +01:00
|
|
|
|
2021-02-02 15:00:29 +01:00
|
|
|
def reopen(
|
|
|
|
self, mw: aqt.AnkiQt, syncer: Any, close_when_done: bool = False
|
|
|
|
) -> None:
|
2020-02-05 03:23:15 +01:00
|
|
|
self._close_when_done = close_when_done
|
2020-02-04 02:41:20 +01:00
|
|
|
self.show()
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2021-02-02 15:00:29 +01:00
|
|
|
def _on_abort(self, *_args: Any) -> None:
|
2020-02-04 00:07:15 +01:00
|
|
|
self._syncer.abort()
|
|
|
|
self.abort_button.setHidden(True)
|
|
|
|
|
|
|
|
def _time_and_text(self, stamp: int, text: str) -> str:
|
|
|
|
asctime = time.asctime(time.localtime(stamp))
|
|
|
|
return f"{asctime}: {text}"
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _entry_to_text(self, entry: LogEntryWithTime) -> str:
|
2020-02-04 03:26:10 +01:00
|
|
|
if isinstance(entry.entry, str):
|
|
|
|
txt = entry.entry
|
2021-02-08 07:40:27 +01:00
|
|
|
elif isinstance(entry.entry, Progress.MediaSync):
|
2020-02-04 00:07:15 +01:00
|
|
|
txt = self._logentry_to_text(entry.entry)
|
|
|
|
else:
|
2020-12-18 07:50:55 +01:00
|
|
|
assert_exhaustive(entry.entry)
|
2020-02-04 00:07:15 +01:00
|
|
|
return self._time_and_text(entry.time, txt)
|
|
|
|
|
2021-02-08 07:40:27 +01:00
|
|
|
def _logentry_to_text(self, e: Progress.MediaSync) -> str:
|
2020-02-16 09:46:51 +01:00
|
|
|
return f"{e.added}, {e.removed}, {e.checked}"
|
2020-02-04 00:07:15 +01:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def _on_log_entry(self, entry: LogEntryWithTime) -> None:
|
2020-02-04 00:07:15 +01:00
|
|
|
self.form.plainTextEdit.appendPlainText(self._entry_to_text(entry))
|
2020-02-04 03:26:10 +01:00
|
|
|
if not self._syncer.is_syncing():
|
|
|
|
self.abort_button.setHidden(True)
|
2020-02-05 03:23:15 +01:00
|
|
|
|
|
|
|
def _on_start_stop(self, running: bool) -> None:
|
|
|
|
if not running and self._close_when_done:
|
|
|
|
aqt.dialogs.markClosed("sync_log")
|
|
|
|
self._close_when_done = False
|
|
|
|
self.close()
|