2019-02-05 04:59:03 +01:00
|
|
|
# Copyright: Ankitects Pty Ltd and contributors
|
2012-12-21 08:51:59 +01:00
|
|
|
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
|
|
|
|
2020-05-30 04:28:22 +02:00
|
|
|
from __future__ import annotations
|
|
|
|
|
2020-07-16 02:12:41 +02:00
|
|
|
import os
|
2021-02-02 14:30:53 +01:00
|
|
|
from concurrent.futures import Future
|
2021-10-03 10:59:42 +02:00
|
|
|
from typing import Callable
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
import aqt
|
2022-02-13 04:40:47 +01:00
|
|
|
import aqt.main
|
2021-04-03 08:00:15 +02:00
|
|
|
from anki.errors import Interrupted, SyncError, SyncErrorKind
|
2021-03-26 05:21:04 +01:00
|
|
|
from anki.lang import without_unicode_isolation
|
2021-01-31 06:55:08 +01:00
|
|
|
from anki.sync import SyncOutput, SyncStatus
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
from anki.sync_pb2 import SyncAuth
|
2021-10-25 06:50:13 +02:00
|
|
|
from anki.utils import plat_desc
|
2022-04-19 09:10:34 +02:00
|
|
|
from aqt import gui_hooks
|
2020-05-30 04:28:22 +02:00
|
|
|
from aqt.qt import (
|
|
|
|
QDialog,
|
|
|
|
QDialogButtonBox,
|
|
|
|
QGridLayout,
|
|
|
|
QLabel,
|
|
|
|
QLineEdit,
|
|
|
|
Qt,
|
|
|
|
QTimer,
|
|
|
|
QVBoxLayout,
|
|
|
|
qconnect,
|
|
|
|
)
|
2021-01-07 05:24:49 +01:00
|
|
|
from aqt.utils import (
|
2022-08-19 02:04:58 +02:00
|
|
|
ask_user_dialog,
|
2021-01-07 05:24:49 +01:00
|
|
|
askUser,
|
|
|
|
disable_help_button,
|
|
|
|
showText,
|
|
|
|
showWarning,
|
2023-10-03 04:57:39 +02:00
|
|
|
tooltip,
|
2021-01-07 05:24:49 +01:00
|
|
|
tr,
|
|
|
|
)
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def get_sync_status(
|
|
|
|
mw: aqt.main.AnkiQt, callback: Callable[[SyncStatus], None]
|
|
|
|
) -> None:
|
2020-05-30 04:28:22 +02:00
|
|
|
auth = mw.pm.sync_auth()
|
|
|
|
if not auth:
|
2021-02-01 14:28:21 +01:00
|
|
|
callback(SyncStatus(required=SyncStatus.NO_CHANGES)) # pylint:disable=no-member
|
2021-02-02 01:41:52 +01:00
|
|
|
return
|
2020-05-30 04:28:22 +02:00
|
|
|
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
def on_future_done(fut: Future[SyncStatus]) -> None:
|
2020-06-02 05:23:01 +02:00
|
|
|
try:
|
|
|
|
out = fut.result()
|
|
|
|
except Exception as e:
|
|
|
|
# swallow errors
|
|
|
|
print("sync status check failed:", str(e))
|
|
|
|
return
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
if out.new_endpoint:
|
|
|
|
mw.pm.set_current_sync_url(out.new_endpoint)
|
2020-06-02 05:23:01 +02:00
|
|
|
callback(out)
|
2020-05-30 04:28:22 +02:00
|
|
|
|
2021-01-31 09:46:43 +01:00
|
|
|
mw.taskman.run_in_background(lambda: mw.col.sync_status(auth), on_future_done)
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def handle_sync_error(mw: aqt.main.AnkiQt, err: Exception) -> None:
|
2020-05-31 03:46:40 +02:00
|
|
|
if isinstance(err, SyncError):
|
2021-04-03 08:00:15 +02:00
|
|
|
if err.kind is SyncErrorKind.AUTH:
|
2020-05-31 03:46:40 +02:00
|
|
|
mw.pm.clear_sync_auth()
|
|
|
|
elif isinstance(err, Interrupted):
|
|
|
|
# no message to show
|
|
|
|
return
|
|
|
|
showWarning(str(err))
|
|
|
|
|
|
|
|
|
2020-05-31 06:43:27 +02:00
|
|
|
def on_normal_sync_timer(mw: aqt.main.AnkiQt) -> None:
|
|
|
|
progress = mw.col.latest_progress()
|
2021-02-08 07:40:27 +01:00
|
|
|
if not progress.HasField("normal_sync"):
|
2020-05-31 06:43:27 +02:00
|
|
|
return
|
2021-02-08 07:40:27 +01:00
|
|
|
sync_progress = progress.normal_sync
|
2020-05-31 06:43:27 +02:00
|
|
|
|
|
|
|
mw.progress.update(
|
2021-02-08 07:40:27 +01:00
|
|
|
label=f"{sync_progress.added}\n{sync_progress.removed}",
|
2020-08-31 05:29:28 +02:00
|
|
|
process=False,
|
2020-05-31 06:43:27 +02:00
|
|
|
)
|
2021-02-08 07:40:27 +01:00
|
|
|
mw.progress.set_title(sync_progress.stage)
|
2020-05-31 06:43:27 +02:00
|
|
|
|
|
|
|
if mw.progress.want_cancel():
|
2021-01-31 09:46:43 +01:00
|
|
|
mw.col.abort_sync()
|
2020-05-31 06:43:27 +02:00
|
|
|
|
|
|
|
|
2020-05-31 02:53:54 +02:00
|
|
|
def sync_collection(mw: aqt.main.AnkiQt, on_done: Callable[[], None]) -> None:
|
2020-05-30 04:28:22 +02:00
|
|
|
auth = mw.pm.sync_auth()
|
2021-11-25 08:47:50 +01:00
|
|
|
if not auth:
|
|
|
|
raise Exception("expected auth")
|
2020-05-30 04:28:22 +02:00
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def on_timer() -> None:
|
2020-05-31 06:43:27 +02:00
|
|
|
on_normal_sync_timer(mw)
|
|
|
|
|
|
|
|
timer = QTimer(mw)
|
|
|
|
qconnect(timer.timeout, on_timer)
|
|
|
|
timer.start(150)
|
|
|
|
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
def on_future_done(fut: Future[SyncOutput]) -> None:
|
2022-08-24 10:37:58 +02:00
|
|
|
# scheduler version may have changed
|
|
|
|
mw.col._load_scheduler()
|
2020-05-31 06:43:27 +02:00
|
|
|
timer.stop()
|
2020-05-30 06:19:21 +02:00
|
|
|
try:
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
out = fut.result()
|
2020-05-31 03:46:40 +02:00
|
|
|
except Exception as err:
|
|
|
|
handle_sync_error(mw, err)
|
2020-05-31 02:53:54 +02:00
|
|
|
return on_done()
|
2020-05-30 06:19:21 +02:00
|
|
|
|
2020-05-30 04:28:22 +02:00
|
|
|
mw.pm.set_host_number(out.host_number)
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
if out.new_endpoint:
|
|
|
|
mw.pm.set_current_sync_url(out.new_endpoint)
|
2020-05-30 12:52:32 +02:00
|
|
|
if out.server_message:
|
|
|
|
showText(out.server_message)
|
2020-05-30 04:28:22 +02:00
|
|
|
if out.required == out.NO_CHANGES:
|
2023-10-03 04:57:39 +02:00
|
|
|
tooltip(parent=mw, msg=tr.sync_collection_complete())
|
2023-09-10 05:22:20 +02:00
|
|
|
# all done; track media progress
|
|
|
|
mw.media_syncer.start_monitoring()
|
2020-05-31 02:53:54 +02:00
|
|
|
return on_done()
|
2012-12-21 08:51:59 +01:00
|
|
|
else:
|
2020-05-31 02:53:54 +02:00
|
|
|
full_sync(mw, out, on_done)
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
mw.taskman.with_progress(
|
2023-09-10 05:22:20 +02:00
|
|
|
lambda: mw.col.sync_collection(auth, mw.pm.media_syncing_enabled()),
|
2020-05-31 02:53:54 +02:00
|
|
|
on_future_done,
|
2021-03-26 04:48:26 +01:00
|
|
|
label=tr.sync_checking(),
|
2020-05-31 03:24:33 +02:00
|
|
|
immediate=True,
|
2020-05-30 04:28:22 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
|
2020-05-31 02:53:54 +02:00
|
|
|
def full_sync(
|
|
|
|
mw: aqt.main.AnkiQt, out: SyncOutput, on_done: Callable[[], None]
|
|
|
|
) -> None:
|
2023-09-10 05:22:20 +02:00
|
|
|
server_usn = out.server_media_usn if mw.pm.media_syncing_enabled() else None
|
2020-05-30 04:28:22 +02:00
|
|
|
if out.required == out.FULL_DOWNLOAD:
|
2023-09-10 05:22:20 +02:00
|
|
|
confirm_full_download(mw, server_usn, on_done)
|
2020-05-30 04:28:22 +02:00
|
|
|
elif out.required == out.FULL_UPLOAD:
|
2023-11-07 02:11:07 +01:00
|
|
|
confirm_full_upload(mw, server_usn, on_done)
|
2020-05-30 04:28:22 +02:00
|
|
|
else:
|
2022-08-19 02:04:58 +02:00
|
|
|
button_labels: list[str] = [
|
|
|
|
tr.sync_upload_to_ankiweb(),
|
|
|
|
tr.sync_download_from_ankiweb(),
|
|
|
|
tr.sync_cancel_button(),
|
|
|
|
]
|
|
|
|
|
|
|
|
def callback(choice: int) -> None:
|
|
|
|
if choice == 0:
|
2023-09-10 05:22:20 +02:00
|
|
|
full_upload(mw, server_usn, on_done)
|
2022-08-19 02:04:58 +02:00
|
|
|
elif choice == 1:
|
2023-09-10 05:22:20 +02:00
|
|
|
full_download(mw, server_usn, on_done)
|
2022-08-19 02:04:58 +02:00
|
|
|
else:
|
|
|
|
on_done()
|
|
|
|
|
|
|
|
ask_user_dialog(
|
|
|
|
tr.sync_conflict_explanation(),
|
|
|
|
callback=callback,
|
|
|
|
buttons=button_labels,
|
|
|
|
default_button=2,
|
|
|
|
)
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
def confirm_full_download(
|
|
|
|
mw: aqt.main.AnkiQt, server_usn: int, on_done: Callable[[], None]
|
|
|
|
) -> None:
|
2020-05-30 04:28:22 +02:00
|
|
|
# confirmation step required, as some users customize their notetypes
|
|
|
|
# in an empty collection, then want to upload them
|
2021-03-26 04:48:26 +01:00
|
|
|
if not askUser(tr.sync_confirm_empty_download()):
|
2020-05-31 02:53:54 +02:00
|
|
|
return on_done()
|
2020-05-30 04:28:22 +02:00
|
|
|
else:
|
2023-09-10 05:22:20 +02:00
|
|
|
mw.closeAllWindows(lambda: full_download(mw, server_usn, on_done))
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
|
2023-11-07 02:11:07 +01:00
|
|
|
def confirm_full_upload(
|
|
|
|
mw: aqt.main.AnkiQt, server_usn: int, on_done: Callable[[], None]
|
|
|
|
) -> None:
|
|
|
|
# confirmation step required, as some users have reported an upload
|
|
|
|
# happening despite having their AnkiWeb collection not being empty
|
|
|
|
# (not reproducible - maybe a compiler bug?)
|
|
|
|
if not askUser(tr.sync_confirm_empty_upload()):
|
|
|
|
return on_done()
|
|
|
|
else:
|
|
|
|
mw.closeAllWindows(lambda: full_upload(mw, server_usn, on_done))
|
|
|
|
|
|
|
|
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
def on_full_sync_timer(mw: aqt.main.AnkiQt, label: str) -> None:
|
2020-05-30 04:28:22 +02:00
|
|
|
progress = mw.col.latest_progress()
|
2021-02-08 07:40:27 +01:00
|
|
|
if not progress.HasField("full_sync"):
|
2020-05-30 04:28:22 +02:00
|
|
|
return
|
2021-02-08 07:40:27 +01:00
|
|
|
sync_progress = progress.full_sync
|
2020-05-30 04:28:22 +02:00
|
|
|
|
2021-02-08 07:40:27 +01:00
|
|
|
if sync_progress.transferred == sync_progress.total:
|
2021-03-26 04:48:26 +01:00
|
|
|
label = tr.sync_checking()
|
2020-05-30 12:52:32 +02:00
|
|
|
mw.progress.update(
|
2021-02-08 07:40:27 +01:00
|
|
|
value=sync_progress.transferred,
|
|
|
|
max=sync_progress.total,
|
2020-05-31 06:43:27 +02:00
|
|
|
process=False,
|
|
|
|
label=label,
|
2020-05-30 12:52:32 +02:00
|
|
|
)
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
if mw.progress.want_cancel():
|
2021-01-31 09:46:43 +01:00
|
|
|
mw.col.abort_sync()
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
def full_download(
|
|
|
|
mw: aqt.main.AnkiQt, server_usn: int, on_done: Callable[[], None]
|
|
|
|
) -> None:
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
label = tr.sync_downloading_from_ankiweb()
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def on_timer() -> None:
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
on_full_sync_timer(mw, label)
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
timer = QTimer(mw)
|
|
|
|
qconnect(timer.timeout, on_timer)
|
|
|
|
timer.start(150)
|
|
|
|
|
2022-04-19 09:10:34 +02:00
|
|
|
# hook needs to be called early, on the main thread
|
|
|
|
gui_hooks.collection_will_temporarily_close(mw.col)
|
|
|
|
|
2021-02-06 10:01:48 +01:00
|
|
|
def download() -> None:
|
2022-04-19 09:10:34 +02:00
|
|
|
mw.create_backup_now()
|
|
|
|
mw.col.close_for_full_sync()
|
2023-09-10 05:22:20 +02:00
|
|
|
mw.col.full_upload_or_download(
|
|
|
|
auth=mw.pm.sync_auth(), server_usn=server_usn, upload=False
|
|
|
|
)
|
2021-02-06 10:01:48 +01:00
|
|
|
|
2021-02-02 14:30:53 +01:00
|
|
|
def on_future_done(fut: Future) -> None:
|
2020-05-30 04:28:22 +02:00
|
|
|
timer.stop()
|
2022-04-19 09:10:34 +02:00
|
|
|
mw.reopen(after_full_sync=True)
|
2020-05-30 04:28:22 +02:00
|
|
|
mw.reset()
|
2017-01-17 08:15:50 +01:00
|
|
|
try:
|
2020-05-30 04:28:22 +02:00
|
|
|
fut.result()
|
2020-05-31 03:46:40 +02:00
|
|
|
except Exception as err:
|
|
|
|
handle_sync_error(mw, err)
|
2023-09-10 05:22:20 +02:00
|
|
|
mw.media_syncer.start_monitoring()
|
2020-05-31 02:53:54 +02:00
|
|
|
return on_done()
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
mw.taskman.with_progress(
|
2021-02-06 10:01:48 +01:00
|
|
|
download,
|
2020-05-31 02:53:54 +02:00
|
|
|
on_future_done,
|
2020-05-30 04:28:22 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
|
2023-09-10 05:22:20 +02:00
|
|
|
def full_upload(
|
|
|
|
mw: aqt.main.AnkiQt, server_usn: int | None, on_done: Callable[[], None]
|
|
|
|
) -> None:
|
2022-04-19 09:10:34 +02:00
|
|
|
gui_hooks.collection_will_temporarily_close(mw.col)
|
2020-05-30 04:28:22 +02:00
|
|
|
mw.col.close_for_full_sync()
|
|
|
|
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
label = tr.sync_uploading_to_ankiweb()
|
|
|
|
|
2021-02-01 14:28:21 +01:00
|
|
|
def on_timer() -> None:
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
on_full_sync_timer(mw, label)
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
timer = QTimer(mw)
|
|
|
|
qconnect(timer.timeout, on_timer)
|
|
|
|
timer.start(150)
|
|
|
|
|
2021-02-02 14:30:53 +01:00
|
|
|
def on_future_done(fut: Future) -> None:
|
2020-05-30 04:28:22 +02:00
|
|
|
timer.stop()
|
2022-04-19 09:10:34 +02:00
|
|
|
mw.reopen(after_full_sync=True)
|
2020-05-30 04:28:22 +02:00
|
|
|
mw.reset()
|
2012-12-21 08:51:59 +01:00
|
|
|
try:
|
2020-05-30 04:28:22 +02:00
|
|
|
fut.result()
|
2020-05-31 03:46:40 +02:00
|
|
|
except Exception as err:
|
|
|
|
handle_sync_error(mw, err)
|
|
|
|
return on_done()
|
2023-09-10 05:22:20 +02:00
|
|
|
mw.media_syncer.start_monitoring()
|
2020-05-31 02:53:54 +02:00
|
|
|
return on_done()
|
2020-05-30 04:28:22 +02:00
|
|
|
|
|
|
|
mw.taskman.with_progress(
|
2023-09-10 05:22:20 +02:00
|
|
|
lambda: mw.col.full_upload_or_download(
|
|
|
|
auth=mw.pm.sync_auth(), server_usn=server_usn, upload=True
|
|
|
|
),
|
2020-05-31 02:53:54 +02:00
|
|
|
on_future_done,
|
2020-05-30 04:28:22 +02:00
|
|
|
)
|
|
|
|
|
2020-05-31 03:24:33 +02:00
|
|
|
|
2020-05-31 02:53:54 +02:00
|
|
|
def sync_login(
|
2021-02-02 14:30:53 +01:00
|
|
|
mw: aqt.main.AnkiQt,
|
|
|
|
on_success: Callable[[], None],
|
|
|
|
username: str = "",
|
|
|
|
password: str = "",
|
2020-05-30 04:28:22 +02:00
|
|
|
) -> None:
|
|
|
|
while True:
|
|
|
|
(username, password) = get_id_and_pass_from_user(mw, username, password)
|
|
|
|
if not username and not password:
|
2012-12-21 08:51:59 +01:00
|
|
|
return
|
2020-05-30 04:28:22 +02:00
|
|
|
if username and password:
|
|
|
|
break
|
|
|
|
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
def on_future_done(fut: Future[SyncAuth]) -> None:
|
2017-01-17 08:15:50 +01:00
|
|
|
try:
|
2020-05-30 04:28:22 +02:00
|
|
|
auth = fut.result()
|
|
|
|
except SyncError as e:
|
2021-04-03 08:00:15 +02:00
|
|
|
if e.kind is SyncErrorKind.AUTH:
|
2020-05-30 04:28:22 +02:00
|
|
|
showWarning(str(e))
|
2020-05-31 02:53:54 +02:00
|
|
|
sync_login(mw, on_success, username, password)
|
2021-01-06 09:51:28 +01:00
|
|
|
else:
|
|
|
|
handle_sync_error(mw, e)
|
|
|
|
return
|
2020-05-31 03:46:40 +02:00
|
|
|
except Exception as err:
|
|
|
|
handle_sync_error(mw, err)
|
2020-05-30 04:28:22 +02:00
|
|
|
return
|
2012-12-21 08:51:59 +01:00
|
|
|
|
2020-05-30 04:28:22 +02:00
|
|
|
mw.pm.set_sync_key(auth.hkey)
|
|
|
|
mw.pm.set_sync_username(username)
|
|
|
|
|
|
|
|
on_success()
|
|
|
|
|
|
|
|
mw.taskman.with_progress(
|
Rework syncing code, and replace local sync server (#2329)
This PR replaces the existing Python-driven sync server with a new one in Rust.
The new server supports both collection and media syncing, and is compatible
with both the new protocol mentioned below, and older clients. A setting has
been added to the preferences screen to point Anki to a local server, and a
similar setting is likely to come to AnkiMobile soon.
Documentation is available here: <https://docs.ankiweb.net/sync-server.html>
In addition to the new server and refactoring, this PR also makes changes to the
sync protocol. The existing sync protocol places payloads and metadata inside a
multipart POST body, which causes a few headaches:
- Legacy clients build the request in a non-deterministic order, meaning the
entire request needs to be scanned to extract the metadata.
- Reqwest's multipart API directly writes the multipart body, without exposing
the resulting stream to us, making it harder to track the progress of the
transfer. We've been relying on a patched version of reqwest for timeouts,
which is a pain to keep up to date.
To address these issues, the metadata is now sent in a HTTP header, with the
data payload sent directly in the body. Instead of the slower gzip, we now
use zstd. The old timeout handling code has been replaced with a new implementation
that wraps the request and response body streams to track progress, allowing us
to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout.
The main other change to the protocol is that one-way syncs no longer need to
downgrade the collection to schema 11 prior to sending.
2023-01-18 03:43:46 +01:00
|
|
|
lambda: mw.col.sync_login(
|
|
|
|
username=username, password=password, endpoint=mw.pm.sync_endpoint()
|
|
|
|
),
|
2020-05-31 02:53:54 +02:00
|
|
|
on_future_done,
|
2020-05-30 04:28:22 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def get_id_and_pass_from_user(
|
2021-02-02 14:30:53 +01:00
|
|
|
mw: aqt.main.AnkiQt, username: str = "", password: str = ""
|
2021-10-03 10:59:42 +02:00
|
|
|
) -> tuple[str, str]:
|
2020-05-30 04:28:22 +02:00
|
|
|
diag = QDialog(mw)
|
|
|
|
diag.setWindowTitle("Anki")
|
2021-01-07 05:24:49 +01:00
|
|
|
disable_help_button(diag)
|
2021-10-05 05:53:01 +02:00
|
|
|
diag.setWindowModality(Qt.WindowModality.WindowModal)
|
2020-05-30 04:28:22 +02:00
|
|
|
vbox = QVBoxLayout()
|
|
|
|
info_label = QLabel(
|
2020-08-24 15:37:13 +02:00
|
|
|
without_unicode_isolation(
|
2021-03-26 05:21:04 +01:00
|
|
|
tr.sync_account_required(link="https://ankiweb.net/account/register")
|
2020-08-24 15:37:13 +02:00
|
|
|
)
|
2020-05-30 04:28:22 +02:00
|
|
|
)
|
|
|
|
info_label.setOpenExternalLinks(True)
|
|
|
|
info_label.setWordWrap(True)
|
|
|
|
vbox.addWidget(info_label)
|
|
|
|
vbox.addSpacing(20)
|
|
|
|
g = QGridLayout()
|
2021-03-26 04:48:26 +01:00
|
|
|
l1 = QLabel(tr.sync_ankiweb_id_label())
|
2020-05-30 04:28:22 +02:00
|
|
|
g.addWidget(l1, 0, 0)
|
|
|
|
user = QLineEdit()
|
|
|
|
user.setText(username)
|
|
|
|
g.addWidget(user, 0, 1)
|
2021-03-26 04:48:26 +01:00
|
|
|
l2 = QLabel(tr.sync_password_label())
|
2020-05-30 04:28:22 +02:00
|
|
|
g.addWidget(l2, 1, 0)
|
|
|
|
passwd = QLineEdit()
|
|
|
|
passwd.setText(password)
|
2021-10-05 05:53:01 +02:00
|
|
|
passwd.setEchoMode(QLineEdit.EchoMode.Password)
|
2020-05-30 04:28:22 +02:00
|
|
|
g.addWidget(passwd, 1, 1)
|
|
|
|
vbox.addLayout(g)
|
2021-10-05 05:53:01 +02:00
|
|
|
bb = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel) # type: ignore
|
|
|
|
bb.button(QDialogButtonBox.StandardButton.Ok).setAutoDefault(True)
|
2020-05-30 04:28:22 +02:00
|
|
|
qconnect(bb.accepted, diag.accept)
|
|
|
|
qconnect(bb.rejected, diag.reject)
|
|
|
|
vbox.addWidget(bb)
|
|
|
|
diag.setLayout(vbox)
|
|
|
|
diag.show()
|
|
|
|
|
2021-10-05 02:01:45 +02:00
|
|
|
accepted = diag.exec()
|
2020-05-30 04:28:22 +02:00
|
|
|
if not accepted:
|
|
|
|
return ("", "")
|
|
|
|
return (user.text().strip(), passwd.text())
|
2020-07-16 02:12:41 +02:00
|
|
|
|
2020-07-16 05:55:53 +02:00
|
|
|
|
2020-07-16 02:12:41 +02:00
|
|
|
# export platform version to syncing code
|
2021-10-25 06:50:13 +02:00
|
|
|
os.environ["PLATFORM"] = plat_desc()
|