1237 lines
44 KiB
Python
1237 lines
44 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright: Damien Elmes <oldanki@ichi2.net>
|
|
# License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
|
|
|
|
"""\
|
|
Synchronisation
|
|
==============================
|
|
|
|
Support for keeping two decks synchronized. Both local syncing and syncing
|
|
over HTTP are supported.
|
|
|
|
Server implements the following calls:
|
|
|
|
getDecks(): return a list of deck names & modtimes
|
|
summary(lastSync): a list of all objects changed after lastSync
|
|
applyPayload(payload): apply any sent changes and return any changed remote
|
|
objects
|
|
finish(): save deck on server after payload applied and response received
|
|
createDeck(name): create a deck on the server
|
|
|
|
Full sync support is not documented yet.
|
|
"""
|
|
__docformat__ = 'restructuredtext'
|
|
|
|
import zlib, re, urllib, urllib2, socket, time, shutil
|
|
from anki.utils import json as simplejson
|
|
import os, base64, httplib, sys, tempfile, httplib, types
|
|
from datetime import date
|
|
import oldanki, oldanki.deck, oldanki.cards
|
|
from oldanki.db import sqlite
|
|
from oldanki.errors import *
|
|
from oldanki.models import Model, FieldModel, CardModel
|
|
from oldanki.facts import Fact, Field
|
|
from oldanki.cards import Card
|
|
from oldanki.stats import Stats, globalStats
|
|
from oldanki.history import CardHistoryEntry
|
|
from oldanki.stats import globalStats
|
|
from oldanki.utils import ids2str, hexifyID, checksum
|
|
from oldanki.media import mediaFiles
|
|
from oldanki.lang import _
|
|
from hooks import runHook
|
|
|
|
if simplejson.__version__ < "1.7.3":
|
|
raise Exception("SimpleJSON must be 1.7.3 or later.")
|
|
|
|
CHUNK_SIZE = 32768
|
|
MIME_BOUNDARY = "Anki-sync-boundary"
|
|
# live
|
|
SYNC_URL = "http://ankiweb.net/sync/"
|
|
SYNC_HOST = "ankiweb.net"; SYNC_PORT = 80
|
|
# testing
|
|
#SYNC_URL = "http://localhost:8001/sync/"
|
|
#SYNC_HOST = "localhost"; SYNC_PORT = 8001
|
|
|
|
|
|
KEYS = ("models", "facts", "cards", "media")
|
|
|
|
##########################################################################
|
|
# Monkey-patch httplib to incrementally send instead of chewing up large
|
|
# amounts of memory, and track progress.
|
|
|
|
sendProgressHook = None
|
|
|
|
def incrementalSend(self, strOrFile):
|
|
if self.sock is None:
|
|
if self.auto_open:
|
|
self.connect()
|
|
else:
|
|
raise NotConnected()
|
|
if self.debuglevel > 0:
|
|
print "send:", repr(str)
|
|
try:
|
|
if (isinstance(strOrFile, str) or
|
|
isinstance(strOrFile, unicode)):
|
|
self.sock.sendall(strOrFile)
|
|
else:
|
|
cnt = 0
|
|
t = time.time()
|
|
while 1:
|
|
if sendProgressHook and time.time() - t > 1:
|
|
sendProgressHook(cnt)
|
|
t = time.time()
|
|
data = strOrFile.read(CHUNK_SIZE)
|
|
cnt += len(data)
|
|
if not data:
|
|
break
|
|
self.sock.sendall(data)
|
|
except socket.error, v:
|
|
if v[0] == 32: # Broken pipe
|
|
self.close()
|
|
raise
|
|
|
|
httplib.HTTPConnection.send = incrementalSend
|
|
|
|
def fullSyncProgressHook(cnt):
|
|
runHook("fullSyncProgress", "fromLocal", cnt)
|
|
|
|
##########################################################################
|
|
|
|
class SyncTools(object):
|
|
|
|
def __init__(self, deck=None):
|
|
self.deck = deck
|
|
self.diffs = {}
|
|
self.serverExcludedTags = []
|
|
self.timediff = 0
|
|
|
|
# Control
|
|
##########################################################################
|
|
|
|
def setServer(self, server):
|
|
self.server = server
|
|
|
|
def sync(self):
|
|
"Sync two decks locally. Reimplement this for finer control."
|
|
if not self.prepareSync(0):
|
|
return
|
|
sums = self.summaries()
|
|
payload = self.genPayload(sums)
|
|
res = self.server.applyPayload(payload)
|
|
self.applyPayloadReply(res)
|
|
self.deck.reset()
|
|
|
|
def prepareSync(self, timediff):
|
|
"Sync setup. True if sync needed."
|
|
self.localTime = self.modified()
|
|
self.remoteTime = self.server.modified()
|
|
if self.localTime == self.remoteTime:
|
|
return False
|
|
l = self._lastSync(); r = self.server._lastSync()
|
|
# set lastSync to the lower of the two sides, and account for slow
|
|
# clocks & assume it took up to 10 seconds for the reply to arrive
|
|
self.deck.lastSync = min(l, r) - timediff - 10
|
|
return True
|
|
|
|
def summaries(self):
|
|
return (self.summary(self.deck.lastSync),
|
|
self.server.summary(self.deck.lastSync))
|
|
|
|
def genPayload(self, summaries):
|
|
(lsum, rsum) = summaries
|
|
self.preSyncRefresh()
|
|
payload = {}
|
|
# first, handle models, facts and cards
|
|
for key in KEYS:
|
|
diff = self.diffSummary(lsum, rsum, key)
|
|
payload["added-" + key] = self.getObjsFromKey(diff[0], key)
|
|
payload["deleted-" + key] = diff[1]
|
|
payload["missing-" + key] = diff[2]
|
|
self.deleteObjsFromKey(diff[3], key)
|
|
# handle the remainder
|
|
if self.localTime > self.remoteTime:
|
|
payload['stats'] = self.bundleStats()
|
|
payload['history'] = self.bundleHistory()
|
|
payload['sources'] = self.bundleSources()
|
|
# finally, set new lastSync and bundle the deck info
|
|
payload['deck'] = self.bundleDeck()
|
|
return payload
|
|
|
|
def applyPayload(self, payload):
|
|
reply = {}
|
|
self.preSyncRefresh()
|
|
# model, facts and cards
|
|
for key in KEYS:
|
|
k = 'added-' + key
|
|
# send back any requested
|
|
if k in payload:
|
|
reply[k] = self.getObjsFromKey(
|
|
payload['missing-' + key], key)
|
|
self.updateObjsFromKey(payload['added-' + key], key)
|
|
self.deleteObjsFromKey(payload['deleted-' + key], key)
|
|
# send back deck-related stuff if it wasn't sent to us
|
|
if not 'deck' in payload:
|
|
reply['stats'] = self.bundleStats()
|
|
reply['history'] = self.bundleHistory()
|
|
reply['sources'] = self.bundleSources()
|
|
# finally, set new lastSync and bundle the deck info
|
|
reply['deck'] = self.bundleDeck()
|
|
else:
|
|
self.updateDeck(payload['deck'])
|
|
self.updateStats(payload['stats'])
|
|
self.updateHistory(payload['history'])
|
|
if 'sources' in payload:
|
|
self.updateSources(payload['sources'])
|
|
self.postSyncRefresh()
|
|
cardIds = [x[0] for x in payload['added-cards']]
|
|
self.deck.updateCardTags(cardIds)
|
|
# rebuild priorities on server
|
|
self.rebuildPriorities(cardIds, self.serverExcludedTags)
|
|
return reply
|
|
|
|
def applyPayloadReply(self, reply):
|
|
# model, facts and cards
|
|
for key in KEYS:
|
|
k = 'added-' + key
|
|
# old version may not send media
|
|
if k in reply:
|
|
self.updateObjsFromKey(reply['added-' + key], key)
|
|
# deck
|
|
if 'deck' in reply:
|
|
self.updateDeck(reply['deck'])
|
|
self.updateStats(reply['stats'])
|
|
self.updateHistory(reply['history'])
|
|
if 'sources' in reply:
|
|
self.updateSources(reply['sources'])
|
|
self.postSyncRefresh()
|
|
# rebuild priorities on client
|
|
cardIds = [x[0] for x in reply['added-cards']]
|
|
self.deck.updateCardTags(cardIds)
|
|
self.rebuildPriorities(cardIds)
|
|
if self.missingFacts() != 0:
|
|
raise Exception(
|
|
"Facts missing after sync. Please run Tools>Advanced>Check DB.")
|
|
|
|
def missingFacts(self):
|
|
return self.deck.s.scalar(
|
|
"select count() from cards where factId "+
|
|
"not in (select id from facts)");
|
|
|
|
def rebuildPriorities(self, cardIds, suspend=[]):
|
|
self.deck.updateAllPriorities(partial=True, dirty=False)
|
|
self.deck.updatePriorities(cardIds, suspend=suspend, dirty=False)
|
|
|
|
def postSyncRefresh(self):
|
|
"Flush changes to DB, and reload object associations."
|
|
self.deck.s.flush()
|
|
self.deck.s.refresh(self.deck)
|
|
self.deck.currentModel
|
|
|
|
def preSyncRefresh(self):
|
|
# ensure global stats are available (queue may not be built)
|
|
self.deck._globalStats = globalStats(self.deck)
|
|
|
|
def payloadChanges(self, payload):
|
|
h = {
|
|
'lf': len(payload['added-facts']['facts']),
|
|
'rf': len(payload['missing-facts']),
|
|
'lc': len(payload['added-cards']),
|
|
'rc': len(payload['missing-cards']),
|
|
'lm': len(payload['added-models']),
|
|
'rm': len(payload['missing-models']),
|
|
}
|
|
if self.localTime > self.remoteTime:
|
|
h['ls'] = _('all')
|
|
h['rs'] = 0
|
|
else:
|
|
h['ls'] = 0
|
|
h['rs'] = _('all')
|
|
return h
|
|
|
|
def payloadChangeReport(self, payload):
|
|
p = self.payloadChanges(payload)
|
|
return _("""\
|
|
<table>
|
|
<tr><td><b>Added/Changed </b></td>
|
|
<td><b>Here </b></td><td><b>Server</b></td></tr>
|
|
<tr><td>Cards</td><td>%(lc)d</td><td>%(rc)d</td></tr>
|
|
<tr><td>Facts</td><td>%(lf)d</td><td>%(rf)d</td></tr>
|
|
<tr><td>Models</td><td>%(lm)d</td><td>%(rm)d</td></tr>
|
|
<tr><td>Stats</td><td>%(ls)s</td><td>%(rs)s</td></tr>
|
|
</table>""") % p
|
|
|
|
# Summaries
|
|
##########################################################################
|
|
|
|
def summary(self, lastSync):
|
|
"Generate a full summary of modtimes for two-way syncing."
|
|
# client may have selected an earlier sync time
|
|
self.deck.lastSync = lastSync
|
|
# ensure we're flushed first
|
|
self.deck.s.flush()
|
|
return {
|
|
# cards
|
|
"cards": self.realLists(self.deck.s.all(
|
|
"select id, modified from cards where modified > :mod",
|
|
mod=lastSync)),
|
|
"delcards": self.realLists(self.deck.s.all(
|
|
"select cardId, deletedTime from cardsDeleted "
|
|
"where deletedTime > :mod", mod=lastSync)),
|
|
# facts
|
|
"facts": self.realLists(self.deck.s.all(
|
|
"select id, modified from facts where modified > :mod",
|
|
mod=lastSync)),
|
|
"delfacts": self.realLists(self.deck.s.all(
|
|
"select factId, deletedTime from factsDeleted "
|
|
"where deletedTime > :mod", mod=lastSync)),
|
|
# models
|
|
"models": self.realLists(self.deck.s.all(
|
|
"select id, modified from models where modified > :mod",
|
|
mod=lastSync)),
|
|
"delmodels": self.realLists(self.deck.s.all(
|
|
"select modelId, deletedTime from modelsDeleted "
|
|
"where deletedTime > :mod", mod=lastSync)),
|
|
# media
|
|
"media": self.realLists(self.deck.s.all(
|
|
"select id, created from media where created > :mod",
|
|
mod=lastSync)),
|
|
"delmedia": self.realLists(self.deck.s.all(
|
|
"select mediaId, deletedTime from mediaDeleted "
|
|
"where deletedTime > :mod", mod=lastSync)),
|
|
}
|
|
|
|
# Diffing
|
|
##########################################################################
|
|
|
|
def diffSummary(self, localSummary, remoteSummary, key):
|
|
# list of ids on both ends
|
|
lexists = localSummary[key]
|
|
ldeleted = localSummary["del"+key]
|
|
rexists = remoteSummary[key]
|
|
rdeleted = remoteSummary["del"+key]
|
|
ldeletedIds = dict(ldeleted)
|
|
rdeletedIds = dict(rdeleted)
|
|
# to store the results
|
|
locallyEdited = []
|
|
locallyDeleted = []
|
|
remotelyEdited = []
|
|
remotelyDeleted = []
|
|
# build a hash of all ids, with value (localMod, remoteMod).
|
|
# deleted/nonexisting cards are marked with a modtime of None.
|
|
ids = {}
|
|
for (id, mod) in rexists:
|
|
ids[id] = [None, mod]
|
|
for (id, mod) in rdeleted:
|
|
ids[id] = [None, None]
|
|
for (id, mod) in lexists:
|
|
if id in ids:
|
|
ids[id][0] = mod
|
|
else:
|
|
ids[id] = [mod, None]
|
|
for (id, mod) in ldeleted:
|
|
if id in ids:
|
|
ids[id][0] = None
|
|
else:
|
|
ids[id] = [None, None]
|
|
# loop through the hash, determining differences
|
|
for (id, (localMod, remoteMod)) in ids.items():
|
|
if localMod and remoteMod:
|
|
# changed/existing on both sides
|
|
if localMod < remoteMod:
|
|
remotelyEdited.append(id)
|
|
elif localMod > remoteMod:
|
|
locallyEdited.append(id)
|
|
elif localMod and not remoteMod:
|
|
# if it's missing on server or newer here, sync
|
|
if (id not in rdeletedIds or
|
|
rdeletedIds[id] < localMod):
|
|
locallyEdited.append(id)
|
|
else:
|
|
remotelyDeleted.append(id)
|
|
elif remoteMod and not localMod:
|
|
# if it's missing locally or newer there, sync
|
|
if (id not in ldeletedIds or
|
|
ldeletedIds[id] < remoteMod):
|
|
remotelyEdited.append(id)
|
|
else:
|
|
locallyDeleted.append(id)
|
|
else:
|
|
if id in ldeletedIds and id not in rdeletedIds:
|
|
locallyDeleted.append(id)
|
|
elif id in rdeletedIds and id not in ldeletedIds:
|
|
remotelyDeleted.append(id)
|
|
return (locallyEdited, locallyDeleted,
|
|
remotelyEdited, remotelyDeleted)
|
|
|
|
# Models
|
|
##########################################################################
|
|
|
|
def getModels(self, ids, updateModified=False):
|
|
return [self.bundleModel(id, updateModified) for id in ids]
|
|
|
|
def bundleModel(self, id, updateModified):
|
|
"Return a model representation suitable for transport."
|
|
mod = self.deck.s.query(Model).get(id)
|
|
# force load of lazy attributes
|
|
mod.fieldModels; mod.cardModels
|
|
m = self.dictFromObj(mod)
|
|
m['fieldModels'] = [self.bundleFieldModel(fm) for fm in m['fieldModels']]
|
|
m['cardModels'] = [self.bundleCardModel(fm) for fm in m['cardModels']]
|
|
if updateModified:
|
|
m['modified'] = time.time()
|
|
return m
|
|
|
|
def bundleFieldModel(self, fm):
|
|
d = self.dictFromObj(fm)
|
|
if 'model' in d: del d['model']
|
|
return d
|
|
|
|
def bundleCardModel(self, cm):
|
|
d = self.dictFromObj(cm)
|
|
if 'model' in d: del d['model']
|
|
return d
|
|
|
|
def updateModels(self, models):
|
|
for model in models:
|
|
local = self.getModel(model['id'])
|
|
# avoid overwriting any existing card/field models
|
|
fms = model['fieldModels']; del model['fieldModels']
|
|
cms = model['cardModels']; del model['cardModels']
|
|
self.applyDict(local, model)
|
|
self.mergeFieldModels(local, fms)
|
|
self.mergeCardModels(local, cms)
|
|
self.deck.s.statement(
|
|
"delete from modelsDeleted where modelId in %s" %
|
|
ids2str([m['id'] for m in models]))
|
|
|
|
def getModel(self, id, create=True):
|
|
"Return a local model with same ID, or create."
|
|
id = int(id)
|
|
for l in self.deck.models:
|
|
if l.id == id:
|
|
return l
|
|
if not create:
|
|
return
|
|
m = Model()
|
|
self.deck.models.append(m)
|
|
return m
|
|
|
|
def mergeFieldModels(self, model, fms):
|
|
ids = []
|
|
for fm in fms:
|
|
local = self.getFieldModel(model, fm)
|
|
self.applyDict(local, fm)
|
|
ids.append(fm['id'])
|
|
for fm in model.fieldModels:
|
|
if fm.id not in ids:
|
|
self.deck.deleteFieldModel(model, fm)
|
|
|
|
def getFieldModel(self, model, remote):
|
|
id = int(remote['id'])
|
|
for fm in model.fieldModels:
|
|
if fm.id == id:
|
|
return fm
|
|
fm = FieldModel()
|
|
model.addFieldModel(fm)
|
|
return fm
|
|
|
|
def mergeCardModels(self, model, cms):
|
|
ids = []
|
|
for cm in cms:
|
|
local = self.getCardModel(model, cm)
|
|
if not 'allowEmptyAnswer' in cm or cm['allowEmptyAnswer'] is None:
|
|
cm['allowEmptyAnswer'] = True
|
|
self.applyDict(local, cm)
|
|
ids.append(cm['id'])
|
|
for cm in model.cardModels:
|
|
if cm.id not in ids:
|
|
self.deck.deleteCardModel(model, cm)
|
|
|
|
def getCardModel(self, model, remote):
|
|
id = int(remote['id'])
|
|
for cm in model.cardModels:
|
|
if cm.id == id:
|
|
return cm
|
|
cm = CardModel()
|
|
model.addCardModel(cm)
|
|
return cm
|
|
|
|
def deleteModels(self, ids):
|
|
for id in ids:
|
|
model = self.getModel(id, create=False)
|
|
if model:
|
|
self.deck.deleteModel(model)
|
|
|
|
# Facts
|
|
##########################################################################
|
|
|
|
def getFacts(self, ids, updateModified=False):
|
|
if updateModified:
|
|
modified = time.time()
|
|
else:
|
|
modified = "modified"
|
|
factIds = ids2str(ids)
|
|
return {
|
|
'facts': self.realLists(self.deck.s.all("""
|
|
select id, modelId, created, %s, tags, spaceUntil, lastCardId from facts
|
|
where id in %s""" % (modified, factIds))),
|
|
'fields': self.realLists(self.deck.s.all("""
|
|
select id, factId, fieldModelId, ordinal, value from fields
|
|
where factId in %s""" % factIds))
|
|
}
|
|
|
|
def updateFacts(self, factsdict):
|
|
facts = factsdict['facts']
|
|
fields = factsdict['fields']
|
|
if not facts:
|
|
return
|
|
# update facts first
|
|
dlist = [{
|
|
'id': f[0],
|
|
'modelId': f[1],
|
|
'created': f[2],
|
|
'modified': f[3],
|
|
'tags': f[4],
|
|
'spaceUntil': f[5] or "",
|
|
'lastCardId': f[6]
|
|
} for f in facts]
|
|
self.deck.s.execute("""
|
|
insert or replace into facts
|
|
(id, modelId, created, modified, tags, spaceUntil, lastCardId)
|
|
values
|
|
(:id, :modelId, :created, :modified, :tags, :spaceUntil, :lastCardId)""", dlist)
|
|
# now fields
|
|
dlist = [{
|
|
'id': f[0],
|
|
'factId': f[1],
|
|
'fieldModelId': f[2],
|
|
'ordinal': f[3],
|
|
'value': f[4]
|
|
} for f in fields]
|
|
# delete local fields since ids may have changed
|
|
self.deck.s.execute(
|
|
"delete from fields where factId in %s" %
|
|
ids2str([f[0] for f in facts]))
|
|
# then update
|
|
self.deck.s.execute("""
|
|
insert into fields
|
|
(id, factId, fieldModelId, ordinal, value)
|
|
values
|
|
(:id, :factId, :fieldModelId, :ordinal, :value)""", dlist)
|
|
self.deck.s.statement(
|
|
"delete from factsDeleted where factId in %s" %
|
|
ids2str([f[0] for f in facts]))
|
|
|
|
def deleteFacts(self, ids):
|
|
self.deck.deleteFacts(ids)
|
|
|
|
# Cards
|
|
##########################################################################
|
|
|
|
def getCards(self, ids):
|
|
return self.realLists(self.deck.s.all("""
|
|
select id, factId, cardModelId, created, modified, tags, ordinal,
|
|
priority, interval, lastInterval, due, lastDue, factor,
|
|
firstAnswered, reps, successive, averageTime, reviewTime, youngEase0,
|
|
youngEase1, youngEase2, youngEase3, youngEase4, matureEase0,
|
|
matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount,
|
|
question, answer, lastFactor, spaceUntil, type, combinedDue, relativeDelay
|
|
from cards where id in %s""" % ids2str(ids)))
|
|
|
|
def updateCards(self, cards):
|
|
if not cards:
|
|
return
|
|
# FIXME: older clients won't send this, so this is temp compat code
|
|
def getType(row):
|
|
if len(row) > 36:
|
|
return row[36]
|
|
if row[15]:
|
|
return 1
|
|
elif row[14]:
|
|
return 0
|
|
return 2
|
|
dlist = [{'id': c[0],
|
|
'factId': c[1],
|
|
'cardModelId': c[2],
|
|
'created': c[3],
|
|
'modified': c[4],
|
|
'tags': c[5],
|
|
'ordinal': c[6],
|
|
'priority': c[7],
|
|
'interval': c[8],
|
|
'lastInterval': c[9],
|
|
'due': c[10],
|
|
'lastDue': c[11],
|
|
'factor': c[12],
|
|
'firstAnswered': c[13],
|
|
'reps': c[14],
|
|
'successive': c[15],
|
|
'averageTime': c[16],
|
|
'reviewTime': c[17],
|
|
'youngEase0': c[18],
|
|
'youngEase1': c[19],
|
|
'youngEase2': c[20],
|
|
'youngEase3': c[21],
|
|
'youngEase4': c[22],
|
|
'matureEase0': c[23],
|
|
'matureEase1': c[24],
|
|
'matureEase2': c[25],
|
|
'matureEase3': c[26],
|
|
'matureEase4': c[27],
|
|
'yesCount': c[28],
|
|
'noCount': c[29],
|
|
'question': c[30],
|
|
'answer': c[31],
|
|
'lastFactor': c[32],
|
|
'spaceUntil': c[33],
|
|
'type': c[34],
|
|
'combinedDue': c[35],
|
|
'rd': getType(c)
|
|
} for c in cards]
|
|
self.deck.s.execute("""
|
|
insert or replace into cards
|
|
(id, factId, cardModelId, created, modified, tags, ordinal,
|
|
priority, interval, lastInterval, due, lastDue, factor,
|
|
firstAnswered, reps, successive, averageTime, reviewTime, youngEase0,
|
|
youngEase1, youngEase2, youngEase3, youngEase4, matureEase0,
|
|
matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount,
|
|
question, answer, lastFactor, spaceUntil, type, combinedDue,
|
|
relativeDelay, isDue)
|
|
values
|
|
(:id, :factId, :cardModelId, :created, :modified, :tags, :ordinal,
|
|
:priority, :interval, :lastInterval, :due, :lastDue, :factor,
|
|
:firstAnswered, :reps, :successive, :averageTime, :reviewTime, :youngEase0,
|
|
:youngEase1, :youngEase2, :youngEase3, :youngEase4, :matureEase0,
|
|
:matureEase1, :matureEase2, :matureEase3, :matureEase4, :yesCount,
|
|
:noCount, :question, :answer, :lastFactor, :spaceUntil,
|
|
:type, :combinedDue, :rd, 0)""", dlist)
|
|
self.deck.s.statement(
|
|
"delete from cardsDeleted where cardId in %s" %
|
|
ids2str([c[0] for c in cards]))
|
|
|
|
def deleteCards(self, ids):
|
|
self.deck.deleteCards(ids)
|
|
|
|
# Deck/stats/history
|
|
##########################################################################
|
|
|
|
def bundleDeck(self):
|
|
# ensure modified is not greater than server time
|
|
if getattr(self, "server", None) and getattr(
|
|
self.server, "timestamp", None):
|
|
self.deck.modified = min(self.deck.modified,self.server.timestamp)
|
|
# and ensure lastSync is greater than modified
|
|
self.deck.lastSync = max(time.time(), self.deck.modified+1)
|
|
d = self.dictFromObj(self.deck)
|
|
del d['Session']
|
|
del d['engine']
|
|
del d['s']
|
|
del d['path']
|
|
del d['syncName']
|
|
del d['version']
|
|
if 'newQueue' in d:
|
|
del d['newQueue']
|
|
del d['failedQueue']
|
|
del d['revQueue']
|
|
# these may be deleted before bundling
|
|
if 'css' in d: del d['css']
|
|
if 'models' in d: del d['models']
|
|
if 'currentModel' in d: del d['currentModel']
|
|
keys = d.keys()
|
|
for k in keys:
|
|
if isinstance(d[k], types.MethodType):
|
|
del d[k]
|
|
d['meta'] = self.realLists(self.deck.s.all("select * from deckVars"))
|
|
return d
|
|
|
|
def updateDeck(self, deck):
|
|
if 'meta' in deck:
|
|
meta = deck['meta']
|
|
for (k,v) in meta:
|
|
self.deck.s.statement("""
|
|
insert or replace into deckVars
|
|
(key, value) values (:k, :v)""", k=k, v=v)
|
|
del deck['meta']
|
|
self.applyDict(self.deck, deck)
|
|
|
|
def bundleStats(self):
|
|
def bundleStat(stat):
|
|
s = self.dictFromObj(stat)
|
|
s['day'] = s['day'].toordinal()
|
|
del s['id']
|
|
return s
|
|
lastDay = date.fromtimestamp(max(0, self.deck.lastSync - 60*60*24))
|
|
ids = self.deck.s.column0(
|
|
"select id from stats where type = 1 and day >= :day", day=lastDay)
|
|
stat = Stats()
|
|
def statFromId(id):
|
|
stat.fromDB(self.deck.s, id)
|
|
return stat
|
|
stats = {
|
|
'global': bundleStat(self.deck._globalStats),
|
|
'daily': [bundleStat(statFromId(id)) for id in ids],
|
|
}
|
|
return stats
|
|
|
|
def updateStats(self, stats):
|
|
stats['global']['day'] = date.fromordinal(stats['global']['day'])
|
|
self.applyDict(self.deck._globalStats, stats['global'])
|
|
self.deck._globalStats.toDB(self.deck.s)
|
|
for record in stats['daily']:
|
|
record['day'] = date.fromordinal(record['day'])
|
|
stat = Stats()
|
|
id = self.deck.s.scalar("select id from stats where "
|
|
"type = :type and day = :day",
|
|
type=1, day=record['day'])
|
|
if id:
|
|
stat.fromDB(self.deck.s, id)
|
|
else:
|
|
stat.create(self.deck.s, 1, record['day'])
|
|
self.applyDict(stat, record)
|
|
stat.toDB(self.deck.s)
|
|
|
|
def bundleHistory(self):
|
|
return self.realLists(self.deck.s.all("""
|
|
select cardId, time, lastInterval, nextInterval, ease, delay,
|
|
lastFactor, nextFactor, reps, thinkingTime, yesCount, noCount
|
|
from reviewHistory where time > :ls""",
|
|
ls=self.deck.lastSync))
|
|
|
|
def updateHistory(self, history):
|
|
dlist = [{'cardId': h[0],
|
|
'time': h[1],
|
|
'lastInterval': h[2],
|
|
'nextInterval': h[3],
|
|
'ease': h[4],
|
|
'delay': h[5],
|
|
'lastFactor': h[6],
|
|
'nextFactor': h[7],
|
|
'reps': h[8],
|
|
'thinkingTime': h[9],
|
|
'yesCount': h[10],
|
|
'noCount': h[11]} for h in history]
|
|
if not dlist:
|
|
return
|
|
self.deck.s.statements("""
|
|
insert or ignore into reviewHistory
|
|
(cardId, time, lastInterval, nextInterval, ease, delay,
|
|
lastFactor, nextFactor, reps, thinkingTime, yesCount, noCount)
|
|
values
|
|
(:cardId, :time, :lastInterval, :nextInterval, :ease, :delay,
|
|
:lastFactor, :nextFactor, :reps, :thinkingTime, :yesCount, :noCount)""",
|
|
dlist)
|
|
|
|
def bundleSources(self):
|
|
return self.realLists(self.deck.s.all("select * from sources"))
|
|
|
|
def updateSources(self, sources):
|
|
for s in sources:
|
|
self.deck.s.statement("""
|
|
insert or replace into sources values
|
|
(:id, :name, :created, :lastSync, :syncPeriod)""",
|
|
id=s[0],
|
|
name=s[1],
|
|
created=s[2],
|
|
lastSync=s[3],
|
|
syncPeriod=s[4])
|
|
|
|
# Media metadata
|
|
##########################################################################
|
|
|
|
def getMedia(self, ids):
|
|
return [tuple(row) for row in self.deck.s.all("""
|
|
select id, filename, size, created, originalPath, description
|
|
from media where id in %s""" % ids2str(ids))]
|
|
|
|
def updateMedia(self, media):
|
|
meta = []
|
|
for m in media:
|
|
# build meta
|
|
meta.append({
|
|
'id': m[0],
|
|
'filename': m[1],
|
|
'size': m[2],
|
|
'created': m[3],
|
|
'originalPath': m[4],
|
|
'description': m[5]})
|
|
# apply metadata
|
|
if meta:
|
|
self.deck.s.statements("""
|
|
insert or replace into media (id, filename, size, created,
|
|
originalPath, description)
|
|
values (:id, :filename, :size, :created, :originalPath,
|
|
:description)""", meta)
|
|
self.deck.s.statement(
|
|
"delete from mediaDeleted where mediaId in %s" %
|
|
ids2str([m[0] for m in media]))
|
|
|
|
def deleteMedia(self, ids):
|
|
sids = ids2str(ids)
|
|
files = self.deck.s.column0(
|
|
"select filename from media where id in %s" % sids)
|
|
self.deck.s.statement("""
|
|
insert into mediaDeleted
|
|
select id, :now from media
|
|
where media.id in %s""" % sids, now=time.time())
|
|
self.deck.s.execute(
|
|
"delete from media where id in %s" % sids)
|
|
|
|
# One-way syncing (sharing)
|
|
##########################################################################
|
|
|
|
def syncOneWay(self, lastSync):
|
|
"Sync two decks one way."
|
|
payload = self.server.genOneWayPayload(lastSync)
|
|
self.applyOneWayPayload(payload)
|
|
self.deck.reset()
|
|
|
|
def syncOneWayDeckName(self):
|
|
return (self.deck.s.scalar("select name from sources where id = :id",
|
|
id=self.server.deckName) or
|
|
hexifyID(int(self.server.deckName)))
|
|
|
|
def prepareOneWaySync(self):
|
|
"Sync setup. True if sync needed. Not used for local sync."
|
|
srcID = self.server.deckName
|
|
(lastSync, syncPeriod) = self.deck.s.first(
|
|
"select lastSync, syncPeriod from sources where id = :id", id=srcID)
|
|
if self.server.modified() <= lastSync:
|
|
return
|
|
self.deck.lastSync = lastSync
|
|
return True
|
|
|
|
def genOneWayPayload(self, lastSync):
|
|
"Bundle all added or changed objects since the last sync."
|
|
p = {}
|
|
# facts
|
|
factIds = self.deck.s.column0(
|
|
"select id from facts where modified > :l", l=lastSync)
|
|
p['facts'] = self.getFacts(factIds, updateModified=True)
|
|
# models
|
|
modelIds = self.deck.s.column0(
|
|
"select id from models where modified > :l", l=lastSync)
|
|
p['models'] = self.getModels(modelIds, updateModified=True)
|
|
# media
|
|
mediaIds = self.deck.s.column0(
|
|
"select id from media where created > :l", l=lastSync)
|
|
p['media'] = self.getMedia(mediaIds)
|
|
# cards
|
|
cardIds = self.deck.s.column0(
|
|
"select id from cards where modified > :l", l=lastSync)
|
|
p['cards'] = self.realLists(self.getOneWayCards(cardIds))
|
|
return p
|
|
|
|
def applyOneWayPayload(self, payload):
|
|
keys = [k for k in KEYS if k != "cards"]
|
|
# model, facts, media
|
|
for key in keys:
|
|
self.updateObjsFromKey(payload[key], key)
|
|
# models need their source tagged
|
|
for m in payload["models"]:
|
|
self.deck.s.statement("update models set source = :s "
|
|
"where id = :id",
|
|
s=self.server.deckName,
|
|
id=m['id'])
|
|
# cards last, handled differently
|
|
t = time.time()
|
|
try:
|
|
self.updateOneWayCards(payload['cards'])
|
|
except KeyError:
|
|
sys.stderr.write("Subscribed to a broken deck. "
|
|
"Try removing your deck subscriptions.")
|
|
t = 0
|
|
# update sync time
|
|
self.deck.s.statement(
|
|
"update sources set lastSync = :t where id = :id",
|
|
id=self.server.deckName, t=t)
|
|
self.deck.modified = time.time()
|
|
|
|
def getOneWayCards(self, ids):
|
|
"The minimum information necessary to generate one way cards."
|
|
return self.deck.s.all(
|
|
"select id, factId, cardModelId, ordinal, created from cards "
|
|
"where id in %s" % ids2str(ids))
|
|
|
|
def updateOneWayCards(self, cards):
|
|
if not cards:
|
|
return
|
|
t = time.time()
|
|
dlist = [{'id': c[0], 'factId': c[1], 'cardModelId': c[2],
|
|
'ordinal': c[3], 'created': c[4], 't': t} for c in cards]
|
|
# add any missing cards
|
|
self.deck.s.statements("""
|
|
insert or ignore into cards
|
|
(id, factId, cardModelId, created, modified, tags, ordinal,
|
|
priority, interval, lastInterval, due, lastDue, factor,
|
|
firstAnswered, reps, successive, averageTime, reviewTime, youngEase0,
|
|
youngEase1, youngEase2, youngEase3, youngEase4, matureEase0,
|
|
matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount,
|
|
question, answer, lastFactor, spaceUntil, isDue, type, combinedDue,
|
|
relativeDelay)
|
|
values
|
|
(:id, :factId, :cardModelId, :created, :t, "", :ordinal,
|
|
1, 0, 0, :created, 0, 2.5,
|
|
0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0,
|
|
0, "", "", 2.5, 0, 0, 2, :t, 2)""", dlist)
|
|
# update q/as
|
|
models = dict(self.deck.s.all("""
|
|
select cards.id, models.id
|
|
from cards, facts, models
|
|
where cards.factId = facts.id
|
|
and facts.modelId = models.id
|
|
and cards.id in %s""" % ids2str([c[0] for c in cards])))
|
|
self.deck.s.flush()
|
|
self.deck.updateCardQACache(
|
|
[(c[0], c[2], c[1], models[c[0]]) for c in cards])
|
|
# rebuild priorities on client
|
|
cardIds = [c[0] for c in cards]
|
|
self.deck.updateCardTags(cardIds)
|
|
self.rebuildPriorities(cardIds)
|
|
|
|
# Tools
|
|
##########################################################################
|
|
|
|
def modified(self):
|
|
return self.deck.modified
|
|
|
|
def _lastSync(self):
|
|
return self.deck.lastSync
|
|
|
|
def unstuff(self, data):
|
|
"Uncompress and convert to unicode."
|
|
return simplejson.loads(unicode(zlib.decompress(data), "utf8"))
|
|
|
|
def stuff(self, data):
|
|
"Convert into UTF-8 and compress."
|
|
return zlib.compress(simplejson.dumps(data))
|
|
|
|
def dictFromObj(self, obj):
|
|
"Return a dict representing OBJ without any hidden db fields."
|
|
return dict([(k,v) for (k,v) in obj.__dict__.items()
|
|
if not k.startswith("_")])
|
|
|
|
def applyDict(self, obj, dict):
|
|
"Apply each element in DICT to OBJ in a way the ORM notices."
|
|
for (k,v) in dict.items():
|
|
setattr(obj, k, v)
|
|
|
|
def realLists(self, result):
|
|
"Convert an SQLAlchemy response into a list of real lists."
|
|
return [list(x) for x in result]
|
|
|
|
def getObjsFromKey(self, ids, key):
|
|
return getattr(self, "get" + key.capitalize())(ids)
|
|
|
|
def deleteObjsFromKey(self, ids, key):
|
|
return getattr(self, "delete" + key.capitalize())(ids)
|
|
|
|
def updateObjsFromKey(self, ids, key):
|
|
return getattr(self, "update" + key.capitalize())(ids)
|
|
|
|
# Full sync
|
|
##########################################################################
|
|
|
|
def needFullSync(self, sums):
|
|
if self.deck.lastSync <= 0:
|
|
return True
|
|
for sum in sums:
|
|
for l in sum.values():
|
|
if len(l) > 1000:
|
|
return True
|
|
if self.deck.s.scalar(
|
|
"select count() from reviewHistory where time > :ls",
|
|
ls=self.deck.lastSync) > 1000:
|
|
return True
|
|
lastDay = date.fromtimestamp(max(0, self.deck.lastSync - 60*60*24))
|
|
if self.deck.s.scalar(
|
|
"select count() from stats where day >= :day",
|
|
day=lastDay) > 100:
|
|
return True
|
|
return False
|
|
|
|
def prepareFullSync(self):
|
|
t = time.time()
|
|
# ensure modified is not greater than server time
|
|
self.deck.modified = min(self.deck.modified, self.server.timestamp)
|
|
self.deck.s.commit()
|
|
self.deck.close()
|
|
fields = {
|
|
"p": self.server.password,
|
|
"u": self.server.username,
|
|
"d": self.server.deckName.encode("utf-8"),
|
|
}
|
|
if self.localTime > self.remoteTime:
|
|
return ("fromLocal", fields, self.deck.path)
|
|
else:
|
|
return ("fromServer", fields, self.deck.path)
|
|
|
|
def fullSync(self):
|
|
ret = self.prepareFullSync()
|
|
if ret[0] == "fromLocal":
|
|
self.fullSyncFromLocal(ret[1], ret[2])
|
|
else:
|
|
self.fullSyncFromServer(ret[1], ret[2])
|
|
|
|
def fullSyncFromLocal(self, fields, path):
|
|
global sendProgressHook
|
|
try:
|
|
# write into a temporary file, since POST needs content-length
|
|
src = open(path, "rb")
|
|
(fd, name) = tempfile.mkstemp(prefix="oldanki")
|
|
tmp = open(name, "w+b")
|
|
# post vars
|
|
for (key, value) in fields.items():
|
|
tmp.write('--' + MIME_BOUNDARY + "\r\n")
|
|
tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key)
|
|
tmp.write('\r\n')
|
|
tmp.write(value)
|
|
tmp.write('\r\n')
|
|
# file header
|
|
tmp.write('--' + MIME_BOUNDARY + "\r\n")
|
|
tmp.write(
|
|
'Content-Disposition: form-data; name="deck"; filename="deck"\r\n')
|
|
tmp.write('Content-Type: application/octet-stream\r\n')
|
|
tmp.write('\r\n')
|
|
# data
|
|
comp = zlib.compressobj()
|
|
while 1:
|
|
data = src.read(CHUNK_SIZE)
|
|
if not data:
|
|
tmp.write(comp.flush())
|
|
break
|
|
tmp.write(comp.compress(data))
|
|
src.close()
|
|
tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
|
|
size = tmp.tell()
|
|
tmp.seek(0)
|
|
# open http connection
|
|
runHook("fullSyncStarted", size)
|
|
headers = {
|
|
'Content-type': 'multipart/form-data; boundary=%s' %
|
|
MIME_BOUNDARY,
|
|
'Content-length': str(size),
|
|
'Host': SYNC_HOST,
|
|
}
|
|
req = urllib2.Request(SYNC_URL + "fullup?v=2", tmp, headers)
|
|
try:
|
|
sendProgressHook = fullSyncProgressHook
|
|
res = urllib2.urlopen(req).read()
|
|
assert res.startswith("OK")
|
|
# update lastSync
|
|
c = sqlite.connect(path)
|
|
c.execute("update decks set lastSync = ?",
|
|
(res[3:],))
|
|
c.commit()
|
|
c.close()
|
|
finally:
|
|
sendProgressHook = None
|
|
tmp.close()
|
|
os.close(fd)
|
|
os.unlink(name)
|
|
finally:
|
|
runHook("fullSyncFinished")
|
|
|
|
def fullSyncFromServer(self, fields, path):
|
|
try:
|
|
runHook("fullSyncStarted", 0)
|
|
fields = urllib.urlencode(fields)
|
|
src = urllib.urlopen(SYNC_URL + "fulldown", fields)
|
|
(fd, tmpname) = tempfile.mkstemp(dir=os.path.dirname(path),
|
|
prefix="fullsync")
|
|
tmp = open(tmpname, "wb")
|
|
decomp = zlib.decompressobj()
|
|
cnt = 0
|
|
while 1:
|
|
data = src.read(CHUNK_SIZE)
|
|
if not data:
|
|
tmp.write(decomp.flush())
|
|
break
|
|
tmp.write(decomp.decompress(data))
|
|
cnt += CHUNK_SIZE
|
|
runHook("fullSyncProgress", "fromServer", cnt)
|
|
src.close()
|
|
tmp.close()
|
|
os.close(fd)
|
|
# if we were successful, overwrite old deck
|
|
os.unlink(path)
|
|
os.rename(tmpname, path)
|
|
# reset the deck name
|
|
c = sqlite.connect(path)
|
|
c.execute("update decks set syncName = ?",
|
|
[checksum(path.encode("utf-8"))])
|
|
c.commit()
|
|
c.close()
|
|
finally:
|
|
runHook("fullSyncFinished")
|
|
|
|
# Local syncing
|
|
##########################################################################
|
|
|
|
|
|
class SyncServer(SyncTools):
|
|
|
|
def __init__(self, deck=None):
|
|
SyncTools.__init__(self, deck)
|
|
|
|
class SyncClient(SyncTools):
|
|
|
|
pass
|
|
|
|
# HTTP proxy: act as a server and direct requests to the real server
|
|
##########################################################################
|
|
|
|
class HttpSyncServerProxy(SyncServer):
|
|
|
|
def __init__(self, user, passwd):
|
|
SyncServer.__init__(self)
|
|
self.decks = None
|
|
self.deckName = None
|
|
self.username = user
|
|
self.password = passwd
|
|
self.protocolVersion = 5
|
|
self.sourcesToCheck = []
|
|
|
|
def connect(self, clientVersion=""):
|
|
"Check auth, protocol & grab deck list."
|
|
if not self.decks:
|
|
import socket
|
|
socket.setdefaulttimeout(30)
|
|
d = self.runCmd("getDecks",
|
|
libanki=oldanki.version,
|
|
client=clientVersion,
|
|
sources=simplejson.dumps(self.sourcesToCheck),
|
|
pversion=self.protocolVersion)
|
|
socket.setdefaulttimeout(None)
|
|
if d['status'] != "OK":
|
|
raise SyncError(type="authFailed", status=d['status'])
|
|
self.decks = d['decks']
|
|
self.timestamp = d['timestamp']
|
|
self.timediff = abs(self.timestamp - time.time())
|
|
|
|
def hasDeck(self, deckName):
|
|
self.connect()
|
|
return deckName in self.decks.keys()
|
|
|
|
def availableDecks(self):
|
|
self.connect()
|
|
return self.decks.keys()
|
|
|
|
def createDeck(self, deckName):
|
|
ret = self.runCmd("createDeck", name=deckName.encode("utf-8"))
|
|
if not ret or ret['status'] != "OK":
|
|
raise SyncError(type="createFailed")
|
|
self.decks[deckName] = [0, 0]
|
|
|
|
def summary(self, lastSync):
|
|
return self.runCmd("summary",
|
|
lastSync=self.stuff(lastSync))
|
|
|
|
def genOneWayPayload(self, lastSync):
|
|
return self.runCmd("genOneWayPayload",
|
|
lastSync=self.stuff(lastSync))
|
|
|
|
def modified(self):
|
|
self.connect()
|
|
return self.decks[self.deckName][0]
|
|
|
|
def _lastSync(self):
|
|
self.connect()
|
|
return self.decks[self.deckName][1]
|
|
|
|
def applyPayload(self, payload):
|
|
return self.runCmd("applyPayload",
|
|
payload=self.stuff(payload))
|
|
|
|
def finish(self):
|
|
assert self.runCmd("finish") == "OK"
|
|
|
|
def runCmd(self, action, **args):
|
|
data = {"p": self.password,
|
|
"u": self.username,
|
|
"v": 2}
|
|
if self.deckName:
|
|
data['d'] = self.deckName.encode("utf-8")
|
|
else:
|
|
data['d'] = None
|
|
data.update(args)
|
|
data = urllib.urlencode(data)
|
|
try:
|
|
f = urllib2.urlopen(SYNC_URL + action, data)
|
|
except (urllib2.URLError, socket.error, socket.timeout,
|
|
httplib.BadStatusLine), e:
|
|
raise SyncError(type="connectionError",
|
|
exc=`e`)
|
|
ret = f.read()
|
|
if not ret:
|
|
raise SyncError(type="noResponse")
|
|
try:
|
|
return self.unstuff(ret)
|
|
except Exception, e:
|
|
raise SyncError(type="connectionError",
|
|
exc=`e`)
|
|
|
|
# HTTP server: respond to proxy requests and return data
|
|
##########################################################################
|
|
|
|
class HttpSyncServer(SyncServer):
|
|
def __init__(self):
|
|
SyncServer.__init__(self)
|
|
self.decks = {}
|
|
self.deck = None
|
|
|
|
def summary(self, lastSync):
|
|
return self.stuff(SyncServer.summary(
|
|
self, float(zlib.decompress(lastSync))))
|
|
|
|
def applyPayload(self, payload):
|
|
return self.stuff(SyncServer.applyPayload(self,
|
|
self.unstuff(payload)))
|
|
|
|
def genOneWayPayload(self, lastSync):
|
|
return self.stuff(SyncServer.genOneWayPayload(
|
|
self, float(zlib.decompress(lastSync))))
|
|
|
|
def getDecks(self, libanki, client, sources, pversion):
|
|
return self.stuff({
|
|
"status": "OK",
|
|
"decks": self.decks,
|
|
"timestamp": time.time(),
|
|
})
|
|
|
|
def createDeck(self, name):
|
|
"Create a deck on the server. Not implemented."
|
|
return self.stuff("OK")
|
|
|
|
# Local media copying
|
|
##########################################################################
|
|
|
|
def copyLocalMedia(src, dst):
|
|
srcDir = src.mediaDir()
|
|
if not srcDir:
|
|
return
|
|
dstDir = dst.mediaDir(create=True)
|
|
files = os.listdir(srcDir)
|
|
# find media references
|
|
used = {}
|
|
for col in ("question", "answer"):
|
|
txt = dst.s.column0("""
|
|
select %(c)s from cards where
|
|
%(c)s like '%%<img %%'
|
|
or %(c)s like '%%[sound:%%'""" % {'c': col})
|
|
for entry in txt:
|
|
for fname in mediaFiles(entry):
|
|
used[fname] = True
|
|
# copy only used media
|
|
for file in files:
|
|
if file not in used:
|
|
continue
|
|
srcfile = os.path.join(srcDir, file)
|
|
dstfile = os.path.join(dstDir, file)
|
|
if not os.path.exists(dstfile):
|
|
try:
|
|
shutil.copy2(srcfile, dstfile)
|
|
except IOError, OSError:
|
|
pass
|