# -*- coding: utf-8 -*- # Copyright: Damien Elmes # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html """\ Synchronisation ============================== Support for keeping two decks synchronized. Both local syncing and syncing over HTTP are supported. Server implements the following calls: getDecks(): return a list of deck names & modtimes summary(lastSync): a list of all objects changed after lastSync applyPayload(payload): apply any sent changes and return any changed remote objects finish(): save deck on server after payload applied and response received createDeck(name): create a deck on the server Full sync support is not documented yet. """ __docformat__ = 'restructuredtext' import zlib, re, urllib, urllib2, socket, time, shutil from anki.utils import json as simplejson import os, base64, httplib, sys, tempfile, httplib, types from datetime import date import oldanki, oldanki.deck, oldanki.cards from oldanki.db import sqlite from oldanki.errors import * from oldanki.models import Model, FieldModel, CardModel from oldanki.facts import Fact, Field from oldanki.cards import Card from oldanki.stats import Stats, globalStats from oldanki.history import CardHistoryEntry from oldanki.stats import globalStats from oldanki.utils import ids2str, hexifyID, checksum from oldanki.media import mediaFiles from oldanki.lang import _ from hooks import runHook if simplejson.__version__ < "1.7.3": raise Exception("SimpleJSON must be 1.7.3 or later.") CHUNK_SIZE = 32768 MIME_BOUNDARY = "Anki-sync-boundary" # live SYNC_URL = "http://ankiweb.net/sync/" SYNC_HOST = "ankiweb.net"; SYNC_PORT = 80 # testing #SYNC_URL = "http://localhost:8001/sync/" #SYNC_HOST = "localhost"; SYNC_PORT = 8001 KEYS = ("models", "facts", "cards", "media") ########################################################################## # Monkey-patch httplib to incrementally send instead of chewing up large # amounts of memory, and track progress. sendProgressHook = None def incrementalSend(self, strOrFile): if self.sock is None: if self.auto_open: self.connect() else: raise NotConnected() if self.debuglevel > 0: print "send:", repr(str) try: if (isinstance(strOrFile, str) or isinstance(strOrFile, unicode)): self.sock.sendall(strOrFile) else: cnt = 0 t = time.time() while 1: if sendProgressHook and time.time() - t > 1: sendProgressHook(cnt) t = time.time() data = strOrFile.read(CHUNK_SIZE) cnt += len(data) if not data: break self.sock.sendall(data) except socket.error, v: if v[0] == 32: # Broken pipe self.close() raise httplib.HTTPConnection.send = incrementalSend def fullSyncProgressHook(cnt): runHook("fullSyncProgress", "fromLocal", cnt) ########################################################################## class SyncTools(object): def __init__(self, deck=None): self.deck = deck self.diffs = {} self.serverExcludedTags = [] self.timediff = 0 # Control ########################################################################## def setServer(self, server): self.server = server def sync(self): "Sync two decks locally. Reimplement this for finer control." if not self.prepareSync(0): return sums = self.summaries() payload = self.genPayload(sums) res = self.server.applyPayload(payload) self.applyPayloadReply(res) self.deck.reset() def prepareSync(self, timediff): "Sync setup. True if sync needed." self.localTime = self.modified() self.remoteTime = self.server.modified() if self.localTime == self.remoteTime: return False l = self._lastSync(); r = self.server._lastSync() # set lastSync to the lower of the two sides, and account for slow # clocks & assume it took up to 10 seconds for the reply to arrive self.deck.lastSync = min(l, r) - timediff - 10 return True def summaries(self): return (self.summary(self.deck.lastSync), self.server.summary(self.deck.lastSync)) def genPayload(self, summaries): (lsum, rsum) = summaries self.preSyncRefresh() payload = {} # first, handle models, facts and cards for key in KEYS: diff = self.diffSummary(lsum, rsum, key) payload["added-" + key] = self.getObjsFromKey(diff[0], key) payload["deleted-" + key] = diff[1] payload["missing-" + key] = diff[2] self.deleteObjsFromKey(diff[3], key) # handle the remainder if self.localTime > self.remoteTime: payload['stats'] = self.bundleStats() payload['history'] = self.bundleHistory() payload['sources'] = self.bundleSources() # finally, set new lastSync and bundle the deck info payload['deck'] = self.bundleDeck() return payload def applyPayload(self, payload): reply = {} self.preSyncRefresh() # model, facts and cards for key in KEYS: k = 'added-' + key # send back any requested if k in payload: reply[k] = self.getObjsFromKey( payload['missing-' + key], key) self.updateObjsFromKey(payload['added-' + key], key) self.deleteObjsFromKey(payload['deleted-' + key], key) # send back deck-related stuff if it wasn't sent to us if not 'deck' in payload: reply['stats'] = self.bundleStats() reply['history'] = self.bundleHistory() reply['sources'] = self.bundleSources() # finally, set new lastSync and bundle the deck info reply['deck'] = self.bundleDeck() else: self.updateDeck(payload['deck']) self.updateStats(payload['stats']) self.updateHistory(payload['history']) if 'sources' in payload: self.updateSources(payload['sources']) self.postSyncRefresh() cardIds = [x[0] for x in payload['added-cards']] self.deck.updateCardTags(cardIds) # rebuild priorities on server self.rebuildPriorities(cardIds, self.serverExcludedTags) return reply def applyPayloadReply(self, reply): # model, facts and cards for key in KEYS: k = 'added-' + key # old version may not send media if k in reply: self.updateObjsFromKey(reply['added-' + key], key) # deck if 'deck' in reply: self.updateDeck(reply['deck']) self.updateStats(reply['stats']) self.updateHistory(reply['history']) if 'sources' in reply: self.updateSources(reply['sources']) self.postSyncRefresh() # rebuild priorities on client cardIds = [x[0] for x in reply['added-cards']] self.deck.updateCardTags(cardIds) self.rebuildPriorities(cardIds) if self.missingFacts() != 0: raise Exception( "Facts missing after sync. Please run Tools>Advanced>Check DB.") def missingFacts(self): return self.deck.s.scalar( "select count() from cards where factId "+ "not in (select id from facts)"); def rebuildPriorities(self, cardIds, suspend=[]): self.deck.updateAllPriorities(partial=True, dirty=False) self.deck.updatePriorities(cardIds, suspend=suspend, dirty=False) def postSyncRefresh(self): "Flush changes to DB, and reload object associations." self.deck.s.flush() self.deck.s.refresh(self.deck) self.deck.currentModel def preSyncRefresh(self): # ensure global stats are available (queue may not be built) self.deck._globalStats = globalStats(self.deck) def payloadChanges(self, payload): h = { 'lf': len(payload['added-facts']['facts']), 'rf': len(payload['missing-facts']), 'lc': len(payload['added-cards']), 'rc': len(payload['missing-cards']), 'lm': len(payload['added-models']), 'rm': len(payload['missing-models']), } if self.localTime > self.remoteTime: h['ls'] = _('all') h['rs'] = 0 else: h['ls'] = 0 h['rs'] = _('all') return h def payloadChangeReport(self, payload): p = self.payloadChanges(payload) return _("""\
Added/Changed    Here   Server
Cards%(lc)d%(rc)d
Facts%(lf)d%(rf)d
Models%(lm)d%(rm)d
Stats%(ls)s%(rs)s
""") % p # Summaries ########################################################################## def summary(self, lastSync): "Generate a full summary of modtimes for two-way syncing." # client may have selected an earlier sync time self.deck.lastSync = lastSync # ensure we're flushed first self.deck.s.flush() return { # cards "cards": self.realLists(self.deck.s.all( "select id, modified from cards where modified > :mod", mod=lastSync)), "delcards": self.realLists(self.deck.s.all( "select cardId, deletedTime from cardsDeleted " "where deletedTime > :mod", mod=lastSync)), # facts "facts": self.realLists(self.deck.s.all( "select id, modified from facts where modified > :mod", mod=lastSync)), "delfacts": self.realLists(self.deck.s.all( "select factId, deletedTime from factsDeleted " "where deletedTime > :mod", mod=lastSync)), # models "models": self.realLists(self.deck.s.all( "select id, modified from models where modified > :mod", mod=lastSync)), "delmodels": self.realLists(self.deck.s.all( "select modelId, deletedTime from modelsDeleted " "where deletedTime > :mod", mod=lastSync)), # media "media": self.realLists(self.deck.s.all( "select id, created from media where created > :mod", mod=lastSync)), "delmedia": self.realLists(self.deck.s.all( "select mediaId, deletedTime from mediaDeleted " "where deletedTime > :mod", mod=lastSync)), } # Diffing ########################################################################## def diffSummary(self, localSummary, remoteSummary, key): # list of ids on both ends lexists = localSummary[key] ldeleted = localSummary["del"+key] rexists = remoteSummary[key] rdeleted = remoteSummary["del"+key] ldeletedIds = dict(ldeleted) rdeletedIds = dict(rdeleted) # to store the results locallyEdited = [] locallyDeleted = [] remotelyEdited = [] remotelyDeleted = [] # build a hash of all ids, with value (localMod, remoteMod). # deleted/nonexisting cards are marked with a modtime of None. ids = {} for (id, mod) in rexists: ids[id] = [None, mod] for (id, mod) in rdeleted: ids[id] = [None, None] for (id, mod) in lexists: if id in ids: ids[id][0] = mod else: ids[id] = [mod, None] for (id, mod) in ldeleted: if id in ids: ids[id][0] = None else: ids[id] = [None, None] # loop through the hash, determining differences for (id, (localMod, remoteMod)) in ids.items(): if localMod and remoteMod: # changed/existing on both sides if localMod < remoteMod: remotelyEdited.append(id) elif localMod > remoteMod: locallyEdited.append(id) elif localMod and not remoteMod: # if it's missing on server or newer here, sync if (id not in rdeletedIds or rdeletedIds[id] < localMod): locallyEdited.append(id) else: remotelyDeleted.append(id) elif remoteMod and not localMod: # if it's missing locally or newer there, sync if (id not in ldeletedIds or ldeletedIds[id] < remoteMod): remotelyEdited.append(id) else: locallyDeleted.append(id) else: if id in ldeletedIds and id not in rdeletedIds: locallyDeleted.append(id) elif id in rdeletedIds and id not in ldeletedIds: remotelyDeleted.append(id) return (locallyEdited, locallyDeleted, remotelyEdited, remotelyDeleted) # Models ########################################################################## def getModels(self, ids, updateModified=False): return [self.bundleModel(id, updateModified) for id in ids] def bundleModel(self, id, updateModified): "Return a model representation suitable for transport." mod = self.deck.s.query(Model).get(id) # force load of lazy attributes mod.fieldModels; mod.cardModels m = self.dictFromObj(mod) m['fieldModels'] = [self.bundleFieldModel(fm) for fm in m['fieldModels']] m['cardModels'] = [self.bundleCardModel(fm) for fm in m['cardModels']] if updateModified: m['modified'] = time.time() return m def bundleFieldModel(self, fm): d = self.dictFromObj(fm) if 'model' in d: del d['model'] return d def bundleCardModel(self, cm): d = self.dictFromObj(cm) if 'model' in d: del d['model'] return d def updateModels(self, models): for model in models: local = self.getModel(model['id']) # avoid overwriting any existing card/field models fms = model['fieldModels']; del model['fieldModels'] cms = model['cardModels']; del model['cardModels'] self.applyDict(local, model) self.mergeFieldModels(local, fms) self.mergeCardModels(local, cms) self.deck.s.statement( "delete from modelsDeleted where modelId in %s" % ids2str([m['id'] for m in models])) def getModel(self, id, create=True): "Return a local model with same ID, or create." id = int(id) for l in self.deck.models: if l.id == id: return l if not create: return m = Model() self.deck.models.append(m) return m def mergeFieldModels(self, model, fms): ids = [] for fm in fms: local = self.getFieldModel(model, fm) self.applyDict(local, fm) ids.append(fm['id']) for fm in model.fieldModels: if fm.id not in ids: self.deck.deleteFieldModel(model, fm) def getFieldModel(self, model, remote): id = int(remote['id']) for fm in model.fieldModels: if fm.id == id: return fm fm = FieldModel() model.addFieldModel(fm) return fm def mergeCardModels(self, model, cms): ids = [] for cm in cms: local = self.getCardModel(model, cm) if not 'allowEmptyAnswer' in cm or cm['allowEmptyAnswer'] is None: cm['allowEmptyAnswer'] = True self.applyDict(local, cm) ids.append(cm['id']) for cm in model.cardModels: if cm.id not in ids: self.deck.deleteCardModel(model, cm) def getCardModel(self, model, remote): id = int(remote['id']) for cm in model.cardModels: if cm.id == id: return cm cm = CardModel() model.addCardModel(cm) return cm def deleteModels(self, ids): for id in ids: model = self.getModel(id, create=False) if model: self.deck.deleteModel(model) # Facts ########################################################################## def getFacts(self, ids, updateModified=False): if updateModified: modified = time.time() else: modified = "modified" factIds = ids2str(ids) return { 'facts': self.realLists(self.deck.s.all(""" select id, modelId, created, %s, tags, spaceUntil, lastCardId from facts where id in %s""" % (modified, factIds))), 'fields': self.realLists(self.deck.s.all(""" select id, factId, fieldModelId, ordinal, value from fields where factId in %s""" % factIds)) } def updateFacts(self, factsdict): facts = factsdict['facts'] fields = factsdict['fields'] if not facts: return # update facts first dlist = [{ 'id': f[0], 'modelId': f[1], 'created': f[2], 'modified': f[3], 'tags': f[4], 'spaceUntil': f[5] or "", 'lastCardId': f[6] } for f in facts] self.deck.s.execute(""" insert or replace into facts (id, modelId, created, modified, tags, spaceUntil, lastCardId) values (:id, :modelId, :created, :modified, :tags, :spaceUntil, :lastCardId)""", dlist) # now fields dlist = [{ 'id': f[0], 'factId': f[1], 'fieldModelId': f[2], 'ordinal': f[3], 'value': f[4] } for f in fields] # delete local fields since ids may have changed self.deck.s.execute( "delete from fields where factId in %s" % ids2str([f[0] for f in facts])) # then update self.deck.s.execute(""" insert into fields (id, factId, fieldModelId, ordinal, value) values (:id, :factId, :fieldModelId, :ordinal, :value)""", dlist) self.deck.s.statement( "delete from factsDeleted where factId in %s" % ids2str([f[0] for f in facts])) def deleteFacts(self, ids): self.deck.deleteFacts(ids) # Cards ########################################################################## def getCards(self, ids): return self.realLists(self.deck.s.all(""" select id, factId, cardModelId, created, modified, tags, ordinal, priority, interval, lastInterval, due, lastDue, factor, firstAnswered, reps, successive, averageTime, reviewTime, youngEase0, youngEase1, youngEase2, youngEase3, youngEase4, matureEase0, matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount, question, answer, lastFactor, spaceUntil, type, combinedDue, relativeDelay from cards where id in %s""" % ids2str(ids))) def updateCards(self, cards): if not cards: return # FIXME: older clients won't send this, so this is temp compat code def getType(row): if len(row) > 36: return row[36] if row[15]: return 1 elif row[14]: return 0 return 2 dlist = [{'id': c[0], 'factId': c[1], 'cardModelId': c[2], 'created': c[3], 'modified': c[4], 'tags': c[5], 'ordinal': c[6], 'priority': c[7], 'interval': c[8], 'lastInterval': c[9], 'due': c[10], 'lastDue': c[11], 'factor': c[12], 'firstAnswered': c[13], 'reps': c[14], 'successive': c[15], 'averageTime': c[16], 'reviewTime': c[17], 'youngEase0': c[18], 'youngEase1': c[19], 'youngEase2': c[20], 'youngEase3': c[21], 'youngEase4': c[22], 'matureEase0': c[23], 'matureEase1': c[24], 'matureEase2': c[25], 'matureEase3': c[26], 'matureEase4': c[27], 'yesCount': c[28], 'noCount': c[29], 'question': c[30], 'answer': c[31], 'lastFactor': c[32], 'spaceUntil': c[33], 'type': c[34], 'combinedDue': c[35], 'rd': getType(c) } for c in cards] self.deck.s.execute(""" insert or replace into cards (id, factId, cardModelId, created, modified, tags, ordinal, priority, interval, lastInterval, due, lastDue, factor, firstAnswered, reps, successive, averageTime, reviewTime, youngEase0, youngEase1, youngEase2, youngEase3, youngEase4, matureEase0, matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount, question, answer, lastFactor, spaceUntil, type, combinedDue, relativeDelay, isDue) values (:id, :factId, :cardModelId, :created, :modified, :tags, :ordinal, :priority, :interval, :lastInterval, :due, :lastDue, :factor, :firstAnswered, :reps, :successive, :averageTime, :reviewTime, :youngEase0, :youngEase1, :youngEase2, :youngEase3, :youngEase4, :matureEase0, :matureEase1, :matureEase2, :matureEase3, :matureEase4, :yesCount, :noCount, :question, :answer, :lastFactor, :spaceUntil, :type, :combinedDue, :rd, 0)""", dlist) self.deck.s.statement( "delete from cardsDeleted where cardId in %s" % ids2str([c[0] for c in cards])) def deleteCards(self, ids): self.deck.deleteCards(ids) # Deck/stats/history ########################################################################## def bundleDeck(self): # ensure modified is not greater than server time if getattr(self, "server", None) and getattr( self.server, "timestamp", None): self.deck.modified = min(self.deck.modified,self.server.timestamp) # and ensure lastSync is greater than modified self.deck.lastSync = max(time.time(), self.deck.modified+1) d = self.dictFromObj(self.deck) del d['Session'] del d['engine'] del d['s'] del d['path'] del d['syncName'] del d['version'] if 'newQueue' in d: del d['newQueue'] del d['failedQueue'] del d['revQueue'] # these may be deleted before bundling if 'css' in d: del d['css'] if 'models' in d: del d['models'] if 'currentModel' in d: del d['currentModel'] keys = d.keys() for k in keys: if isinstance(d[k], types.MethodType): del d[k] d['meta'] = self.realLists(self.deck.s.all("select * from deckVars")) return d def updateDeck(self, deck): if 'meta' in deck: meta = deck['meta'] for (k,v) in meta: self.deck.s.statement(""" insert or replace into deckVars (key, value) values (:k, :v)""", k=k, v=v) del deck['meta'] self.applyDict(self.deck, deck) def bundleStats(self): def bundleStat(stat): s = self.dictFromObj(stat) s['day'] = s['day'].toordinal() del s['id'] return s lastDay = date.fromtimestamp(max(0, self.deck.lastSync - 60*60*24)) ids = self.deck.s.column0( "select id from stats where type = 1 and day >= :day", day=lastDay) stat = Stats() def statFromId(id): stat.fromDB(self.deck.s, id) return stat stats = { 'global': bundleStat(self.deck._globalStats), 'daily': [bundleStat(statFromId(id)) for id in ids], } return stats def updateStats(self, stats): stats['global']['day'] = date.fromordinal(stats['global']['day']) self.applyDict(self.deck._globalStats, stats['global']) self.deck._globalStats.toDB(self.deck.s) for record in stats['daily']: record['day'] = date.fromordinal(record['day']) stat = Stats() id = self.deck.s.scalar("select id from stats where " "type = :type and day = :day", type=1, day=record['day']) if id: stat.fromDB(self.deck.s, id) else: stat.create(self.deck.s, 1, record['day']) self.applyDict(stat, record) stat.toDB(self.deck.s) def bundleHistory(self): return self.realLists(self.deck.s.all(""" select cardId, time, lastInterval, nextInterval, ease, delay, lastFactor, nextFactor, reps, thinkingTime, yesCount, noCount from reviewHistory where time > :ls""", ls=self.deck.lastSync)) def updateHistory(self, history): dlist = [{'cardId': h[0], 'time': h[1], 'lastInterval': h[2], 'nextInterval': h[3], 'ease': h[4], 'delay': h[5], 'lastFactor': h[6], 'nextFactor': h[7], 'reps': h[8], 'thinkingTime': h[9], 'yesCount': h[10], 'noCount': h[11]} for h in history] if not dlist: return self.deck.s.statements(""" insert or ignore into reviewHistory (cardId, time, lastInterval, nextInterval, ease, delay, lastFactor, nextFactor, reps, thinkingTime, yesCount, noCount) values (:cardId, :time, :lastInterval, :nextInterval, :ease, :delay, :lastFactor, :nextFactor, :reps, :thinkingTime, :yesCount, :noCount)""", dlist) def bundleSources(self): return self.realLists(self.deck.s.all("select * from sources")) def updateSources(self, sources): for s in sources: self.deck.s.statement(""" insert or replace into sources values (:id, :name, :created, :lastSync, :syncPeriod)""", id=s[0], name=s[1], created=s[2], lastSync=s[3], syncPeriod=s[4]) # Media metadata ########################################################################## def getMedia(self, ids): return [tuple(row) for row in self.deck.s.all(""" select id, filename, size, created, originalPath, description from media where id in %s""" % ids2str(ids))] def updateMedia(self, media): meta = [] for m in media: # build meta meta.append({ 'id': m[0], 'filename': m[1], 'size': m[2], 'created': m[3], 'originalPath': m[4], 'description': m[5]}) # apply metadata if meta: self.deck.s.statements(""" insert or replace into media (id, filename, size, created, originalPath, description) values (:id, :filename, :size, :created, :originalPath, :description)""", meta) self.deck.s.statement( "delete from mediaDeleted where mediaId in %s" % ids2str([m[0] for m in media])) def deleteMedia(self, ids): sids = ids2str(ids) files = self.deck.s.column0( "select filename from media where id in %s" % sids) self.deck.s.statement(""" insert into mediaDeleted select id, :now from media where media.id in %s""" % sids, now=time.time()) self.deck.s.execute( "delete from media where id in %s" % sids) # One-way syncing (sharing) ########################################################################## def syncOneWay(self, lastSync): "Sync two decks one way." payload = self.server.genOneWayPayload(lastSync) self.applyOneWayPayload(payload) self.deck.reset() def syncOneWayDeckName(self): return (self.deck.s.scalar("select name from sources where id = :id", id=self.server.deckName) or hexifyID(int(self.server.deckName))) def prepareOneWaySync(self): "Sync setup. True if sync needed. Not used for local sync." srcID = self.server.deckName (lastSync, syncPeriod) = self.deck.s.first( "select lastSync, syncPeriod from sources where id = :id", id=srcID) if self.server.modified() <= lastSync: return self.deck.lastSync = lastSync return True def genOneWayPayload(self, lastSync): "Bundle all added or changed objects since the last sync." p = {} # facts factIds = self.deck.s.column0( "select id from facts where modified > :l", l=lastSync) p['facts'] = self.getFacts(factIds, updateModified=True) # models modelIds = self.deck.s.column0( "select id from models where modified > :l", l=lastSync) p['models'] = self.getModels(modelIds, updateModified=True) # media mediaIds = self.deck.s.column0( "select id from media where created > :l", l=lastSync) p['media'] = self.getMedia(mediaIds) # cards cardIds = self.deck.s.column0( "select id from cards where modified > :l", l=lastSync) p['cards'] = self.realLists(self.getOneWayCards(cardIds)) return p def applyOneWayPayload(self, payload): keys = [k for k in KEYS if k != "cards"] # model, facts, media for key in keys: self.updateObjsFromKey(payload[key], key) # models need their source tagged for m in payload["models"]: self.deck.s.statement("update models set source = :s " "where id = :id", s=self.server.deckName, id=m['id']) # cards last, handled differently t = time.time() try: self.updateOneWayCards(payload['cards']) except KeyError: sys.stderr.write("Subscribed to a broken deck. " "Try removing your deck subscriptions.") t = 0 # update sync time self.deck.s.statement( "update sources set lastSync = :t where id = :id", id=self.server.deckName, t=t) self.deck.modified = time.time() def getOneWayCards(self, ids): "The minimum information necessary to generate one way cards." return self.deck.s.all( "select id, factId, cardModelId, ordinal, created from cards " "where id in %s" % ids2str(ids)) def updateOneWayCards(self, cards): if not cards: return t = time.time() dlist = [{'id': c[0], 'factId': c[1], 'cardModelId': c[2], 'ordinal': c[3], 'created': c[4], 't': t} for c in cards] # add any missing cards self.deck.s.statements(""" insert or ignore into cards (id, factId, cardModelId, created, modified, tags, ordinal, priority, interval, lastInterval, due, lastDue, factor, firstAnswered, reps, successive, averageTime, reviewTime, youngEase0, youngEase1, youngEase2, youngEase3, youngEase4, matureEase0, matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount, question, answer, lastFactor, spaceUntil, isDue, type, combinedDue, relativeDelay) values (:id, :factId, :cardModelId, :created, :t, "", :ordinal, 1, 0, 0, :created, 0, 2.5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 2.5, 0, 0, 2, :t, 2)""", dlist) # update q/as models = dict(self.deck.s.all(""" select cards.id, models.id from cards, facts, models where cards.factId = facts.id and facts.modelId = models.id and cards.id in %s""" % ids2str([c[0] for c in cards]))) self.deck.s.flush() self.deck.updateCardQACache( [(c[0], c[2], c[1], models[c[0]]) for c in cards]) # rebuild priorities on client cardIds = [c[0] for c in cards] self.deck.updateCardTags(cardIds) self.rebuildPriorities(cardIds) # Tools ########################################################################## def modified(self): return self.deck.modified def _lastSync(self): return self.deck.lastSync def unstuff(self, data): "Uncompress and convert to unicode." return simplejson.loads(unicode(zlib.decompress(data), "utf8")) def stuff(self, data): "Convert into UTF-8 and compress." return zlib.compress(simplejson.dumps(data)) def dictFromObj(self, obj): "Return a dict representing OBJ without any hidden db fields." return dict([(k,v) for (k,v) in obj.__dict__.items() if not k.startswith("_")]) def applyDict(self, obj, dict): "Apply each element in DICT to OBJ in a way the ORM notices." for (k,v) in dict.items(): setattr(obj, k, v) def realLists(self, result): "Convert an SQLAlchemy response into a list of real lists." return [list(x) for x in result] def getObjsFromKey(self, ids, key): return getattr(self, "get" + key.capitalize())(ids) def deleteObjsFromKey(self, ids, key): return getattr(self, "delete" + key.capitalize())(ids) def updateObjsFromKey(self, ids, key): return getattr(self, "update" + key.capitalize())(ids) # Full sync ########################################################################## def needFullSync(self, sums): if self.deck.lastSync <= 0: return True for sum in sums: for l in sum.values(): if len(l) > 1000: return True if self.deck.s.scalar( "select count() from reviewHistory where time > :ls", ls=self.deck.lastSync) > 1000: return True lastDay = date.fromtimestamp(max(0, self.deck.lastSync - 60*60*24)) if self.deck.s.scalar( "select count() from stats where day >= :day", day=lastDay) > 100: return True return False def prepareFullSync(self): t = time.time() # ensure modified is not greater than server time self.deck.modified = min(self.deck.modified, self.server.timestamp) self.deck.s.commit() self.deck.close() fields = { "p": self.server.password, "u": self.server.username, "d": self.server.deckName.encode("utf-8"), } if self.localTime > self.remoteTime: return ("fromLocal", fields, self.deck.path) else: return ("fromServer", fields, self.deck.path) def fullSync(self): ret = self.prepareFullSync() if ret[0] == "fromLocal": self.fullSyncFromLocal(ret[1], ret[2]) else: self.fullSyncFromServer(ret[1], ret[2]) def fullSyncFromLocal(self, fields, path): global sendProgressHook try: # write into a temporary file, since POST needs content-length src = open(path, "rb") (fd, name) = tempfile.mkstemp(prefix="oldanki") tmp = open(name, "w+b") # post vars for (key, value) in fields.items(): tmp.write('--' + MIME_BOUNDARY + "\r\n") tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key) tmp.write('\r\n') tmp.write(value) tmp.write('\r\n') # file header tmp.write('--' + MIME_BOUNDARY + "\r\n") tmp.write( 'Content-Disposition: form-data; name="deck"; filename="deck"\r\n') tmp.write('Content-Type: application/octet-stream\r\n') tmp.write('\r\n') # data comp = zlib.compressobj() while 1: data = src.read(CHUNK_SIZE) if not data: tmp.write(comp.flush()) break tmp.write(comp.compress(data)) src.close() tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n') size = tmp.tell() tmp.seek(0) # open http connection runHook("fullSyncStarted", size) headers = { 'Content-type': 'multipart/form-data; boundary=%s' % MIME_BOUNDARY, 'Content-length': str(size), 'Host': SYNC_HOST, } req = urllib2.Request(SYNC_URL + "fullup?v=2", tmp, headers) try: sendProgressHook = fullSyncProgressHook res = urllib2.urlopen(req).read() assert res.startswith("OK") # update lastSync c = sqlite.connect(path) c.execute("update decks set lastSync = ?", (res[3:],)) c.commit() c.close() finally: sendProgressHook = None tmp.close() os.close(fd) os.unlink(name) finally: runHook("fullSyncFinished") def fullSyncFromServer(self, fields, path): try: runHook("fullSyncStarted", 0) fields = urllib.urlencode(fields) src = urllib.urlopen(SYNC_URL + "fulldown", fields) (fd, tmpname) = tempfile.mkstemp(dir=os.path.dirname(path), prefix="fullsync") tmp = open(tmpname, "wb") decomp = zlib.decompressobj() cnt = 0 while 1: data = src.read(CHUNK_SIZE) if not data: tmp.write(decomp.flush()) break tmp.write(decomp.decompress(data)) cnt += CHUNK_SIZE runHook("fullSyncProgress", "fromServer", cnt) src.close() tmp.close() os.close(fd) # if we were successful, overwrite old deck os.unlink(path) os.rename(tmpname, path) # reset the deck name c = sqlite.connect(path) c.execute("update decks set syncName = ?", [checksum(path.encode("utf-8"))]) c.commit() c.close() finally: runHook("fullSyncFinished") # Local syncing ########################################################################## class SyncServer(SyncTools): def __init__(self, deck=None): SyncTools.__init__(self, deck) class SyncClient(SyncTools): pass # HTTP proxy: act as a server and direct requests to the real server ########################################################################## class HttpSyncServerProxy(SyncServer): def __init__(self, user, passwd): SyncServer.__init__(self) self.decks = None self.deckName = None self.username = user self.password = passwd self.protocolVersion = 5 self.sourcesToCheck = [] def connect(self, clientVersion=""): "Check auth, protocol & grab deck list." if not self.decks: import socket socket.setdefaulttimeout(30) d = self.runCmd("getDecks", libanki=oldanki.version, client=clientVersion, sources=simplejson.dumps(self.sourcesToCheck), pversion=self.protocolVersion) socket.setdefaulttimeout(None) if d['status'] != "OK": raise SyncError(type="authFailed", status=d['status']) self.decks = d['decks'] self.timestamp = d['timestamp'] self.timediff = abs(self.timestamp - time.time()) def hasDeck(self, deckName): self.connect() return deckName in self.decks.keys() def availableDecks(self): self.connect() return self.decks.keys() def createDeck(self, deckName): ret = self.runCmd("createDeck", name=deckName.encode("utf-8")) if not ret or ret['status'] != "OK": raise SyncError(type="createFailed") self.decks[deckName] = [0, 0] def summary(self, lastSync): return self.runCmd("summary", lastSync=self.stuff(lastSync)) def genOneWayPayload(self, lastSync): return self.runCmd("genOneWayPayload", lastSync=self.stuff(lastSync)) def modified(self): self.connect() return self.decks[self.deckName][0] def _lastSync(self): self.connect() return self.decks[self.deckName][1] def applyPayload(self, payload): return self.runCmd("applyPayload", payload=self.stuff(payload)) def finish(self): assert self.runCmd("finish") == "OK" def runCmd(self, action, **args): data = {"p": self.password, "u": self.username, "v": 2} if self.deckName: data['d'] = self.deckName.encode("utf-8") else: data['d'] = None data.update(args) data = urllib.urlencode(data) try: f = urllib2.urlopen(SYNC_URL + action, data) except (urllib2.URLError, socket.error, socket.timeout, httplib.BadStatusLine), e: raise SyncError(type="connectionError", exc=`e`) ret = f.read() if not ret: raise SyncError(type="noResponse") try: return self.unstuff(ret) except Exception, e: raise SyncError(type="connectionError", exc=`e`) # HTTP server: respond to proxy requests and return data ########################################################################## class HttpSyncServer(SyncServer): def __init__(self): SyncServer.__init__(self) self.decks = {} self.deck = None def summary(self, lastSync): return self.stuff(SyncServer.summary( self, float(zlib.decompress(lastSync)))) def applyPayload(self, payload): return self.stuff(SyncServer.applyPayload(self, self.unstuff(payload))) def genOneWayPayload(self, lastSync): return self.stuff(SyncServer.genOneWayPayload( self, float(zlib.decompress(lastSync)))) def getDecks(self, libanki, client, sources, pversion): return self.stuff({ "status": "OK", "decks": self.decks, "timestamp": time.time(), }) def createDeck(self, name): "Create a deck on the server. Not implemented." return self.stuff("OK") # Local media copying ########################################################################## def copyLocalMedia(src, dst): srcDir = src.mediaDir() if not srcDir: return dstDir = dst.mediaDir(create=True) files = os.listdir(srcDir) # find media references used = {} for col in ("question", "answer"): txt = dst.s.column0(""" select %(c)s from cards where %(c)s like '%%