handle duplicate keys in schema 11 deckconf

rslib/src/deckconf/schema11.rs

@@ -34,7 +34,8 @@ pub struct DeckConfSchema11 {
     dynamic: bool,
 
     // 2021 scheduler options: these were not in schema 11, but we need to persist them
-    // so the settings are not lost on upgrade/downgrade
+    // so the settings are not lost on upgrade/downgrade.
+    // NOTE: if adding new ones, make sure to update clear_other_duplicates()
     #[serde(default)]
     new_mix: i32,
     #[serde(default)]
@@ -278,7 +279,7 @@ impl From<DeckConfSchema11> for DeckConf {
     }
 }
 
-// schema 15 -> schema 11
+// latest schema -> schema 11
 impl From<DeckConf> for DeckConfSchema11 {
     fn from(c: DeckConf) -> DeckConfSchema11 {
         // split extra json up
@@ -290,6 +291,7 @@ impl From<DeckConf> for DeckConfSchema11 {
             top_other = Default::default();
         } else {
             top_other = serde_json::from_slice(&c.inner.other).unwrap_or_default();
+            clear_other_duplicates(&mut top_other);
             if let Some(new) = top_other.remove("new") {
                 let val: HashMap<String, Value> = serde_json::from_value(new).unwrap_or_default();
                 new_other = val;
@@ -359,3 +361,21 @@ impl From<DeckConf> for DeckConfSchema11 {
         }
     }
 }
+
+fn clear_other_duplicates(top_other: &mut HashMap<String, Value>) {
+    // Older clients may have received keys from a newer client when
+    // syncing, which get bundled into `other`. If they then upgrade, then
+    // downgrade their collection to schema11, serde will serialize the
+    // new default keys, but then add them again from `other`, leading
+    // to the keys being duplicated in the resulting json - which older
+    // clients then can't read. So we need to strip out any new keys we
+    // add.
+    for key in &[
+        "newMix",
+        "newPerDayMinimum",
+        "interdayLearningMix",
+        "reviewOrder",
+    ] {
+        top_other.remove(*key);
+    }
+}

rslib/src/storage/deckconf/mod.rs

@@ -9,6 +9,7 @@ use crate::{
 };
 use prost::Message;
 use rusqlite::{params, Row, NO_PARAMS};
+use serde_json::Value;
 use std::collections::HashMap;
 
 fn row_to_deckconf(row: &Row) -> Result<DeckConf> {
@@ -139,13 +140,20 @@ impl SqliteStorage {
     }
 
     pub(super) fn upgrade_deck_conf_to_schema14(&self) -> Result<()> {
-        let conf = self
-            .db
-            .query_row_and_then("select dconf from col", NO_PARAMS, |row| {
-                let conf: Result<HashMap<DeckConfID, DeckConfSchema11>> =
-                    serde_json::from_str(row.get_raw(0).as_str()?).map_err(Into::into);
-                conf
-            })?;
+        let conf: HashMap<DeckConfID, DeckConfSchema11> =
+            self.db
+                .query_row_and_then("select dconf from col", NO_PARAMS, |row| -> Result<_> {
+                    let text = row.get_raw(0).as_str()?;
+                    // try direct parse
+                    serde_json::from_str(text)
+                        .or_else(|_| {
+                            // failed, and could be caused by duplicate keys. Serialize into
+                            // a value first to discard them, then try again
+                            let conf: Value = serde_json::from_str(text)?;
+                            serde_json::from_value(conf)
+                        })
+                        .map_err(Into::into)
+                })?;
         for (_, mut conf) in conf.into_iter() {
             self.add_deck_conf_schema14(&mut conf)?;
         }