Add APIs for IO mask editing (#2758)

* Add simple mask editor add-on API

* Signal completed mask editor image loading to Python

* Add API methods for querying mask editor state, fix formatting

* Use event forwarding to propagate image loaded event

Should fix mobile support by moving all bridgeCommand calls to `NoteEditor.svelte`

* Add shape classes to mask editor API

---------

Co-authored-by: Glutanimate <glutanimate@users.noreply.github.com>

qt/aqt/editor.py

@@ -493,6 +493,16 @@ require("anki/ui").loaded.then(() => require("anki/NoteEditor").instances[0].too
                 EditorState(int(new_state_id)), EditorState(int(old_state_id))
             )
 
+        elif cmd.startswith("ioImageLoaded"):
+            (_, path_or_nid_data) = cmd.split(":", 1)
+            path_or_nid = json.loads(path_or_nid_data)
+            if self.addMode:
+                gui_hooks.editor_mask_editor_did_load_image(self, path_or_nid)
+            else:
+                gui_hooks.editor_mask_editor_did_load_image(
+                    self, NoteId(int(path_or_nid))
+                )
+
         elif cmd in self._links:
             return self._links[cmd](self)
 

qt/tools/genhooks_gui.py

@@ -1155,6 +1155,15 @@ gui_hooks.webview_did_inject_style_into_page.append(mytest)
         doc="""Called when the input state of the editor changes, e.g. when
         switching to an image occlusion note type.""",
     ),
+    Hook(
+        name="editor_mask_editor_did_load_image",
+        args=["editor: aqt.editor.Editor", "path_or_nid: str | anki.notes.NoteId"],
+        doc="""Called when the image occlusion mask editor has completed
+        loading an image.
+        
+        When adding new notes `path_or_nid` will be the path to the image file.
+        When editing existing notes `path_or_nid` will be the note id.""",
+    ),
     # Tag
     ###################
     Hook(name="tag_editor_did_process_key", args=["tag_edit: TagEdit", "evt: QEvent"]),

ts/editor/NoteEditor.svelte

@@ -42,7 +42,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 <script lang="ts">
     import { bridgeCommand } from "@tslib/bridgecommand";
     import * as tr from "@tslib/ftl";
-    import { resetIOImage } from "image-occlusion/mask-editor";
+    import { type ImageLoadedEvent, resetIOImage } from "image-occlusion/mask-editor";
     import { onMount, tick } from "svelte";
     import { get, writable } from "svelte/store";
 
@@ -425,7 +425,13 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
             // new image is being added
             if (isIOImageLoaded) {
-                resetIOImage(options.mode.imagePath);
+                resetIOImage(options.mode.imagePath, (event: ImageLoadedEvent) =>
+                    onImageLoaded(
+                        new CustomEvent("image-loaded", {
+                            detail: event,
+                        }),
+                    ),
+                );
             }
         } else {
             const clozeNote = get(fieldStores[ioFields.occlusions]);
@@ -497,6 +503,14 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
         return false;
     }
 
+    // Signal image occlusion image loading to Python
+    function onImageLoaded(event: CustomEvent<ImageLoadedEvent>) {
+        const detail = event.detail;
+        bridgeCommand(
+            `ioImageLoaded:${JSON.stringify(detail.path || detail.noteId?.toString())}`,
+        );
+    }
+
     // Signal editor UI state changes to add-ons
 
     let editorState: EditorState = EditorState.Initial;
@@ -638,6 +652,7 @@ the AddCards dialog) should be implemented in the user of this component.
             <ImageOcclusionPage
                 mode={imageOcclusionMode}
                 on:change={updateIONoteInEditMode}
+                on:image-loaded={onImageLoaded}
             />
         </div>
     {/if}

ts/image-occlusion/ImageOcclusionPage.svelte

@@ -42,7 +42,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
     </div>
 
     <div hidden={activeTabValue != 1}>
-        <MasksEditor {mode} on:change />
+        <MasksEditor {mode} on:change on:image-loaded />
     </div>
 
     <div hidden={activeTabValue != 2}>

ts/image-occlusion/MaskEditor.svelte

@@ -19,11 +19,13 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
     import type { IOMode } from "./lib";
     import {
+        type ImageLoadedEvent,
         setCanvasZoomRatio,
         setupMaskEditor,
         setupMaskEditorForEdit,
     } from "./mask-editor";
     import Toolbar from "./Toolbar.svelte";
+    import { MaskEditorAPI } from "./tools/api";
 
     export let mode: IOMode;
     const iconSize = 80;
@@ -32,12 +34,20 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
     const startingTool = mode.kind === "add" ? "draw-rectangle" : "cursor";
     $: canvas = null;
 
+    $: {
+        globalThis.maskEditor = canvas ? new MaskEditorAPI(canvas) : null;
+    }
+
     const dispatch = createEventDispatcher();
 
     function onChange() {
         dispatch("change", { canvas });
     }
 
+    function onImageLoaded({ path, noteId }: ImageLoadedEvent) {
+        dispatch("image-loaded", { path, noteId });
+    }
+
     $: $changeSignal, onChange();
 
     function init(node) {
@@ -51,13 +61,17 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
         instance.pause();
 
         if (mode.kind == "add") {
-            setupMaskEditor(mode.imagePath, instance, onChange).then((canvas1) => {
-                canvas = canvas1;
-            });
+            setupMaskEditor(mode.imagePath, instance, onChange, onImageLoaded).then(
+                (canvas1) => {
+                    canvas = canvas1;
+                },
+            );
         } else {
-            setupMaskEditorForEdit(mode.noteId, instance, onChange).then((canvas1) => {
-                canvas = canvas1;
-            });
+            setupMaskEditorForEdit(mode.noteId, instance, onChange, onImageLoaded).then(
+                (canvas1) => {
+                    canvas = canvas1;
+                },
+            );
         }
     }
 

ts/image-occlusion/mask-editor.ts

@@ -16,10 +16,16 @@ import { enableSelectable, moveShapeToCanvasBoundaries } from "./tools/lib";
 import { undoStack } from "./tools/tool-undo-redo";
 import type { Size } from "./types";
 
+export interface ImageLoadedEvent {
+    path?: string;
+    noteId?: bigint;
+}
+
 export const setupMaskEditor = async (
     path: string,
     instance: PanZoom,
     onChange: () => void,
+    onImageLoaded: (event: ImageLoadedEvent) => void,
 ): Promise<fabric.Canvas> => {
     const imageData = await getImageForOcclusion({ path });
     const canvas = initCanvas(onChange);
@@ -35,6 +41,7 @@ export const setupMaskEditor = async (
         image.width = size.width;
         setCanvasZoomRatio(canvas, instance);
         undoStack.reset();
+        onImageLoaded({ path });
     };
 
     return canvas;
@@ -44,6 +51,7 @@ export const setupMaskEditorForEdit = async (
     noteId: number,
     instance: PanZoom,
     onChange: () => void,
+    onImageLoaded: (event: ImageLoadedEvent) => void,
 ): Promise<fabric.Canvas> => {
     const clozeNoteResponse = await getImageOcclusionNote({ noteId: BigInt(noteId) });
     const kind = clozeNoteResponse.value?.case;
@@ -79,6 +87,7 @@ export const setupMaskEditorForEdit = async (
         undoStack.reset();
         window.requestAnimationFrame(() => {
             image.style.visibility = "visible";
+            onImageLoaded({ noteId: BigInt(noteId) });
         });
     };
 
@@ -143,7 +152,7 @@ function containerSize(): Size {
     };
 }
 
-export async function resetIOImage(path) {
+export async function resetIOImage(path: string, onImageLoaded: (event: ImageLoadedEvent) => void) {
     const imageData = await getImageForOcclusion({ path });
     const image = document.getElementById("image") as HTMLImageElement;
     image.src = getImageData(imageData.data!);
@@ -158,6 +167,7 @@ export async function resetIOImage(path) {
         canvas.setHeight(size.height);
         image.height = size.height;
         image.width = size.width;
+        onImageLoaded({ path });
     };
 }
 globalThis.resetIOImage = resetIOImage;

ts/image-occlusion/shapes/index.ts

@@ -1,6 +1,7 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
+export type { ShapeOrShapes } from "./base";
 export { Shape } from "./base";
 export { Ellipse } from "./ellipse";
 export { extractShapesFromRenderedClozes } from "./from-cloze";

ts/image-occlusion/shapes/to-cloze.ts

@@ -34,7 +34,7 @@ export function exportShapesToClozeDeletions(occludeInactive: boolean): {
 /** Gather all Fabric shapes, and convert them into BaseShapes or
  * BaseShape[]s.
  */
-function baseShapesFromFabric(occludeInactive: boolean): ShapeOrShapes[] {
+export function baseShapesFromFabric(occludeInactive: boolean): ShapeOrShapes[] {
     const canvas = globalThis.canvas as Canvas;
     makeMaskTransparent(canvas, false);
     const activeObject = canvas.getActiveObject();

ts/image-occlusion/tools/add-from-cloze.ts

@@ -2,36 +2,22 @@
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
 import type { GetImageOcclusionNoteResponse_ImageOcclusion } from "@tslib/anki/image_occlusion_pb";
-import { fabric } from "fabric";
+import type { fabric } from "fabric";
 import { extractShapesFromClozedField } from "image-occlusion/shapes/from-cloze";
 
-import type { Size } from "../types";
-import { addBorder, disableRotation, enableUniformScaling } from "./lib";
+import { addShape, addShapeGroup } from "./from-shapes";
+import { redraw } from "./lib";
 
 export const addShapesToCanvasFromCloze = (
     canvas: fabric.Canvas,
     occlusions: GetImageOcclusionNoteResponse_ImageOcclusion[],
 ): void => {
-    const size: Size = canvas;
     for (const shapeOrShapes of extractShapesFromClozedField(occlusions)) {
         if (Array.isArray(shapeOrShapes)) {
-            const group = new fabric.Group();
-            shapeOrShapes.map((shape) => {
-                const fabricShape = shape.toFabric(size);
-                addBorder(fabricShape);
-                group.addWithUpdate(fabricShape);
-                disableRotation(group);
-            });
-            canvas.add(group);
+            addShapeGroup(canvas, shapeOrShapes);
         } else {
-            const shape = shapeOrShapes.toFabric(size);
-            addBorder(shape);
-            disableRotation(shape);
-            if (shape.type === "i-text") {
-                enableUniformScaling(canvas, shape);
-            }
-            canvas.add(shape);
+            addShape(canvas, shapeOrShapes);
         }
     }
-    canvas.requestRenderAll();
+    redraw(canvas);
 };

ts/image-occlusion/tools/api.ts

@@ -0,0 +1,54 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import type { fabric } from "fabric";
+import { baseShapesFromFabric, exportShapesToClozeDeletions } from "image-occlusion/shapes/to-cloze";
+
+import type { ShapeOrShapes } from "../shapes";
+import { Ellipse, Polygon, Rectangle, Shape, Text } from "../shapes";
+import { addShape, addShapeGroup } from "./from-shapes";
+import { clear, redraw } from "./lib";
+
+interface ClozeExportResult {
+    clozes: string;
+    cardCount: number;
+}
+
+export class MaskEditorAPI {
+    readonly Shape = Shape;
+    readonly Rectangle = Rectangle;
+    readonly Ellipse = Ellipse;
+    readonly Polygon = Polygon;
+    readonly Text = Text;
+
+    readonly canvas: fabric.Canvas;
+
+    constructor(canvas) {
+        this.canvas = canvas;
+    }
+
+    addShape(shape: Shape): void {
+        addShape(this.canvas, shape);
+    }
+
+    addShapeGroup(shapes: Shape[]): void {
+        addShapeGroup(this.canvas, shapes);
+    }
+
+    getClozes(occludeInactive: boolean): ClozeExportResult {
+        const { clozes, noteCount: cardCount } = exportShapesToClozeDeletions(occludeInactive);
+        return { clozes, cardCount };
+    }
+
+    getShapes(occludeInactive: boolean): ShapeOrShapes[] {
+        return baseShapesFromFabric(occludeInactive);
+    }
+
+    redraw(): void {
+        redraw(this.canvas);
+    }
+
+    clear(): void {
+        clear(this.canvas);
+    }
+}

ts/image-occlusion/tools/from-shapes.ts

@@ -0,0 +1,34 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import { fabric } from "fabric";
+import type { Shape } from "image-occlusion/shapes";
+
+import { addBorder, disableRotation, enableUniformScaling } from "./lib";
+
+export const addShape = (
+    canvas: fabric.Canvas,
+    shape: Shape,
+): void => {
+    const fabricShape = shape.toFabric(canvas);
+    addBorder(fabricShape);
+    disableRotation(fabricShape);
+    if (fabricShape.type === "i-text") {
+        enableUniformScaling(canvas, fabricShape);
+    }
+    canvas.add(fabricShape);
+};
+
+export const addShapeGroup = (
+    canvas: fabric.Canvas,
+    shapes: Shape[],
+): void => {
+    const group = new fabric.Group();
+    shapes.map((shape) => {
+        const fabricShape = shape.toFabric(canvas);
+        addBorder(fabricShape);
+        group.addWithUpdate(fabricShape);
+        disableRotation(group);
+    });
+    canvas.add(group);
+};

ts/image-occlusion/tools/lib.ts

@@ -60,7 +60,7 @@ export const groupShapes = (canvas: fabric.Canvas): void => {
     }
 
     canvas.getActiveObject().toGroup();
-    canvas.requestRenderAll();
+    redraw(canvas);
 };
 
 export const unGroupShapes = (canvas: fabric.Canvas): void => {
@@ -80,7 +80,7 @@ export const unGroupShapes = (canvas: fabric.Canvas): void => {
         canvas.add(item);
     });
 
-    canvas.requestRenderAll();
+    redraw(canvas);
 };
 
 export const zoomIn = (instance: PanZoom): void => {
@@ -137,7 +137,7 @@ const pasteItem = (canvas: fabric.Canvas): void => {
         _clipboard.top += 10;
         _clipboard.left += 10;
         canvas.setActiveObject(clonedObj);
-        canvas.requestRenderAll();
+        redraw(canvas);
     });
 };
 
@@ -258,3 +258,11 @@ export function enableUniformScaling(canvas: fabric.Canvas, obj: fabric.Object):
 export function addBorder(obj: fabric.Object): void {
     obj.stroke = BORDER_COLOR;
 }
+
+export const redraw = (canvas: fabric.Canvas): void => {
+    canvas.requestRenderAll();
+};
+
+export const clear = (canvas: fabric.Canvas): void => {
+    canvas.clear();
+};