import throttle from "lodash/throttle"; import { destroy, detach, flow, getEnv, getParent, getRoot, isAlive, onSnapshot, types } from "mobx-state-tree"; import { ff } from "@humansignal/core"; import { errorBuilder } from "../../core/DataValidator/ConfigValidator"; import { guidGenerator } from "../../core/Helpers"; import { Hotkey } from "../../core/Hotkey"; import TimeTraveller from "../../core/TimeTraveller"; import Tree, { TRAVERSE_STOP } from "../../core/Tree"; import Types from "../../core/Types"; import Area from "../../regions/Area"; import Result from "../../regions/Result"; import Utils from "../../utils"; import { FF_DEV_1284, FF_DEV_3391, FF_LLM_EPIC, FF_LSDV_3009, FF_LSDV_4583, FF_REVIEWER_FLOW, isFF, } from "../../utils/feature-flags"; import { delay, isDefined } from "../../utils/utilities"; import { CommentStore } from "../Comment/CommentStore"; import RegionStore from "../RegionStore"; import RelationStore from "../RelationStore"; import { UserExtended } from "../UserStore"; import { LinkingModes } from "./LinkingModes"; const hotkeys = Hotkey("Annotations", "Annotations"); /** * Omit value fields from the object. * * This should fix a problem with wrong region type detection caused by overlapping fields from a result and from an area. * The current problem is related to the text field of richtext region that could appear in the result for the textarea as well. * As these fields have the same name and different types it could make the region to be detected as a classification region instead of richtext, * and we may miss its displaying. * * For now, it is mostly related to the rich text region with per region textareas. * The problem may appear when we get wrong order of results for deserialization. * The other reason may be the omitted main result (We declare that all the data that we need to restore the main region is also contained in the per-region results). * @example Wrong order: * [{ * "id": "id_1", * "from_name": "comment", * "to_name": "text", * "type": "textarea", * "value": { * "start": 0, * "end": 11, * "text": ["A comment for the region], * }, * }, * { * "id": "id_1", * "from_name": "labels", * "to_name": "text", * "type": "labels", * "value": { * "start": 0, * "end": 11, * "labels": ["Label 1"], * "text": "Just a text", * }, * }] * * @example Omitted main result: * [{ * "id": "only_per_region_textarea", * "from_name": "comment", * "to_name": "text", * "type": "textarea", * "value": { * "start": 0, * "end": 11, * "labels": ["Label 1"], * "text": ["A comment for the region"], * }, * }] * * @param value {Object} object to fix * @returns {Object} new object without value fields */ const omitValueFields = ff.isActive(ff.FF_CUSTOM_TAGS) ? (value) => { // @todo describe that we only omit `text` from TextArea if (Array.isArray(value.text)) { const { text: _, ...newValue } = value; return newValue; } return value; } : (value) => { const newValue = { ...value }; Result.properties.value.propertyNames.forEach((propName) => { delete newValue[propName]; }); return newValue; }; const TrackedState = types.model("TrackedState", { areas: types.map(Area), relationStore: types.optional(RelationStore, {}), }); // Create a union type that can handle both user references and frozen user objects const UserOrReference = types.union({ dispatcher: (snapshot) => { // If it's a number, it's a reference to a user ID if (typeof snapshot === "number") { return types.safeReference(UserExtended); } // If it's a full user object, store it as frozen to avoid duplicate instances if (snapshot && typeof snapshot === "object" && (snapshot.firstName || snapshot.email || snapshot.username)) { return types.frozen(); } // Default to reference for any other case return types.safeReference(UserExtended); }, cases: { frozen: types.frozen(), reference: types.safeReference(UserExtended), }, }); const _Annotation = types .model("AnnotationBase", { id: types.identifier, // @todo this value used `guidGenerator(5)` as default value before // @todo but it calculates once, so all the annotations have the same pk // @todo why don't use only `id`? // @todo reverted back to wrong type; maybe it breaks all the deserialisation pk: types.maybeNull(types.string), selected: types.optional(types.boolean, false), type: types.enumeration(["annotation", "prediction", "history"]), createdDate: types.optional(types.string, Utils.UDate.currentISODate()), createdAgo: types.maybeNull(types.string), createdBy: types.optional(types.string, "Admin"), user: types.optional(types.maybeNull(UserOrReference), null), score: types.maybeNull(types.number), parent_prediction: types.maybeNull(types.integer), parent_annotation: types.maybeNull(types.integer), last_annotation_history: types.maybeNull(types.integer), comment_count: types.maybeNull(types.integer), unresolved_comment_count: types.maybeNull(types.integer), loadedDate: types.optional(types.Date, () => new Date()), leadTime: types.maybeNull(types.number), // @todo use types.Date draftSaved: types.maybe(types.string), // created by user during this session userGenerate: types.optional(types.boolean, true), sentUserGenerate: types.optional(types.boolean, false), localUpdate: types.optional(types.boolean, false), ground_truth: types.optional(types.boolean, false), skipped: false, // This field stores all data that affects undo/redo history // It should contain real objects to be able to work with them through snapshots // Annotation will use getters to get them at the top level // This data is never redefined directly, it's empty at the start trackedState: types.optional(TrackedState, {}), history: types.optional(TimeTraveller, { targetPath: "../trackedState" }), dragMode: types.optional(types.boolean, false), editable: types.optional(types.boolean, true), readonly: types.optional(types.boolean, false), suggestions: types.map(Area), regionStore: types.optional(RegionStore, { regions: [], }), isDrawing: types.optional(types.boolean, false), commentStore: types.optional(CommentStore, { comments: [], }), ...(isFF(FF_DEV_3391) ? { root: Types.allModelsTypes() } : {}), }) .views((self) => ({ get areas() { return self.trackedState.areas; }, get relationStore() { return self.trackedState.relationStore; }, })) .preProcessSnapshot((sn) => { // sn.draft = Boolean(sn.draft); const user = sn.user ?? sn.completed_by ?? undefined; let root; const updateIds = (item) => { const children = item.children?.map(updateIds); const imageEntities = item.imageEntities?.map(updateIds); let updatedItem = item; if (children) updatedItem = { ...updatedItem, children }; if (imageEntities) updatedItem = { ...updatedItem, imageEntities }; if (updatedItem.id) updatedItem = { ...updatedItem, id: `${updatedItem.name ?? updatedItem.id}@${sn.id}` }; // @todo fallback for tags with name as id: // if (item.name) item = { ...item, name: item.name + "@" + sn.id }; // @todo soon no such tags should left return updatedItem; }; if (isFF(FF_DEV_3391)) { root = updateIds(sn.root.toJSON()); } const getCreatedBy = (snapshot) => { if (snapshot.type === "prediction") { const modelVersion = snapshot.model_version?.trim() ?? ""; return modelVersion || "Admin"; } return snapshot.createdBy ?? "Admin"; }; const getCreatedAt = (snapshot) => { return snapshot.draft_created_at ?? snapshot.created_at ?? snapshot.createdDate; }; return { ...sn, ...(isFF(FF_DEV_3391) ? { root } : {}), user, editable: sn.editable ?? sn.type === "annotation", createdBy: getCreatedBy(sn), createdDate: getCreatedAt(sn), ground_truth: sn.honeypot ?? sn.ground_truth ?? false, skipped: sn.skipped || sn.was_cancelled, acceptedState: sn.accepted_state ?? sn.acceptedState ?? null, }; }) .views((self) => isFF(FF_DEV_3391) ? {} : { get root() { return self.list.root; }, get names() { return self.list.names; }, get toNames() { return self.list.toNames; }, }, ) .views((self) => ({ get store() { return getRoot(self); }, get list() { return getParent(self, 2); }, get objects() { // Without correct validation toname may be null for control tags so we need to check isObjectTag instead of it return Array.from(self.names.values()).filter((tag) => tag.isObjectTag); }, get regions() { return Array.from(self.areas.values()); }, get lastSelectedRegion() { return self.selectedRegions[self.selectedRegions.length - 1]; }, get results() { const results = []; if (isAlive(self)) self.areas.forEach((a) => a.results.forEach((r) => results.push(r))); return results; }, get serialized() { // Dirty hack to force MST track changes self.areas.toJSON(); return self.results .map((r) => r.serialize()) .filter(Boolean) .concat(self.relationStore.serialize()); }, get serializedSelection() { // Dirty hack to force MST track changes self.areas.toJSON(); const selectedResults = []; self.areas.forEach((a) => { if (!a.inSelection) return; a.results.forEach((r) => { selectedResults.push(r); }); }); return selectedResults.map((r) => r.serialize()).filter(Boolean); }, get highlightedNode() { return self.regionStore.selection.highlighted; }, get hasSelection() { return self.regionStore.hasSelection; }, get selectionSize() { return self.regionStore.selection.size; }, get selectedRegions() { return Array.from(self.regionStore.selection.selected.values()); }, get selectedDrawingRegions() { return Array.from(self.regionStore.selection.drawingSelected.values()); }, // existing annotation which can be updated get exists() { const dataExists = (self.userGenerate && self.sentUserGenerate) || isDefined(self.versions.result); const pkExists = isDefined(self.pk); return dataExists && pkExists; }, get hasSuggestionsSupport() { return self.objects.some((obj) => { return obj.supportSuggestions; }); }, get isNonEditableDraft() { const isKnownUsers = !!self.user && !!self.store.user; // If we do not know what user created draft // and who we are, then, we shouldn't prevent the ability to edit annotation // because we can't predict is it our draft or not. // It most probably could be relevant for standalone `lsf` if (!isKnownUsers) return false; // If there is no `pk` than there is no annotation in DataBase const isDraft = self.pk === null; const isNonEditable = self.user.id !== self.store.user.id; return isDraft && isNonEditable; }, isReadOnly() { return self.isNonEditableDraft || self.readonly || !self.editable; }, })) .volatile(() => ({ hidden: false, draftId: 0, draftSelected: false, autosaveDelay: 5000, isDraftSaving: false, // This flag indicates that we are accepting suggestions right now (an accepting is started and not finished yet) isSuggestionsAccepting: false, submissionStarted: 0, versions: {}, resultSnapshot: "", })) .volatile(() => isFF(FF_DEV_3391) ? { names: new Map(), toNames: new Map(), ids: new Map(), } : {}, ) .views((self) => ({ // experiment to display review buttons in Quick View get canBeReviewed() { const store = self.store; return ( isFF(FF_REVIEWER_FLOW) && // not a current user — we can only review others' annotations self.user?.email && store.user?.email !== self.user?.email && // we have this only in LSE getEnv(self).events.hasEvent("acceptAnnotation") && // Quick View — we don't have View All in Label Stream store.hasInterface("annotations:view-all") && // skipped annotations can't be reviewed !self.skipped && // annotation was submitted already !isNaN(self.pk) ); }, })) .actions((self) => ({ reinitHistory(force = true) { self.history.reinit(force); self.autosave?.cancel(); if (self.type === "annotation") self.setInitialValues(); }, setEditable(val) { self.editable = val; }, setReadonly(val) { self.readonly = val; }, setIsDrawing(isDrawing) { self.isDrawing = isDrawing; }, setUnresolvedCommentCount(val) { self.unresolved_comment_count = val; }, setCommentCount(val) { self.comment_count = val; }, setGroundTruth(value, ivokeEvent = true) { const root = getRoot(self); if (root && root !== self && ivokeEvent) { const as = root.annotationStore; const assignGroundTruths = (p) => { if (self !== p) p.setGroundTruth(false, false); }; as.predictions.forEach(assignGroundTruths); as.annotations.forEach(assignGroundTruths); } self.ground_truth = value; if (ivokeEvent) { getEnv(self).events.invoke("groundTruth", self.store, self, value); } }, sendUserGenerate() { self.sentUserGenerate = true; }, setLocalUpdate(value) { self.localUpdate = value; }, setDragMode(val) { self.dragMode = val; }, updatePersonalKey(value) { self.pk = value; getRoot(self).addAnnotationToTaskHistory?.(self.pk); }, toggleVisibility(visible) { self.hidden = visible === undefined ? !self.hidden : !visible; }, setHighlightedNode() { // moved to selectArea and others }, selectArea(area) { if (self.highlightedNode === area) return; // if (current) current.setSelected(false); self.regionStore.highlight(area); // area.setSelected(true); }, toggleRegionSelection(area, isSelected) { self.regionStore.toggleSelection(area, isSelected); }, selectAreas(areas) { self.unselectAreas(); self.extendSelectionWith(areas); }, extendSelectionWith(areas) { for (const area of Array.isArray(areas) ? areas : [areas]) { self.regionStore.toggleSelection(area, true); } }, unselectArea(area) { if (self.highlightedNode !== area) return; // area.setSelected(false); self.regionStore.toggleSelection(area, false); }, unselectAreas() { if (!self.selectionSize) return; self.regionStore.clearSelection(); }, lockSelectedRegions() { for (const region of self.selectedRegions) { region.setLocked(!region.locked); } }, hideSelectedRegions() { for (const region of self.selectedRegions) { region.toggleHidden(); } }, deleteSelectedRegions() { for (const region of self.selectedRegions) { region.deleteRegion(); } }, unselectStates() { self.names.forEach((tag) => tag.unselectAll?.()); }, /** * @param {boolean} tryToKeepStates don't unselect labels if such setting is enabled */ unselectAll(tryToKeepStates = false) { const keepStates = tryToKeepStates && self.store.settings.continuousLabeling; self.unselectAreas(); if (!keepStates) self.unselectStates(); }, removeArea(area) { destroy(area); }, deleteAllRegions({ deleteReadOnly = false } = {}) { let regions = Array.from(self.areas.values()); // remove everything unconditionally if (deleteReadOnly) { self.unselectAll(true); self.setIsDrawing(false); self.relationStore.deleteAllRelations(); for (const r of regions) { r.destroyRegion?.(); destroy(r); } self.updateObjects(); return; } if (deleteReadOnly === false) regions = regions.filter((r) => r.readonly === false); for (const r of regions) { r.deleteRegion(); } self.updateObjects(); }, addRegion(reg) { self.regionStore.unselectAll(true); if (self.isLinkingMode) { self.addLinkedRegion(reg); self.stopLinkingMode(); } }, validate() { let ok = true; self.traverseTree((node) => { ok = node.validate?.(); if (ok === false) { return TRAVERSE_STOP; } }); // should be true or false return ok ?? true; }, traverseTree(cb) { return Tree.traverseTree(self.root, cb); }, /** * */ beforeSend() { self.traverseTree((node) => { node?.beforeSend?.(); }); self.stopLinkingMode(); self.unselectAll(); }, /** * Delete region * @param {*} region */ deleteRegion(region) { if (region.isReadOnly()) return; const { regions } = self.regionStore; // move all children into the parent region of the given one const children = regions.filter((r) => r.parentID === region.id); if (children) { for (const r of children) { r.setParentID(region.parentID); } } if (!region.classification) getEnv(self).events.invoke("entityDelete", region); self.relationStore.deleteNodeRelation(region); if (region.type === "polygonregion" || region.type === "vectorregion") { detach(region); } destroy(region); // If the annotation was in a drawing state and the user deletes it, we need to reset the drawing state // to avoid the user being stuck in a drawing state self.setIsDrawing(false); }, deleteArea(area) { destroy(area); }, undo() { const { history, regionStore } = self; if (history?.canUndo) { let stopDrawingAfterNextUndo = false; const selectedIds = regionStore.selectedIds; const currentRegion = regionStore.findRegion( selectedIds[selectedIds.length - 1] ?? regionStore.regions[regionStore.regions.length - 1]?.id, ); if (currentRegion?.type === "polygonregion") { const points = currentRegion?.points?.length ?? 0; stopDrawingAfterNextUndo = points <= 1; } else if (currentRegion?.type === "vectorregion") { const vertices = currentRegion?.vertices?.length ?? 0; stopDrawingAfterNextUndo = vertices <= 1; } history.undo(); regionStore.selectRegionsByIds(selectedIds); if (stopDrawingAfterNextUndo) { currentRegion.setDrawing(false); self.setIsDrawing(false); } } }, redo() { const { history, regionStore } = self; if (history?.canRedo) { const selectedIds = regionStore.selectedIds; history.redo(); regionStore.selectRegionsByIds(selectedIds); } }, /** * update some fragile parts after snapshot manipulations (undo/redo) * * @param {boolean} [force=true] force update will unselect all regions */ updateObjects(force = true) { // Some async or lazy mode operations (ie. Images lazy load) need to reinitHistory without removing state selections if (force) self.unselectAll(); self.names.forEach((tag) => tag.needsUpdate?.()); self.updateAppearenceFromState(); const areas = Array.from(self.areas.values()); // It should find just one unfinished region, but just in case we work with array const filtered = areas.filter((area) => area.isDrawing); // Update UI to reflect the state of an unfinished region in case if it exists if (filtered.length) self.regionStore.selection._updateResultsFromRegions(filtered); }, updateAppearenceFromState() { self.areas.forEach((area) => area.updateAppearenceFromState?.()); }, setInitialValues() { //