From e32ef921bfe6436e9f553b92b1629b8b47e49413 Mon Sep 17 00:00:00 2001 From: sid597 Date: Wed, 19 Aug 2026 23:16:40 +0530 Subject: [PATCH 1/3] ENG-2154 Undecorate Obsidian node titles into core_title on publish --- apps/obsidian/src/utils/conceptConversion.ts | 6 ++++++ apps/obsidian/src/utils/syncDgNodesToSupabase.ts | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/obsidian/src/utils/conceptConversion.ts b/apps/obsidian/src/utils/conceptConversion.ts index 339311234..931ff36ab 100644 --- a/apps/obsidian/src/utils/conceptConversion.ts +++ b/apps/obsidian/src/utils/conceptConversion.ts @@ -10,6 +10,7 @@ import type { DiscourseNodeInVault } from "./getDiscourseNodes"; import type { LocalConceptDataInput } from "@repo/database/inputTypes"; import type { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase"; import type { Json } from "@repo/database/dbTypes"; +import { extractContentFromTitle } from "./extractContentFromTitle"; /** * Get extra data (author, timestamps) from file metadata @@ -160,12 +161,17 @@ export const discourseRelationTripleSchemaToLocalConcept = ({ export const discourseNodeInstanceToLocalConcept = ( context: SupabaseContext, nodeData: ObsidianDiscourseNodeData, + nodeTypesById: Record, ): LocalConceptDataInput => { const extraData = getNodeExtraData(nodeData.file, context.userId); const { nodeInstanceId, nodeTypeId, importedFromRid, ...otherData } = nodeData.frontmatter; const literal_content: Record = { label: nodeData.file.basename, + core_title: extractContentFromTitle( + nodeTypesById[nodeData.nodeTypeId]?.format ?? "", + nodeData.file.basename, + ), source_data: otherData as unknown as Json, }; if (importedFromRid && typeof importedFromRid === "string") diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index e8021f346..246a7b4f0 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -594,7 +594,7 @@ const convertDgToSupabaseConcepts = async ({ .filter((n) => !!n); const nodeInstanceToLocalConcepts = nodesSince.map((node) => { - return discourseNodeInstanceToLocalConcept(context, node); + return discourseNodeInstanceToLocalConcept(context, node, nodeTypesById); }); const relationInstancesData = await loadRelations(plugin); From 64fa0aef81ba07e42fb663c144ae168de1987e3e Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 23 Aug 2026 13:45:31 +0530 Subject: [PATCH 2/3] ENG-2155 Migrate existing database titles to undecorated core_title --- .../src/utils/syncDgNodesToSupabase.ts | 86 ++++++++++--- .../utils/__tests__/coreTitleBackfill.test.ts | 92 ++++++++++++++ apps/roam/src/utils/coreTitleBackfill.ts | 44 +++++++ apps/roam/src/utils/syncDgNodesToSupabase.ts | 114 ++++++++++++++---- .../lib/__tests__/coreTitleBackfill.test.ts | 42 +++++++ .../database/src/lib/coreTitleBackfill.ts | 23 ++++ 6 files changed, 364 insertions(+), 37 deletions(-) create mode 100644 apps/roam/src/utils/__tests__/coreTitleBackfill.test.ts create mode 100644 apps/roam/src/utils/coreTitleBackfill.ts create mode 100644 packages/database/src/lib/__tests__/coreTitleBackfill.test.ts create mode 100644 packages/database/src/lib/coreTitleBackfill.ts diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 246a7b4f0..b762c7373 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -30,6 +30,10 @@ import { isAcceptedSchema } from "./typeUtils"; import { getTemplatePluginInfo } from "./templates"; import { difference } from "@repo/utils/setOperations"; import { getAllPages } from "@repo/database/lib/pagination"; +import { + CORE_TITLE_PROBE_SELECT, + partitionByCoreTitle, +} from "@repo/database/lib/coreTitleBackfill"; const DEFAULT_TIME = "1970-01-01"; export type ChangeType = "title" | "content"; @@ -231,6 +235,17 @@ type BuildChangedNodesOptions = { fullSync?: boolean; }; +type CoreTitleBackfillCounts = { + backfilled: number; + skipped: number; + orphaned: number; +}; + +type BuildChangedNodesResult = { + changedNodes: ObsidianDiscourseNodeData[]; + coreTitleBackfill: CoreTitleBackfillCounts | null; +}; + const mergeChangeTypes = ( base: ChangeType[], additional: ChangeType[], @@ -320,15 +335,31 @@ const detectNodeChanges = ( return changeTypes; }; +const noticeCoreTitleBackfill = ({ + backfilled, + skipped, + orphaned, +}: CoreTitleBackfillCounts): void => { + if (backfilled === 0 && orphaned === 0) return; + const messages = [ + `Backfilled core title for ${backfilled} node${backfilled === 1 ? "" : "s"}.`, + `${skipped} already had one.`, + ]; + if (orphaned > 0) { + messages.push(`${orphaned} not found in this vault.`); + } + new Notice(messages.join(" "), 5000); +}; + const buildChangedNodesFromNodes = async ({ nodes, supabaseClient, context, changeTypesByPath, fullSync = false, -}: BuildChangedNodesOptions): Promise => { +}: BuildChangedNodesOptions): Promise => { if (nodes.length === 0) { - return []; + return { changedNodes: [], coreTitleBackfill: null }; } const nodeInstanceIds = nodes.map((node) => node.nodeInstanceId); @@ -344,11 +375,14 @@ const buildChangedNodesFromNodes = async ({ ); const changedNodes: ObsidianDiscourseNodeData[] = []; let missingConcepts: Set | undefined; + let coreTitleProbe: + | { missingCoreTitleIds: Set; skipped: number; orphaned: number } + | undefined; if (fullSync) { const existingConceptIds = await getAllPages( supabaseClient .from("my_concepts") - .select("source_local_id") + .select(CORE_TITLE_PROBE_SELECT) .eq("space_id", context.spaceId) .eq("is_relation", false) .eq("is_schema", false) @@ -369,6 +403,13 @@ const buildChangedNodesFromNodes = async ({ .filter((id) => id !== null), ); missingConcepts = difference(nodeIds, dbConceptIds); + const { missingCoreTitleIds, withCoreTitleCount } = + partitionByCoreTitle(existingConceptIds); + coreTitleProbe = { + missingCoreTitleIds, + skipped: withCoreTitleCount, + orphaned: difference(missingCoreTitleIds, nodeIds).size, + }; } } @@ -389,7 +430,8 @@ const buildChangedNodesFromNodes = async ({ if ( finalChangeTypes.length === 0 && - !missingConcepts?.has(node.nodeInstanceId) + !missingConcepts?.has(node.nodeInstanceId) && + !coreTitleProbe?.missingCoreTitleIds.has(node.nodeInstanceId) ) { continue; } @@ -405,7 +447,18 @@ const buildChangedNodesFromNodes = async ({ }); } - return changedNodes; + return { + changedNodes, + coreTitleBackfill: coreTitleProbe + ? { + backfilled: changedNodes.filter((node) => + coreTitleProbe.missingCoreTitleIds.has(node.nodeInstanceId), + ).length, + skipped: coreTitleProbe.skipped, + orphaned: coreTitleProbe.orphaned, + } + : null, + }; }; export const syncAllNodesAndRelations = async ( @@ -426,14 +479,15 @@ export const syncAllNodesAndRelations = async ( const allNodes = await collectDiscourseNodesFromVault(plugin, true); - const changedNodeInstances = relationsOnly - ? [] - : await buildChangedNodesFromNodes({ - nodes: allNodes, - supabaseClient, - context, - fullSync: true, - }); + const { changedNodes: changedNodeInstances, coreTitleBackfill } = + relationsOnly + ? { changedNodes: [], coreTitleBackfill: null } + : await buildChangedNodesFromNodes({ + nodes: allNodes, + supabaseClient, + context, + fullSync: true, + }); const accountLocalId = plugin.settings.accountLocalId; if (!accountLocalId) { @@ -458,6 +512,10 @@ export const syncAllNodesAndRelations = async ( fullSync: true, }); + if (coreTitleBackfill !== null) { + noticeCoreTitleBackfill(coreTitleBackfill); + } + // When synced nodes are already published, ensure non-text assets are in storage. await syncPublishedNodesAssets(plugin, changedNodeInstances); } catch (error) { @@ -935,7 +993,7 @@ export const syncDiscourseNodeChanges = async ( return; } - const changedNodes = await buildChangedNodesFromNodes({ + const { changedNodes } = await buildChangedNodesFromNodes({ nodes: dgNodesInVault, supabaseClient, context, diff --git a/apps/roam/src/utils/__tests__/coreTitleBackfill.test.ts b/apps/roam/src/utils/__tests__/coreTitleBackfill.test.ts new file mode 100644 index 000000000..b8e7a436a --- /dev/null +++ b/apps/roam/src/utils/__tests__/coreTitleBackfill.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import { + buildCoreTitleBackfill, + mergeNodesBySourceLocalId, +} from "../coreTitleBackfill"; +import type { RoamDiscourseNodeData } from "../getAllDiscourseNodesSince"; + +const node = (sourceLocalId: string): RoamDiscourseNodeData => ({ + author_local_id: "author", + author_name: "Author", + source_local_id: sourceLocalId, + created: "1", + last_modified: "2", + text: `CLM - ${sourceLocalId}`, + type: "claim-type", +}); + +describe("buildCoreTitleBackfill", () => { + it("forces in local nodes whose row has no core_title", () => { + const backfill = buildCoreTitleBackfill({ + conceptRows: [ + { source_local_id: "a", core_title: null }, + { source_local_id: "b", core_title: "already set" }, + ], + localNodes: [node("a"), node("b")], + }); + + expect(backfill.nodesToBackfill.map((n) => n.source_local_id)).toEqual([ + "a", + ]); + expect(backfill.withCoreTitleCount).toBe(1); + expect(backfill.orphanedIds).toEqual([]); + }); + + it("reports rows that are no longer in the graph as orphaned", () => { + const backfill = buildCoreTitleBackfill({ + conceptRows: [ + { source_local_id: "a", core_title: null }, + { source_local_id: "gone", core_title: null }, + ], + localNodes: [node("a")], + }); + + expect(backfill.nodesToBackfill.map((n) => n.source_local_id)).toEqual([ + "a", + ]); + expect(backfill.orphanedIds).toEqual(["gone"]); + }); + + it("skips rows without a source_local_id", () => { + const backfill = buildCoreTitleBackfill({ + conceptRows: [{ source_local_id: null, core_title: null }], + localNodes: [node("a")], + }); + + expect(backfill.nodesToBackfill).toEqual([]); + expect(backfill.withCoreTitleCount).toBe(0); + expect(backfill.orphanedIds).toEqual([]); + }); + + it("is a no-op once every row has a core_title", () => { + const backfill = buildCoreTitleBackfill({ + conceptRows: [ + { source_local_id: "a", core_title: "a" }, + { source_local_id: "b", core_title: "b" }, + ], + localNodes: [node("a"), node("b")], + }); + + expect(backfill.nodesToBackfill).toEqual([]); + expect(backfill.withCoreTitleCount).toBe(2); + expect(backfill.orphanedIds).toEqual([]); + }); +}); + +describe("mergeNodesBySourceLocalId", () => { + it("appends nodes that are not already in the batch", () => { + const merged = mergeNodesBySourceLocalId( + [node("a")], + [node("b"), node("c")], + ); + + expect(merged.map((n) => n.source_local_id)).toEqual(["a", "b", "c"]); + }); + + it("keeps the original node when both batches hold the same id", () => { + const original = node("a"); + const merged = mergeNodesBySourceLocalId([original], [node("a")]); + + expect(merged).toEqual([original]); + }); +}); diff --git a/apps/roam/src/utils/coreTitleBackfill.ts b/apps/roam/src/utils/coreTitleBackfill.ts new file mode 100644 index 000000000..3d6dfd93a --- /dev/null +++ b/apps/roam/src/utils/coreTitleBackfill.ts @@ -0,0 +1,44 @@ +import { difference } from "@repo/utils/setOperations"; +import { + partitionByCoreTitle, + type CoreTitleProbeRow, +} from "@repo/database/lib/coreTitleBackfill"; +import { type RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; + +export type CoreTitleBackfill = { + nodesToBackfill: RoamDiscourseNodeData[]; + withCoreTitleCount: number; + orphanedIds: string[]; +}; + +export const buildCoreTitleBackfill = ({ + conceptRows, + localNodes, +}: { + conceptRows: CoreTitleProbeRow[]; + localNodes: RoamDiscourseNodeData[]; +}): CoreTitleBackfill => { + const { missingCoreTitleIds, withCoreTitleCount } = + partitionByCoreTitle(conceptRows); + const localIds = new Set(localNodes.map((node) => node.source_local_id)); + return { + nodesToBackfill: localNodes.filter((node) => + missingCoreTitleIds.has(node.source_local_id), + ), + withCoreTitleCount, + orphanedIds: [...difference(missingCoreTitleIds, localIds)], + }; +}; + +export const mergeNodesBySourceLocalId = ( + nodes: RoamDiscourseNodeData[], + additionalNodes: RoamDiscourseNodeData[], +): RoamDiscourseNodeData[] => { + const nodesById = new Map(nodes.map((node) => [node.source_local_id, node])); + for (const node of additionalNodes) { + if (!nodesById.has(node.source_local_id)) { + nodesById.set(node.source_local_id, node); + } + } + return [...nodesById.values()]; +}; diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index cc0525355..f0cf630c2 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -26,6 +26,12 @@ import { } from "./convertRoamNodeToFullContent"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import { intersection } from "@repo/utils/setOperations"; +import { CORE_TITLE_PROBE_SELECT } from "@repo/database/lib/coreTitleBackfill"; +import { + buildCoreTitleBackfill, + mergeNodesBySourceLocalId, + type CoreTitleBackfill, +} from "./coreTitleBackfill"; import type { Json, Enums } from "@repo/database/dbTypes"; import { render as renderToast } from "roamjs-components/components/Toast"; import internalError from "~/utils/internalError"; @@ -870,6 +876,35 @@ export const setSyncActivity = (active: boolean) => { } }; +const reportCoreTitleBackfill = ({ + backfilled, + skipped, + orphaned, +}: { + backfilled: number; + skipped: number; + orphaned: number; +}): void => { + posthog.capture("Sync core_title backfill", { + backfilled, + skipped, + orphaned, + }); + if (backfilled === 0 && orphaned === 0) return; + const messages = [ + `Backfilled core title for ${backfilled} node${backfilled === 1 ? "" : "s"}.`, + `${skipped} already had one.`, + ]; + if (orphaned > 0) { + messages.push(`${orphaned} not found in this graph.`); + } + renderToast({ + id: "core-title-backfill", + intent: orphaned > 0 ? "warning" : "success", + content: messages.join(" "), + }); +}; + const getAllMissingOrNewDiscourseNodes = async ({ supabaseClient, spaceId, @@ -880,9 +915,12 @@ const getAllMissingOrNewDiscourseNodes = async ({ spaceId: number; since: number | undefined; nodeTypes: DiscourseNode[]; -}): Promise => { +}): Promise<{ + nodes: RoamDiscourseNodeData[]; + coreTitleBackfill: CoreTitleBackfill | null; +}> => { const allNodes = await getAllDiscourseNodesSince(undefined, nodeTypes); - if (since === undefined) return allNodes; + if (since === undefined) return { nodes: allNodes, coreTitleBackfill: null }; const newNodes = await getAllDiscourseNodesSince(since, nodeTypes); const existingContentIdsReq = await getAllPages( supabaseClient @@ -896,7 +934,7 @@ const getAllMissingOrNewDiscourseNodes = async ({ const existingConceptIdsReq = await getAllPages( supabaseClient .from("my_concepts") - .select("source_local_id") + .select(CORE_TITLE_PROBE_SELECT) .eq("space_id", spaceId) .eq("is_relation", false) .eq("is_schema", false) @@ -911,10 +949,16 @@ const getAllMissingOrNewDiscourseNodes = async ({ ), ...newNodes.map((n) => n.source_local_id), ]); - return [ - ...newNodes, - ...allNodes.filter((n) => !existingIds.has(n.source_local_id)), - ]; + return { + nodes: [ + ...newNodes, + ...allNodes.filter((n) => !existingIds.has(n.source_local_id)), + ], + coreTitleBackfill: buildCoreTitleBackfill({ + conceptRows: existingConceptIdsReq, + localNodes: allNodes, + }), + }; }; const getSharedNodeInstanceSourceLocalIds = async ({ @@ -1209,21 +1253,28 @@ export const createOrUpdateDiscourseEmbedding = async ( (n) => n.backedBy === "user", ); - const changedNodeInstances = await measureSyncPhase({ - phase: isInitialSync - ? "getAllMissingOrNewDiscourseNodes" - : "getAllDiscourseNodesSince", - phases, - operation: () => - isInitialSync - ? getAllMissingOrNewDiscourseNodes({ - supabaseClient: activeSupabaseClient, - spaceId: activeContext.spaceId, - since: sinceTime, - nodeTypes: allDgNodeTypes, - }) - : getAllDiscourseNodesSince(sinceTime, allDgNodeTypes), - }); + const { nodes: changedNodeInstances, coreTitleBackfill } = + await measureSyncPhase({ + phase: isInitialSync + ? "getAllMissingOrNewDiscourseNodes" + : "getAllDiscourseNodesSince", + phases, + operation: async () => + isInitialSync + ? getAllMissingOrNewDiscourseNodes({ + supabaseClient: activeSupabaseClient, + spaceId: activeContext.spaceId, + since: sinceTime, + nodeTypes: allDgNodeTypes, + }) + : { + nodes: await getAllDiscourseNodesSince( + sinceTime, + allDgNodeTypes, + ), + coreTitleBackfill: null, + }, + }); const sharedSourceLocalIds = await measureSyncPhase({ phase: "getSharedNodeInstanceSourceLocalIds", phases, @@ -1238,6 +1289,16 @@ export const createOrUpdateDiscourseEmbedding = async ( sharedSourceLocalIds.has(node.source_local_id), ) : changedNodeInstances; + const nodesToBackfillCoreTitle = ( + coreTitleBackfill?.nodesToBackfill ?? [] + ).filter( + (node) => + !sharedNodesOnlySync || sharedSourceLocalIds.has(node.source_local_id), + ); + const conceptNodesToSync = mergeNodesBySourceLocalId( + nodeInstancesToSync, + nodesToBackfillCoreTitle, + ); const sharedSourceLocalIdsToBackfill = await measureSyncPhase({ phase: "getSharedSourceLocalIdsMissingFullContent", phases, @@ -1316,7 +1377,7 @@ export const createOrUpdateDiscourseEmbedding = async ( phases, operation: () => convertDgToSupabaseConcepts({ - nodesSince: nodeInstancesToSync, + nodesSince: conceptNodesToSync, since: sinceTime, allNodeTypes: allDgNodeTypes, sharedNodeTypeIds, @@ -1324,6 +1385,13 @@ export const createOrUpdateDiscourseEmbedding = async ( context: activeContext, }), }); + if (coreTitleBackfill !== null) { + reportCoreTitleBackfill({ + backfilled: nodesToBackfillCoreTitle.length, + skipped: coreTitleBackfill.withCoreTitleCount, + orphaned: coreTitleBackfill.orphanedIds.length, + }); + } await measureSyncPhase({ phase: "cleanupOrphanedNodes", phases, diff --git a/packages/database/src/lib/__tests__/coreTitleBackfill.test.ts b/packages/database/src/lib/__tests__/coreTitleBackfill.test.ts new file mode 100644 index 000000000..6dfa6e33b --- /dev/null +++ b/packages/database/src/lib/__tests__/coreTitleBackfill.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { partitionByCoreTitle } from "../coreTitleBackfill"; + +describe("partitionByCoreTitle", () => { + it("treats a null core_title as missing", () => { + const { missingCoreTitleIds, withCoreTitleCount } = partitionByCoreTitle([ + { source_local_id: "a", core_title: null }, + { source_local_id: "b", core_title: "already set" }, + ]); + + expect([...missingCoreTitleIds]).toEqual(["a"]); + expect(withCoreTitleCount).toBe(1); + }); + + it("counts an empty core_title as present", () => { + const { missingCoreTitleIds, withCoreTitleCount } = partitionByCoreTitle([ + { source_local_id: "a", core_title: "" }, + ]); + + expect([...missingCoreTitleIds]).toEqual([]); + expect(withCoreTitleCount).toBe(1); + }); + + it("ignores rows without a source_local_id", () => { + const { missingCoreTitleIds, withCoreTitleCount } = partitionByCoreTitle([ + { source_local_id: null, core_title: null }, + { source_local_id: null, core_title: "set" }, + ]); + + expect([...missingCoreTitleIds]).toEqual([]); + expect(withCoreTitleCount).toBe(0); + }); + + it("returns nothing to backfill for an empty probe", () => { + const { missingCoreTitleIds, withCoreTitleCount } = partitionByCoreTitle( + [], + ); + + expect([...missingCoreTitleIds]).toEqual([]); + expect(withCoreTitleCount).toBe(0); + }); +}); diff --git a/packages/database/src/lib/coreTitleBackfill.ts b/packages/database/src/lib/coreTitleBackfill.ts new file mode 100644 index 000000000..c563983dc --- /dev/null +++ b/packages/database/src/lib/coreTitleBackfill.ts @@ -0,0 +1,23 @@ +export const CORE_TITLE_PROBE_SELECT = + "source_local_id, core_title:literal_content->>core_title"; + +export type CoreTitleProbeRow = { + source_local_id: string | null; + core_title: string | null; +}; + +export const partitionByCoreTitle = ( + rows: CoreTitleProbeRow[], +): { missingCoreTitleIds: Set; withCoreTitleCount: number } => { + const missingCoreTitleIds = new Set(); + let withCoreTitleCount = 0; + for (const row of rows) { + if (row.source_local_id === null) continue; + if (row.core_title === null) { + missingCoreTitleIds.add(row.source_local_id); + } else { + withCoreTitleCount += 1; + } + } + return { missingCoreTitleIds, withCoreTitleCount }; +}; From cc583e0d6817d6b097aabc0d390663a14d884a7f Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 23 Aug 2026 16:28:50 +0530 Subject: [PATCH 3/3] Keep author_id through the core_title backfill and tighten its report getNodeExtraData returned the Roam creator uid under author_uid, a key the concept_local_input type does not have, so every instance pushed by the sync lost its author_id on upsert; re-pushing every pre-core_title row would have blanked the authors publish had set. The key is now author_local_id, which _local_concept_to_db_concept resolves against PlatformAccount. The Roam toast now states how many rows wait for sync to be enabled instead of implying the pass was complete, names the leftover rows as no longer matching a discourse node, and carries the explicit 5s timeout its siblings use; the backfill result holds the leftover count rather than ids nobody reads. Obsidian's asset pass runs only for nodes with a detected file change, so backfill-only nodes cost no file-reference round-trips. --- .../src/utils/syncDgNodesToSupabase.ts | 9 +++++++-- .../utils/__tests__/coreTitleBackfill.test.ts | 8 ++++---- apps/roam/src/utils/conceptConversion.ts | 6 +++--- apps/roam/src/utils/coreTitleBackfill.ts | 4 ++-- apps/roam/src/utils/syncDgNodesToSupabase.ts | 18 +++++++++++++++--- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index b762c7373..a696c6d69 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -346,7 +346,9 @@ const noticeCoreTitleBackfill = ({ `${skipped} already had one.`, ]; if (orphaned > 0) { - messages.push(`${orphaned} not found in this vault.`); + messages.push( + `${orphaned} no longer match a discourse node in this vault.`, + ); } new Notice(messages.join(" "), 5000); }; @@ -517,7 +519,10 @@ export const syncAllNodesAndRelations = async ( } // When synced nodes are already published, ensure non-text assets are in storage. - await syncPublishedNodesAssets(plugin, changedNodeInstances); + await syncPublishedNodesAssets( + plugin, + changedNodeInstances.filter((node) => node.changeTypes.length > 0), + ); } catch (error) { console.error("syncAllNodesAndRelations: Process failed:", error); throw error; diff --git a/apps/roam/src/utils/__tests__/coreTitleBackfill.test.ts b/apps/roam/src/utils/__tests__/coreTitleBackfill.test.ts index b8e7a436a..59eeddb45 100644 --- a/apps/roam/src/utils/__tests__/coreTitleBackfill.test.ts +++ b/apps/roam/src/utils/__tests__/coreTitleBackfill.test.ts @@ -29,7 +29,7 @@ describe("buildCoreTitleBackfill", () => { "a", ]); expect(backfill.withCoreTitleCount).toBe(1); - expect(backfill.orphanedIds).toEqual([]); + expect(backfill.orphanedCount).toBe(0); }); it("reports rows that are no longer in the graph as orphaned", () => { @@ -44,7 +44,7 @@ describe("buildCoreTitleBackfill", () => { expect(backfill.nodesToBackfill.map((n) => n.source_local_id)).toEqual([ "a", ]); - expect(backfill.orphanedIds).toEqual(["gone"]); + expect(backfill.orphanedCount).toBe(1); }); it("skips rows without a source_local_id", () => { @@ -55,7 +55,7 @@ describe("buildCoreTitleBackfill", () => { expect(backfill.nodesToBackfill).toEqual([]); expect(backfill.withCoreTitleCount).toBe(0); - expect(backfill.orphanedIds).toEqual([]); + expect(backfill.orphanedCount).toBe(0); }); it("is a no-op once every row has a core_title", () => { @@ -69,7 +69,7 @@ describe("buildCoreTitleBackfill", () => { expect(backfill.nodesToBackfill).toEqual([]); expect(backfill.withCoreTitleCount).toBe(2); - expect(backfill.orphanedIds).toEqual([]); + expect(backfill.orphanedCount).toBe(0); }); }); diff --git a/apps/roam/src/utils/conceptConversion.ts b/apps/roam/src/utils/conceptConversion.ts index 35c8f76a5..47d02803a 100644 --- a/apps/roam/src/utils/conceptConversion.ts +++ b/apps/roam/src/utils/conceptConversion.ts @@ -13,7 +13,7 @@ import getPageTitleByPageUid from "roamjs-components/queries/getPageTitleByPageU const getNodeExtraData = ( node_uid: string, ): { - author_uid: string; + author_local_id: string; created: string; last_modified: string; page_uid: string; @@ -50,7 +50,7 @@ const getNodeExtraData = ( const created = new Date(created_t).toISOString(); const last_modified = new Date(last_modified_t).toISOString(); return { - author_uid, + author_local_id: author_uid, created, last_modified, page_uid, @@ -202,7 +202,7 @@ export const discourseRelationDataToLocalConcept = ( const created = new Date( Math.max(...nodeData.map((nd) => new Date(nd.created).getTime())), ).toISOString(); - const author_local_id: string = nodeData[0].author_uid; // take any one; again until I get the relation object + const author_local_id: string = nodeData[0].author_local_id; // take any one; again until I get the relation object return { space_id: context.spaceId, source_local_id: relationUid, diff --git a/apps/roam/src/utils/coreTitleBackfill.ts b/apps/roam/src/utils/coreTitleBackfill.ts index 3d6dfd93a..8794aee43 100644 --- a/apps/roam/src/utils/coreTitleBackfill.ts +++ b/apps/roam/src/utils/coreTitleBackfill.ts @@ -8,7 +8,7 @@ import { type RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; export type CoreTitleBackfill = { nodesToBackfill: RoamDiscourseNodeData[]; withCoreTitleCount: number; - orphanedIds: string[]; + orphanedCount: number; }; export const buildCoreTitleBackfill = ({ @@ -26,7 +26,7 @@ export const buildCoreTitleBackfill = ({ missingCoreTitleIds.has(node.source_local_id), ), withCoreTitleCount, - orphanedIds: [...difference(missingCoreTitleIds, localIds)], + orphanedCount: difference(missingCoreTitleIds, localIds).size, }; }; diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index f0cf630c2..044fd6b2a 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -878,30 +878,39 @@ export const setSyncActivity = (active: boolean) => { const reportCoreTitleBackfill = ({ backfilled, + deferred, skipped, orphaned, }: { backfilled: number; + deferred: number; skipped: number; orphaned: number; }): void => { posthog.capture("Sync core_title backfill", { backfilled, + deferred, skipped, orphaned, }); - if (backfilled === 0 && orphaned === 0) return; + if (backfilled === 0 && deferred === 0 && orphaned === 0) return; const messages = [ `Backfilled core title for ${backfilled} node${backfilled === 1 ? "" : "s"}.`, `${skipped} already had one.`, ]; + if (deferred > 0) { + messages.push(`${deferred} waiting for sync to be enabled.`); + } if (orphaned > 0) { - messages.push(`${orphaned} not found in this graph.`); + messages.push( + `${orphaned} no longer match a discourse node in this graph.`, + ); } renderToast({ id: "core-title-backfill", intent: orphaned > 0 ? "warning" : "success", content: messages.join(" "), + timeout: 5000, }); }; @@ -1388,8 +1397,11 @@ export const createOrUpdateDiscourseEmbedding = async ( if (coreTitleBackfill !== null) { reportCoreTitleBackfill({ backfilled: nodesToBackfillCoreTitle.length, + deferred: + coreTitleBackfill.nodesToBackfill.length - + nodesToBackfillCoreTitle.length, skipped: coreTitleBackfill.withCoreTitleCount, - orphaned: coreTitleBackfill.orphanedIds.length, + orphaned: coreTitleBackfill.orphanedCount, }); } await measureSyncPhase({