diff --git a/apps/roam/src/utils/__tests__/conceptConversion.test.ts b/apps/roam/src/utils/__tests__/conceptConversion.test.ts index cde435348..98420d07a 100644 --- a/apps/roam/src/utils/__tests__/conceptConversion.test.ts +++ b/apps/roam/src/utils/__tests__/conceptConversion.test.ts @@ -45,4 +45,10 @@ describe("discourseNodeSchemaToLocalConcept", () => { template: "* Evidence\n", }); }); + + it("carries the type author as author_local_id", () => { + stubRoamQuery(); + const concept = discourseNodeSchemaToLocalConcept(context, claimSchema); + expect(concept.author_local_id).toBe("author-1"); + }); }); diff --git a/apps/roam/src/utils/__tests__/schemaFormatBackfill.test.ts b/apps/roam/src/utils/__tests__/schemaFormatBackfill.test.ts new file mode 100644 index 000000000..1ec5aae52 --- /dev/null +++ b/apps/roam/src/utils/__tests__/schemaFormatBackfill.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import type { DiscourseNode } from "~/utils/getDiscourseNodes"; +import { buildSchemaFormatBackfill } from "~/utils/schemaFormatBackfill"; + +const nodeType = (type: string): DiscourseNode => ({ + type, + text: "Claim", + shortcut: "C", + specification: [], + backedBy: "user", + canvasSettings: {}, + format: "[[CLM]] - {content}", +}); + +describe("buildSchemaFormatBackfill", () => { + it("targets rows without a format that match a local node type", () => { + const result = buildSchemaFormatBackfill({ + conceptRows: [ + { source_local_id: "schema-1", format: null }, + { source_local_id: "schema-2", format: "[[QUE]] - {content}" }, + ], + nodeTypes: [nodeType("schema-1"), nodeType("schema-2")], + }); + expect(result.nodeTypeIdsToBackfill).toEqual(new Set(["schema-1"])); + expect(result.withFormatCount).toBe(1); + expect(result.orphanedCount).toBe(0); + }); + + it("treats an empty format as already set", () => { + const result = buildSchemaFormatBackfill({ + conceptRows: [{ source_local_id: "schema-1", format: "" }], + nodeTypes: [nodeType("schema-1")], + }); + expect(result.nodeTypeIdsToBackfill.size).toBe(0); + expect(result.withFormatCount).toBe(1); + }); + + it("reports rows with no matching local node type as orphaned", () => { + const result = buildSchemaFormatBackfill({ + conceptRows: [ + { source_local_id: "gone-schema", format: null }, + { source_local_id: "schema-1", format: null }, + ], + nodeTypes: [nodeType("schema-1")], + }); + expect(result.nodeTypeIdsToBackfill).toEqual(new Set(["schema-1"])); + expect(result.orphanedCount).toBe(1); + }); + + it("ignores rows without a source_local_id", () => { + const result = buildSchemaFormatBackfill({ + conceptRows: [{ source_local_id: null, format: null }], + nodeTypes: [nodeType("schema-1")], + }); + expect(result.nodeTypeIdsToBackfill.size).toBe(0); + expect(result.withFormatCount).toBe(0); + expect(result.orphanedCount).toBe(0); + }); + + it("returns zero counts for an empty probe", () => { + const result = buildSchemaFormatBackfill({ + conceptRows: [], + nodeTypes: [nodeType("schema-1")], + }); + expect(result.nodeTypeIdsToBackfill.size).toBe(0); + expect(result.withFormatCount).toBe(0); + expect(result.orphanedCount).toBe(0); + }); +}); diff --git a/apps/roam/src/utils/conceptConversion.ts b/apps/roam/src/utils/conceptConversion.ts index 8db580d6f..820d169f2 100644 --- a/apps/roam/src/utils/conceptConversion.ts +++ b/apps/roam/src/utils/conceptConversion.ts @@ -13,7 +13,7 @@ import getPageTitleByPageUid from "roamjs-components/queries/getPageTitleByPageU const getNodeExtraData = ( node_uid: string, ): { - author_uid: string; + author_local_id: string; created: string; last_modified: string; page_uid: string; @@ -50,7 +50,7 @@ const getNodeExtraData = ( const created = new Date(created_t).toISOString(); const last_modified = new Date(last_modified_t).toISOString(); return { - author_uid, + author_local_id: author_uid, created, last_modified, page_uid, @@ -196,7 +196,7 @@ export const discourseRelationDataToLocalConcept = ( const created = new Date( Math.max(...nodeData.map((nd) => new Date(nd.created).getTime())), ).toISOString(); - const author_local_id: string = nodeData[0].author_uid; // take any one; again until I get the relation object + const author_local_id: string = nodeData[0].author_local_id; // take any one; again until I get the relation object return { space_id: context.spaceId, source_local_id: relationUid, diff --git a/apps/roam/src/utils/schemaFormatBackfill.ts b/apps/roam/src/utils/schemaFormatBackfill.ts new file mode 100644 index 000000000..1088331fa --- /dev/null +++ b/apps/roam/src/utils/schemaFormatBackfill.ts @@ -0,0 +1,41 @@ +import { difference, intersection } from "@repo/utils/setOperations"; +import { type DiscourseNode } from "./getDiscourseNodes"; + +export const SCHEMA_FORMAT_PROBE_SELECT = + "source_local_id, format:literal_content->>format"; + +type SchemaFormatProbeRow = { + source_local_id: string | null; + format: string | null; +}; + +export type SchemaFormatBackfill = { + nodeTypeIdsToBackfill: Set; + withFormatCount: number; + orphanedCount: number; +}; + +export const buildSchemaFormatBackfill = ({ + conceptRows, + nodeTypes, +}: { + conceptRows: SchemaFormatProbeRow[]; + nodeTypes: DiscourseNode[]; +}): SchemaFormatBackfill => { + const missingFormatIds = new Set(); + let withFormatCount = 0; + for (const row of conceptRows) { + if (row.source_local_id === null) continue; + if (row.format === null) { + missingFormatIds.add(row.source_local_id); + } else { + withFormatCount += 1; + } + } + const localTypeIds = new Set(nodeTypes.map((nodeType) => nodeType.type)); + return { + nodeTypeIdsToBackfill: intersection(missingFormatIds, localTypeIds), + withFormatCount, + orphanedCount: difference(missingFormatIds, localTypeIds).size, + }; +}; diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index af20f4566..afbf085fa 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -25,6 +25,11 @@ import { } from "./convertRoamNodeToFullContent"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import { intersection } from "@repo/utils/setOperations"; +import { + buildSchemaFormatBackfill, + SCHEMA_FORMAT_PROBE_SELECT, + type SchemaFormatBackfill, +} from "./schemaFormatBackfill"; import type { Json, Enums } from "@repo/database/dbTypes"; import { render as renderToast } from "roamjs-components/components/Toast"; import internalError from "~/utils/internalError"; @@ -634,6 +639,7 @@ export const convertDgToSupabaseConcepts = async ({ since, allNodeTypes, sharedNodeTypeIds = new Set(), + backfillNodeTypeIds = new Set(), supabaseClient, context, }: { @@ -641,6 +647,7 @@ export const convertDgToSupabaseConcepts = async ({ since: number | undefined; allNodeTypes: DiscourseNode[]; sharedNodeTypeIds?: ReadonlySet; + backfillNodeTypeIds?: ReadonlySet; supabaseClient: DGSupabaseClient; context: SupabaseContext; }) => { @@ -650,7 +657,10 @@ export const convertDgToSupabaseConcepts = async ({ ); allNodeTypes.forEach((nodeType) => { - if (sharedNodeTypeIds.has(nodeType.type)) { + if ( + sharedNodeTypeIds.has(nodeType.type) || + backfillNodeTypeIds.has(nodeType.type) + ) { nodeTypesByUid.set(nodeType.type, nodeType); } }); @@ -1069,6 +1079,62 @@ const getSharedRoamNodesWithFullContentUpdatesSince = async ({ }); }; +const probeSchemaFormatBackfill = async ({ + supabaseClient, + spaceId, + nodeTypes, +}: { + supabaseClient: DGSupabaseClient; + spaceId: number; + nodeTypes: DiscourseNode[]; +}): Promise => { + const probeRows = await getAllPages( + supabaseClient + .from("my_concepts") + .select(SCHEMA_FORMAT_PROBE_SELECT) + .eq("space_id", spaceId) + .eq("is_schema", true) + .eq("is_relation", false) + .order("id"), + 1000, + ); + if (!Array.isArray(probeRows)) throw probeRows; + return buildSchemaFormatBackfill({ + conceptRows: probeRows, + nodeTypes, + }); +}; + +const reportSchemaFormatBackfill = ({ + backfilled, + skipped, + orphaned, +}: { + backfilled: number; + skipped: number; + orphaned: number; +}): void => { + posthog.capture("Sync schema format backfill", { + backfilled, + skipped, + orphaned, + }); + if (backfilled === 0 && orphaned === 0) return; + const messages = [ + `Backfilled format for ${backfilled} node type${backfilled === 1 ? "" : "s"}.`, + `${skipped} already had one.`, + ]; + if (orphaned > 0) { + messages.push(`${orphaned} no longer match a node type in this graph.`); + } + renderToast({ + id: "schema-format-backfill", + intent: orphaned > 0 ? "warning" : "success", + content: messages.join(" "), + timeout: 5000, + }); +}; + export const createOrUpdateDiscourseEmbedding = async ( showToast = false, ): Promise => { @@ -1202,6 +1268,19 @@ export const createOrUpdateDiscourseEmbedding = async ( (n) => n.backedBy === "user", ); + const schemaFormatBackfill = isInitialSync + ? await measureSyncPhase({ + phase: "probeSchemaFormatBackfill", + phases, + operation: () => + probeSchemaFormatBackfill({ + supabaseClient: activeSupabaseClient, + spaceId: activeContext.spaceId, + nodeTypes: allDgNodeTypes, + }), + }) + : null; + const changedNodeInstances = await measureSyncPhase({ phase: isInitialSync ? "getAllMissingOrNewDiscourseNodes" @@ -1313,10 +1392,18 @@ export const createOrUpdateDiscourseEmbedding = async ( since: sinceTime, allNodeTypes: allDgNodeTypes, sharedNodeTypeIds, + backfillNodeTypeIds: schemaFormatBackfill?.nodeTypeIdsToBackfill, supabaseClient: activeSupabaseClient, context: activeContext, }), }); + if (schemaFormatBackfill !== null) { + reportSchemaFormatBackfill({ + backfilled: schemaFormatBackfill.nodeTypeIdsToBackfill.size, + skipped: schemaFormatBackfill.withFormatCount, + orphaned: schemaFormatBackfill.orphanedCount, + }); + } await measureSyncPhase({ phase: "cleanupOrphanedNodes", phases,