ENG-2175 Backfill format on existing node type schema rows

apps/roam/src/utils/__tests__/conceptConversion.test.ts

@@ -45,4 +45,10 @@ describe("discourseNodeSchemaToLocalConcept", () => {
       template: "* Evidence\n",
     });
   });
+
+  it("carries the type author as author_local_id", () => {
+    stubRoamQuery();
+    const concept = discourseNodeSchemaToLocalConcept(context, claimSchema);
+    expect(concept.author_local_id).toBe("author-1");
+  });
 });

apps/roam/src/utils/__tests__/schemaFormatBackfill.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, it } from "vitest";
+import type { DiscourseNode } from "~/utils/getDiscourseNodes";
+import { buildSchemaFormatBackfill } from "~/utils/schemaFormatBackfill";
+
+const nodeType = (type: string): DiscourseNode => ({
+  type,
+  text: "Claim",
+  shortcut: "C",
+  specification: [],
+  backedBy: "user",
+  canvasSettings: {},
+  format: "[[CLM]] - {content}",
+});
+
+describe("buildSchemaFormatBackfill", () => {
+  it("targets rows without a format that match a local node type", () => {
+    const result = buildSchemaFormatBackfill({
+      conceptRows: [
+        { source_local_id: "schema-1", format: null },
+        { source_local_id: "schema-2", format: "[[QUE]] - {content}" },
+      ],
+      nodeTypes: [nodeType("schema-1"), nodeType("schema-2")],
+    });
+    expect(result.nodeTypeIdsToBackfill).toEqual(new Set(["schema-1"]));
+    expect(result.withFormatCount).toBe(1);
+    expect(result.orphanedCount).toBe(0);
+  });
+
+  it("treats an empty format as already set", () => {
+    const result = buildSchemaFormatBackfill({
+      conceptRows: [{ source_local_id: "schema-1", format: "" }],
+      nodeTypes: [nodeType("schema-1")],
+    });
+    expect(result.nodeTypeIdsToBackfill.size).toBe(0);
+    expect(result.withFormatCount).toBe(1);
+  });
+
+  it("reports rows with no matching local node type as orphaned", () => {
+    const result = buildSchemaFormatBackfill({
+      conceptRows: [
+        { source_local_id: "gone-schema", format: null },
+        { source_local_id: "schema-1", format: null },
+      ],
+      nodeTypes: [nodeType("schema-1")],
+    });
+    expect(result.nodeTypeIdsToBackfill).toEqual(new Set(["schema-1"]));
+    expect(result.orphanedCount).toBe(1);
+  });
+
+  it("ignores rows without a source_local_id", () => {
+    const result = buildSchemaFormatBackfill({
+      conceptRows: [{ source_local_id: null, format: null }],
+      nodeTypes: [nodeType("schema-1")],
+    });
+    expect(result.nodeTypeIdsToBackfill.size).toBe(0);
+    expect(result.withFormatCount).toBe(0);
+    expect(result.orphanedCount).toBe(0);
+  });
+
+  it("returns zero counts for an empty probe", () => {
+    const result = buildSchemaFormatBackfill({
+      conceptRows: [],
+      nodeTypes: [nodeType("schema-1")],
+    });
+    expect(result.nodeTypeIdsToBackfill.size).toBe(0);
+    expect(result.withFormatCount).toBe(0);
+    expect(result.orphanedCount).toBe(0);
+  });
+});

apps/roam/src/utils/conceptConversion.ts

@@ -13,7 +13,7 @@ import getPageTitleByPageUid from "roamjs-components/queries/getPageTitleByPageU
 const getNodeExtraData = (
   node_uid: string,
 ): {
-  author_uid: string;
+  author_local_id: string;
   created: string;
   last_modified: string;
   page_uid: string;
@@ -50,7 +50,7 @@ const getNodeExtraData = (
   const created = new Date(created_t).toISOString();
   const last_modified = new Date(last_modified_t).toISOString();
   return {
-    author_uid,
+    author_local_id: author_uid,
     created,
     last_modified,
     page_uid,
@@ -196,7 +196,7 @@ export const discourseRelationDataToLocalConcept = (
   const created = new Date(
     Math.max(...nodeData.map((nd) => new Date(nd.created).getTime())),
   ).toISOString();
-  const author_local_id: string = nodeData[0].author_uid; // take any one; again until I get the relation object
+  const author_local_id: string = nodeData[0].author_local_id; // take any one; again until I get the relation object
   return {
     space_id: context.spaceId,
     source_local_id: relationUid,

apps/roam/src/utils/schemaFormatBackfill.ts

@@ -0,0 +1,41 @@
+import { difference, intersection } from "@repo/utils/setOperations";
+import { type DiscourseNode } from "./getDiscourseNodes";
+
+export const SCHEMA_FORMAT_PROBE_SELECT =
+  "source_local_id, format:literal_content->>format";
+
+type SchemaFormatProbeRow = {
+  source_local_id: string | null;
+  format: string | null;
+};
+
+export type SchemaFormatBackfill = {
+  nodeTypeIdsToBackfill: Set<string>;
+  withFormatCount: number;
+  orphanedCount: number;
+};
+
+export const buildSchemaFormatBackfill = ({
+  conceptRows,
+  nodeTypes,
+}: {
+  conceptRows: SchemaFormatProbeRow[];
+  nodeTypes: DiscourseNode[];
+}): SchemaFormatBackfill => {
+  const missingFormatIds = new Set<string>();
+  let withFormatCount = 0;
+  for (const row of conceptRows) {
+    if (row.source_local_id === null) continue;
+    if (row.format === null) {
+      missingFormatIds.add(row.source_local_id);
+    } else {
+      withFormatCount += 1;
+    }
+  }
+  const localTypeIds = new Set(nodeTypes.map((nodeType) => nodeType.type));
+  return {
+    nodeTypeIdsToBackfill: intersection(missingFormatIds, localTypeIds),
+    withFormatCount,
+    orphanedCount: difference(missingFormatIds, localTypeIds).size,
+  };
+};

apps/roam/src/utils/syncDgNodesToSupabase.ts

@@ -25,6 +25,11 @@ import {
 } from "./convertRoamNodeToFullContent";
 import type { DGSupabaseClient } from "@repo/database/lib/client";
 import { intersection } from "@repo/utils/setOperations";
+import {
+  buildSchemaFormatBackfill,
+  SCHEMA_FORMAT_PROBE_SELECT,
+  type SchemaFormatBackfill,
+} from "./schemaFormatBackfill";
 import type { Json, Enums } from "@repo/database/dbTypes";
 import { render as renderToast } from "roamjs-components/components/Toast";
 import internalError from "~/utils/internalError";
@@ -634,13 +639,15 @@ export const convertDgToSupabaseConcepts = async ({
   since,
   allNodeTypes,
   sharedNodeTypeIds = new Set<string>(),
+  backfillNodeTypeIds = new Set<string>(),
   supabaseClient,
   context,
 }: {
   nodesSince: RoamDiscourseNodeData[];
   since: number | undefined;
   allNodeTypes: DiscourseNode[];
   sharedNodeTypeIds?: ReadonlySet<string>;
+  backfillNodeTypeIds?: ReadonlySet<string>;
   supabaseClient: DGSupabaseClient;
   context: SupabaseContext;
 }) => {
@@ -650,7 +657,10 @@ export const convertDgToSupabaseConcepts = async ({
   );
 
   allNodeTypes.forEach((nodeType) => {
-    if (sharedNodeTypeIds.has(nodeType.type)) {
+    if (
+      sharedNodeTypeIds.has(nodeType.type) ||
+      backfillNodeTypeIds.has(nodeType.type)
+    ) {
       nodeTypesByUid.set(nodeType.type, nodeType);
     }
   });
@@ -1069,6 +1079,62 @@ const getSharedRoamNodesWithFullContentUpdatesSince = async ({
   });
 };
 
+const probeSchemaFormatBackfill = async ({
+  supabaseClient,
+  spaceId,
+  nodeTypes,
+}: {
+  supabaseClient: DGSupabaseClient;
+  spaceId: number;
+  nodeTypes: DiscourseNode[];
+}): Promise<SchemaFormatBackfill> => {
+  const probeRows = await getAllPages(
+    supabaseClient
+      .from("my_concepts")
+      .select(SCHEMA_FORMAT_PROBE_SELECT)
+      .eq("space_id", spaceId)
+      .eq("is_schema", true)
+      .eq("is_relation", false)
+      .order("id"),
+    1000,
+  );
+  if (!Array.isArray(probeRows)) throw probeRows;
+  return buildSchemaFormatBackfill({
+    conceptRows: probeRows,
+    nodeTypes,
+  });
+};
+
+const reportSchemaFormatBackfill = ({
+  backfilled,
+  skipped,
+  orphaned,
+}: {
+  backfilled: number;
+  skipped: number;
+  orphaned: number;
+}): void => {
+  posthog.capture("Sync schema format backfill", {
+    backfilled,
+    skipped,
+    orphaned,
+  });
+  if (backfilled === 0 && orphaned === 0) return;
+  const messages = [
+    `Backfilled format for ${backfilled} node type${backfilled === 1 ? "" : "s"}.`,
+    `${skipped} already had one.`,
+  ];
+  if (orphaned > 0) {
+    messages.push(`${orphaned} no longer match a node type in this graph.`);
+  }
+  renderToast({
+    id: "schema-format-backfill",
+    intent: orphaned > 0 ? "warning" : "success",
+    content: messages.join(" "),
+    timeout: 5000,
+  });
+};
+
 export const createOrUpdateDiscourseEmbedding = async (
   showToast = false,
 ): Promise<void> => {
@@ -1202,6 +1268,19 @@ export const createOrUpdateDiscourseEmbedding = async (
       (n) => n.backedBy === "user",
     );
 
+    const schemaFormatBackfill = isInitialSync
+      ? await measureSyncPhase({
+          phase: "probeSchemaFormatBackfill",
+          phases,
+          operation: () =>
+            probeSchemaFormatBackfill({
+              supabaseClient: activeSupabaseClient,
+              spaceId: activeContext.spaceId,
+              nodeTypes: allDgNodeTypes,
+            }),
+        })
+      : null;
+
     const changedNodeInstances = await measureSyncPhase({
       phase: isInitialSync
         ? "getAllMissingOrNewDiscourseNodes"
@@ -1313,10 +1392,18 @@ export const createOrUpdateDiscourseEmbedding = async (
           since: sinceTime,
           allNodeTypes: allDgNodeTypes,
           sharedNodeTypeIds,
+          backfillNodeTypeIds: schemaFormatBackfill?.nodeTypeIdsToBackfill,
           supabaseClient: activeSupabaseClient,
           context: activeContext,
         }),
     });
+    if (schemaFormatBackfill !== null) {
+      reportSchemaFormatBackfill({
+        backfilled: schemaFormatBackfill.nodeTypeIdsToBackfill.size,
+        skipped: schemaFormatBackfill.withFormatCount,
+        orphaned: schemaFormatBackfill.orphanedCount,
+      });
+    }
     await measureSyncPhase({
       phase: "cleanupOrphanedNodes",
       phases,