fix: unblock CLI generator when local aaz repo misses command models or versions

docs/pages/usage/cli_generator.md

@@ -67,7 +67,7 @@ The **Unregistered** commands will be used for generate code only, but will not
 
 ### Remove commands
 
-When you unpick some commands in selection and regenerate the code, the unpicked commands will be removed.
+When you unpick some commands in selection and regenerate the code by **Generate All**, the unpicked commands will be removed. **Generate Edit Only** never deletes generated code, it only rewrites the commands you edited.
 
 ![remove_commands](../../assets/recordings/cli_generator/remove_commands.gif)
 
@@ -86,7 +86,14 @@ The sub folders of **aaz** in the module/extension represent each profiles. For
 
 ### Miss command models
 
-When you open a module/extension, you may encounter `Miss command groups in aaz:...` error. That's because in your module/extension, some commands generated but in your local `aaz` repo the related command models don't exist. It can be resolved by two ways:
+When you open a module/extension, some commands may be generated in your module/extension while the related command models don't exist in your local `aaz` repo, either the command model itself or the version generated in the module is missing. Commands whose model is missing are not displayed in the command tree, commands whose version is missing fall back to the latest version in your local `aaz` repo. A warning listing them is displayed in the generate dialog.
+
+Generation is not blocked:
+
+- **Generate Edit Only** keeps the code of those commands untouched.
+- **Generate All** regenerates the whole `aaz` folder from your local `aaz` repo, so the code of those commands is **deleted** or **regenerated with another version**.
+
+It can be resolved by two ways:
 
 #### Sync the latest change of `aaz` repo in upstream
 

src/aaz_dev/cli/controller/az_atomic_profile_builder.py

@@ -128,6 +128,8 @@ def _build_command_from_aaz(self, *names, version_name, load_cfg=True):
             if v.name == version_name:
                 version = v
                 break
+        if not version and not load_cfg and aaz_cmd.versions:
+            version = aaz_cmd.versions[0]
         if not version:
             raise ResourceNotFind("Version '{}' of command '{}' not exist in AAZ".format(version_name, ' '.join(names)))
 

src/aaz_dev/cli/controller/az_module_manager.py

@@ -71,10 +71,11 @@ def load_module(self, mod_name):
     def update_module(self, mod_name, profiles, **kwargs):
         aaz_folder = self.get_aaz_path(mod_name)
         generators = {}
-        atomic_builder = AzAtomicProfileBuilder(mod_name=mod_name, by_patch=kwargs.pop('by_patch', False))
+        by_patch = kwargs.pop('by_patch', False)
+        atomic_builder = AzAtomicProfileBuilder(mod_name=mod_name, by_patch=by_patch)
         for profile_name, profile in profiles.items():
             profile = atomic_builder(profile)
-            generators[profile_name] = AzProfileGenerator(aaz_folder, profile)
+            generators[profile_name] = AzProfileGenerator(aaz_folder, profile, by_patch=by_patch)
         for generator in generators.values():
             generator.generate()
         for generator in generators.values():

src/aaz_dev/cli/controller/az_profile_generator.py

@@ -11,10 +11,11 @@
 class AzProfileGenerator:
     """Used to generate atomic layer command group"""
 
-    def __init__(self, aaz_folder, profile):
+    def __init__(self, aaz_folder, profile, by_patch=False):
         self.aaz_folder = aaz_folder
         self.profile = profile
         self.profile_folder_name = profile.profile_folder_name
+        self._by_patch = by_patch
         self._removed_folders = set()
         self._removed_files = set()
         self._modified_files = {}
@@ -90,7 +91,8 @@ def _generate_by_command_group(self, profile_folder_name, command_group):
         for name in del_folders:
             self._delete_folder(profile_folder_name, *command_group_folder_names, name)
 
-        files = set()
+        # in patch mode files on disk are kept, so they must stay imported by __init__.py
+        files = set(cur_files) if self._by_patch else set()
         if command_group.commands:
             for command in command_group.commands.values():
                 assert command.names[:-1] == command_group.names, f"Invalid command name: {command.names}"
@@ -171,12 +173,16 @@ def _get_path(self, *names):
         return os.path.join(self.aaz_folder, *names)
 
     def _delete_folder(self, *names):
+        if self._by_patch:
+            return
         path = self._get_path(*names)
         if os.path.exists(path):
             assert os.path.isdir(path), f'Invalid folder path {path}'
             self._removed_folders.add(path)
 
     def _delete_file(self, *names):
+        if self._by_patch:
+            return
         path = self._get_path(*names)
         if os.path.exists(path):
             assert os.path.isfile(path), f'Invalid file path {path}'

src/aaz_dev/cli/tests/test_codegen_regressions.py

@@ -6,13 +6,57 @@
 from unittest.mock import patch
 
 from cli.api import _cmds
+from cli.controller.az_atomic_profile_builder import AzAtomicProfileBuilder
 from cli.controller.az_module_manager import AzModuleManager
+from cli.controller.az_profile_generator import AzProfileGenerator
 from swagger.model.specs import SwaggerSpecs, TypeSpecResourceProvider
 from utils.config import Config
 from utils.plane import PlaneEnum
 
 
 class CodegenRegressionTest(TestCase):
+    def test_patch_generation_preserves_files_missing_from_aaz(self):
+        with TemporaryDirectory() as folder:
+            group_folder = Path(folder) / "latest" / "test"
+            group_folder.mkdir(parents=True)
+            stale_file = group_folder / "_stale.py"
+            stale_file.write_text("stale\n", encoding="utf-8")
+            (group_folder / "_current.py").write_text("current\n", encoding="utf-8")
+            (group_folder / "__init__.py").write_text("", encoding="utf-8")
+            (group_folder / "__cmd_group.py").write_text("", encoding="utf-8")
+            profile = SimpleNamespace(profile_folder_name="latest", command_groups=None)
+            command_group = SimpleNamespace(
+                names=["test"],
+                command_groups=None,
+                commands={"current": SimpleNamespace(names=["test", "current"], cfg=None)},
+                wait_command=None,
+                register_info=None,
+                help=SimpleNamespace(short="Test", long=None),
+            )
+
+            generator = AzProfileGenerator(folder, profile, by_patch=True)
+            generator._generate_by_command_group("latest", command_group)
+            generator.save()
+
+            self.assertTrue(stale_file.exists())
+            self.assertIn("from ._stale import *", (group_folder / "__init__.py").read_text(encoding="utf-8"))
+
+    def test_patch_generation_accepts_removed_command_version(self):
+        with patch("cli.controller.az_atomic_profile_builder.AAZSpecsManager"):
+            builder = AzAtomicProfileBuilder("test", by_patch=True)
+        old_version = SimpleNamespace(
+            name="new", stage=None, examples=None,
+            resources=[SimpleNamespace(to_primitive=lambda: {"plane": PlaneEnum.Mgmt, "id": "/test"})],
+        )
+        builder._aaz_spec_manager = SimpleNamespace(find_command=lambda *args: SimpleNamespace(
+            versions=[old_version],
+            help=SimpleNamespace(short="Test", lines=None),
+        ))
+
+        command = builder._build_command_from_aaz("test", "show", version_name="old", load_cfg=False)
+
+        self.assertEqual(command.version, "new")
+
     def test_generate_rejects_incomplete_selections_before_updating_cli(self):
         resources = {"/test": {"v1": object()}}
         good = SimpleNamespace(name="Good", default_tag="v1", get_resource_map_by_tag=lambda _: resources)

src/web/src/__tests__/unit/cli/CLIModGeneratorProfileCommandTree.test.ts

@@ -1,13 +1,60 @@
 import { describe, it, expect } from "vitest";
 import {
   ProfileCommandTree,
+  collectMissingVersionsInAaz,
+  decodeProfileCTCommand,
   initializeCommandTreeByModView,
   exportModViewProfile,
 } from "../../../views/cli/utils/commandTreeInitialization";
 import { CLIModViewProfile } from "../../../views/cli/interfaces";
 import { CLISpecsSimpleCommandTree } from "../../../views/cli/components/CLIModuleGenerator";
 
 describe("CLIModGeneratorProfileCommandTree", () => {
+  it("should replace a removed selected version with the current aaz version", () => {
+    const command = decodeProfileCTCommand(
+      {
+        names: ["test", "show"],
+        help: { short: "Test" },
+        versions: [{ name: "new", stage: "Stable", resources: [] }],
+      },
+      true,
+      false,
+      true,
+      "old",
+    );
+
+    expect(command.selectedVersion).toBe("new");
+    expect(command.missingVersionInAaz).toBe("old");
+    expect(
+      collectMissingVersionsInAaz({
+        name: "test-profile",
+        commandGroups: {
+          test: { id: "test", names: ["test"], commands: { show: command }, loading: false, selected: true },
+        },
+      }),
+    ).toEqual(["az test show (old -> new)"]);
+  });
+
+  it("should keep a selected version that still exists in aaz", () => {
+    const command = decodeProfileCTCommand(
+      {
+        names: ["test", "show"],
+        help: { short: "Test" },
+        versions: [
+          { name: "new", stage: "Stable", resources: [] },
+          { name: "old", stage: "Stable", resources: [] },
+        ],
+      },
+      true,
+      false,
+      true,
+      "old",
+    );
+
+    expect(command.selectedVersion).toBe("old");
+    expect(command.missingVersionInAaz).toBeUndefined();
+  });
+
   describe("initializeCommandTreeByModView", () => {
     it("should initialize command tree with empty profile", () => {
       const profileName = "test-profile";
@@ -89,7 +136,7 @@ describe("CLIModGeneratorProfileCommandTree", () => {
       expect(result.commandGroups["test-group"].commands!["test-command"].registered).toBe(true);
     });
 
-    it("should throw error for missing command groups in aaz", () => {
+    it("should ignore command groups missing from aaz", () => {
       const profileName = "test-profile";
       const view: CLIModViewProfile = {
         name: "test-profile",
@@ -108,9 +155,8 @@ describe("CLIModGeneratorProfileCommandTree", () => {
         },
       };
 
-      expect(() => {
-        initializeCommandTreeByModView(profileName, view, simpleTree);
-      }).toThrow("Miss command groups in aaz: `az missing-group`");
+      expect(initializeCommandTreeByModView(profileName, view, simpleTree).commandGroups).toEqual({});
+      expect(initializeCommandTreeByModView(profileName, view, simpleTree).missingInAaz).toEqual(["az missing-group"]);
     });
   });
 

src/web/src/views/cli/components/CommandItem.tsx

@@ -74,6 +74,7 @@ const CommandItem: React.FC<CommandItemProps> = memo(({ command, onUpdateCommand
         return {
           ...oldCommand,
           selectedVersion: version,
+          missingVersionInAaz: undefined,
           modified: true,
         };
       });

src/web/src/views/cli/components/GenerateDialog.tsx

@@ -2,13 +2,19 @@ import { Alert, Button, Dialog, DialogActions, DialogContent, DialogTitle } from
 import { cliApi, errorHandlerApi } from "../../../services";
 import { useAsyncOperation } from "../../../services/hooks";
 import { AsyncOperationBanner } from "../../../components";
-import { exportModViewProfile, type ProfileCommandTree } from "../utils/commandTreeInitialization";
+import {
+  collectMissingVersionsInAaz,
+  exportModViewProfile,
+  type ProfileCommandTree,
+} from "../utils/commandTreeInitialization";
 import { type CLIModViewProfiles } from "../interfaces";
 
 interface ProfileCommandTrees {
   [name: string]: ProfileCommandTree;
 }
 
+const preview = (names: string[]) => names.slice(0, 3).join(", ") + (names.length > 3 ? ", ..." : "");
+
 interface GenerateDialogProps {
   repoName: string;
   moduleName: string;
@@ -63,13 +69,32 @@ const GenerateDialog = (props: GenerateDialogProps) => {
 
   const isLoading = updateAllOperation.loading || updateModifiedOperation.loading;
   const error = updateAllOperation.error || updateModifiedOperation.error;
+  const trees = Object.values(props.profileCommandTrees);
+  const missingInAaz = trees.flatMap((tree) => tree.missingInAaz ?? []);
+  const missingVersionsInAaz = trees.flatMap(collectMissingVersionsInAaz);
 
   return (
     <Dialog disableEscapeKeyDown open={props.open}>
       <DialogTitle>{!isLoading && `Generate CLI commands for ${props.moduleName} module?`}</DialogTitle>
       <DialogContent>
         <AsyncOperationBanner operation={updateAllOperation} />
         <AsyncOperationBanner operation={updateModifiedOperation} />
+        {!isLoading && (missingInAaz.length > 0 || missingVersionsInAaz.length > 0) && (
+          <Alert variant="outlined" severity="warning" sx={{ whiteSpace: "pre-line" }}>
+            {[
+              missingInAaz.length > 0 &&
+                `${missingInAaz.length} generated command(s)/group(s) have no command model in the local aaz repo, ` +
+                  `'Generate All' deletes their code: ${preview(missingInAaz)}`,
+              missingVersionsInAaz.length > 0 &&
+                `${missingVersionsInAaz.length} generated command(s) have no model of their version in the local aaz ` +
+                  `repo, 'Generate All' regenerates them with another version: ${preview(missingVersionsInAaz)}`,
+              "'Generate Edited Only' keeps them untouched.",
+              "See: https://azure.github.io/aaz-dev-tools/pages/usage/cli-generator/#miss-command-models.",
+            ]
+              .filter(Boolean)
+              .join("\n")}
+          </Alert>
+        )}
         {error && (
           <Alert variant="filled" severity="error">
             {" "}