[AKS] Container Insights azure monitor profile & OTLP GA

.gitignore

@@ -30,6 +30,7 @@ build/
 TestResults/
 test_failures.txt
 zone*export.txt
+report.*.json
 
 # Credentials
 credentials_real.json

linter_exclusions.yml

@@ -297,6 +297,39 @@ aks create:
     container_storage_version:
       rule_exclusions:
         - option_length_too_long
+    enable_azure_monitor_logs:
+      rule_exclusions:
+        - option_length_too_long
+    enable_prometheus_metrics_scraping:
+      rule_exclusions:
+        - option_length_too_long
+    disable_prometheus_metrics_scraping:
+      rule_exclusions:
+        - option_length_too_long
+    enable_opentelemetry_metrics:
+      rule_exclusions:
+        - option_length_too_long
+    disable_opentelemetry_metrics:
+      rule_exclusions:
+        - option_length_too_long
+    opentelemetry_metrics_port_http:
+      rule_exclusions:
+        - option_length_too_long
+    opentelemetry_metrics_port_grpc:
+      rule_exclusions:
+        - option_length_too_long
+    enable_opentelemetry_logs_traces:
+      rule_exclusions:
+        - option_length_too_long
+    disable_opentelemetry_logs_traces:
+      rule_exclusions:
+        - option_length_too_long
+    opentelemetry_logs_traces_port_http:
+      rule_exclusions:
+        - option_length_too_long
+    opentelemetry_logs_traces_port_grpc:
+      rule_exclusions:
+        - option_length_too_long
 aks enable-addons:
   parameters:
     workspace_resource_id:
@@ -386,6 +419,48 @@ aks update:
     container_storage_version:
       rule_exclusions:
         - option_length_too_long
+    enable_azure_monitor_logs:
+      rule_exclusions:
+        - option_length_too_long
+    disable_azure_monitor_logs:
+      rule_exclusions:
+        - option_length_too_long
+    enable_msi_auth_for_monitoring:
+      rule_exclusions:
+        - option_length_too_long
+    data_collection_settings:
+      rule_exclusions:
+        - option_length_too_long
+    enable_prometheus_metrics_scraping:
+      rule_exclusions:
+        - option_length_too_long
+    disable_prometheus_metrics_scraping:
+      rule_exclusions:
+        - option_length_too_long
+    enable_opentelemetry_metrics:
+      rule_exclusions:
+        - option_length_too_long
+    disable_opentelemetry_metrics:
+      rule_exclusions:
+        - option_length_too_long
+    opentelemetry_metrics_port_http:
+      rule_exclusions:
+        - option_length_too_long
+    opentelemetry_metrics_port_grpc:
+      rule_exclusions:
+        - option_length_too_long
+    enable_opentelemetry_logs_traces:
+      rule_exclusions:
+        - option_length_too_long
+    disable_opentelemetry_logs_traces:
+      rule_exclusions:
+        - option_length_too_long
+    opentelemetry_logs_traces_port_http:
+      rule_exclusions:
+        - option_length_too_long
+    opentelemetry_logs_traces_port_grpc:
+      rule_exclusions:
+        - option_length_too_long
 aks update-credentials:
   parameters:
     aad_server_app_secret:

src/azure-cli/HISTORY.rst

@@ -24,6 +24,24 @@ Release History
 * `az aks nodepool rollback`: Show an accurate warning when only the node OS upgrade channel is enabled (#33854)
 * Implement enable/disable flags for user-defined scheduler configuration (#33934)
 * `az aks update`: Fix Azure Container Storage configuration detection for lowercase and boolean extension settings (#33938)
+* `az aks create`, `az aks update`: Add `--enable-azure-monitor-logs` to onboard Container Insights through the Azure Monitor profile using managed identity authentication
+* `az aks update`: Add `--disable-azure-monitor-logs` to offboard Container Insights
+* `az aks create`, `az aks update`: Add `--syslog-port`, `--enable-prometheus-metrics-scraping` and `--disable-prometheus-metrics-scraping` to tune the Azure Monitor Container Insights configuration
+* `az aks create`, `az aks update`: Add `--enable-opentelemetry-metrics`, `--disable-opentelemetry-metrics`, `--opentelemetry-metrics-port-http` and `--opentelemetry-metrics-port-grpc` for the OpenTelemetry metrics receiver
+* `az aks create`, `az aks update`: Add `--enable-opentelemetry-logs-traces`, `--disable-opentelemetry-logs-traces`, `--opentelemetry-logs-traces-port-http` and `--opentelemetry-logs-traces-port-grpc` for the OpenTelemetry logs and traces receiver
+* `az aks create`, `az aks update`: Write container network logs to `azureMonitorProfile.containerInsights.containerNetworkLogs` instead of the monitoring addon configuration, and reject `--enable-container-network-logs` on clusters using legacy shared key authentication
+* `az aks create`, `az aks update`, `az aks enable-addons`: Deprecate `--enable-msi-auth-for-monitoring` in favor of `--enable-azure-monitor-logs`
+* `az aks create`, `az aks update`: Reject `--enable-azure-monitor-logs` on clusters using service principal authentication, since the Azure Monitor profile onboards with managed identity only
+* `az aks update`: Reject `--enable-azure-monitor-logs` when Azure Monitor logs is already enabled on the cluster, matching `az aks enable-addons -a monitoring`. Run `--disable-azure-monitor-logs` first to change the configuration
+* `az aks update`: `--disable-azure-monitor-logs` now removes the data collection rule association and resets the Container Insights settings (syslog port, Prometheus scraping and container network logs) back to their defaults, and asks for confirmation when OpenTelemetry logs and traces are enabled
+* `az aks update`: Fix `--enable-azure-monitor-logs` not creating the data collection rule and association unless the Log Analytics workspace changed, which left the agent running with no data collection rule attached so no logs were ingested
+* `az aks update`: Create the data collection rule and association before the cluster update when enabling with `--enable-azure-monitor-logs`, matching `az aks enable-addons -a monitoring`. Provisioning them afterwards meant the agent started before the association existed and then stayed idle for several minutes before restarting once the configuration arrived
+* `az aks update`: `--disable-azure-monitor-metrics` now also disables OpenTelemetry metrics, since they are collected through the managed Prometheus pipeline, and asks for confirmation first unless `--yes` is specified
+* `az aks update`: Fix the OpenTelemetry port flags (`--opentelemetry-metrics-port-http`, `--opentelemetry-metrics-port-grpc`, `--opentelemetry-logs-traces-port-http` and `--opentelemetry-logs-traces-port-grpc`) being silently ignored when supplied on their own to change a port on an already enabled receiver, and report an error instead of doing nothing when the matching receiver is not enabled or is being disabled in the same command
+* `az aks update`: Fix `--enable-syslog` updating only the cluster and never re-provisioning the data collection rule, which left the DCR without the syslog data source so no syslog was ingested
+* `az aks update`: Fix `--data-collection-settings` and `--ampls-resource-id` being silently ignored, as neither re-provisioned the data collection rule that carries them
+* `az aks update`: Collect every monitoring disable confirmation before any of them deletes collection resources. Combining `--disable-azure-monitor-metrics` with `--disable-azure-monitor-logs` used to delete the metrics collection resources before asking about logs, so declining that prompt aborted the command with metrics still enabled on the cluster but its data collection objects and recording rules already removed
+* `az aks create`, `az aks update`: Fix the `--data-collection-settings` size limit being applied to the file path instead of the settings it holds, which let an oversized file through to fail the data collection rule call with `Request Header Fields Too Large`
 
 **App Config**
 

src/azure-cli/azure/cli/command_modules/acs/_consts.py

@@ -164,6 +164,15 @@
 CONST_MONITORING_LOG_ANALYTICS_WORKSPACE_RESOURCE_ID = "logAnalyticsWorkspaceResourceID"
 CONST_MONITORING_USING_AAD_MSI_AUTH = "useAADAuth"
 
+# container network logs (azureMonitorProfile.containerInsights.containerNetworkLogs)
+CONST_CONTAINER_NETWORK_LOGS_ENABLED = "Enabled"
+CONST_CONTAINER_NETWORK_LOGS_DISABLED = "Disabled"
+# legacy omsagent addon config key, superseded by containerNetworkLogs on the Azure Monitor
+# profile path. Only read, to keep recognizing clusters onboarded before the switch.
+CONST_MONITORING_ENABLE_RETINA_NETWORK_FLAGS = "enableRetinaNetworkFlags"
+# server-side default for azureMonitorProfile.containerInsights.syslogPort
+CONST_CONTAINER_INSIGHTS_DEFAULT_SYSLOG_PORT = 28330
+
 # virtual node
 CONST_VIRTUAL_NODE_ADDON_NAME = "aciConnector"
 CONST_VIRTUAL_NODE_SUBNET_NAME = "SubnetName"

src/azure-cli/azure/cli/command_modules/acs/_help.py

@@ -334,6 +334,22 @@
   - name: --enable-high-log-scale-mode
     type: bool
     short-summary: Enable High Log Scale Mode for Container Logs. Auto-enabled when --enable-container-network-logs is specified.
+  - name: --enable-azure-monitor-logs
+    type: bool
+    short-summary: Enable Azure Monitor logs (Container Insights) for the cluster using managed identity authentication.
+    long-summary: |
+        Configures Container Insights through the cluster's Azure Monitor profile instead of the monitoring addon.
+        Cannot be combined with "--enable-addons monitoring" or with "--enable-msi-auth-for-monitoring".
+        Requires the cluster to use a managed identity; clusters created with service principal authentication are not supported.
+  - name: --syslog-port
+    type: int
+    short-summary: TCP port that the Azure Monitor agent listens on for syslog data. Requires --enable-azure-monitor-logs.
+  - name: --enable-prometheus-metrics-scraping
+    type: bool
+    short-summary: Enable Prometheus metrics scraping by the Azure Monitor agent. Requires --enable-azure-monitor-logs.
+  - name: --disable-prometheus-metrics-scraping
+    type: bool
+    short-summary: Disable Prometheus metrics scraping by the Azure Monitor agent. Requires --enable-azure-monitor-logs.
   - name: --sku
     type: string
     short-summary: Specify SKU name for managed clusters. Use '--sku base' enables a base managed cluster. Use '--sku automatic' enables an automatic managed cluster.
@@ -568,6 +584,30 @@
   - name: --enable-azure-monitor-app-monitoring
     type: bool
     short-summary: Enable Azure Monitor Application Monitoring auto-instrumentation for a Kubernetes cluster.
+  - name: --enable-opentelemetry-metrics
+    type: bool
+    short-summary: Enable the OpenTelemetry (OTLP) metrics receiver. Requires --enable-azure-monitor-metrics.
+  - name: --disable-opentelemetry-metrics
+    type: bool
+    short-summary: Disable the OpenTelemetry (OTLP) metrics receiver.
+  - name: --opentelemetry-metrics-port-http
+    type: int
+    short-summary: HTTP/protobuf port for the OpenTelemetry metrics receiver.
+  - name: --opentelemetry-metrics-port-grpc
+    type: int
+    short-summary: gRPC port for the OpenTelemetry metrics receiver.
+  - name: --enable-opentelemetry-logs-traces
+    type: bool
+    short-summary: Enable the OpenTelemetry (OTLP) logs and traces receiver. Requires --enable-azure-monitor-logs.
+  - name: --disable-opentelemetry-logs-traces
+    type: bool
+    short-summary: Disable the OpenTelemetry (OTLP) logs and traces receiver.
+  - name: --opentelemetry-logs-traces-port-http
+    type: int
+    short-summary: HTTP/protobuf port for the OpenTelemetry logs and traces receiver.
+  - name: --opentelemetry-logs-traces-port-grpc
+    type: int
+    short-summary: gRPC port for the OpenTelemetry logs and traces receiver.
   - name: --nodepool-taints
     type: string
     short-summary: The node taints for all node pool.
@@ -756,6 +796,14 @@
     text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-keda
   - name: Create a kubernetes cluster with the Azure Monitor managed service for Prometheus integration enabled.
     text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-azure-monitor-metrics
+  - name: Create a kubernetes cluster with Azure Monitor logs (Container Insights) enabled.
+    text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-azure-monitor-logs
+  - name: Create a kubernetes cluster with Azure Monitor logs enabled and syslog collected on a custom port.
+    text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-azure-monitor-logs --enable-syslog --syslog-port 28330
+  - name: Create a kubernetes cluster with the OpenTelemetry logs and traces receiver enabled.
+    text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-azure-monitor-logs --enable-opentelemetry-logs-traces --opentelemetry-logs-traces-port-grpc 4317
+  - name: Create a kubernetes cluster with the OpenTelemetry metrics receiver enabled.
+    text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-azure-monitor-metrics --enable-opentelemetry-metrics --opentelemetry-metrics-port-grpc 4319
   - name: Create a kubernetes cluster with vertical pod autoscaler enaled.
     text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-vpa
   - name: create a kubernetes cluster with a Capacity Reservation Group(CRG) ID.
@@ -1116,6 +1164,10 @@
   - name: --disable-azure-monitor-metrics
     type: bool
     short-summary: Disable Azure Monitor Metrics Profile. This will delete all DCRA's associated with the cluster, any linked DCRs with the data stream = prometheus-stream and the recording rule groups created by the addon for this AKS cluster.
+    long-summary: |
+        If OpenTelemetry metrics are enabled, they are disabled as well, since they are collected
+        through the managed Prometheus pipeline. Confirmation is requested first unless "--yes" is
+        specified.
   - name: --enable-control-plane-metrics --enable-cp-metrics
     type: bool
     short-summary: Enable collection of Azure Monitor managed Prometheus control plane metrics for managed cluster components (controlplane-apiserver and controlplane-etcd targets by default). Requires Azure Monitor metrics to be enabled (already enabled or via --enable-azure-monitor-metrics).
@@ -1128,6 +1180,68 @@
   - name: --disable-azure-monitor-app-monitoring
     type: bool
     short-summary: Disable Azure Monitor Application Monitoring auto-instrumentation for a Kubernetes cluster.
+  - name: --enable-azure-monitor-logs
+    type: bool
+    short-summary: Enable Azure Monitor logs (Container Insights) for the cluster using managed identity authentication.
+    long-summary: |
+        Configures Container Insights through the cluster's Azure Monitor profile instead of the monitoring addon.
+        Clusters still using legacy shared key authentication must first migrate to managed identity authentication.
+        Requires the cluster to use a managed identity; clusters using service principal authentication are not supported.
+        Fails if Azure Monitor logs is already enabled on the cluster. To change the configuration, run "az aks update --disable-azure-monitor-logs" first.
+  - name: --disable-azure-monitor-logs
+    type: bool
+    short-summary: Disable Azure Monitor logs (Container Insights) for the cluster.
+    long-summary: |
+        Disables Container Insights, removes the data collection rule association, and resets the Container Insights settings (syslog port, Prometheus scraping and container network logs) back to their defaults. The workspace is left recorded on the profile but is unused while disabled, and is replaced on the next enable.
+        If OpenTelemetry logs and traces are enabled they are disabled as well, and confirmation is requested first unless "--yes" is specified.
+  - name: --workspace-resource-id
+    type: string
+    short-summary: The resource ID of an existing Log Analytics Workspace to use for storing monitoring data. If not specified, uses the default Log Analytics Workspace if it exists, otherwise creates one.
+  - name: --enable-msi-auth-for-monitoring
+    type: bool
+    short-summary: Enable Managed Identity Auth for Monitoring addon.
+  - name: --enable-syslog
+    type: bool
+    short-summary: Enable syslog data collection for Monitoring addon.
+  - name: --data-collection-settings
+    type: string
+    short-summary: Path to JSON file containing data collection settings for Monitoring addon.
+  - name: --ampls-resource-id
+    type: string
+    short-summary: Resource ID of Azure Monitor Private Link scope for Monitoring Addon.
+  - name: --syslog-port
+    type: int
+    short-summary: TCP port that the Azure Monitor agent listens on for syslog data. Requires Azure Monitor logs to be enabled.
+  - name: --enable-prometheus-metrics-scraping
+    type: bool
+    short-summary: Enable Prometheus metrics scraping by the Azure Monitor agent. Requires Azure Monitor logs to be enabled.
+  - name: --disable-prometheus-metrics-scraping
+    type: bool
+    short-summary: Disable Prometheus metrics scraping by the Azure Monitor agent. Requires Azure Monitor logs to be enabled.
+  - name: --enable-opentelemetry-metrics
+    type: bool
+    short-summary: Enable the OpenTelemetry (OTLP) metrics receiver. Requires Azure Monitor metrics to be enabled.
+  - name: --disable-opentelemetry-metrics
+    type: bool
+    short-summary: Disable the OpenTelemetry (OTLP) metrics receiver.
+  - name: --opentelemetry-metrics-port-http
+    type: int
+    short-summary: HTTP/protobuf port for the OpenTelemetry metrics receiver.
+  - name: --opentelemetry-metrics-port-grpc
+    type: int
+    short-summary: gRPC port for the OpenTelemetry metrics receiver.
+  - name: --enable-opentelemetry-logs-traces
+    type: bool
+    short-summary: Enable the OpenTelemetry (OTLP) logs and traces receiver. Requires Azure Monitor logs to be enabled.
+  - name: --disable-opentelemetry-logs-traces
+    type: bool
+    short-summary: Disable the OpenTelemetry (OTLP) logs and traces receiver.
+  - name: --opentelemetry-logs-traces-port-http
+    type: int
+    short-summary: HTTP/protobuf port for the OpenTelemetry logs and traces receiver.
+  - name: --opentelemetry-logs-traces-port-grpc
+    type: int
+    short-summary: gRPC port for the OpenTelemetry logs and traces receiver.
   - name: --nodepool-taints
     type: string
     short-summary: The node taints for all node pool.
@@ -1255,6 +1369,16 @@
 examples:
   - name: Reconcile the cluster back to its current state.
     text: az aks update -g MyResourceGroup -n MyManagedCluster
+  - name: Enable Azure Monitor logs (Container Insights) on an existing cluster.
+    text: az aks update -g MyResourceGroup -n MyManagedCluster --enable-azure-monitor-logs
+  - name: Disable Azure Monitor logs (Container Insights) on an existing cluster.
+    text: az aks update -g MyResourceGroup -n MyManagedCluster --disable-azure-monitor-logs
+  - name: Change the syslog port used by Azure Monitor logs on an existing cluster.
+    text: az aks update -g MyResourceGroup -n MyManagedCluster --enable-syslog --syslog-port 28330
+  - name: Enable the OpenTelemetry logs and traces receiver on an existing cluster.
+    text: az aks update -g MyResourceGroup -n MyManagedCluster --enable-opentelemetry-logs-traces --opentelemetry-logs-traces-port-grpc 4317
+  - name: Enable the OpenTelemetry metrics receiver on an existing cluster.
+    text: az aks update -g MyResourceGroup -n MyManagedCluster --enable-opentelemetry-metrics --opentelemetry-metrics-port-grpc 4319
   - name: Update a kubernetes cluster with standard SKU load balancer to use two AKS created IPs for the load balancer outbound connection usage.
     text: az aks update -g MyResourceGroup -n MyManagedCluster --load-balancer-managed-outbound-ip-count 2
   - name: Update a kubernetes cluster with standard SKU load balancer to use the provided public IPs for the load balancer outbound connection usage.