From 072f528f23fae9f6321c676cfe268d0cd5cf6b81 Mon Sep 17 00:00:00 2001 From: Thomas Cooper Date: Mon, 3 Aug 2026 16:56:31 +0100 Subject: [PATCH] Add StreamsHub MCP overlays * Add StreamsHub MCP component using upstream kustomization base * Add overlays for core + mcp and core + metrics + mcp * Add docs for the new overlays * Update install and other scripts for new overlays Assisted-By: Claude Opus 4.6 (1M context) Signed-off-by: Thomas Cooper --- .../mcp/stack/kafka-access/kustomization.yaml | 13 +++ components/mcp/stack/kustomization.yaml | 6 + .../mcp/stack/strimzi-mcp/kustomization.yaml | 14 +++ docs/overlays/mcp-metrics.md | 106 ++++++++++++++++++ docs/overlays/mcp.md | 101 +++++++++++++++++ docs/prerequisites.md | 4 +- install.sh | 51 +++++++-- overlays/mcp-metrics/base/kustomization.yaml | 5 + overlays/mcp-metrics/stack/kustomization.yaml | 18 +++ .../mcp-metrics/stack/mcp-podmonitor.yaml | 18 +++ .../stack/mcp-prometheus-patch.yaml | 15 +++ overlays/mcp/base/kustomization.yaml | 5 + overlays/mcp/stack/kustomization.yaml | 6 + uninstall.sh | 2 +- update-version.sh | 17 ++- 15 files changed, 369 insertions(+), 12 deletions(-) create mode 100644 components/mcp/stack/kafka-access/kustomization.yaml create mode 100644 components/mcp/stack/kustomization.yaml create mode 100644 components/mcp/stack/strimzi-mcp/kustomization.yaml create mode 100644 docs/overlays/mcp-metrics.md create mode 100644 docs/overlays/mcp.md create mode 100644 overlays/mcp-metrics/base/kustomization.yaml create mode 100644 overlays/mcp-metrics/stack/kustomization.yaml create mode 100644 overlays/mcp-metrics/stack/mcp-podmonitor.yaml create mode 100644 overlays/mcp-metrics/stack/mcp-prometheus-patch.yaml create mode 100644 overlays/mcp/base/kustomization.yaml create mode 100644 overlays/mcp/stack/kustomization.yaml diff --git a/components/mcp/stack/kafka-access/kustomization.yaml b/components/mcp/stack/kafka-access/kustomization.yaml new file mode 100644 index 0000000..39d5152 --- /dev/null +++ b/components/mcp/stack/kafka-access/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: kafka + +resources: + - https://raw.githubusercontent.com/streamshub/streamshub-mcp/v0.1.0/install/strimzi-mcp/optional/role-sensitive.yaml + - https://raw.githubusercontent.com/streamshub/streamshub-mcp/v0.1.0/install/strimzi-mcp/optional/rolebinding-sensitive.yaml + +labels: + - pairs: + app.kubernetes.io/part-of: streamshub-developer-quickstart + includeSelectors: false diff --git a/components/mcp/stack/kustomization.yaml b/components/mcp/stack/kustomization.yaml new file mode 100644 index 0000000..7b05ec1 --- /dev/null +++ b/components/mcp/stack/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - strimzi-mcp + - kafka-access diff --git a/components/mcp/stack/strimzi-mcp/kustomization.yaml b/components/mcp/stack/strimzi-mcp/kustomization.yaml new file mode 100644 index 0000000..fea60b8 --- /dev/null +++ b/components/mcp/stack/strimzi-mcp/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - https://github.com/streamshub/streamshub-mcp//install/strimzi-mcp/base?ref=v0.1.0 + +labels: + - pairs: + app.kubernetes.io/part-of: streamshub-developer-quickstart + includeSelectors: false + +images: + - name: quay.io/streamshub/strimzi-mcp + newTag: "0.1.0" diff --git a/docs/overlays/mcp-metrics.md b/docs/overlays/mcp-metrics.md new file mode 100644 index 0000000..d0db397 --- /dev/null +++ b/docs/overlays/mcp-metrics.md @@ -0,0 +1,106 @@ ++++ +title = 'MCP + Metrics' +weight = 3 +cpu_total = '3 CPU cores' +memory_total = '5.5 GiB' ++++ + +The MCP + Metrics overlay combines the core stack with both Prometheus monitoring and the StreamsHub MCP server. +The MCP server is configured to use Prometheus as its centralized metrics provider. + +## Quick-Start Install + +```shell +curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/install.sh | OVERLAY=mcp-metrics bash +``` + +## Manual Install + +If you prefer step-by-step control, the MCP + Metrics overlay uses `overlays/mcp-metrics`: + +```shell +# Phase 1 — Operators and CRDs (includes Prometheus Operator) +kubectl apply --server-side --force-conflicts -k 'https://github.com/streamshub/developer-quickstart//overlays/mcp-metrics/base?ref=main' + +# Optionally, wait for the operators to be ready +kubectl wait --for=condition=Available deployment/prometheus-operator -n monitoring --timeout=120s +kubectl wait --for=condition=Available deployment/strimzi-cluster-operator -n strimzi --timeout=120s +kubectl wait --for=condition=Available deployment/apicurio-registry-operator -n apicurio-registry --timeout=120s +kubectl wait --for=condition=Available deployment/streamshub-console-operator -n streamshub-console --timeout=120s + +# Phase 2 — Operands, MCP server, Prometheus instance and monitors +kubectl apply -k 'https://github.com/streamshub/developer-quickstart//overlays/mcp-metrics/stack?ref=main' +``` + +## Uninstall + +```shell +curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/uninstall.sh | OVERLAY=mcp-metrics bash +``` + +## What Gets Added + +On top of the components provided by the core overlay, the MCP + Metrics overlay adds: + +| Component | Namespace | Description | +|------------------------|------------------|----------------------------------------------------------| +| Prometheus Operator | `monitoring` | Manages Prometheus instances and monitors | +| Prometheus instance | `monitoring` | Collects and stores metrics (400Mi memory, 1 replica) | +| PodMonitors | `monitoring` | Scrape targets for Kafka brokers and Strimzi operators | +| ServiceMonitor | `monitoring` | Scrape target for StreamsHub Console operator | +| StreamsHub MCP server | `streamshub-mcp` | MCP server configured with Prometheus metrics provider | +| MCP PodMonitor | `monitoring` | Scrape target for MCP server metrics | +| Sensitive Role | `kafka` | Grants access to TLS certificates and pod metrics | + +The overlay patches existing resources: + +- Kafka — enables the [Strimzi Metrics Reporter](https://strimzi.io/docs/operators/latest/deploying#proc-metrics-kafka-str) on the `dev-cluster` +- Console — adds Prometheus as a metrics data source so the Console UI displays Kafka metrics +- MCP server — configured to use Prometheus as centralized metrics provider instead of direct pod scraping + +## Resource Requirements + +The MCP + Metrics overlay requires at least {{< param cpu_total >}} and {{< param memory_total >}} of allocatable cluster resources. +This includes the resources for the [core](core.md) stack plus the additional components listed above. + +## Accessing the MCP Server + +Port-forward to the MCP server: + +```shell +kubectl port-forward -n streamshub-mcp svc/streamshub-strimzi-mcp 8085:8080 +``` + +The MCP endpoint is available at `http://localhost:8085/mcp`. + +## Accessing Prometheus + +Port-forward to the Prometheus UI: + +```shell +kubectl port-forward -n monitoring svc/prometheus-operated 9090:9090 +``` + +Open [http://localhost:9090](http://localhost:9090) and navigate to **Status > Targets** to verify all scrape targets are up, including the MCP server. + +## Verify the Installation + +Confirm all components are running: + +```shell +kubectl get deployment -n streamshub-mcp streamshub-strimzi-mcp +kubectl get prometheus -n monitoring +kubectl get podmonitor -n monitoring +``` + +The MCP server deployment should show `1/1` ready replicas. Prometheus should reach `Available` status. + +## Troubleshooting + +### MCP Server Not Starting + +See the [MCP overlay troubleshooting](mcp.md#troubleshooting) section. + +### Metrics Not Appearing + +See the [Metrics overlay troubleshooting](metrics.md#metrics-not-appearing) section. diff --git a/docs/overlays/mcp.md b/docs/overlays/mcp.md new file mode 100644 index 0000000..0ef6dab --- /dev/null +++ b/docs/overlays/mcp.md @@ -0,0 +1,101 @@ ++++ +title = 'MCP' +weight = 2 +cpu_total = '3 CPU cores' +memory_total = '5 GiB' ++++ + +The MCP overlay extends the core stack with the StreamsHub MCP server, giving AI assistants (Claude Code, VS Code Copilot, etc.) read-only access to the Strimzi-managed Kafka cluster. + +## Quick-Start Install + +```shell +curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/install.sh | OVERLAY=mcp bash +``` + +## Manual Install + +If you prefer step-by-step control, the MCP overlay uses `overlays/mcp` instead of `overlays/core`: + +```shell +# Phase 1 — Operators and CRDs +kubectl apply --server-side --force-conflicts -k 'https://github.com/streamshub/developer-quickstart//overlays/mcp/base?ref=main' + +# Optionally, wait for the operators to be ready +kubectl wait --for=condition=Available deployment/strimzi-cluster-operator -n strimzi --timeout=120s +kubectl wait --for=condition=Available deployment/apicurio-registry-operator -n apicurio-registry --timeout=120s +kubectl wait --for=condition=Available deployment/streamshub-console-operator -n streamshub-console --timeout=120s + +# Phase 2 — Operands and MCP server +kubectl apply -k 'https://github.com/streamshub/developer-quickstart//overlays/mcp/stack?ref=main' +``` + +## Uninstall + +```shell +curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/uninstall.sh | OVERLAY=mcp bash +``` + +## What Gets Added + +On top of the components provided by the core overlay, the MCP overlay adds: + +| Component | Namespace | Description | +|------------------------|------------------|----------------------------------------------------------| +| StreamsHub MCP server | `streamshub-mcp` | MCP server for AI-assisted Kafka cluster management | +| Sensitive Role | `kafka` | Grants access to TLS certificates and pod metrics | + +The MCP server provides read-only tools for inspecting and troubleshooting Strimzi-managed Kafka clusters via the [Model Context Protocol](https://modelcontextprotocol.io/). + +## Resource Requirements + +The MCP overlay requires at least {{< param cpu_total >}} and {{< param memory_total >}} of allocatable cluster resources. +This includes the resources for the [core](core.md) stack plus the MCP server listed above. + +## Accessing the MCP Server + +Port-forward to the MCP server: + +```shell +kubectl port-forward -n streamshub-mcp svc/streamshub-strimzi-mcp 8085:8080 +``` + +The MCP endpoint is available at `http://localhost:8085/mcp`. + +## Verify the Installation + +Confirm the MCP server is running: + +```shell +kubectl get deployment -n streamshub-mcp streamshub-strimzi-mcp +``` + +The deployment should show `1/1` ready replicas. Check the health endpoint: + +```shell +kubectl port-forward -n streamshub-mcp svc/streamshub-strimzi-mcp 8085:8080 & +curl -s http://localhost:8085/q/health/ready +``` + +## Troubleshooting + +### MCP Server Not Starting + +If the MCP server deployment is not becoming ready: + +```shell +# Check pod status +kubectl get pods -n streamshub-mcp + +# Check logs +kubectl logs -n streamshub-mcp deployment/streamshub-strimzi-mcp + +# Verify Strimzi CRDs are installed +kubectl get crd kafkas.kafka.strimzi.io +``` + +**Common causes:** + +- Strimzi operator not installed — the MCP server requires Strimzi CRDs to exist +- Insufficient cluster resources — check node resource availability +- Image pull errors — verify the cluster can pull from `quay.io` diff --git a/docs/prerequisites.md b/docs/prerequisites.md index 87b0ddd..110d1da 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -49,10 +49,10 @@ nodes: node-labels: "ingress-ready=true" extraPortMappings: - containerPort: 80 - hostPort: 80 + hostPort: 8080 protocol: TCP - containerPort: 443 - hostPort: 443 + hostPort: 8443 protocol: TCP EOF ``` diff --git a/install.sh b/install.sh index 94719bb..abc5c87 100755 --- a/install.sh +++ b/install.sh @@ -88,10 +88,19 @@ main() { stack_path="overlays/${OVERLAY}/stack" fi + # Determine which optional features are included in this overlay + local has_metrics=false + local has_mcp=false + case "${OVERLAY}" in + metrics) has_metrics=true ;; + mcp) has_mcp=true ;; + mcp-metrics) has_metrics=true; has_mcp=true ;; + esac + # Compute total steps based on overlay and verification local total_steps=5 - if [ "${OVERLAY}" = "metrics" ]; then - total_steps=6 + if [ "$has_metrics" = true ]; then + total_steps=$((total_steps + 1)) fi if [ -z "${SKIP_VERIFY}" ]; then total_steps=$((total_steps + 1)) @@ -127,8 +136,8 @@ main() { kubectl apply --server-side --force-conflicts -k "${base_url}" echo "" - # --- Step: Wait for prometheus-operator (metrics overlay only) --- - if [ "${OVERLAY}" = "metrics" ]; then + # --- Step: Wait for prometheus-operator (metrics overlays only) --- + if [ "$has_metrics" = true ]; then step=$((step + 1)) info "Step ${step}/${total_steps}: Waiting for prometheus-operator to be ready (timeout: ${TIMEOUT})..." kubectl wait --for=condition=Available deployment/prometheus-operator \ @@ -203,7 +212,7 @@ main() { verify_failed=true fi - if [ "${OVERLAY}" = "metrics" ]; then + if [ "$has_metrics" = true ]; then info " Waiting for Prometheus..." if kubectl wait --for=condition=Available prometheus.monitoring.coreos.com/prometheus \ -n monitoring --timeout="${TIMEOUT}" 2>/dev/null; then @@ -214,6 +223,17 @@ main() { fi fi + if [ "$has_mcp" = true ]; then + info " Waiting for StreamsHub MCP server..." + if kubectl wait --for=condition=Available deployment/streamshub-strimzi-mcp \ + -n streamshub-mcp --timeout="${TIMEOUT}" 2>/dev/null; then + info " StreamsHub MCP server is ready" + else + warn " StreamsHub MCP server did not become ready within ${TIMEOUT}" + verify_failed=true + fi + fi + echo "" fi @@ -234,26 +254,38 @@ main() { echo " - Apicurio Registry instance (namespace: apicurio-registry, storage: in-memory)" echo " - StreamsHub Console operator (namespace: streamshub-console)" echo " - StreamsHub Console instance (namespace: streamshub-console)" - if [ "${OVERLAY}" = "metrics" ]; then + if [ "$has_metrics" = true ]; then echo " - Prometheus operator (namespace: monitoring)" echo " - Prometheus instance (namespace: monitoring)" echo " - Kafka metrics (PodMonitors) (namespace: monitoring)" fi + if [ "$has_mcp" = true ]; then + echo " - StreamsHub MCP server (namespace: streamshub-mcp)" + fi echo "" if [ -z "${SKIP_VERIFY}" ] && [ "${verify_failed}" != "true" ]; then echo "Access the Console:" echo " kubectl port-forward -n streamshub-console svc/streamshub-console-console-service 8090:80" echo " Then open http://localhost:8090" + if [ "$has_mcp" = true ]; then + echo "" + echo "Access the MCP server:" + echo " kubectl port-forward -n streamshub-mcp svc/streamshub-strimzi-mcp 8085:8080" + echo " MCP endpoint: http://localhost:8085/mcp" + fi echo "" elif [ -z "${SKIP_VERIFY}" ] && [ "${verify_failed}" = "true" ]; then echo "Check operand status:" echo " kubectl get kafka -n kafka" echo " kubectl get apicurioregistry3 -n apicurio-registry" echo " kubectl get console -n streamshub-console" - if [ "${OVERLAY}" = "metrics" ]; then + if [ "$has_metrics" = true ]; then echo " kubectl get prometheus -n monitoring" fi + if [ "$has_mcp" = true ]; then + echo " kubectl get deployment -n streamshub-mcp streamshub-strimzi-mcp" + fi echo "" echo "Some resources may still be starting up. Re-check after a few minutes." echo "" @@ -265,10 +297,13 @@ main() { echo " kubectl get apicurioregistry3 -n apicurio-registry" echo " kubectl get deployment -n streamshub-console console-operator" echo " kubectl get console -n streamshub-console" - if [ "${OVERLAY}" = "metrics" ]; then + if [ "$has_metrics" = true ]; then echo " kubectl get prometheus -n monitoring" echo " kubectl get podmonitor -n monitoring" fi + if [ "$has_mcp" = true ]; then + echo " kubectl get deployment -n streamshub-mcp streamshub-strimzi-mcp" + fi echo "" echo "Note: It may take several minutes for all resources to become ready." echo "" diff --git a/overlays/mcp-metrics/base/kustomization.yaml b/overlays/mcp-metrics/base/kustomization.yaml new file mode 100644 index 0000000..9d7cb1c --- /dev/null +++ b/overlays/mcp-metrics/base/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../metrics/base diff --git a/overlays/mcp-metrics/stack/kustomization.yaml b/overlays/mcp-metrics/stack/kustomization.yaml new file mode 100644 index 0000000..3c38332 --- /dev/null +++ b/overlays/mcp-metrics/stack/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../components/core/stack + - ../../../components/metrics/stack + - ../../../components/mcp/stack + +resources: + - mcp-podmonitor.yaml + +patches: + - target: + group: apps + version: v1 + kind: Deployment + name: streamshub-strimzi-mcp + path: mcp-prometheus-patch.yaml diff --git a/overlays/mcp-metrics/stack/mcp-podmonitor.yaml b/overlays/mcp-metrics/stack/mcp-podmonitor.yaml new file mode 100644 index 0000000..76d3a5f --- /dev/null +++ b/overlays/mcp-metrics/stack/mcp-podmonitor.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: strimzi-mcp-metrics + namespace: monitoring + labels: + app: strimzi + app.kubernetes.io/part-of: streamshub-developer-quickstart +spec: + namespaceSelector: + matchNames: + - streamshub-mcp + selector: + matchLabels: + app.kubernetes.io/name: strimzi-mcp + podMetricsEndpoints: + - path: /q/metrics + port: http diff --git a/overlays/mcp-metrics/stack/mcp-prometheus-patch.yaml b/overlays/mcp-metrics/stack/mcp-prometheus-patch.yaml new file mode 100644 index 0000000..7437186 --- /dev/null +++ b/overlays/mcp-metrics/stack/mcp-prometheus-patch.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: streamshub-strimzi-mcp + namespace: streamshub-mcp +spec: + template: + spec: + containers: + - name: strimzi-mcp + env: + - name: MCP_METRICS_PROVIDER + value: "streamshub-prometheus" + - name: QUARKUS_REST_CLIENT_PROMETHEUS_URL + value: "http://prometheus-operated.monitoring.svc:9090" diff --git a/overlays/mcp/base/kustomization.yaml b/overlays/mcp/base/kustomization.yaml new file mode 100644 index 0000000..c78dfeb --- /dev/null +++ b/overlays/mcp/base/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../core/base diff --git a/overlays/mcp/stack/kustomization.yaml b/overlays/mcp/stack/kustomization.yaml new file mode 100644 index 0000000..725ab51 --- /dev/null +++ b/overlays/mcp/stack/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../components/core/stack + - ../../../components/mcp/stack diff --git a/uninstall.sh b/uninstall.sh index 372e102..5cc38b8 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -485,7 +485,7 @@ main() { fi # Remove prometheus-operator if metrics overlay was used - if [ "${OVERLAY}" = "metrics" ]; then + if [ "${OVERLAY}" = "metrics" ] || [ "${OVERLAY}" = "mcp-metrics" ]; then info " Removing Prometheus operator..." local prom_url prom_url=$(kustomize_url "components/metrics/base/prometheus-operator") diff --git a/update-version.sh b/update-version.sh index 40b3007..4d33571 100755 --- a/update-version.sh +++ b/update-version.sh @@ -79,6 +79,16 @@ setup_prometheus_operator() { ) } +setup_streamshub_mcp() { + COMPONENT_LABEL="StreamsHub MCP" + GITHUB_REPO="streamshub/streamshub-mcp" + VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+' + COMPONENT_FILES=( + "${SCRIPT_DIR}/components/mcp/stack/strimzi-mcp/kustomization.yaml" + "${SCRIPT_DIR}/components/mcp/stack/kafka-access/kustomization.yaml" + ) +} + usage() { cat < @@ -90,6 +100,7 @@ Components: apicurio-registry Apicurio Registry Operator streamshub-console StreamsHub Console Operator prometheus-operator Prometheus Operator (metrics overlay) + streamshub-mcp StreamsHub MCP Server (mcp overlay) Arguments: component The component to update @@ -110,6 +121,7 @@ Examples: $(basename "$0") apicurio-registry 3.1.8 # Update Apicurio Registry $(basename "$0") streamshub-console 0.12.0 # Update StreamsHub Console $(basename "$0") prometheus-operator 0.90.0 # Update Prometheus Operator + $(basename "$0") streamshub-mcp 0.2.0 # Update StreamsHub MCP Server EOF exit 0 } @@ -130,9 +142,12 @@ setup_component() { prometheus-operator) setup_prometheus_operator ;; + streamshub-mcp) + setup_streamshub_mcp + ;; *) error "Unknown component: ${component}" - error "Valid components: strimzi, apicurio-registry, streamshub-console, prometheus-operator" + error "Valid components: strimzi, apicurio-registry, streamshub-console, prometheus-operator, streamshub-mcp" exit 1 ;; esac