From 2b4070e2e80444e6bee28eb6aeaa625842f61d33 Mon Sep 17 00:00:00 2001 From: "Srivastava, Piyush" Date: Fri, 7 Aug 2026 11:30:04 +0530 Subject: [PATCH] feature/CSTACKEX-237: Pilot changes for scalebenmark framework --- private-cicd/benchmark/ontap/.gitignore | 8 + private-cicd/benchmark/ontap/README.md | 233 +++++++++++++ private-cicd/benchmark/ontap/USAGE.md | 179 ++++++++++ .../benchmark_storage_pool_concurrency.py | 208 ++++++++++++ .../benchmark_storage_pool_sequential.py | 198 +++++++++++ .../ontap/benchmark_vm_instance_combined.py | 123 +++++++ .../benchmark_vm_instance_concurrency.py | 220 ++++++++++++ .../ontap/benchmark_vm_instance_sequential.py | 211 ++++++++++++ .../benchmark/ontap/cloudstack_client.py | 159 +++++++++ .../benchmark/ontap/config.example.yaml | 124 +++++++ private-cicd/benchmark/ontap/render_report.py | 227 +++++++++++++ private-cicd/benchmark/ontap/requirements.txt | 2 + private-cicd/benchmark/ontap/results/.gitkeep | 0 .../benchmark/ontap/storage_pool_common.py | 218 ++++++++++++ .../benchmark/ontap/vm_instance_common.py | 319 ++++++++++++++++++ 15 files changed, 2429 insertions(+) create mode 100644 private-cicd/benchmark/ontap/.gitignore create mode 100644 private-cicd/benchmark/ontap/README.md create mode 100644 private-cicd/benchmark/ontap/USAGE.md create mode 100644 private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py create mode 100644 private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py create mode 100644 private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py create mode 100644 private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py create mode 100644 private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py create mode 100644 private-cicd/benchmark/ontap/cloudstack_client.py create mode 100644 private-cicd/benchmark/ontap/config.example.yaml create mode 100644 private-cicd/benchmark/ontap/render_report.py create mode 100644 private-cicd/benchmark/ontap/requirements.txt create mode 100644 private-cicd/benchmark/ontap/results/.gitkeep create mode 100644 private-cicd/benchmark/ontap/storage_pool_common.py create mode 100644 private-cicd/benchmark/ontap/vm_instance_common.py diff --git a/private-cicd/benchmark/ontap/.gitignore b/private-cicd/benchmark/ontap/.gitignore new file mode 100644 index 000000000000..2e639cd15227 --- /dev/null +++ b/private-cicd/benchmark/ontap/.gitignore @@ -0,0 +1,8 @@ +config.yaml +config_sanity.yaml +.venv/ +__pycache__/ +*.pyc +results/*.csv +results/*.md +results/*.log diff --git a/private-cicd/benchmark/ontap/README.md b/private-cicd/benchmark/ontap/README.md new file mode 100644 index 000000000000..a0e3e0eb5265 --- /dev/null +++ b/private-cicd/benchmark/ontap/README.md @@ -0,0 +1,233 @@ + +# ONTAP plugin - storage pool benchmark (Step 1) + +> Looking for a quick "what does script X do / how do I run it" cheat +> sheet instead of the full design write-up below? See [USAGE.md](USAGE.md). + +Downstream-only benchmarking tool (not for Apache upstream) that drives +`createStoragePool` / `deleteStoragePool` over the CloudStack HTTP/REST API +to reproduce the **Storage Pool** rows of the Sequential Scale Matrix (5.1) +and Parallel/Concurrency Matrix (6.1) from the Confluence page +["ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix"](https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350). + +This is step 1 of the automation follow-up (storage pool only). VM instance +benchmarks are now also covered (see below); volume/snapshot benchmarks will +be added as separate scripts later, reusing `cloudstack_client.py`. + +The storage-pool benchmark is split into **two independent scripts** (on +purpose — a sequential run is cheap/safe and worth reviewing before deciding +to launch a concurrency run, which is the one most likely to stress the +mgmt-server/plugin job queue): + +- `benchmark_storage_pool_sequential.py` — Section 5.1 (5.1.1 / 5.1.2) +- `benchmark_storage_pool_concurrency.py` — Section 6.1 (6.1.1 / 6.1.2) + +Both share the same createStoragePool/deleteStoragePool call shape, CSV +formats, and cleanup logic via `storage_pool_common.py`. Run them with the +**same `--run-id`** to accumulate both into a single +`results/summary_.csv` (and therefore a single `render_report.py` +output covering 5.1.x and 6.1.x together). + +The VM-instance benchmark is split into **three scripts** sharing +`vm_instance_common.py` (same call shape/CSV formats/cleanup logic): + +- `benchmark_vm_instance_sequential.py` — Section 5.2 (5.2.1 / 5.2.2) +- `benchmark_vm_instance_concurrency.py` — Section 6.2 (6.2.1 / 6.2.2) +- `benchmark_vm_instance_combined.py` — runs both of the above back to back + under one shared `--run-id`, for when you already trust the config/ + environment and just want the full 5.2.x + 6.2.x matrix in one invocation + +See "VM instance benchmark" below for prerequisites and usage. + +## What it does + +1. **Sequential scale test (5.1.1 / 5.1.2)** — for each protocol (NFS3, + iSCSI), creates storage pools one at a time up to N=30, logging every + single create call, and reports cumulative-total/avg-per-op timing at + checkpoints N = 1, 5, 10, 20, 30. Then deletes them one at a time and + reports the same at "N remaining" = 30, 20, 10, 5, 1. +2. **Concurrency test (6.1.1 / 6.1.2)** — for each concurrency level + C = 2, 5, 10, 20, 30 and each protocol, creates C pools in parallel + (`ThreadPoolExecutor`), records wall-clock time for the whole batch plus + success/failure/avg-per-op, then deletes the same C pools in parallel. +3. Writes **every** individual API call (start/end timestamp, duration, + success/failure, pool id, error) to `results/raw_ops_.csv`, and a + checkpoint-level roll-up to `results/summary_.csv`. +4. `render_report.py` turns those two CSVs into markdown tables that match + the Confluence page's table layout (including min/avg/p95/max stats for + the "Results Log" section), ready to paste back into the page. + +## Setup + +```bash +cd private-cicd/benchmark/ontap +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +cp config.example.yaml config.yaml +``` + +Edit `config.yaml`: + +- `cloudstack.*` — API URL, admin username/password of your lab's + management server. +- `infrastructure.*` — zone/pod/cluster UUIDs to create pools under. +- `ontap.nfs3` / `ontap.iscsi` — connection details of the ONTAP SVM(s) you + want to benchmark against for each protocol. Delete a block if you only + have one protocol available. +- `benchmark.*` — checkpoints, concurrency levels, pool name prefix, output + directory. Defaults match the Confluence page (N/C = 1,5,10,20,30 / + 2,5,10,20,30). + +`config.yaml` and everything under `results/` are gitignored — never commit +real lab IPs/credentials or run output. + +## Running + +```bash +# Sanity-check either script/config without hitting a real management server: +python3 benchmark_storage_pool_sequential.py --config config.yaml --dry-run +python3 benchmark_storage_pool_concurrency.py --config config.yaml --dry-run + +# Sequential scale matrix, both protocols, up to config's sequential_checkpoints: +python3 benchmark_storage_pool_sequential.py --config config.yaml + +# Sequential scale matrix, NFS3 only: +python3 benchmark_storage_pool_sequential.py --config config.yaml --protocol nfs3 + +# Concurrency matrix, both protocols, up to config's concurrency_levels (e.g. 30): +python3 benchmark_storage_pool_concurrency.py --config config.yaml + +# Concurrency matrix, iSCSI only, overriding the levels to run: +python3 benchmark_storage_pool_concurrency.py --config config.yaml --protocol iscsi --levels 2,5,10,20,30 + +# Use the SAME --run-id across both scripts to combine 5.1.x + 6.1.x into one +# summary_.csv / report (run sequential first, review it, then decide +# whether to proceed with concurrency under the same run id): +python3 benchmark_storage_pool_sequential.py --config config.yaml --run-id RUN-0001 +python3 benchmark_storage_pool_concurrency.py --config config.yaml --run-id RUN-0001 +``` + +At the end of a real run each script automatically checks for and deletes any +storage pool whose name still contains the run id (safety net for pools that +were created but never got torn down because of a mid-run crash). Pass +`--skip-cleanup` to disable that and inspect the pools yourself. + +If a run crashes hard (script killed, network partition, etc.) and left +pools behind, recover with either script's `--cleanup-only` (they share the +same cleanup logic): + +```bash +# Deletes every pool whose name contains "bench_ontap" (the default prefix) +python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only + +# Or scope it to one specific run: +python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only RUN_20260722_101500 +``` + +## Rendering the Confluence-ready report + +```bash +python3 render_report.py --run-id RUN-20260722-101500 \ + --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.15.1 +``` + +This prints markdown tables for 5.1.1, 5.1.2, 6.1.1, 6.1.2, and a set of +pre-filled rows for the page's Section 9 "Results Log" table (with Min/Avg/ +P95/Max computed from the raw per-op log), and also saves them to +`results/report_.md`. + +## VM instance benchmark + +Drives `deployVirtualMachine` / `destroyVirtualMachine` to reproduce the +**VM Instance** rows of the same Confluence page's Sequential Scale Matrix +(5.2) and Parallel/Concurrency Matrix (6.2). Every VM gets a root disk (from +the service offering) AND a data disk (from the disk offering), both landing +on the same tagged storage pool. + +### Prerequisites (one-time setup, per protocol) + +Unlike the storage-pool benchmark's transient pools, the VM-instance +benchmark needs a **persistent, pre-created** `bench_vm_` pool plus +matching offerings - it does not create/destroy the pool itself: + +1. A storage pool (e.g. `bench_vm_nfs3_pool`) with a distinct storage tag + (e.g. `bench_vm_nfs3`), sized comfortably below your aggregate's capacity. +2. A service offering (root disk) and a disk offering (data disk), both + tagged with that same storage tag, so root + data land on the same pool. + Set the service offering's `rootdisksize` (GB) to comfortably exceed the + template's actual `qemu-img` **virtual** size (not its sparse/download + file size) - too small silently "succeeds" on NFS but hard-fails on iSCSI + with `qemu-img: Cannot grow device files` (see Confluence Issue #2). +3. Fill in `vm_bench.templateid` / `networkid` / per-protocol + `serviceofferingid` / `diskofferingid` in `config.yaml` with the above. + +### Running + +```bash +# Sanity-check without hitting a real management server: +python3 benchmark_vm_instance_sequential.py --config config.yaml --dry-run +python3 benchmark_vm_instance_concurrency.py --config config.yaml --dry-run + +# Sequential scale matrix (5.2.1/5.2.2), both protocols: +python3 benchmark_vm_instance_sequential.py --config config.yaml + +# Concurrency matrix (6.2.1/6.2.2), iSCSI only, custom levels: +python3 benchmark_vm_instance_concurrency.py --config config.yaml --protocol iscsi --levels 1,2,5,10 + +# Both matrices in one invocation, combined into a single summary/report: +python3 benchmark_vm_instance_combined.py --config config.yaml + +# Or run the two standalone scripts under the SAME --run-id to combine them +# instead (lets you review the sequential results before committing to +# concurrency): +python3 benchmark_vm_instance_sequential.py --config config.yaml --run-id RUN-0001 +python3 benchmark_vm_instance_concurrency.py --config config.yaml --run-id RUN-0001 + +python3 render_report.py --run-id RUN-0001 --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm +``` + +All three scripts share the same `--skip-cleanup` / `--cleanup-only` safety +net as the storage-pool scripts (see above), via `vm_instance_common.py`. +Failed `deployVirtualMachine` calls are logged but their VM/disk are +intentionally left in place (not auto-destroyed) so you can inspect exactly +what got left behind - use `--cleanup-only` once you're done. + +## Notes / caveats + +- Auth is session-key based (`login` -> `sessionkey` + `JSESSIONID` cookie), + matching the approach documented on the Confluence API page. API + key/secret signed requests are not implemented (add to + `cloudstack_client.py` if your lab requires it). +- Timing is measured end-to-end from the caller's perspective, including any + async job polling (`queryAsyncJobResult`) — this is the number an operator + or automation would actually observe, not raw server-side processing time. +- The single `CloudStackClient`/`requests.Session` is shared across threads + during concurrency tests. We never mutate session state after login, so + this is safe; if you prefer full isolation, instantiate one + `CloudStackClient` per thread instead. +- `createStoragePool`/`deleteStoragePool`/`updateStoragePool` are + synchronous in this CloudStack version (no `jobid` in the response); + `enableStorageMaintenance`/`cancelStorageMaintenance` are async — the + client polls `queryAsyncJobResult` automatically for any response that + does contain a `jobid`, so the script works either way. +- This script intentionally covers **create/delete only** (per the current + ask). Maintenance mode and enable/disable pool are documented in + `cloudstack_client.py`'s API surface but not yet wired into a benchmark + phase — flag if you want those added to the matrix too. diff --git a/private-cicd/benchmark/ontap/USAGE.md b/private-cicd/benchmark/ontap/USAGE.md new file mode 100644 index 000000000000..f62dd3c89e01 --- /dev/null +++ b/private-cicd/benchmark/ontap/USAGE.md @@ -0,0 +1,179 @@ + +# Usage Reference + +Quick reference for **what each file does** and **how to run it**. For the +"why" behind the design (what each benchmark measures, how it maps to the +Confluence test matrix, known caveats) see [README.md](README.md) instead - +this file is deliberately just a lookup table / cheat sheet. + +## 1. Prerequisites + +| # | Requirement | Notes | +|---|---|---| +| 1 | Python 3.9+ | No other version-specific features used; anything 3.9-3.12 should work. | +| 2 | Network access to a CloudStack management server's API (`/client/api`) | Admin credentials with permission to create/delete storage pools, service/disk offerings, and deploy/destroy VMs. | +| 3 | Network access (HTTPS/443) from **your machine** to the ONTAP cluster's management LIF | Only needed if you run the storage-pool scripts' `--cleanup-only` helper functions that talk to ONTAP directly for orphan checks; the benchmark scripts themselves only ever call the CloudStack API, never ONTAP directly. | +| 4 | A CloudStack zone/pod/cluster with at least one KVM host, and the ONTAP plugin's SVM(s) reachable from the CloudStack management server | See README.md's "Test Environment" section on the Confluence page for the exact lab topology this was built/tested against. | +| 5 | For the VM-instance scripts only: a pre-created `bench_vm_` storage pool + matching service/disk offerings | One-time setup, see [Prerequisites (VM instance benchmark)](#4-vm-instance-benchmark-scripts) below. | + +### One-time environment setup + +```bash +cd private-cicd/benchmark/ontap +python3 -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements.txt +cp config.example.yaml config.yaml +``` + +Then edit `config.yaml` with your lab's real API URL, credentials, zone/pod/ +cluster IDs, and ONTAP connection details (see inline comments in +`config.example.yaml` for what each field means and why). + +> **About `.venv/`:** this is a completely standard, disposable Python +> virtual environment (`python3 -m venv .venv` + `pip install -r +> requirements.txt`, which just pulls in `requests` and `PyYAML` plus their +> small set of transitive dependencies). It is **never committed** (it's in +> `.gitignore`) because it is: +> - **Machine/OS-specific** - it stores an absolute path back to the Python +> interpreter that created it (`.venv/pyvenv.cfg`) and can contain +> platform-compiled binary wheels (e.g. a `.so`/`.pyd` file bundled with +> one of the dependencies) that won't load on a different OS/architecture. +> - **Python-version-specific** - the `lib/python3.9/...` directory layout +> is tied to whatever interpreter version created it. +> - **Fully reproducible** - anyone can regenerate an equivalent one in a +> few seconds with the two commands above; there is nothing hand-tuned or +> irreplaceable in it. +> +> If you ever see `.venv/` show up in `git status` as untracked, it means +> `.gitignore` isn't being picked up (e.g. it was created before the +> `.gitignore` entry was added) - just confirm `.venv/` is listed in +> `.gitignore` and it will stop appearing. + +## 2. Shared library modules (not run directly) + +| File | Purpose | +|---|---| +| `cloudstack_client.py` | Minimal CloudStack HTTP/REST client (session-key login, generic `call()`, automatic `queryAsyncJobResult` polling). Used by every benchmark script. | +| `storage_pool_common.py` | Shared helpers for the storage-pool scripts: `createStoragePool`/`deleteStoragePool` request building, CSV logging (`RawLogger`, `append_summary_csv`), cleanup-by-filter, config loading. | +| `vm_instance_common.py` | Shared helpers for the VM-instance scripts: `deployVirtualMachine`/`destroyVirtualMachine` request building (incl. force-purging data disks and `Destroy`-state volumes so ONTAP space is reclaimed immediately instead of waiting for CloudStack's 24h cleanup delay), CSV logging, cleanup-by-filter, config loading. | + +## 3. Storage-pool benchmark scripts + +Reproduces Confluence sections 5.1 (sequential) and 6.1 (concurrency) - +`createStoragePool` / `deleteStoragePool` timing at scale. + +| Script | What it does | Typical command | +|---|---|---| +| `benchmark_storage_pool_sequential.py` | Creates storage pools one at a time (per protocol) up to N=30, then deletes them one at a time, logging every call and reporting checkpoint totals at N = 1, 5, 10, 20, 30. | `python3 benchmark_storage_pool_sequential.py --config config.yaml` | +| `benchmark_storage_pool_concurrency.py` | For each concurrency level C (default 2, 5, 10, 20, 30), creates C pools in parallel via a thread pool, records wall-clock time for the whole batch, then deletes the same C pools in parallel. | `python3 benchmark_storage_pool_concurrency.py --config config.yaml` | + +Common flags (both scripts): + +| Flag | Meaning | +|---|---| +| `--config PATH` | Config YAML to use (default `config.yaml`). | +| `--protocol {nfs3,iscsi,both}` | Restrict the run to one protocol (default `both`). | +| `--run-id ID` | Reuse a specific run id (auto-generated otherwise) - use the **same id** across the sequential and concurrency scripts to merge both into one `summary_.csv`/report. | +| `--dry-run` | Simulate timings with no real API calls - use this first to sanity-check your config. | +| `--skip-cleanup` | Skip the automatic post-run sweep for leftover pools from this run id. | +| `--cleanup-only [FILTER]` | Don't run the benchmark - just delete every pool whose name contains `FILTER` (default: config's `pool_name_prefix`) and exit. Use this to recover after a crashed run. | +| `--levels 2,5,10` | *(concurrency script only)* Override the concurrency levels to run instead of the config's `concurrency_levels`. | + +## 4. VM-instance benchmark scripts + +Reproduces Confluence sections 5.2 (sequential) and 6.2 (concurrency) - +`deployVirtualMachine` / `destroyVirtualMachine` timing at scale, with each +VM getting both a root disk and a data disk on the same tagged pool. + +### Prerequisites (one-time, per protocol - not created/destroyed by the scripts) + +1. A persistent storage pool (e.g. `bench_vm_nfs3_pool`) with a distinct + storage tag (e.g. `bench_vm_nfs3`), sized comfortably below your + aggregate's capacity. +2. A service offering (root disk) and a disk offering (data disk), both + tagged with that same storage tag, so root + data land on the same pool. + Set the service offering's `rootdisksize` (GB) to comfortably exceed the + template's actual `qemu-img` **virtual** size - too small silently + "succeeds" on NFS but hard-fails on iSCSI with `qemu-img: Cannot grow + device files`. +3. Fill in `vm_bench.templateid` / `networkid` / per-protocol + `serviceofferingid` / `diskofferingid` in `config.yaml`. + +### Scripts + +| Script | What it does | Typical command | +|---|---|---| +| `benchmark_vm_instance_sequential.py` | Deploys VMs one at a time (per protocol) up to N=30, then destroys them one at a time, reporting checkpoint totals at N = 1, 2, 5, 10, 20, 30. | `python3 benchmark_vm_instance_sequential.py --config config.yaml` | +| `benchmark_vm_instance_concurrency.py` | For each concurrency level C, deploys C VMs in parallel, records wall-clock time, then destroys the same C VMs in parallel. | `python3 benchmark_vm_instance_concurrency.py --config config.yaml` | +| `benchmark_vm_instance_combined.py` | Runs the sequential matrix immediately followed by the concurrency matrix, under one shared run id, in a single invocation. Use once you already trust your config/environment. | `python3 benchmark_vm_instance_combined.py --config config.yaml` | + +Common flags: same as the storage-pool scripts above (`--config`, +`--protocol`, `--run-id`, `--dry-run`, `--skip-cleanup`, `--cleanup-only +[FILTER]`; `--levels` on the concurrency/combined scripts). Failed +`deployVirtualMachine` calls are intentionally **left in place** (not +auto-destroyed) so you can inspect what got left behind - clean them up +with `--cleanup-only` once done. + +## 5. Report rendering + +| Script | What it does | Typical command | +|---|---|---| +| `render_report.py` | Turns a run's `raw_ops_.csv` + `summary_.csv` into Confluence-ready markdown tables (matches the page's section layout, plus Min/Avg/P95/Max rows for the Results Log). Writes to stdout and to `results/report_.md`. | `python3 render_report.py --run-id RUN-0001 --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.17.1` | + +For VM-instance runs (which write `raw_ops_vm_.csv` / +`summary_vm_.csv` instead of the storage-pool scripts' `raw_ops_`/ +`summary_` prefix), pass the matching prefixes: + +```bash +python3 render_report.py --run-id RUN-0001 \ + --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm +``` + +| Flag | Meaning | +|---|---| +| `--run-id ID` | **Required.** The run id used by the benchmark script(s). | +| `--output-dir DIR` | Where the CSVs live / the report gets written (default `results`). | +| `--cloudstack-build STR` | Label only, stamped into the Results Log rows. | +| `--ontap-version STR` | Label only, stamped into the Results Log rows. | +| `--run-date YYYY-MM-DD` | Defaults to today. | +| `--raw-prefix STR` | CSV filename prefix for raw per-op data (default `raw_ops`; use `raw_ops_vm` for VM-instance runs). | +| `--summary-prefix STR` | CSV filename prefix for checkpoint summaries (default `summary`; use `summary_vm` for VM-instance runs). | +| `--report-suffix STR` | Suffix appended to the output report filename (e.g. `_vm`) to avoid clobbering a storage-pool report with the same run id. | + +## 6. Config files at a glance + +| File | Committed? | Purpose | +|---|---|---| +| `config.example.yaml` | Yes | Sanitized template - copy to `config.yaml` and fill in your lab's real values. | +| `config.yaml` | **No** (gitignored) | Your real, lab-specific config (real IPs/credentials) - never commit this. | +| `config_sanity.yaml` | **No** (gitignored) | Optional scratch config some contributors keep locally for quick one-off dry-runs; not part of the checked-in tooling. | + +## 7. Output files at a glance + +All written under `results/` (gitignored - these are run artifacts, not +source): + +| Pattern | Produced by | Contents | +|---|---|---| +| `raw_ops_.csv` | storage-pool scripts | One row per individual API call (timestamps, duration, success/failure, error). | +| `raw_ops_vm_.csv` | VM-instance scripts | Same, for VM deploy/destroy calls. | +| `summary_.csv` / `summary_vm_.csv` | both | One row per checkpoint/concurrency level (totals, averages, success/failure counts). | +| `report_.md` / `report_vm_.md` | `render_report.py` | Confluence-ready markdown tables generated from the two CSVs above. | diff --git a/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py b/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py new file mode 100644 index 000000000000..f097dcd97330 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Concurrency storage-pool benchmark for the NetApp ONTAP CloudStack plugin. + +Drives createStoragePool / deleteStoragePool in parallel (ThreadPoolExecutor) +over the CloudStack HTTP/REST API and reproduces Section 6.1 - Parallel/ +Concurrency Matrix (C = 2,5,10,20,30 workers) from the +"ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix" page: + + 6.1.1 Parallel pool creation + 6.1.2 Parallel pool deletion + +See benchmark_storage_pool_sequential.py for the sequential scale matrix +(kept as a separate script/entry point on purpose - concurrency runs are the +ones most likely to hit mgmt-server/plugin job-queue saturation, so they're +easy to run, review, and re-run independently of the sequential matrix). + +Every individual API call is logged to results/raw_ops_.csv, and a +per-checkpoint roll-up (matching the columns of the Confluence tables) is +appended to results/summary_.csv. Use render_report.py afterwards to +turn the summary CSV into paste-ready markdown for the Confluence page. Using +the same --run-id here and in benchmark_storage_pool_sequential.py combines +both into a single summary_.csv / report. + +Usage: + python3 benchmark_storage_pool_concurrency.py --config config.yaml --protocol both + python3 benchmark_storage_pool_concurrency.py --config config.yaml --levels 2,5,10 + python3 benchmark_storage_pool_concurrency.py --config config.yaml --dry-run + python3 benchmark_storage_pool_concurrency.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import statistics +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +from storage_pool_common import ( + append_summary_csv, + cleanup_by_filter, + create_pool, + delete_pool, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + resolve_protocols, +) + + +def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels): + infra = cfg["infrastructure"] + ontap_cfg = cfg["ontap"][protocol_key] + prefix = cfg["benchmark"]["pool_name_prefix"] + + for level in levels: + print(f"\n=== [6.1.1] Parallel CREATE - protocol={protocol_key} C={level} ===") + names = [f"{prefix}_c{level}_{protocol_key}_{run_id}_{i:03d}" for i in range(1, level + 1)] + + t0 = time.perf_counter() + results = [] + if dry_run: + results = [fake_create(n) for n in names] + else: + with ThreadPoolExecutor(max_workers=level) as ex: + futures = {ex.submit(create_pool, client, n, infra, ontap_cfg): n for n in names} + for fut in as_completed(futures): + results.append(fut.result()) + wall = time.perf_counter() - t0 + + for idx, r in enumerate(results, start=1): + raw_logger.log(run_id, "concurrent_create", "6.1.1", protocol_key, level, idx, r) + + succ = [r for r in results if r.success] + fail = [r for r in results if not r.success] + avg = statistics.mean([r.duration_sec for r in succ]) if succ else 0 + notes = "Watch mgmt-server job-queue/thread-pool saturation as a confound" if level >= 30 else "" + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_create", "test_id": "6.1.1", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall, 3), "avg_time_sec": round(avg, 3), + "success_count": len(succ), "failure_count": len(fail), "notes": notes, + }) + print(f" wall-clock={wall:.3f}s success={len(succ)} failure={len(fail)} avg/op={avg:.3f}s") + for r in fail: + print(f" !! {r.pool_name}: {r.error}") + + print(f"\n=== [6.1.2] Parallel DELETE - protocol={protocol_key} C={len(succ)} ===") + t0 = time.perf_counter() + del_results = [] + if dry_run: + del_results = [fake_delete(r.pool_id, r.pool_name) for r in succ] + elif succ: + with ThreadPoolExecutor(max_workers=len(succ)) as ex: + futures = {ex.submit(delete_pool, client, r.pool_id, r.pool_name): r for r in succ} + for fut in as_completed(futures): + del_results.append(fut.result()) + wall_del = time.perf_counter() - t0 + + for idx, r in enumerate(del_results, start=1): + raw_logger.log(run_id, "concurrent_delete", "6.1.2", protocol_key, level, idx, r) + + succ_d = [r for r in del_results if r.success] + fail_d = [r for r in del_results if not r.success] + avg_d = statistics.mean([r.duration_sec for r in succ_d]) if succ_d else 0 + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_delete", "test_id": "6.1.2", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall_del, 3), "avg_time_sec": round(avg_d, 3), + "success_count": len(succ_d), "failure_count": len(fail_d), "notes": "", + }) + print(f" wall-clock={wall_del:.3f}s success={len(succ_d)} failure={len(fail_d)} avg/op={avg_d:.3f}s") + for r in fail_d: + print(f" !! {r.pool_name}: {r.error}") + if delay: + time.sleep(delay) + + +def parse_levels(raw, default_levels): + if not raw: + return sorted(default_levels) + return sorted(int(x.strip()) for x in raw.split(",") if x.strip()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument( + "--levels", default=None, + help="Comma-separated concurrency levels to run, e.g. '2,5,10' " + "(default: config.benchmark.concurrency_levels, typically up to 30)", + ) + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete all storage pools whose name contains FILTER (default: config pool_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["benchmark"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["benchmark"]["pool_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["benchmark"].get("inter_op_delay_sec", 0) + output_dir = cfg["benchmark"].get("output_dir", "results") + levels = parse_levels(args.levels, cfg["benchmark"]["concurrency_levels"]) + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: concurrency") + print(f"Concurrency levels: {levels}") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay, levels) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + f" --output-dir {output_dir}") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned pools remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py b/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py new file mode 100644 index 000000000000..b103596ab7d8 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Sequential storage-pool benchmark for the NetApp ONTAP CloudStack plugin. + +Drives createStoragePool / deleteStoragePool one at a time over the CloudStack +HTTP/REST API (see https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350) +and reproduces Section 5.1 - Sequential Scale Matrix (N = 1,5,10,20,30) from the +"ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix" page: + + 5.1.1 Sequential create (1 pool at a time, cumulative to N) + 5.1.2 Sequential delete (1 pool at a time, from N remaining) + +See benchmark_storage_pool_concurrency.py for the parallel/concurrency matrix +(kept as a separate script/entry point on purpose, so a sequential run can be +kicked off and reviewed independently before committing to a concurrency run). + +Every individual API call is logged to results/raw_ops_.csv, and a +per-checkpoint roll-up (matching the columns of the Confluence tables) is +appended to results/summary_.csv. Use render_report.py afterwards to +turn the summary CSV into paste-ready markdown for the Confluence page. Using +the same --run-id here and in benchmark_storage_pool_concurrency.py combines +both into a single summary_.csv / report. + +Usage: + python3 benchmark_storage_pool_sequential.py --config config.yaml --protocol both + python3 benchmark_storage_pool_sequential.py --config config.yaml --dry-run + python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import time + +from storage_pool_common import ( + append_summary_csv, + cleanup_by_filter, + create_pool, + delete_pool, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + resolve_protocols, +) + + +def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay): + infra = cfg["infrastructure"] + ontap_cfg = cfg["ontap"][protocol_key] + checkpoints = sorted(cfg["benchmark"]["sequential_checkpoints"]) + max_n = max(checkpoints) + prefix = cfg["benchmark"]["pool_name_prefix"] + + created = [] + create_durations = [] + print(f"\n=== [5.1.1] Sequential CREATE - protocol={protocol_key} up to N={max_n} ===") + for i in range(1, max_n + 1): + # ONTAP volume names only allow alphanumeric + underscore (no hyphens). + name = f"{prefix}_seq_{protocol_key}_{run_id}_{i:03d}" + result = fake_create(name) if dry_run else create_pool(client, name, infra, ontap_cfg) + raw_logger.log(run_id, "sequential_create", "5.1.1", protocol_key, max_n, i, result) + status = "OK" if result.success else "FAIL" + print(f" [{i:>3}/{max_n}] create {name} -> {status} ({result.duration_sec:.3f}s)") + if result.success: + created.append((name, result.pool_id)) + create_durations.append(result.duration_sec) + else: + print(f" !! {result.error}") + if i in checkpoints: + total = sum(create_durations) + avg = total / len(create_durations) if create_durations else 0 + summary_rows.append({ + "run_id": run_id, "phase": "sequential_create", "test_id": "5.1.1", + "protocol": protocol_key, "checkpoint": i, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": len(created), "failure_count": i - len(created), "notes": "", + }) + print(f" >> checkpoint N={i}: total={total:.3f}s avg={avg:.3f}s/op " + f"success={len(created)} failure={i - len(created)}") + if delay: + time.sleep(delay) + + total_created = len(created) + print(f"\n=== [5.1.2] Sequential DELETE - protocol={protocol_key} from N={total_created} remaining ===") + if total_created in checkpoints: + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "5.1.2", + "protocol": protocol_key, "checkpoint": total_created, + "total_time_sec": 0, "avg_time_sec": 0, "success_count": 0, "failure_count": 0, + "notes": "baseline - no deletes issued yet", + }) + delete_durations = [] + deleted_ok = 0 + for idx, (name, pool_id) in enumerate(created, start=1): + result = fake_delete(pool_id, name) if dry_run else delete_pool(client, pool_id, name) + remaining = total_created - idx + raw_logger.log(run_id, "sequential_delete", "5.1.2", protocol_key, total_created, idx, result) + status = "OK" if result.success else "FAIL" + print(f" [{idx:>3}/{total_created}] delete {name} -> {status} " + f"({result.duration_sec:.3f}s) remaining={remaining}") + if result.success: + delete_durations.append(result.duration_sec) + deleted_ok += 1 + else: + print(f" !! {result.error}") + if remaining in checkpoints: + total = sum(delete_durations) + avg = total / len(delete_durations) if delete_durations else 0 + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "5.1.2", + "protocol": protocol_key, "checkpoint": remaining, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": deleted_ok, "failure_count": idx - deleted_ok, "notes": "", + }) + print(f" >> checkpoint remaining={remaining}: total={total:.3f}s avg={avg:.3f}s/op " + f"success={deleted_ok} failure={idx - deleted_ok}") + if delay: + time.sleep(delay) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete all storage pools whose name contains FILTER (default: config pool_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["benchmark"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["benchmark"]["pool_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["benchmark"].get("inter_op_delay_sec", 0) + output_dir = cfg["benchmark"].get("output_dir", "results") + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: sequential") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + f" --output-dir {output_dir}") + print(f" (or run benchmark_storage_pool_concurrency.py --run-id {run_id} first to add 6.1.x to the same report)") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned pools remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py new file mode 100644 index 000000000000..86e0430038c3 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Combined (sequential + concurrency) VM-instance benchmark for the NetApp +ONTAP CloudStack plugin. + +Convenience wrapper that runs benchmark_vm_instance_sequential.py's Section +5.2 scale matrix immediately followed by benchmark_vm_instance_concurrency.py's +Section 6.2 parallel matrix, for each protocol, under a single shared +--run-id - equivalent to running both scripts back to back, but in one +invocation and one combined results/summary_vm_.csv. + +Prefer the two standalone scripts (benchmark_vm_instance_sequential.py / +benchmark_vm_instance_concurrency.py) when you want to review the sequential +results before deciding whether to proceed with a concurrency run - this +combined script is for when you already trust the config/environment and just +want the full 5.2.x + 6.2.x matrix in one go. + +Usage: + python3 benchmark_vm_instance_combined.py --config config.yaml --protocol both + python3 benchmark_vm_instance_combined.py --config config.yaml --dry-run + python3 benchmark_vm_instance_combined.py --config config.yaml --cleanup-only +""" + +import argparse +import os + +from benchmark_vm_instance_concurrency import parse_levels, run_concurrency +from benchmark_vm_instance_sequential import run_sequential +from vm_instance_common import ( + append_summary_csv, + cleanup_by_filter, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + resolve_protocols, +) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument( + "--levels", default=None, + help="Comma-separated concurrency levels to run, e.g. '2,5,10' " + "(default: config.vm_bench.concurrency_levels, typically up to 30)", + ) + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy all VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["vm_bench"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["vm_bench"]["vm_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) + output_dir = cfg["vm_bench"].get("output_dir", "results") + levels = parse_levels(args.levels, cfg["vm_bench"]["concurrency_levels"]) + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: sequential + concurrency (combined)") + print(f"Concurrency levels: {levels}") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay) + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay, levels) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_vm_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + + f" --output-dir {output_dir} --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned VMs remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py new file mode 100644 index 000000000000..c002e56b4d2b --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Concurrency VM-instance benchmark for the NetApp ONTAP CloudStack plugin. + +Drives deployVirtualMachine / destroyVirtualMachine in parallel +(ThreadPoolExecutor) over the CloudStack HTTP/REST API against pre- +provisioned bench_vm_nfs3 / bench_vm_iscsi storage pools (see README.md +"VM instance benchmark prerequisites") and reproduces Section 6.2 - Parallel/ +Concurrency Matrix from the "ONTAP Plugin - CloudStack Operations, Scale & +Parallel Test Matrix" page: + + 6.2.1 Parallel VM creation + 6.2.2 Parallel VM deletion + +Each deployed VM gets a root disk (from the service offering) AND one data +disk (from the disk offering) landing on the SAME tagged storage pool. + +See benchmark_vm_instance_sequential.py for the sequential scale matrix, or +benchmark_vm_instance_combined.py to run both in one invocation (kept as +separate scripts/entry points on purpose - concurrency runs are the ones most +likely to hit mgmt-server/plugin job-queue saturation or pool capacity limits, +so they're easy to run, review, and re-run independently of the sequential +matrix). + +Every individual API call is logged to results/raw_ops_vm_.csv, and a +per-checkpoint roll-up (matching the columns of the Confluence tables) is +appended to results/summary_vm_.csv. Use render_report.py afterwards +to turn the summary CSV into paste-ready markdown. Using the same --run-id +here and in benchmark_vm_instance_sequential.py combines both into a single +summary_vm_.csv / report. + +Usage: + python3 benchmark_vm_instance_concurrency.py --config config.yaml --protocol both + python3 benchmark_vm_instance_concurrency.py --config config.yaml --levels 2,5,10 + python3 benchmark_vm_instance_concurrency.py --config config.yaml --dry-run + python3 benchmark_vm_instance_concurrency.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import statistics +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +from vm_instance_common import ( + append_summary_csv, + cleanup_by_filter, + deploy_vm, + destroy_vm, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + report_failed_creates, + resolve_protocols, +) + + +def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels): + infra = cfg["infrastructure"] + vm_cfg = cfg["vm_bench"] + proto_cfg = vm_cfg["protocols"][protocol_key] + prefix = vm_cfg["vm_name_prefix"] + + for level in levels: + print(f"\n=== [6.2.1] Parallel CREATE - protocol={protocol_key} C={level} ===") + names = [f"{prefix}-c{level}-{protocol_key}-{run_id}-{i:03d}".replace("_", "-") + for i in range(1, level + 1)] + + t0 = time.perf_counter() + results = [] + if dry_run: + results = [fake_create(n) for n in names] + else: + with ThreadPoolExecutor(max_workers=level) as ex: + futures = {ex.submit(deploy_vm, client, n, infra, vm_cfg, proto_cfg): n for n in names} + for fut in as_completed(futures): + results.append(fut.result()) + wall = time.perf_counter() - t0 + + for idx, r in enumerate(results, start=1): + raw_logger.log(run_id, "concurrent_create", "6.2.1", protocol_key, level, idx, r) + + succ = [r for r in results if r.success] + fail = [r for r in results if not r.success] + avg = statistics.mean([r.duration_sec for r in succ]) if succ else 0 + notes = "Watch mgmt-server job-queue/thread-pool + single-host KVM saturation as a confound" if level >= 20 else "" + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_create", "test_id": "6.2.1", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall, 3), "avg_time_sec": round(avg, 3), + "success_count": len(succ), "failure_count": len(fail), "notes": notes, + }) + print(f" wall-clock={wall:.3f}s success={len(succ)} failure={len(fail)} avg/op={avg:.3f}s") + for r in fail: + print(f" !! {r.vm_name}: {r.error}") + report_failed_creates([(r.vm_name, r.vm_id) for r in fail]) + + print(f"\n=== [6.2.2] Parallel DELETE - protocol={protocol_key} C={len(succ)} ===") + t0 = time.perf_counter() + del_results = [] + if dry_run: + del_results = [fake_delete(r.vm_id, r.vm_name) for r in succ] + elif succ: + with ThreadPoolExecutor(max_workers=len(succ)) as ex: + futures = {ex.submit(destroy_vm, client, r.vm_id, r.vm_name): r for r in succ} + for fut in as_completed(futures): + del_results.append(fut.result()) + wall_del = time.perf_counter() - t0 + + for idx, r in enumerate(del_results, start=1): + raw_logger.log(run_id, "concurrent_delete", "6.2.2", protocol_key, level, idx, r) + + succ_d = [r for r in del_results if r.success] + fail_d = [r for r in del_results if not r.success] + avg_d = statistics.mean([r.duration_sec for r in succ_d]) if succ_d else 0 + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_delete", "test_id": "6.2.2", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall_del, 3), "avg_time_sec": round(avg_d, 3), + "success_count": len(succ_d), "failure_count": len(fail_d), "notes": "", + }) + print(f" wall-clock={wall_del:.3f}s success={len(succ_d)} failure={len(fail_d)} avg/op={avg_d:.3f}s") + for r in fail_d: + print(f" !! {r.vm_name}: {r.error}") + if delay: + time.sleep(delay) + + +def parse_levels(raw, default_levels): + if not raw: + return sorted(default_levels) + return sorted(int(x.strip()) for x in raw.split(",") if x.strip()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument( + "--levels", default=None, + help="Comma-separated concurrency levels to run, e.g. '2,5,10' " + "(default: config.vm_bench.concurrency_levels, typically up to 30)", + ) + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy all VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["vm_bench"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["vm_bench"]["vm_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) + output_dir = cfg["vm_bench"].get("output_dir", "results") + levels = parse_levels(args.levels, cfg["vm_bench"]["concurrency_levels"]) + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: concurrency") + print(f"Concurrency levels: {levels}") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay, levels) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_vm_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + + f" --output-dir {output_dir} --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned VMs remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py new file mode 100644 index 000000000000..2890c99cfae7 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Sequential VM-instance benchmark for the NetApp ONTAP CloudStack plugin. + +Drives deployVirtualMachine / destroyVirtualMachine one at a time over the +CloudStack HTTP/REST API against pre-provisioned bench_vm_nfs3 / bench_vm_iscsi +storage pools (see README.md "VM instance benchmark prerequisites") and +reproduces Section 5.2 - Sequential Scale Matrix from the "ONTAP Plugin - +CloudStack Operations, Scale & Parallel Test Matrix" page: + + 5.2.1 Sequential create (1 VM at a time, cumulative to N) + 5.2.2 Sequential delete (1 VM at a time, from N remaining) + +Each deployed VM gets a root disk (from the service offering) AND one data +disk (from the disk offering) landing on the SAME tagged storage pool. + +See benchmark_vm_instance_concurrency.py for the parallel/concurrency matrix, +or benchmark_vm_instance_combined.py to run both in one invocation (kept as +separate scripts/entry points on purpose, so a sequential run can be kicked +off and reviewed independently before committing to a concurrency run). + +Every individual API call is logged to results/raw_ops_vm_.csv, and a +per-checkpoint roll-up (matching the columns of the Confluence tables) is +appended to results/summary_vm_.csv. Use render_report.py afterwards +to turn the summary CSV into paste-ready markdown. Using the same --run-id +here and in benchmark_vm_instance_concurrency.py combines both into a single +summary_vm_.csv / report. + +Usage: + python3 benchmark_vm_instance_sequential.py --config config.yaml --protocol both + python3 benchmark_vm_instance_sequential.py --config config.yaml --dry-run + python3 benchmark_vm_instance_sequential.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import time + +from vm_instance_common import ( + append_summary_csv, + cleanup_by_filter, + deploy_vm, + destroy_vm, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + report_failed_creates, + resolve_protocols, +) + + +def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay): + infra = cfg["infrastructure"] + vm_cfg = cfg["vm_bench"] + proto_cfg = vm_cfg["protocols"][protocol_key] + checkpoints = sorted(vm_cfg["sequential_checkpoints"]) + max_n = max(checkpoints) + prefix = vm_cfg["vm_name_prefix"] + + created = [] + failed = [] + create_durations = [] + print(f"\n=== [5.2.1] Sequential CREATE - protocol={protocol_key} up to N={max_n} ===") + for i in range(1, max_n + 1): + # CloudStack VM names are used as hostnames (RFC1123): letters, digits, + # and "-" only (no "_", the opposite of ONTAP's volume-name rule). + name = f"{prefix}-seq-{protocol_key}-{run_id}-{i:03d}".replace("_", "-") + result = fake_create(name) if dry_run else deploy_vm(client, name, infra, vm_cfg, proto_cfg) + raw_logger.log(run_id, "sequential_create", "5.2.1", protocol_key, max_n, i, result) + status = "OK" if result.success else "FAIL" + print(f" [{i:>3}/{max_n}] create {name} -> {status} ({result.duration_sec:.3f}s)") + if result.success: + created.append((name, result.vm_id)) + create_durations.append(result.duration_sec) + else: + print(f" !! {result.error}") + failed.append((name, result.vm_id)) + if i in checkpoints: + total = sum(create_durations) + avg = total / len(create_durations) if create_durations else 0 + summary_rows.append({ + "run_id": run_id, "phase": "sequential_create", "test_id": "5.2.1", + "protocol": protocol_key, "checkpoint": i, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": len(created), "failure_count": i - len(created), "notes": "", + }) + print(f" >> checkpoint N={i}: total={total:.3f}s avg={avg:.3f}s/op " + f"success={len(created)} failure={i - len(created)}") + if delay: + time.sleep(delay) + + report_failed_creates(failed) + + total_created = len(created) + print(f"\n=== [5.2.2] Sequential DELETE - protocol={protocol_key} from N={total_created} remaining ===") + if total_created in checkpoints: + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "5.2.2", + "protocol": protocol_key, "checkpoint": total_created, + "total_time_sec": 0, "avg_time_sec": 0, "success_count": 0, "failure_count": 0, + "notes": "baseline - no deletes issued yet", + }) + delete_durations = [] + deleted_ok = 0 + for idx, (name, vm_id) in enumerate(created, start=1): + result = fake_delete(vm_id, name) if dry_run else destroy_vm(client, vm_id, name) + remaining = total_created - idx + raw_logger.log(run_id, "sequential_delete", "5.2.2", protocol_key, total_created, idx, result) + status = "OK" if result.success else "FAIL" + print(f" [{idx:>3}/{total_created}] delete {name} -> {status} " + f"({result.duration_sec:.3f}s) remaining={remaining}") + if result.success: + delete_durations.append(result.duration_sec) + deleted_ok += 1 + else: + print(f" !! {result.error}") + if remaining in checkpoints: + total = sum(delete_durations) + avg = total / len(delete_durations) if delete_durations else 0 + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "5.2.2", + "protocol": protocol_key, "checkpoint": remaining, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": deleted_ok, "failure_count": idx - deleted_ok, "notes": "", + }) + print(f" >> checkpoint remaining={remaining}: total={total:.3f}s avg={avg:.3f}s/op " + f"success={deleted_ok} failure={idx - deleted_ok}") + if delay: + time.sleep(delay) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy all VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["vm_bench"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["vm_bench"]["vm_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) + output_dir = cfg["vm_bench"].get("output_dir", "results") + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: sequential") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_vm_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + + f" --output-dir {output_dir} --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm") + print(f" (or run benchmark_vm_instance_concurrency.py --run-id {run_id} first to add 6.2.x to the same report)") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned VMs remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/cloudstack_client.py b/private-cicd/benchmark/ontap/cloudstack_client.py new file mode 100644 index 000000000000..7de56435fd8d --- /dev/null +++ b/private-cicd/benchmark/ontap/cloudstack_client.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Minimal CloudStack HTTP/REST API client used by the ONTAP plugin benchmark +scripts (private-cicd/benchmark/ontap/). + +Only session-key based auth (login -> sessionkey + JSESSIONID cookie) is +implemented, matching the approach documented in: +https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350/CloudStack+API + +Every call is timed end-to-end (including async job polling, since that is +part of the wall-clock cost an operator/automation actually pays) and the +elapsed time is handed back to the caller so benchmark scripts can log it. +""" + +import logging +import time + +import requests + +log = logging.getLogger("cloudstack_client") + + +class CloudStackAPIError(Exception): + """Raised when CloudStack returns an errorcode/errortext, or the HTTP + call itself fails validation.""" + + def __init__(self, command, detail): + self.command = command + self.detail = detail + super().__init__(f"{command} failed: {detail}") + + +class CloudStackClient: + """Thin wrapper around the CloudStack query API. + + A single instance (and its underlying requests.Session) is safe to share + across threads for concurrency benchmarks: we never mutate session state + after login, and urllib3's connection pool is designed for concurrent + use. + """ + + def __init__( + self, + api_url, + username, + password, + verify_ssl=True, + http_timeout_sec=30, + job_timeout_sec=300, + job_poll_interval_sec=1.5, + ): + self.api_url = api_url.rstrip("/") + self.http_timeout_sec = http_timeout_sec + self.job_timeout_sec = job_timeout_sec + self.job_poll_interval_sec = job_poll_interval_sec + + self.session = requests.Session() + self.session.verify = verify_ssl + if not verify_ssl: + requests.packages.urllib3.disable_warnings( + requests.packages.urllib3.exceptions.InsecureRequestWarning + ) + + self.sessionkey = None + self._login(username, password) + + def _login(self, username, password): + params = { + "command": "login", + "username": username, + "password": password, + "response": "json", + } + resp = self.session.post(self.api_url, data=params, timeout=self.http_timeout_sec) + resp.raise_for_status() + data = resp.json() + login_resp = data.get("loginresponse") + if not login_resp or "sessionkey" not in login_resp: + raise CloudStackAPIError("login", data) + self.sessionkey = login_resp["sessionkey"] + log.info("Logged in as %s, session established", username) + + def call(self, command, params=None, poll_async=True): + """Issue one CloudStack API command. + + Returns a tuple (payload, elapsed_sec). `elapsed_sec` covers the + initial HTTP round trip AND any async job polling (i.e. the full + wall-clock time a caller would observe waiting for the operation to + finish). + """ + req_params = dict(params or {}) + req_params["command"] = command + req_params["response"] = "json" + req_params["sessionkey"] = self.sessionkey + + start = time.perf_counter() + resp = self.session.post(self.api_url, data=req_params, timeout=self.http_timeout_sec) + resp.raise_for_status() + data = resp.json() + + # Most commands wrap their payload as {"response": {...}}, but a + # few (e.g. enableStorageMaintenance -> prepareprimarystorageformaintenanceresponse) + # use a legacy internal command name instead. Since CloudStack always wraps + # the payload in exactly one top-level key, fall back to "the sole value" + # rather than assuming the key name matches the command. + resp_key = f"{command.lower()}response" + if resp_key in data: + payload = data[resp_key] + elif len(data) == 1: + payload = next(iter(data.values())) + else: + payload = data + + if isinstance(payload, dict) and "errorcode" in payload and "jobid" not in payload: + raise CloudStackAPIError(command, payload.get("errortext", payload)) + + if poll_async and isinstance(payload, dict) and "jobid" in payload: + payload = self._poll_job(payload["jobid"]) + + elapsed = time.perf_counter() - start + return payload, elapsed + + def _poll_job(self, jobid): + deadline = time.time() + self.job_timeout_sec + while time.time() < deadline: + params = { + "command": "queryAsyncJobResult", + "jobid": jobid, + "response": "json", + "sessionkey": self.sessionkey, + } + resp = self.session.post(self.api_url, data=params, timeout=self.http_timeout_sec) + resp.raise_for_status() + job = resp.json().get("queryasyncjobresultresponse", {}) + status = job.get("jobstatus", 0) + if status == 1: + return job.get("jobresult", job) + if status == 2: + raise CloudStackAPIError( + "queryAsyncJobResult", job.get("jobresult", job) + ) + time.sleep(self.job_poll_interval_sec) + raise TimeoutError(f"Async job {jobid} did not complete within {self.job_timeout_sec}s") diff --git a/private-cicd/benchmark/ontap/config.example.yaml b/private-cicd/benchmark/ontap/config.example.yaml new file mode 100644 index 000000000000..1c2ae3ff23ca --- /dev/null +++ b/private-cicd/benchmark/ontap/config.example.yaml @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Copy this file to config.yaml and fill in your lab's real values. +# config.yaml is gitignored - never commit real credentials/IPs. + +cloudstack: + api_url: "http://10.193.56.12:8080/client/api" + username: "admin" + password: "password" + verify_ssl: false # CloudStack mgmt server usually plain http/self-signed + # createStoragePool is synchronous from the client's POV and internally polls + # ONTAP jobs for up to 60s (StorageStrategy.jobPollForSuccess: 30 retries * + # 2000ms) before returning. Under C=20-30 concurrency, createStoragePool can + # legitimately take close to that full 60s. A 30s client timeout was firing + # "Read timed out" on calls that actually succeeded moments later on the + # server (confirmed via real CloudStack pool records found + deleted during + # cleanup) - so keep meaningful headroom above the plugin's poll budget. + http_timeout_sec: 90 # per HTTP request + job_timeout_sec: 300 # max wait for an async job (enable/cancel maintenance) + job_poll_interval_sec: 1.5 + +# Zone/pod/cluster the pools will be created under. scope=cluster needs +# podid+clusterid; scope=zone only needs zoneid. +infrastructure: + zoneid: "91659187-b00b-4997-8ae3-8955c05a7d48" + podid: "004494cc-8a24-4519-9cd4-5e68b2eff03a" + clusterid: "23788acb-1a63-45a9-8c61-5ed8f5374bc6" + scope: "cluster" + +# One entry per protocol you want to benchmark. Keys must be "nfs3"/"iscsi" +# to match --protocol on the CLI. Remove a block if you don't have that +# backend available. +# NOTE: the ONTAP plugin (StorageStrategy.java) builds its REST API base URL +# from storageIP for EVERY call (aggregates/volumes/svm/jobs/network/san/nas) - +# it never falls back to `url`. storageIP MUST answer HTTPS/443 with the ONTAP +# REST service (i.e. it needs the same management-https access as the mgmt +# LIF). A pure data-only LIF (NFS/iSCSI only, no REST) will fail with +# "errorcode":530 / "Connection refused" on createStoragePool. If your data +# LIFs don't have REST enabled, point storageIP at the mgmt LIF instead (as +# below) rather than the actual data LIF. +ontap: + nfs3: + provider: "NetApp ONTAP" + url: "https://10.0.0.10" # ONTAP mgmt LIF URL + storageIP: "10.0.0.10" # must answer HTTPS/443 (see NOTE above) + svmName: "vs0" + protocol: "NFS3" + username: "admin" + password_b64: "cGFzc3dvcmQ=" # base64-encoded ONTAP password (this is "password" - replace it) + capacitybytes: 10737418240 # 10 GiB per pool + tags: "bench_nfs3" + iscsi: + provider: "NetApp ONTAP" + url: "https://10.0.0.11" + storageIP: "10.0.0.11" # must answer HTTPS/443 (see NOTE above) + svmName: "vs0" + protocol: "ISCSI" + username: "admin" + password_b64: "cGFzc3dvcmQ=" + capacitybytes: 10737418240 + tags: "bench_iscsi" + +benchmark: + pool_name_prefix: "bench_ontap" # ONTAP volume names disallow "-"; use "_" only + sequential_checkpoints: [1, 5, 10, 20, 30] + concurrency_levels: [2, 5, 10, 20, 30] + # Pause between deleteStoragePool calls to give the mgmt server GC/ONTAP a + # beat; set to 0 to hammer as fast as possible. + inter_op_delay_sec: 0 + output_dir: "results" + +# VM instance benchmark (benchmark_vm_instance_sequential.py / +# benchmark_vm_instance_concurrency.py / benchmark_vm_instance_combined.py). Requires: +# - a template (templateid) +# - a network (networkid) +# - per protocol: a service offering (root disk) + disk offering (data disk) +# whose storage `tags` both match a persistent bench_vm_ pool, so +# root and data disk land on the same pool. These are one-time setup +# (createServiceOffering / createDiskOffering / createStoragePool) - not +# created/destroyed by the benchmark itself, unlike the storage-pool +# benchmark's transient pools. +# NOTE: watch aggregate capacity - StorageStrategy.java's aggregate-selection +# pre-check requires available space >= requested size even though volumes +# are created with guarantee=none (thin). On small/vsim aggregates this can +# reject valid requests; keep pool/disk sizes modest until that's addressed. +vm_bench: + templateid: "1af26eaa-8985-4b1c-ae8e-d39b7984b8f3" + networkid: "5e6f4c59-dc01-4245-8c0f-2c1c9639b37d" + # CloudStack VM names are used as hostnames (RFC1123): letters/digits/"-" only, + # no "_" (the opposite of ONTAP's volume-name rule used by pool_name_prefix above). + vm_name_prefix: "bench-vm" + sequential_checkpoints: [1, 5, 10, 20] + concurrency_levels: [2, 5, 10] + inter_op_delay_sec: 0 + output_dir: "results" + protocols: + nfs3: + # rootdisksize (GB, set on the offering) must be >= the template's + # actual qemu-img virtual size, not its download/file size - e.g. the + # stock Ubuntu 22.04 cloud image download is ~660MB on disk but has a + # 2.2 GiB virtual size. Undersizing this "works" on NFS (the backing + # file just grows past its nominal size) but hard-fails on iSCSI with + # "qemu-img: Cannot grow device files" since a LUN is a fixed-size + # block device that cannot be grown during the template copy. + serviceofferingid: "ac54f38c-5fd3-401b-b784-2c2163b08274" + diskofferingid: "e27813c0-6e8b-4375-8603-14c8ade75246" + iscsi: + serviceofferingid: "481feb00-16d8-4127-9937-cba3bec8a88b" + diskofferingid: "bf057b78-c5db-4ce4-820b-d7bd78f8ae69" diff --git a/private-cicd/benchmark/ontap/render_report.py b/private-cicd/benchmark/ontap/render_report.py new file mode 100644 index 000000000000..85935598df7a --- /dev/null +++ b/private-cicd/benchmark/ontap/render_report.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Turn benchmark_storage_pool_sequential.py / benchmark_storage_pool_concurrency.py's +raw_ops_.csv / summary_.csv into markdown tables that match the +layout of the storage-pool sections +(5.1.1, 5.1.2, 6.1.1, 6.1.2) and the Results Log (section 9) of the +"ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix" +Confluence page, ready to paste back in. + +Usage: + python3 render_report.py --run-id RUN-20260722-120000 \ + --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.15.1 +""" + +import argparse +import csv +import math +import os +from collections import defaultdict + +PROTOCOL_ORDER = ["nfs3", "iscsi"] +PROTOCOL_LABEL = {"nfs3": "NFS3", "iscsi": "iSCSI"} + +# test_id -> (section title, sequential N-column label, category for section 9) +SECTION_DEFS = { + "5.1.1": ("5.1.1 Sequential create (1 pool at a time, cumulative to N)", "N (pools)", "Storage Pool"), + "5.1.2": ("5.1.2 Sequential delete (1 pool at a time, from N remaining)", "N (pools remaining)", "Storage Pool"), + "6.1.1": ("6.1.1 Parallel pool creation", None, "Storage Pool"), + "6.1.2": ("6.1.2 Parallel pool deletion", None, "Storage Pool"), + "5.2.1": ("5.2.1 Sequential create (1 VM at a time, cumulative to N)", "N (VMs)", "VM Instance"), + "5.2.2": ("5.2.2 Sequential delete (1 VM at a time, from N remaining)", "N (VMs remaining)", "VM Instance"), + "6.2.1": ("6.2.1 Parallel VM creation", None, "VM Instance"), + "6.2.2": ("6.2.2 Parallel VM deletion", None, "VM Instance"), +} + + +def percentile(sorted_values, pct): + if not sorted_values: + return None + if len(sorted_values) == 1: + return sorted_values[0] + k = (len(sorted_values) - 1) * (pct / 100.0) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_values[int(k)] + d0 = sorted_values[f] * (c - k) + d1 = sorted_values[c] * (k - f) + return d0 + d1 + + +def load_csv(path): + if not os.path.exists(path): + return [] + with open(path, newline="") as f: + return list(csv.DictReader(f)) + + +def stats_for(durations): + if not durations: + return None, None, None, None + durations = sorted(durations) + return ( + round(durations[0], 3), + round(sum(durations) / len(durations), 3), + round(percentile(durations, 95), 3), + round(durations[-1], 3), + ) + + +def raw_durations(raw_rows, phase, protocol, predicate): + return [ + float(r["duration_sec"]) for r in raw_rows + if r["phase"] == phase and r["protocol"] == protocol + and r["success"] == "True" and predicate(r) + ] + + +def render_sequential_table(summary_rows, raw_rows, phase, test_id, n_label): + checkpoints = sorted({int(r["checkpoint"]) for r in summary_rows if r["test_id"] == test_id}) + by_key = {(r["protocol"], int(r["checkpoint"])): r for r in summary_rows if r["test_id"] == test_id} + + lines = [f"| {n_label} | NFS3 Total (s) | NFS3 Avg/op (s) | iSCSI Total (s) | iSCSI Avg/op (s) | Notes |", + "|---|---|---|---|---|---|"] + for cp in checkpoints: + cells = [str(cp)] + notes = [] + for proto in PROTOCOL_ORDER: + row = by_key.get((proto, cp)) + if row: + cells.append(row["total_time_sec"]) + cells.append(row["avg_time_sec"]) + if row.get("notes") and row["notes"] not in notes: + notes.append(row["notes"]) + else: + cells.append("") + cells.append("") + cells.append("; ".join(notes)) + lines.append("| " + " | ".join(cells) + " |") + return "\n".join(lines) + + +def render_concurrency_table(summary_rows, test_id): + rows = [r for r in summary_rows if r["test_id"] == test_id] + rows.sort(key=lambda r: (int(r["checkpoint"]), PROTOCOL_ORDER.index(r["protocol"]) + if r["protocol"] in PROTOCOL_ORDER else 99)) + lines = ["| Concurrency (C) | Protocol | Total Wall-clock (s) | Success | Failure | Avg Time/pool (s) | Notes |", + "|---|---|---|---|---|---|---|"] + for r in rows: + lines.append( + f"| {r['checkpoint']} | {PROTOCOL_LABEL.get(r['protocol'], r['protocol'])} | " + f"{r['total_time_sec']} | {r['success_count']} | {r['failure_count']} | " + f"{r['avg_time_sec']} | {r.get('notes', '')} |" + ) + return "\n".join(lines) + + +def build_results_log_rows(summary_rows, raw_rows, run_id, cs_build, ontap_version, run_date): + header = ("| Run Date | Run ID | CloudStack Build | ONTAP Version | Category | Test ID / Section | " + "Scale (N) / Concurrency (C) | Protocol | Min (s) | Avg (s) | P95 (s) | Max (s) | " + "Success Rate | Notes |") + sep = "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|" + lines = [header, sep] + + delete_total_created = {} + for r in raw_rows: + if r["phase"] == "sequential_delete": + delete_total_created[r["protocol"]] = int(r["scale_or_concurrency"]) + + for row in summary_rows: + phase, test_id, protocol, checkpoint = row["phase"], row["test_id"], row["protocol"], int(row["checkpoint"]) + if phase == "sequential_create": + durations = raw_durations(raw_rows, phase, protocol, lambda r: int(r["index"]) <= checkpoint) + scale_label = f"N={checkpoint}" + elif phase == "sequential_delete": + total_created = delete_total_created.get(protocol, checkpoint) + deleted_so_far = total_created - checkpoint + durations = raw_durations(raw_rows, phase, protocol, lambda r: int(r["index"]) <= deleted_so_far) + scale_label = f"N={checkpoint}" + else: # concurrent_create / concurrent_delete + durations = raw_durations(raw_rows, phase, protocol, lambda r: int(r["scale_or_concurrency"]) == checkpoint) + scale_label = f"C={checkpoint}" + + mn, avg, p95, mx = stats_for(durations) + total_ops = int(row["success_count"]) + int(row["failure_count"]) + success_rate = f"{(int(row['success_count']) / total_ops * 100):.0f}%" if total_ops else "-" + category = SECTION_DEFS.get(test_id, (None, None, "Storage Pool"))[2] + lines.append( + f"| {run_date} | {run_id} | {cs_build} | {ontap_version} | {category} | {test_id} | " + f"{scale_label} | {PROTOCOL_LABEL.get(protocol, protocol)} | " + f"{mn if mn is not None else '-'} | {avg if avg is not None else '-'} | " + f"{p95 if p95 is not None else '-'} | {mx if mx is not None else '-'} | " + f"{success_rate} | {row.get('notes', '')} |" + ) + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--run-id", required=True) + parser.add_argument("--output-dir", default="results") + parser.add_argument("--cloudstack-build", default="unknown") + parser.add_argument("--ontap-version", default="unknown") + parser.add_argument("--run-date", default=None, help="Defaults to today's date (YYYY-MM-DD)") + parser.add_argument("--raw-prefix", default="raw_ops", + help="Filename prefix for the raw ops CSV, e.g. raw_ops_vm for the benchmark_vm_instance_* scripts") + parser.add_argument("--summary-prefix", default="summary", + help="Filename prefix for the summary CSV, e.g. summary_vm for the benchmark_vm_instance_* scripts") + parser.add_argument("--report-suffix", default="", + help="Optional suffix to disambiguate the output report filename, e.g. _vm") + args = parser.parse_args() + + import datetime + run_date = args.run_date or datetime.date.today().isoformat() + + summary_path = os.path.join(args.output_dir, f"{args.summary_prefix}_{args.run_id}.csv") + raw_path = os.path.join(args.output_dir, f"{args.raw_prefix}_{args.run_id}.csv") + summary_rows = load_csv(summary_path) + raw_rows = load_csv(raw_path) + if not summary_rows: + raise SystemExit(f"No summary rows found at {summary_path}") + + # Render whichever sections actually have data in this run, in a stable order. + present_test_ids = sorted({r["test_id"] for r in summary_rows if r["test_id"] in SECTION_DEFS}) + sections = [] + for test_id in present_test_ids: + title, n_label, _category = SECTION_DEFS[test_id] + phase = { + "5.1.1": "sequential_create", "5.1.2": "sequential_delete", + "5.2.1": "sequential_create", "5.2.2": "sequential_delete", + }.get(test_id) + if n_label is not None: + sections.append(f"### {title}\n\n" + + render_sequential_table(summary_rows, raw_rows, phase, test_id, n_label)) + else: + sections.append(f"### {title}\n\n" + render_concurrency_table(summary_rows, test_id)) + sections.append("### 9. Results Log rows for this run\n\n" + + build_results_log_rows(summary_rows, raw_rows, args.run_id, args.cloudstack_build, + args.ontap_version, run_date)) + + report = f"# Benchmark report - {args.run_id}\n\n" + "\n\n".join(sections) + "\n" + print(report) + + report_path = os.path.join(args.output_dir, f"report{args.report_suffix}_{args.run_id}.md") + with open(report_path, "w") as f: + f.write(report) + print(f"\nSaved to {report_path}") + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/requirements.txt b/private-cicd/benchmark/ontap/requirements.txt new file mode 100644 index 000000000000..34304aee700c --- /dev/null +++ b/private-cicd/benchmark/ontap/requirements.txt @@ -0,0 +1,2 @@ +requests>=2.31.0 +PyYAML>=6.0 diff --git a/private-cicd/benchmark/ontap/results/.gitkeep b/private-cicd/benchmark/ontap/results/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/private-cicd/benchmark/ontap/storage_pool_common.py b/private-cicd/benchmark/ontap/storage_pool_common.py new file mode 100644 index 000000000000..e01c50c0e69d --- /dev/null +++ b/private-cicd/benchmark/ontap/storage_pool_common.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Shared helpers for the storage-pool benchmark scripts +(benchmark_storage_pool_sequential.py / benchmark_storage_pool_concurrency.py). + +Kept in one module (rather than duplicated per script) so the createStoragePool/ +deleteStoragePool call shape, CSV formats, and cleanup logic can't drift between +the two - only the sequential-vs-concurrent driving logic differs per script. +""" + +import csv +import dataclasses +import datetime +import os +import random +import sys +import time + +try: + import yaml +except ImportError: + sys.exit("PyYAML is required: pip install -r requirements.txt") + +from cloudstack_client import CloudStackAPIError, CloudStackClient + +RAW_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "scale_or_concurrency", + "index", "pool_name", "pool_id", "success", "duration_sec", + "start_ts", "end_ts", "error", +] + +SUMMARY_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "checkpoint", + "total_time_sec", "avg_time_sec", "success_count", "failure_count", "notes", +] + + +@dataclasses.dataclass +class OpResult: + success: bool + duration_sec: float + start_ts: str + end_ts: str + pool_name: str + pool_id: str = None + error: str = None + + +def now_iso(): + return datetime.datetime.utcnow().isoformat(timespec="milliseconds") + "Z" + + +def new_run_id(): + # ONTAP volume names only allow alphanumeric + underscore (no hyphens), and + # pool names flow straight through to the ONTAP volume name, so avoid "-" here. + return datetime.datetime.utcnow().strftime("RUN_%Y%m%d_%H%M%S") + + +def build_create_pool_params(name, infra_cfg, ontap_cfg): + params = { + "name": name, + "zoneid": infra_cfg["zoneid"], + "scope": infra_cfg.get("scope", "cluster"), + "provider": ontap_cfg["provider"], + "url": ontap_cfg["url"], + "managed": "true", + "details[0].username": ontap_cfg["username"], + "details[0].password": ontap_cfg["password_b64"], + "details[0].svmName": ontap_cfg["svmName"], + "details[0].protocol": ontap_cfg["protocol"], + "details[0].storageIP": ontap_cfg["storageIP"], + } + if infra_cfg.get("scope", "cluster") == "cluster": + params["podid"] = infra_cfg["podid"] + params["clusterid"] = infra_cfg["clusterid"] + if ontap_cfg.get("capacitybytes"): + params["capacitybytes"] = str(ontap_cfg["capacitybytes"]) + if ontap_cfg.get("tags"): + params["tags"] = ontap_cfg["tags"] + return params + + +def create_pool(client, name, infra_cfg, ontap_cfg): + start_ts = now_iso() + t0 = time.perf_counter() + try: + payload, elapsed = client.call( + "createStoragePool", build_create_pool_params(name, infra_cfg, ontap_cfg) + ) + pool = payload.get("storagepool", payload) if isinstance(payload, dict) else {} + pool_id = pool.get("id") if isinstance(pool, dict) else None + return OpResult(True, elapsed, start_ts, now_iso(), name, pool_id=pool_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + return OpResult(False, elapsed, start_ts, now_iso(), name, error=str(exc)) + + +def delete_pool(client, pool_id, name, forced=True): + """deleteStoragePool requires the pool to already be in Maintenance state + (CloudStack error 431 otherwise), so this enables maintenance first and + waits for that async job before issuing the actual delete. Both steps are + included in the reported duration since that is the real wall-clock cost + of retiring a pool.""" + start_ts = now_iso() + t0 = time.perf_counter() + try: + client.call("enableStorageMaintenance", {"id": pool_id}) + payload, _ = client.call( + "deleteStoragePool", {"id": pool_id, "forced": "true" if forced else "false"} + ) + elapsed = time.perf_counter() - t0 + success = payload.get("success", True) if isinstance(payload, dict) else True + return OpResult(bool(success), elapsed, start_ts, now_iso(), name, pool_id=pool_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + return OpResult(False, elapsed, start_ts, now_iso(), name, pool_id=pool_id, error=str(exc)) + + +def fake_create(name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(0.5, 3.0), now_iso(), now_iso(), name, pool_id=f"dryrun-{name}") + + +def fake_delete(pool_id, name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(0.3, 2.0), now_iso(), now_iso(), name, pool_id=pool_id) + + +class RawLogger: + def __init__(self, path): + self.file = open(path, "a", newline="") + self.writer = csv.DictWriter(self.file, fieldnames=RAW_FIELDNAMES) + if self.file.tell() == 0: + self.writer.writeheader() + + def log(self, run_id, phase, test_id, protocol, scale_or_concurrency, index, result: OpResult): + self.writer.writerow({ + "run_id": run_id, "phase": phase, "test_id": test_id, "protocol": protocol, + "scale_or_concurrency": scale_or_concurrency, "index": index, + "pool_name": result.pool_name, "pool_id": result.pool_id or "", + "success": result.success, "duration_sec": round(result.duration_sec, 4), + "start_ts": result.start_ts, "end_ts": result.end_ts, "error": result.error or "", + }) + self.file.flush() + + def close(self): + self.file.close() + + +def append_summary_csv(path, rows): + """Appends checkpoint rows to the run's summary CSV, writing the header only + if the file is new/empty. Append (not overwrite) so that running the + sequential and concurrency scripts separately under the *same* --run-id + accumulates into one combined summary_.csv that render_report.py + can render as a single report, instead of the second script clobbering the + first script's rows.""" + is_new = not os.path.exists(path) or os.path.getsize(path) == 0 + with open(path, "a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=SUMMARY_FIELDNAMES) + if is_new: + writer.writeheader() + writer.writerows(rows) + + +def cleanup_by_filter(client, name_filter): + payload, _ = client.call("listStoragePools", {}, poll_async=False) + pools = payload.get("storagepool", []) if isinstance(payload, dict) else [] + matches = [p for p in pools if name_filter in p.get("name", "")] + print(f"Found {len(matches)} pool(s) whose name contains '{name_filter}'") + for p in matches: + try: + if p.get("state") != "Maintenance": + client.call("enableStorageMaintenance", {"id": p["id"]}) + client.call("deleteStoragePool", {"id": p["id"], "forced": "true"}) + print(f" deleted {p['name']} ({p['id']})") + except Exception as exc: # noqa: BLE001 + print(f" FAILED to delete {p['name']}: {exc}") + + +def load_config(path): + with open(path) as f: + return yaml.safe_load(f) + + +def resolve_protocols(cfg, requested): + available = list(cfg.get("ontap", {}).keys()) + if requested == "both": + return available + if requested not in available: + sys.exit(f"Protocol '{requested}' not found in config.ontap (available: {available})") + return [requested] + + +def make_cloudstack_client(cfg): + cs_cfg = cfg["cloudstack"] + return CloudStackClient( + cs_cfg["api_url"], cs_cfg["username"], cs_cfg["password"], + verify_ssl=cs_cfg.get("verify_ssl", True), + http_timeout_sec=cs_cfg.get("http_timeout_sec", 30), + job_timeout_sec=cs_cfg.get("job_timeout_sec", 300), + job_poll_interval_sec=cs_cfg.get("job_poll_interval_sec", 1.5), + ) diff --git a/private-cicd/benchmark/ontap/vm_instance_common.py b/private-cicd/benchmark/ontap/vm_instance_common.py new file mode 100644 index 000000000000..b671f3c576dc --- /dev/null +++ b/private-cicd/benchmark/ontap/vm_instance_common.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Shared helpers for the VM-instance benchmark scripts +(benchmark_vm_instance_sequential.py / benchmark_vm_instance_concurrency.py / +benchmark_vm_instance_combined.py). + +Kept in one module (rather than duplicated per script) so the +deployVirtualMachine/destroyVirtualMachine call shape, CSV formats, and +cleanup logic can't drift between the three - only the sequential-vs- +concurrent-vs-both driving logic differs per script. +""" + +import csv +import dataclasses +import datetime +import os +import random +import re +import sys +import time + +try: + import yaml +except ImportError: + sys.exit("PyYAML is required: pip install -r requirements.txt") + +from cloudstack_client import CloudStackAPIError, CloudStackClient + +RAW_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "scale_or_concurrency", + "index", "vm_name", "vm_id", "success", "duration_sec", + "start_ts", "end_ts", "error", +] + +SUMMARY_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "checkpoint", + "total_time_sec", "avg_time_sec", "success_count", "failure_count", "notes", +] + + +@dataclasses.dataclass +class OpResult: + success: bool + duration_sec: float + start_ts: str + end_ts: str + vm_name: str + vm_id: str = None + error: str = None + + +def now_iso(): + return datetime.datetime.utcnow().isoformat(timespec="milliseconds") + "Z" + + +def new_run_id(): + return datetime.datetime.utcnow().strftime("RUN_%Y%m%d_%H%M%S") + + +def build_deploy_vm_params(name, infra_cfg, vm_cfg, proto_cfg): + return { + "name": name, + "displayname": name, + "zoneid": infra_cfg["zoneid"], + "templateid": vm_cfg["templateid"], + "networkids": vm_cfg["networkid"], + "serviceofferingid": proto_cfg["serviceofferingid"], + "diskofferingid": proto_cfg["diskofferingid"], + "startvm": "true", + } + + +_UUID_RE = re.compile(r'"uuid"\s*:\s*"([0-9a-fA-F-]{36})"') + + +def _extract_vm_id_from_error(error_text): + """When deployVirtualMachine's async job fails deep in orchestration (e.g. the + ENOSPC-driven "Unable to orchestrate the start of VM instance" failures - see + Confluence Issue #10), CloudStack has ALREADY created the VM (and its ROOT/DATA + volume records) before the failure - it just never finished starting it. The + job's errortext embeds that VM's uuid (e.g. '...{"instanceName":"i-2-107-VM", + "uuid":"b8f4..."}.'), so scrape it out here so the leftover VM/volume can at + least be identified and reported for manual inspection (see + report_failed_creates()) instead of being silently invisible.""" + if not error_text: + return None + m = _UUID_RE.search(error_text) + return m.group(1) if m else None + + +def deploy_vm(client, name, infra_cfg, vm_cfg, proto_cfg): + start_ts = now_iso() + t0 = time.perf_counter() + try: + payload, elapsed = client.call( + "deployVirtualMachine", build_deploy_vm_params(name, infra_cfg, vm_cfg, proto_cfg) + ) + vm = payload.get("virtualmachine", payload) if isinstance(payload, dict) else {} + vm_id = vm.get("id") if isinstance(vm, dict) else None + return OpResult(True, elapsed, start_ts, now_iso(), name, vm_id=vm_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + error = str(exc) + return OpResult(False, elapsed, start_ts, now_iso(), name, + vm_id=_extract_vm_id_from_error(error), error=error) + + +def _force_purge_destroy_state_volumes(client, volume_ids, max_wait_sec=10): + """destroyVirtualMachine(expunge=true) only guarantees the VM/volume DB records + move to "Destroy" state - actual physical removal from the backing storage + (ONTAP LUN/file deletion) is deferred to CloudStack's periodic storage-cleanup + background thread, which by default only *considers* volumes for real deletion + after `storage.cleanup.delay` (86400s / 24h) has elapsed, even though the + cleanup thread itself polls every `storage.cleanup.interval` (40s by default). + Confirmed directly against our management server via listConfigurations. + + That means every "successfully destroyed" VM's ROOT/DATA disks still + physically occupy space on the ONTAP pool for up to 24h unless something + calls deleteVolume() on them explicitly - which force-deletes a Destroy-state + volume immediately, bypassing the delay. Without this, repeated benchmark + runs progressively fill up the dedicated bench_vm_ pool with + "deleted" disks, compounding the ENOSPC failures in Confluence Issue #10. + """ + deadline = time.time() + max_wait_sec + remaining = set(volume_ids) + while remaining and time.time() < deadline: + still_pending = set() + for vol_id in remaining: + try: + vol_payload, _ = client.call("listVolumes", {"id": vol_id}, poll_async=False) + vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] + if not vols: + continue # already fully gone + state = vols[0].get("state") + if state == "Destroy": + client.call("deleteVolume", {"id": vol_id}) + elif state not in ("Expunged",): + still_pending.add(vol_id) # not yet transitioned to Destroy - retry shortly + except (CloudStackAPIError, TimeoutError, Exception): # noqa: BLE001 + still_pending.add(vol_id) + remaining = still_pending + if remaining: + time.sleep(1) + + +def destroy_vm(client, vm_id, name, expunge=True): + """destroyVirtualMachine with expunge=true only reclaims the ROOT disk; + data disks are merely DETACHED (left behind as free-floating DATADISK + volumes) unless their ids are explicitly passed via `volumeids`. So this + looks up any attached data disks first and includes them, otherwise every + VM with a data disk leaks a LUN/file on the backing storage pool. + + It also force-purges any ROOT/DATA volumes still left in "Destroy" state + afterwards (see _force_purge_destroy_state_volumes) so this VM's disks are + truly gone from ONTAP before the next benchmark op runs, instead of + lingering for CloudStack's 24h background cleanup delay. + """ + start_ts = now_iso() + t0 = time.perf_counter() + try: + vol_payload, _ = client.call( + "listVolumes", {"virtualmachineid": vm_id}, poll_async=False + ) + all_vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] + all_volume_ids = [v["id"] for v in all_vols] + data_disk_ids = [v["id"] for v in all_vols if v.get("type") == "DATADISK"] + + params = {"id": vm_id, "expunge": "true" if expunge else "false"} + if data_disk_ids: + params["volumeids"] = ",".join(data_disk_ids) + payload, _ = client.call("destroyVirtualMachine", params) + + if expunge and all_volume_ids: + _force_purge_destroy_state_volumes(client, all_volume_ids) + + elapsed = time.perf_counter() - t0 + success = payload.get("success", True) if isinstance(payload, dict) else True + return OpResult(bool(success), elapsed, start_ts, now_iso(), name, vm_id=vm_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + return OpResult(False, elapsed, start_ts, now_iso(), name, vm_id=vm_id, error=str(exc)) + + +def fake_create(name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(3.0, 15.0), now_iso(), now_iso(), name, vm_id=f"dryrun-{name}") + + +def fake_delete(vm_id, name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(2.0, 8.0), now_iso(), now_iso(), name, vm_id=vm_id) + + +class RawLogger: + def __init__(self, path): + self.file = open(path, "a", newline="") + self.writer = csv.DictWriter(self.file, fieldnames=RAW_FIELDNAMES) + if self.file.tell() == 0: + self.writer.writeheader() + + def log(self, run_id, phase, test_id, protocol, scale_or_concurrency, index, result: OpResult): + self.writer.writerow({ + "run_id": run_id, "phase": phase, "test_id": test_id, "protocol": protocol, + "scale_or_concurrency": scale_or_concurrency, "index": index, + "vm_name": result.vm_name, "vm_id": result.vm_id or "", + "success": result.success, "duration_sec": round(result.duration_sec, 4), + "start_ts": result.start_ts, "end_ts": result.end_ts, "error": result.error or "", + }) + self.file.flush() + + def close(self): + self.file.close() + + +def append_summary_csv(path, rows): + """Appends checkpoint rows to the run's summary CSV, writing the header only + if the file is new/empty. Append (not overwrite) so that running the + sequential and concurrency scripts separately under the *same* --run-id + accumulates into one combined summary_vm_.csv that render_report.py + can render as a single report, instead of the second script clobbering the + first script's rows.""" + is_new = not os.path.exists(path) or os.path.getsize(path) == 0 + with open(path, "a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=SUMMARY_FIELDNAMES) + if is_new: + writer.writeheader() + writer.writerows(rows) + + +def report_failed_creates(failed): + """Failed deployVirtualMachine calls (e.g. the ENOSPC "Unable to orchestrate + the start of VM instance" failures in Confluence Issue #10) still leave a real + VM + ROOT/DATA volume record behind in CloudStack - they just never finished + starting. These are intentionally NOT auto-destroyed here (unlike the + `created` list, which the benchmark itself deletes as part of 5.2.2/6.2.2) so + the leftover VM/volume artifacts stay inspectable afterwards - e.g. via + `listVolumes --state Destroy` on the bench_vm_ pool - to confirm + exactly what got left behind and why. Clean them up manually (or via + --cleanup-only) once you're done inspecting them.""" + failed_with_id = [(name, vm_id) for name, vm_id in failed if vm_id] + if not failed_with_id: + return + print(f"\n=== {len(failed_with_id)} failed-create VM(s) left in place for inspection (not cleaned up) ===") + for name, vm_id in failed_with_id: + print(f" {name} ({vm_id})") + + +def cleanup_by_filter(client, name_filter): + payload, _ = client.call("listVirtualMachines", {}, poll_async=False) + vms = payload.get("virtualmachine", []) if isinstance(payload, dict) else [] + matches = [v for v in vms if name_filter in v.get("name", "")] + print(f"Found {len(matches)} VM(s) whose name contains '{name_filter}'") + for v in matches: + try: + result = destroy_vm(client, v["id"], v["name"]) + if result.success: + print(f" destroyed {v['name']} ({v['id']}) [incl. any data disks]") + else: + print(f" FAILED to destroy {v['name']}: {result.error}") + except Exception as exc: # noqa: BLE001 + print(f" FAILED to destroy {v['name']}: {exc}") + + # Data disks can also end up orphaned (unattached) from prior runs whose + # VM was already destroyed without this volumeids fix. Volume names (e.g. + # "DATA-25") don't carry the run id/prefix, so instead of matching + # name_filter here, sweep any unattached data disk left on our dedicated + # bench_vm_* pools - nothing else legitimately lives there. + vol_payload, _ = client.call("listVolumes", {"type": "DATADISK"}, poll_async=False) + vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] + orphan_vols = [v for v in vols if not v.get("virtualmachineid") and v.get("storage", "").startswith("bench_vm")] + if orphan_vols: + print(f"Found {len(orphan_vols)} orphaned unattached DATADISK volume(s) on matching pools") + for v in orphan_vols: + try: + client.call("deleteVolume", {"id": v["id"]}) + print(f" deleted orphan volume {v['name']} ({v['id']})") + except Exception as exc: # noqa: BLE001 + print(f" FAILED to delete orphan volume {v['name']}: {exc}") + + +def load_config(path): + with open(path) as f: + return yaml.safe_load(f) + + +def resolve_protocols(cfg, requested): + available = list(cfg.get("vm_bench", {}).get("protocols", {}).keys()) + if requested == "both": + return available + if requested not in available: + sys.exit(f"Protocol '{requested}' not found in config.vm_bench.protocols (available: {available})") + return [requested] + + +def make_cloudstack_client(cfg): + cs_cfg = cfg["cloudstack"] + return CloudStackClient( + cs_cfg["api_url"], cs_cfg["username"], cs_cfg["password"], + verify_ssl=cs_cfg.get("verify_ssl", True), + http_timeout_sec=cs_cfg.get("http_timeout_sec", 30), + job_timeout_sec=cs_cfg.get("job_timeout_sec", 300), + job_poll_interval_sec=cs_cfg.get("job_poll_interval_sec", 1.5), + )